diff --git a/.gitignore b/.gitignore index bf6a22fc6..957306f03 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *.patch _build *.pt +*.vscode* +.DS_Store +generated_rtd* diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 86% rename from .readthedocs.yml rename to .readthedocs.yaml index e2c25ef5e..16f057784 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yaml @@ -4,10 +4,13 @@ version: 2 +build: + image: latest + sphinx: configuration: doc/rtd/conf.py python: - version: 3.7 + version: 3.8 install: - requirements: doc/rtd/requirements.txt diff --git a/.travis.yml b/.travis.yml index 909717b1d..72b7b9552 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ env: - BUILD_TYPE=Debug - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - - TECA_DATA_REVISION=101 + - TECA_DATA_REVISION=117 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=TRUE - DOCKER_IMAGE=ubuntu IMAGE_VERSION=20.04 IMAGE_NAME=ubuntu_20_04 REQUIRE_NETCDF_MPI=FALSE diff --git a/CMake/teca_app.cmake b/CMake/teca_app.cmake index fbaa0e923..82bb06234 100644 --- a/CMake/teca_app.cmake +++ b/CMake/teca_app.cmake @@ -31,6 +31,7 @@ function (teca_add_app app_name) teca_system teca_core teca_data teca_io teca_alg ${APP_LIBS}) endif() + set_target_properties(${app_name} PROPERTIES APP_TYPE C++) install(TARGETS ${app_name} RUNTIME DESTINATION ${BIN_PREFIX}) else() message(STATUS "command line application ${app_name} -- disabled") diff --git a/CMake/teca_python.cmake b/CMake/teca_python.cmake index 6ee3f7397..b94dcb80c 100644 --- a/CMake/teca_python.cmake +++ b/CMake/teca_python.cmake @@ -60,6 +60,8 @@ function (teca_add_python_app app_name) if (NOT APP_SOURCES) set(APP_SOURCES "${app_name}.in") endif() + add_custom_target(${app_name}) + set_target_properties(${app_name} PROPERTIES APP_TYPE Python) teca_py_install_apps(${APP_SOURCES}) else() message(STATUS "command line application ${app_name} -- disabled") diff --git a/CMake/teca_test.cmake b/CMake/teca_test.cmake index 375c3524f..346eaf319 100644 --- a/CMake/teca_test.cmake +++ b/CMake/teca_test.cmake @@ -51,3 +51,9 @@ function (teca_add_test T_NAME) endif() endif() endfunction() + +function (teca_add_app_test T_NAME T_TARGET) + if (TARGET ${T_TARGET}) + teca_add_test(${T_NAME} ${ARGV}) + endif() +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index c1ecb1fbb..4eb5ff0ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -387,26 +387,50 @@ if (BUILD_TESTING) # figure out how many cores we can use for parallel tests set(TECA_TEST_CORES 0 CACHE STRING "Max number of cores for use in parallel tests") + + # by default assume 2 hyperthreads per core, if this is not + # the case override here + set(HYPERTHREADS_PER_CORE 2 CACHE STRING + "The number of hyperthreads per core.") + + # use CMake to get the number of logical cores. includes hyperthreads + # in the count. if (TECA_TEST_CORES LESS 1) ProcessorCount(LOGICAL_CORES) if (LOGICAL_CORES EQUAL 0) - set(LOGICAL_CORES 4) + message(FATAL_ERROR "Failed to detect the number of cores. " + "Set TECA_TEST_CORES") endif() else() - math(EXPR LOGICAL_CORES "${TECA_TEST_CORES}*2") + math(EXPR LOGICAL_CORES "${TECA_TEST_CORES}*${HYPERTHREADS_PER_CORE}") endif() - math(EXPR PHYSICAL_CORES "${LOGICAL_CORES}/2") - if (PHYSICAL_CORES LESS 3) - set(TEST_CORES 2) - set(HALF_TEST_CORES 2) - set(TWICE_TEST_CORES 4) - else() - set(TEST_CORES ${PHYSICAL_CORES}) - math(EXPR HALF_TEST_CORES "${TEST_CORES}/2") - set(TWICE_TEST_CORES ${LOGICAL_CORES}) + + # adjust count for hyperthreads. + math(EXPR PHYSICAL_CORES "${LOGICAL_CORES}/${HYPERTHREADS_PER_CORE}") + if (PHYSICAL_CORES LESS 1) + message(FATAL_ERROR "Invalid CPU configuration. " + "LOGICAL_CORES=${LOGICAL_CORES} HYPERTHREADS_PER_CORE=" + "${HYPERTHREADS_PER_CORE}") endif() + + # set the number of cores to use for pure MPI or purely threaded tests + set(TEST_CORES ${PHYSICAL_CORES}) message(STATUS "regression testing -- enabled (${TEST_CORES} cores).") + # set the number of cores to use for MPI + threads tests. if there are too + # few physical cores then disable hybrid parallel tests + math(EXPR HALF_TEST_CORES "${TEST_CORES}/2") + if (HALF_TEST_CORES LESS 2) + message(STATUS "Hybrid parallel tests -- disabled.") + set(TEST_MPI_THREADS OFF) + else() + message(STATUS "Hybrid parallel tests -- enabled.") + set(TEST_MPI_THREADS ON) + endif() + + # set the number of cores for oversubscription/streaming tests + math(EXPR TWICE_TEST_CORES "${TEST_CORES}*2") + add_subdirectory(test) else() message(STATUS "regression testing -- disbaled") diff --git a/README.md b/README.md index 8e0485626..a10c5ad87 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ - + + +
+ +
PyPI version @@ -8,7 +12,7 @@ TECA is a collection of climate analysis algorithms geared toward extreme event detection and tracking implemented in a scalable parallel framework. The code has been successfully deployed and run at massive scales on current DOE supercomputers. TECA's core is written in modern C++ and exploits MPI + X parallelism where X is one of threads, OpenMP, or GPUs. The framework supports a number of parallel design patterns including distributed data parallelism and map-reduce. While modern C++ delivers the highest performance, Python bindings make the code approachable and easy to use. ### Documentation -The [TECA User's Guide](https://teca.readthedocs.io/en/latest/) is the authorotative source for documentation on topics such as [installing TECA](https://teca.readthedocs.io/en/latest/installation.html), running TECA's [command line applications](https://teca.readthedocs.io/en/latest/applications.html), and [Python development](https://teca.readthedocs.io/en/latest/python.html). +The [TECA User's Guide](https://teca.readthedocs.io/en/latest/) is the authorotative source for documentation on topics such as [installing TECA](https://teca.readthedocs.io/en/latest/installation.html), running TECA's [command line applications](https://teca.readthedocs.io/en/latest/applications.html), and [Python development](https://teca.readthedocs.io/en/latest/python.html). The TECA source code is documented on our [Doxygen site](https://teca.readthedocs.io/en/latest/doxygen/index.html). ### Tutorials The [TECA tutorials](https://sourceforge.net/p/teca/TECA_tutorials) subversion repository contains slides from previous tutorials. diff --git a/alg/CMakeLists.txt b/alg/CMakeLists.txt index 48038978d..d8f7e1734 100644 --- a/alg/CMakeLists.txt +++ b/alg/CMakeLists.txt @@ -21,16 +21,20 @@ set(teca_alg_cxx_srcs teca_component_statistics.cxx teca_derived_quantity.cxx teca_descriptive_statistics.cxx + teca_elevation_mask.cxx teca_evaluate_expression.cxx teca_face_to_cell_centering.cxx teca_geography.cxx + teca_indexed_dataset_cache.cxx teca_integrated_vapor_transport.cxx + teca_integrated_water_vapor.cxx teca_l2_norm.cxx teca_latitude_damper.cxx teca_laplacian.cxx teca_mask.cxx teca_normalize_coordinates.cxx teca_parser.cxx + teca_rename_variables.cxx teca_table_calendar.cxx teca_table_reduce.cxx teca_table_region_mask.cxx @@ -41,7 +45,8 @@ set(teca_alg_cxx_srcs teca_tc_classify.cxx teca_tc_wind_radii.cxx teca_tc_trajectory.cxx - teca_temporal_average.cxx + teca_simple_moving_average.cxx + teca_unpack_data.cxx teca_valid_value_mask.cxx teca_variant_array_operand.cxx teca_vertical_coordinate_transform.cxx diff --git a/alg/teca_2d_component_area.cxx b/alg/teca_2d_component_area.cxx index 285d40f52..9bdcbac60 100644 --- a/alg/teca_2d_component_area.cxx +++ b/alg/teca_2d_component_area.cxx @@ -132,11 +132,13 @@ void teca_2d_component_area::get_properties_description( "name of the varibale containing region labels") TECA_POPTS_GET(int, prefix, contiguous_component_ids, "when the region label ids start at 0 and are consecutive " - "this flag enables use of an optimization (0)") + "this flag enables use of an optimization") TECA_POPTS_GET(long, prefix, background_id, - "the label id that corresponds to the background (-1)") + "the label id that corresponds to the background") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -144,6 +146,8 @@ void teca_2d_component_area::get_properties_description( void teca_2d_component_area::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, component_variable) TECA_POPTS_SET(opts, int, prefix, contiguous_component_ids) TECA_POPTS_SET(opts, long, prefix, background_id) diff --git a/alg/teca_2d_component_area.h b/alg/teca_2d_component_area.h index 5a9d57e6a..54f4a2e42 100644 --- a/alg/teca_2d_component_area.h +++ b/alg/teca_2d_component_area.h @@ -10,40 +10,40 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_2d_component_area) -/// an algorithm that computes the area of labeled regions +/// An algorithm that computes the areas of labeled regions /** -Given a set of labels on a Cartesian mesh, the algorithm computes the area of -each region. Regions are identified by assigning a unique integer value to each -mesh point that belongs in the region. The component_variable property names -the variable containing the region labels. - -if the region labels start at 0 and are contiguous then an optimization can be -used. Set contiguous_component_ids property to enable the optimization. Note that -TECA's connected component labeler assigns the background (i.e. cells not inside -the segmentation) the label 0. One can identify the background region and area -via this label. When processing data generated outside of TECA it might be -necessary to supply the background label. Use -2 if there is no background. - -the input dataset is passed through and the results of the calculations are -stored in the output dataset metadata in the following keys: - - number_of_components - number of component ids for which area was - computed. Note that this can include a background - component i.e. for cells outside of the segmentation. - - component_ids - a vector containing the label of each component. This is - always starts with 0, where the label 0 identifies cells - out side of the segmentation, and ranges up to - number_of_components - 1, where the labels from 1 up to - number_of_components - 1 identify connected regions of - cells inside the segmentation. - - component_area - a vector containing the area for the corresponding entry - in the component_ids array. - - background_id - the label used for cells outside of the segmentation, - i.e. the background. This can be used to skip processing - of the background when desirable. + * Given a set of labels on a Cartesian mesh, the algorithm computes the area + * of each region. Regions are identified by assigning a unique integer value + * to each mesh point that belongs in the region. The component_variable + * property names the variable containing the region labels. + * + * if the region labels start at 0 and are contiguous then an optimization can + * be used. Set contiguous_component_ids property to enable the optimization. + * Note that TECA's connected component labeler assigns the background (i.e. + * cells not inside the segmentation) the label 0. One can identify the + * background region and area via this label. When processing data generated + * outside of TECA it might be necessary to supply the background label. Use -2 + * if there is no background. + * + * the input dataset is passed through and the results of the calculations are + * stored in the output dataset metadata in the following keys: + * + * | name | description | + * | ---- | ----------- | + * | number_of_components | number of component ids for which area was | + * | | computed. Note that this can include a background | + * | | component i.e. for cells outside of the segmentation. | + * | component_ids | a vector containing the label of each component. This is | + * | | always starts with 0, where the label 0 identifies | + * | | cells out side of the segmentation, and ranges up | + * | | to number_of_components - 1, where the labels from | + * | | 1 up to number_of_components - 1 identify | + * | | connected regions of cells inside the segmentation. | + * | component_area | a vector containing the area for the corresponding | + * | | entry in the component_ids array. | + * | background_id | the label used for cells outside of the segmentation, | + * | | i.e. the background. This can be used to skip processing | + * | | of the background when desirable. | */ class teca_2d_component_area : public teca_algorithm { @@ -53,23 +53,40 @@ class teca_2d_component_area : public teca_algorithm TECA_ALGORITHM_CLASS_NAME(teca_2d_component_area) ~teca_2d_component_area(); - // report/initialize to/from Boost program options objects. + /** @name program_options + * report/initialize to/from Boost program options objects. + */ + ///@{ TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() + ///@} - // set the name of the input array + /** @name component_variable + * Sets the name of the array containing component labels to compute the + * area of. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_variable) + ///@} - // set this only if you know for certain that label ids are contiguous and - // start at 0. this enables use of a faster implementation. + /** @name contiguous_component_ids + * set this only if you know for certain that label ids are contiguous and + * start at 0. this enables use of a faster implementation. + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, contiguous_component_ids) - - // set this to override the component label used for background. By default - // this is set to -1 to indicate that the value should be obtained from the - // metadata key `background_id`. Note that TECA's connected component - // labeler uses the id 0 for the background and passes this in a metadata - // key and as a result no action is required. + ///@} + + /** @name background_id + * set this to override the component label used for background. By default + * this is set to -1 to indicate that the value should be obtained from the + * metadata key `background_id`. Note that TECA's connected component + * labeler uses the id 0 for the background and passes this in a metadata + * key and as a result no action is required. + */ + ///@{ TECA_ALGORITHM_PROPERTY(long, background_id) + ///@} protected: teca_2d_component_area(); diff --git a/alg/teca_apply_binary_mask.cxx b/alg/teca_apply_binary_mask.cxx index 4ec27d4df..0e2d9aa8b 100644 --- a/alg/teca_apply_binary_mask.cxx +++ b/alg/teca_apply_binary_mask.cxx @@ -1,26 +1,44 @@ #include "teca_apply_binary_mask.h" -#include "teca_mesh.h" +#include "teca_cartesian_mesh.h" #include "teca_array_collection.h" #include "teca_variant_array.h" #include "teca_metadata.h" -#include "teca_mesh.h" +#include "teca_array_attributes.h" +#include "teca_mpi_util.h" #include #include -#include #include -using std::deque; -using std::vector; -using std::set; +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +//#define TECA_DEBUG + using std::cerr; using std::endl; -//#define TECA_DEBUG +namespace internal +{ +// output = mask*input +template +void apply_mask(var_t * __restrict__ output, + const mask_t * __restrict__ mask, + const var_t * __restrict__ input, + unsigned long n) +{ + for (size_t i = 0; i < n; ++i) + output[i] = mask[i]*input[i]; +} +} // -------------------------------------------------------------------------- -teca_apply_binary_mask::teca_apply_binary_mask() : mask_variable("") +teca_apply_binary_mask::teca_apply_binary_mask() : + mask_variable(""), output_variable_prefix("masked_") { this->set_number_of_input_connections(1); this->set_number_of_output_ports(1); @@ -30,6 +48,125 @@ teca_apply_binary_mask::teca_apply_binary_mask() : mask_variable("") teca_apply_binary_mask::~teca_apply_binary_mask() {} +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_apply_binary_mask::get_properties_description( + const std::string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_apply_binary_mask":prefix)); + + opts.add_options() + TECA_POPTS_MULTI_GET(std::vector, prefix, masked_variables, + "A list of variables to apply the mask to.") + TECA_POPTS_GET(std::string, prefix, mask_variable, + "The name of the variable containing the mask values.") + TECA_POPTS_GET(std::string, prefix, output_variable_prefix, + "A string prepended to the output variable names. If empty the" + " input variables will be replaced by their masked results") + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_apply_binary_mask::set_properties( + const std::string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, std::vector, prefix, masked_variables) + TECA_POPTS_SET(opts, std::string, prefix, mask_variable) + TECA_POPTS_SET(opts, std::string, prefix, output_variable_prefix) +} +#endif + +// -------------------------------------------------------------------------- +std::string teca_apply_binary_mask::get_output_variable_name(std::string input_var) +{ + return this->output_variable_prefix + input_var; +} + +// -------------------------------------------------------------------------- +void teca_apply_binary_mask::get_output_variable_names( + std::vector &names) +{ + int n_inputs = this->masked_variables.size(); + for (int i = 0; i < n_inputs; ++i) + { + names.push_back( + this->get_output_variable_name(this->masked_variables[i])); + } +} + +// -------------------------------------------------------------------------- +teca_metadata teca_apply_binary_mask::get_output_metadata( + unsigned int port, + const std::vector &input_md) +{ +#ifdef TECA_DEBUG + cerr << teca_parallel_id() + << "teca_apply_binary_mask::get_output_metadata" << endl; +#endif + (void)port; + + // check that the input variables have been specified. + // this is likely a user error. + if (this->masked_variables.empty() && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_WARNING("Nothing to do, masked_variables have not" + " been specified.") + } + + // add in the array we will generate + teca_metadata out_md(input_md[0]); + + // get the attributes + teca_metadata attributes; + out_md.get("attributes", attributes); + + // construct the list of output variable names + for (auto& input_var : masked_variables) + { + std::string output_var = this->get_output_variable_name(input_var); + + // add the varible to the list of output variables + out_md.append("variables", output_var); + + // insert attributes to enable this variable to be written by the CF writer + teca_metadata input_atts; + if (attributes.get(input_var, input_atts)) + { + TECA_WARNING("Failed to get attributes for \"" << input_var + << "\". Writing the result will not be possible") + } + else + { + // copy the attributes from the input. this will capture the + // data type, size, units, etc. + teca_array_attributes output_atts(input_atts); + + // update description and long name + output_atts.description = input_var + + " multiplied by " + this->mask_variable; + + output_atts.long_name.clear(); + + // update the array attributes + attributes.set(output_var, (teca_metadata)output_atts); + } + + } + + // update the attributes + out_md.set("attributes", attributes); + + return out_md; +} + // -------------------------------------------------------------------------- std::vector teca_apply_binary_mask::get_upstream_request( unsigned int port, const std::vector &input_md, @@ -42,9 +179,9 @@ std::vector teca_apply_binary_mask::get_upstream_request( (void) port; (void) input_md; - vector up_reqs; + std::vector up_reqs; - // get the name of the array to request + // get the name of the mask array if (this->mask_variable.empty()) { TECA_ERROR("A mask variable was not specified") @@ -55,11 +192,36 @@ std::vector teca_apply_binary_mask::get_upstream_request( // add in what we need teca_metadata req(request); std::set arrays; + if (req.has("arrays")) req.get("arrays", arrays); + arrays.insert(this->mask_variable); - if (!this->mask_arrays.empty()) - arrays.insert(this->mask_arrays.begin(), this->mask_arrays.end()); + + // check that the input variables have been specified. + // this is likely a user error. + if (this->masked_variables.empty() && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_WARNING("Nothing to do, masked_variables have not" + " been specified.") + } + + // request the arrays to mask + for (auto& input_var : masked_variables) + { + // request the needed variable + arrays.insert(input_var); + + // intercept request for our output if the variable will have a new name + std::string out_var = this->get_output_variable_name(input_var); + if (out_var != input_var) + { + arrays.erase(out_var); + } + } + + // update the list of arrays to request req.set("arrays", arrays); // send up @@ -79,68 +241,80 @@ const_p_teca_dataset teca_apply_binary_mask::execute( (void)request; // get the input - const_p_teca_mesh in_mesh = - std::dynamic_pointer_cast(input_data[0]); + const_p_teca_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); if (!in_mesh) { - TECA_ERROR("empty input, or not a mesh") + TECA_ERROR("Failed to apply mask. Dataset is not a teca_mesh") return nullptr; } - // create output and copy metadata, coordinates, etc + // create the output mesh, pass everything through + // masked arrays are added or replaced below p_teca_mesh out_mesh = - std::dynamic_pointer_cast(in_mesh->new_instance()); - out_mesh->copy(in_mesh); + std::static_pointer_cast + (std::const_pointer_cast(in_mesh)->new_shallow_copy()); - // get the mask array + // check that a masking variable has been provided if (this->mask_variable.empty()) { - TECA_ERROR("A mask variable was not specified") + TECA_ERROR("The mask_variable name was not specified") return nullptr; } - p_teca_array_collection arrays = out_mesh->get_point_arrays(); - - p_teca_variant_array mask_array = arrays->get(this->mask_variable); + // get the mask array + const_p_teca_variant_array mask_array + = in_mesh->get_point_arrays()->get(this->mask_variable); if (!mask_array) { - TECA_ERROR("mask variable \"" << this->mask_variable - << "\" is not in the input") + TECA_ERROR("The mask_variable \"" << this->mask_variable + << "\" was requested but is not present in the input data.") return nullptr; } // apply the mask - unsigned long nelem = mask_array->size(); + NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, + mask_array.get(), _MASK, - NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, - mask_array.get(), _1, + // loop over input variables + for (auto& input_var : masked_variables) + { + std::string output_var = this->get_output_variable_name(input_var); - NT_1 *pmask = static_cast(mask_array.get())->get(); + // get the input array + const_p_teca_variant_array input_array + = in_mesh->get_point_arrays()->get(input_var); + if (!input_array) + { + TECA_ERROR("The masked_variable \"" << input_var + << "\" was requested but is not present in the input data.") + return nullptr; + } - unsigned int narrays = arrays->size(); - for (unsigned int i = 0; i < narrays; ++i) - { - // if the user provided a list, restrict masking to that - // list. and if not, mask everything - if (!this->mask_arrays.empty() && - !std::count(this->mask_arrays.begin(), - this->mask_arrays.end(), arrays->get_name(i))) - continue; + // allocate the output array + size_t n = input_array->size(); - p_teca_variant_array array = arrays->get(i); + p_teca_variant_array output_array = input_array->new_instance(n); - NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, - array.get(), _2, + //output_array->resize(n); - NT_2 *parray = static_cast(array.get())->get(); + // do the mask calculation + NESTED_TEMPLATE_DISPATCH( + teca_variant_array_impl, + output_array.get(), _VAR, - for (unsigned long q = 0; q < nelem; ++q) - { - parray[q] *= static_cast(pmask[q]); - } + internal::apply_mask( + dynamic_cast(output_array.get())->get(), + static_cast(mask_array.get())->get(), + static_cast(input_array.get())->get(), + n); ) + + out_mesh->get_point_arrays()->set( + output_var, output_array); } - ) + ) + return out_mesh; } diff --git a/alg/teca_apply_binary_mask.h b/alg/teca_apply_binary_mask.h index f30a1081e..ceeed4040 100644 --- a/alg/teca_apply_binary_mask.h +++ b/alg/teca_apply_binary_mask.h @@ -10,12 +10,26 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_apply_binary_mask) -/// an algorithm that applies a binary mask multiplicatively +/// Applies a mask to a given list of variables /** -an algorithm that applies a binary mask multiplicatively to all -arrays in the input dataset. where mask is 1 values are passed -through, where mask is 0 values are removed. -*/ + * Given a mask variable, this routine applies the mask to a list of input + * variables. + * + * The mask variable can either be binary, or it can represent a probability + * ranging from 0 to 1. For mask variable `mask` and input variable `var`, this + * algorithm computes `mask * var` and sends the resulting array downstream; this + * masking operation is applied for all variables in the input list. + * + * A potential use-case for this algorithm is masking quantities like + * precipitation by the probability of atmospheric river presence; the average + * of this masked precipitation variable gives the average precipitation due to + * atmospheric rivers. + * + * The output variable names are given a prefix to distinguish them from the + * upstream versions. E.g., if the algorithm property `output_variable_prefix` is set + * to 'ar_', and the variable being masked is 'precip', then the output array + * name is 'ar_precip'. + */ class teca_apply_binary_mask : public teca_algorithm { public: @@ -24,19 +38,48 @@ class teca_apply_binary_mask : public teca_algorithm TECA_ALGORITHM_CLASS_NAME(teca_apply_binary_mask) ~teca_apply_binary_mask(); - // set the name of the output array + // report/initialize to/from Boost program options + // objects. + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + + /** @name mask_variable + * set the name of the variable containing the mask values + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, mask_variable) + ///@} + + /** @name masked_variable + * A list of of variables to apply the mask to. If empty no arrays will be + * requested, and no variables will be masked + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(std::string, masked_variable) + ///@} - // set the arrays to mask. if empty no arrays will be - // requested, but all present will be masked - TECA_ALGORITHM_VECTOR_PROPERTY(std::string, mask_array) + /** @name output_variable_prefix + * A prefix for the names of the variables that have been masked. If this + * is empty masked data replaces its input, otherwise input data is + * preserved and masked data is added. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, output_variable_prefix) + ///@} + + /** helper that constructs and returns the result variable names taking + * into account he list of masked_variables and the output_variable_prefix. + * use this to know what variables will be produced. + */ + void get_output_variable_names(std::vector &names); protected: teca_apply_binary_mask(); private: - //teca_metadata get_output_metadata(unsigned int port, - // const std::vector &input_md) override; + teca_metadata get_output_metadata( + unsigned int port, + const std::vector &input_md) override; std::vector get_upstream_request( unsigned int port, const std::vector &input_md, @@ -46,9 +89,14 @@ class teca_apply_binary_mask : public teca_algorithm const std::vector &input_data, const teca_metadata &request) override; + // helper that given and input variable name constructs the result variable + // name taking into account the output_variable_prefix + std::string get_output_variable_name(std::string input_var); + private: std::string mask_variable; - std::vector mask_arrays; + std::vector masked_variables; + std::string output_variable_prefix; }; #endif diff --git a/alg/teca_ar_detect.cxx b/alg/teca_ar_detect.cxx deleted file mode 100644 index 23b848c3b..000000000 --- a/alg/teca_ar_detect.cxx +++ /dev/null @@ -1,1347 +0,0 @@ -#include "teca_ar_detect.h" - -#include "teca_cartesian_mesh.h" -#include "teca_variant_array.h" -#include "teca_table.h" -#include "teca_calendar.h" -#include "teca_coordinate_util.h" - -#include -#include -#include -#include -#include - -#if defined(TECA_HAS_BOOST) -#include -#endif - -using std::cerr; -using std::endl; - -#define TECA_DEBUG 0 -#if TECA_DEBUG > 0 -#include "teca_cartesian_mesh_writer.h" -#include "teca_programmable_algorithm.h" -int write_mesh( - const const_p_teca_cartesian_mesh &mesh, - const const_p_teca_variant_array &vapor, - const const_p_teca_variant_array &thres, - const const_p_teca_variant_array &ccomp, - const const_p_teca_variant_array &lsmask, - const std::string &base_name); -#endif - -// a description of the atmospheric river -struct atmospheric_river -{ - atmospheric_river() : - pe(false), length(0.0), - min_width(0.0), max_width(0.0), - end_top_lat(0.0), end_top_lon(0.0), - end_bot_lat(0.0), end_bot_lon(0.0) - {} - - bool pe; - double length; - double min_width; - double max_width; - double end_top_lat; - double end_top_lon; - double end_bot_lat; - double end_bot_lon; -}; - -std::ostream &operator<<(std::ostream &os, const atmospheric_river &ar) -{ - os << " type=" << (ar.pe ? "PE" : "AR") - << " length=" << ar.length - << " width=" << ar.min_width << ", " << ar.max_width - << " bounds=" << ar.end_bot_lon << ", " << ar.end_bot_lat << ", " - << ar.end_top_lon << ", " << ar.end_top_lat; - return os; -} - -unsigned sauf(const unsigned nrow, const unsigned ncol, unsigned int *image); - -bool ar_detect( - const_p_teca_variant_array lat, - const_p_teca_variant_array lon, - const_p_teca_variant_array land_sea_mask, - p_teca_unsigned_int_array con_comp, - unsigned long n_comp, - double river_start_lat, - double river_start_lon, - double river_end_lat_low, - double river_end_lon_low, - double river_end_lat_high, - double river_end_lon_high, - double percent_in_mesh, - double river_width, - double river_length, - double land_threshold_low, - double land_threshold_high, - atmospheric_river &ar); - -// set locations in the output where the input array -// has values within the low high range. -template -void threshold( - const T *input, unsigned int *output, - size_t n_vals, T low, T high) -{ - for (size_t i = 0; i < n_vals; ++i) - output[i] = ((input[i] >= low) && (input[i] <= high)) ? 1 : 0; -} - - - -// -------------------------------------------------------------------------- -teca_ar_detect::teca_ar_detect() : - water_vapor_variable("prw"), - land_sea_mask_variable(""), - low_water_vapor_threshold(20), - high_water_vapor_threshold(75), - search_lat_low(19.0), - search_lon_low(180.0), - search_lat_high(56.0), - search_lon_high(250.0), - river_start_lat_low(18.0), - river_start_lon_low(180.0), - river_end_lat_low(29.0), - river_end_lon_low(233.0), - river_end_lat_high(56.0), - river_end_lon_high(238.0), - percent_in_mesh(5.0), - river_width(1250.0), - river_length(2000.0), - land_threshold_low(1.0), - land_threshold_high(std::numeric_limits::max()) -{ - this->set_number_of_input_connections(1); - this->set_number_of_output_ports(1); -} - -// -------------------------------------------------------------------------- -teca_ar_detect::~teca_ar_detect() -{} - -#if defined(TECA_HAS_BOOST) -// -------------------------------------------------------------------------- -void teca_ar_detect::get_properties_description( - const std::string &prefix, options_description &opts) -{ - options_description ard_opts("Options for " - + (prefix.empty()?"teca_ar_detect":prefix)); - - ard_opts.add_options() - TECA_POPTS_GET(std::string, prefix, water_vapor_variable, - "name of variable containing water vapor values") - TECA_POPTS_GET(double, prefix, low_water_vapor_threshold, - "low water vapor threshold") - TECA_POPTS_GET(double, prefix, high_water_vapor_threshold, - "high water vapor threshold") - TECA_POPTS_GET(double, prefix, search_lat_low, - "search space low latitude") - TECA_POPTS_GET(double, prefix, search_lon_low, - "search space low longitude") - TECA_POPTS_GET(double, prefix, search_lat_high, - "search space high latitude") - TECA_POPTS_GET(double, prefix, search_lon_high, - "search space high longitude") - TECA_POPTS_GET(double, prefix, river_start_lat_low, - "latitude used to classify as AR or PE") - TECA_POPTS_GET(double, prefix, river_start_lon_low, - "longitude used to classify as AR or PE") - TECA_POPTS_GET(double, prefix, river_end_lat_low, - "CA coastal region low latitude") - TECA_POPTS_GET(double, prefix, river_end_lon_low, - "CA coastal region low longitude") - TECA_POPTS_GET(double, prefix, river_end_lat_high, - "CA coastal region high latitude") - TECA_POPTS_GET(double, prefix, river_end_lon_high, - "CA coastal region high longitude") - TECA_POPTS_GET(double, prefix, percent_in_mesh, - "size of river in relation to search space area") - TECA_POPTS_GET(double, prefix, river_width, - "minimum river width") - TECA_POPTS_GET(double, prefix, river_length, - "minimum river length") - TECA_POPTS_GET(std::string, prefix, land_sea_mask_variable, - "name of variable containing land-sea mask values") - TECA_POPTS_GET(double, prefix, land_threshold_low, - "low land value") - TECA_POPTS_GET(double, prefix, land_threshold_high, - "high land value") - ; - - opts.add(ard_opts); -} - -// -------------------------------------------------------------------------- -void teca_ar_detect::set_properties( - const std::string &prefix, variables_map &opts) -{ - TECA_POPTS_SET(opts, std::string, prefix, water_vapor_variable) - TECA_POPTS_SET(opts, double, prefix, low_water_vapor_threshold) - TECA_POPTS_SET(opts, double, prefix, high_water_vapor_threshold) - TECA_POPTS_SET(opts, double, prefix, search_lat_low) - TECA_POPTS_SET(opts, double, prefix, search_lon_low) - TECA_POPTS_SET(opts, double, prefix, search_lat_high) - TECA_POPTS_SET(opts, double, prefix, search_lon_high) - TECA_POPTS_SET(opts, double, prefix, river_start_lat_low) - TECA_POPTS_SET(opts, double, prefix, river_start_lon_low) - TECA_POPTS_SET(opts, double, prefix, river_end_lat_low) - TECA_POPTS_SET(opts, double, prefix, river_end_lon_low) - TECA_POPTS_SET(opts, double, prefix, river_end_lat_high) - TECA_POPTS_SET(opts, double, prefix, river_end_lon_high) - TECA_POPTS_SET(opts, double, prefix, percent_in_mesh) - TECA_POPTS_SET(opts, double, prefix, river_width) - TECA_POPTS_SET(opts, double, prefix, river_length) - TECA_POPTS_SET(opts, std::string, prefix, land_sea_mask_variable) - TECA_POPTS_SET(opts, double, prefix, land_threshold_low) - TECA_POPTS_SET(opts, double, prefix, land_threshold_high) -} - -#endif - -// -------------------------------------------------------------------------- -teca_metadata teca_ar_detect::get_output_metadata( - unsigned int port, - const std::vector &input_md) -{ -#if TECA_DEBUG > 1 - cerr << teca_parallel_id() - << "teca_ar_detect::get_output_metadata" << endl; -#endif - (void)port; - - teca_metadata output_md(input_md[0]); - return output_md; -} - -// -------------------------------------------------------------------------- -std::vector teca_ar_detect::get_upstream_request( - unsigned int port, - const std::vector &input_md, - const teca_metadata &request) -{ -#if TECA_DEBUG > 1 - cerr << teca_parallel_id() - << "teca_ar_detect::get_upstream_request" << endl; -#endif - (void)port; - - std::vector up_reqs; - - teca_metadata md = input_md[0]; - - // locate the extents of the user supplied region of - // interest - teca_metadata coords; - if (md.get("coordinates", coords)) - { - TECA_ERROR("metadata is missing \"coordinates\"") - return up_reqs; - } - - p_teca_variant_array lat; - p_teca_variant_array lon; - if (!(lat = coords.get("y")) || !(lon = coords.get("x"))) - { - TECA_ERROR("metadata missing lat lon coordinates") - return up_reqs; - } - - std::vector bounds = {this->search_lon_low, - this->search_lon_high, this->search_lat_low, - this->search_lat_high, 0.0, 0.0}; - - // build the request - std::vector arrays; - request.get("arrays", arrays); - arrays.push_back(this->water_vapor_variable); - if (!this->land_sea_mask_variable.empty()) - arrays.push_back(this->land_sea_mask_variable); - - teca_metadata up_req(request); - up_req.set("arrays", arrays); - up_req.set("bounds", bounds); - - up_reqs.push_back(up_req); - return up_reqs; -} - -// -------------------------------------------------------------------------- -const_p_teca_dataset teca_ar_detect::execute( - unsigned int port, - const std::vector &input_data, - const teca_metadata &request) -{ -#if TECA_DEBUG > 1 - cerr << teca_parallel_id() << "teca_ar_detect::execute"; - this->to_stream(cerr); - cerr << endl; -#endif - (void)port; - (void)request; - - // get the input dataset - const_p_teca_cartesian_mesh mesh - = std::dynamic_pointer_cast(input_data[0]); - if (!mesh) - { - TECA_ERROR("invalid input. teca_cartesian_mesh is required") - return nullptr; - } - - // get coordinate arrays - const_p_teca_variant_array lat = mesh->get_y_coordinates(); - const_p_teca_variant_array lon = mesh->get_x_coordinates(); - - if (!lon || !lat) - { - TECA_ERROR("invalid mesh. missing lat lon coordinates") - return nullptr; - } - - // get land sea mask - const_p_teca_variant_array land_sea_mask; - if (this->land_sea_mask_variable.empty() || - !(land_sea_mask = mesh->get_point_arrays()->get(this->land_sea_mask_variable))) - { - // input doesn't have it, generate a stand in such - // that land fall criteria will evaluate true - size_t n = lat->size()*lon->size(); - p_teca_double_array lsm = teca_double_array::New(n, this->land_threshold_low); - land_sea_mask = lsm; - } - - // get the mesh extents - std::vector extent; - mesh->get_extent(extent); - - unsigned long num_rows = extent[3] - extent[2] + 1; - unsigned long num_cols = extent[1] - extent[0] + 1; - unsigned long num_rc = num_rows*num_cols; - - // get water vapor data - const_p_teca_variant_array water_vapor - = mesh->get_point_arrays()->get(this->water_vapor_variable); - - if (!water_vapor) - { - TECA_ERROR( - << "Dataset missing water vapor variable \"" - << this->water_vapor_variable << "\"") - return nullptr; - } - - p_teca_table event = teca_table::New(); - - event->declare_columns( - "time", double(), "time_step", long(), - "length", double(), "min width", double(), - "max width", double(), "end_top_lat", double(), - "end_top_lon", double(), "end_bot_lat", double(), - "end_bot_lon", double(), "type", std::string()); - - // get calendar - std::string calendar; - mesh->get_calendar(calendar); - event->set_calendar(calendar); - - // get units - std::string units; - mesh->get_time_units(units); - event->set_time_units(units); - - // get time step - unsigned long time_step; - mesh->get_time_step(time_step); - - // get offset of the current timestep - double time = 0.0; - mesh->get_time(time); - - TEMPLATE_DISPATCH( - const teca_variant_array_impl, - water_vapor.get(), - - const NT *p_wv = dynamic_cast(water_vapor.get())->get(); - - // threshold - p_teca_unsigned_int_array con_comp - = teca_unsigned_int_array::New(num_rc, 0); - - unsigned int *p_con_comp = con_comp->get(); - - threshold(p_wv, p_con_comp, num_rc, - static_cast(this->low_water_vapor_threshold), - static_cast(this->high_water_vapor_threshold)); - -#if TECA_DEBUG > 0 - p_teca_variant_array thresh = con_comp->new_copy(); -#endif - - // label - int num_comp = sauf(num_rows, num_cols, p_con_comp); - -#if TECA_DEBUG > 0 - write_mesh(mesh, water_vapor, thresh, con_comp, - land_sea_mask, "ar_mesh_%t%.%e%"); -#endif - - // detect ar - atmospheric_river ar; - if (num_comp && - ar_detect(lat, lon, land_sea_mask, con_comp, num_comp, - this->river_start_lat_low, this->river_start_lon_low, - this->river_end_lat_low, this->river_end_lon_low, - this->river_end_lat_high, this->river_end_lon_high, - this->percent_in_mesh, this->river_width, - this->river_length, this->land_threshold_low, - this->land_threshold_high, ar)) - { -#if TECA_DEBUG > 0 - cerr << teca_parallel_id() << " event detected " << time_step << endl; -#endif - event << time << time_step - << ar.length << ar.min_width << ar.max_width - << ar.end_top_lat << ar.end_top_lon - << ar.end_bot_lat << ar.end_bot_lon - << std::string(ar.pe ? "PE" : "AR"); - } - ) - - return event; -} - -// -------------------------------------------------------------------------- -void teca_ar_detect::to_stream(std::ostream &os) const -{ - os << " water_vapor_variable=" << this->water_vapor_variable - << " land_sea_mask_variable=" << this->land_sea_mask_variable - << " low_water_vapor_threshold=" << this->low_water_vapor_threshold - << " high_water_vapor_threshold=" << this->high_water_vapor_threshold - << " river_start_lon_low=" << this->river_start_lon_low - << " river_start_lat_low=" << this->river_start_lat_low - << " river_end_lon_low=" << this->river_end_lon_low - << " river_end_lat_low=" << this->river_end_lat_low - << " river_end_lon_high=" << this->river_end_lon_high - << " river_end_lat_high=" << this->river_end_lat_high - << " percent_in_mesh=" << this->percent_in_mesh - << " river_width=" << this->river_width - << " river_length=" << this->river_length - << " land_threshodl_low=" << this->land_threshold_low - << " land_threshodl_high=" << this->land_threshold_high; -} - - -// Code borrowed from John Wu's sauf.cpp -// Find the minimal value starting @arg ind. -inline unsigned afind(const std::vector& equiv, - const unsigned ind) -{ - unsigned ret = ind; - while (equiv[ret] < ret) - { - ret = equiv[ret]; - } - return ret; -} - -// Set the values starting with @arg ind. -inline void aset(std::vector& equiv, - const unsigned ind, const unsigned val) -{ - unsigned i = ind; - while (equiv[i] < i) - { - unsigned j = equiv[i]; - equiv[i] = val; - i = j; - } - equiv[i] = val; -} - -/* -* Purpose: Scan with Array-based Union-Find -* Return vals: Number of connected components -* Description: SAUF -- Scan with Array-based Union-Find. -* This is an implementation that follows the decision try to minimize -* number of neighbors visited and uses the array-based union-find -* algorithms to minimize work on the union-find data structure. It works -* with each pixel/cell of the 2D binary image individually. -* The 2D binary image is passed to sauf as a unsigned*. On input, the -* zero value is treated as the background, and non-zero is treated as -* object. On successful completion of this function, the non-zero values -* in array image is replaced by its label. -* The return value is the number of components found. -*/ -unsigned sauf(const unsigned nrow, const unsigned ncol, unsigned *image) -{ - const unsigned ncells = ncol * nrow; - const unsigned ncp1 = ncol + 1; - const unsigned ncm1 = ncol - 1; - std::vector equiv; // equivalence array - unsigned nextLabel = 1; - - equiv.reserve(ncol); - equiv.push_back(0); - - // the first cell of the first line - if (*image != 0) - { - *image = nextLabel; - equiv.push_back(nextLabel); - ++ nextLabel; - } - // first row of cells - for (unsigned i = 1; i < ncol; ++ i) - { - if (image[i] != 0) - { - if (image[i-1] != 0) - { - image[i] = image[i-1]; - } - else - { - equiv.push_back(nextLabel); - image[i] = nextLabel; - ++ nextLabel; - } - } - } - - // scan the rest of lines, check neighbor b first - for (unsigned j = ncol; j < ncells; j += ncol) - { - unsigned nc, nd, k, l; - - // the first point of the line has two neighbors, and the two - // neighbors must have at most one label (recorded as nc) - if (image[j] != 0) - { - if (image[j-ncm1] != 0) - nc = image[j-ncm1]; - else if (image[j-ncol] != 0) - nc = image[j-ncol]; - else - nc = nextLabel; - if (nc != nextLabel) { // use existing label - nc = equiv[nc]; - image[j] = nc; - } - else { // need a new label - equiv.push_back(nc); - image[j] = nc; - ++ nextLabel; - } - } - - // the rest of the line - for (unsigned i = j+1; i < j+ncol; ++ i) - { - if (image[i] != 0) { - if (image[i-ncol] != 0) { - nc = image[i-ncol]; - l = afind(equiv, nc); - aset(equiv, nc, l); - image[i] = l; - } - else if (i-ncm1 " - << equiv[equiv[i]] << std::endl; -#endif - equiv[i] = equiv[equiv[i]]; - } - else { // change to the next smallest unused label -#if defined(_DEBUG) || defined(DEBUG) - std::cout << i << " final " << equiv[i] << " ==> " - << nextLabel << std::endl; -#endif - equiv[i] = nextLabel; - ++ nextLabel; - } - } - - if (nextLabel < nequiv) {// relabel all cells to their minimal labels - for (unsigned i = 0; i < ncells; ++ i) - image[i] = equiv[image[i]]; - } - -#if defined(_DEBUG) || defined(DEBUG) - std::cout << "sauf(" << nrow << ", " << ncol << ") assigned " - << nextLabel-1 << " label" << (nextLabel>2?"s":"") - << ", used " << nequiv << " provisional labels" - << std::endl; -#endif - return nextLabel-1; -} - -// do any of the detected points meet the river start -// criteria. retrun true if so. -template -bool river_start_criteria_lat( - const std::vector &con_comp_r, - const T *p_lat, - T river_start_lat) -{ - unsigned long n = con_comp_r.size(); - for (unsigned long q = 0; q < n; ++q) - { - if (p_lat[con_comp_r[q]] >= river_start_lat) - return true; - } - return false; -} - -// do any of the detected points meet the river start -// criteria. retrun true if so. -template -bool river_start_criteria_lon( - const std::vector &con_comp_c, - const T *p_lon, - T river_start_lon) -{ - unsigned long n = con_comp_c.size(); - for (unsigned long q = 0; q < n; ++q) - { - if (p_lon[con_comp_c[q]] >= river_start_lon) - return true; - } - return false; -} - -// helper return true if the start criteria is -// met, and classifies the ar as PE if it starts -// in the bottom boundary. -template -bool river_start_criteria( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon, - T start_lat, - T start_lon, - atmospheric_river &ar) -{ - return - ((ar.pe = river_start_criteria_lat(con_comp_r, p_lat, start_lat)) - || river_start_criteria_lon(con_comp_c, p_lon, start_lon)); -} - -// do any of the detected points meet the river end -// criteria? (ie. does it hit the west coasts?) if so -// store a bounding box covering the river and return -// true. -template -bool river_end_criteria( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon, - T river_end_lat_low, - T river_end_lon_low, - T river_end_lat_high, - T river_end_lon_high, - atmospheric_river &ar) -{ - bool end_criteria = false; - - std::vector end_col_idx; - - unsigned int count = con_comp_r.size(); - for (unsigned int i = 0; i < count; ++i) - { - // approximate land mask boundaries for the western coast of the US, - T lon_val = p_lon[con_comp_c[i]]; - if ((lon_val >= river_end_lon_low) && (lon_val <= river_end_lon_high)) - end_col_idx.push_back(i); - } - - // look for rows that fall between lat boundaries - T top_lat = T(); - T top_lon = T(); - T bot_lat = T(); - T bot_lon = T(); - - bool top_touch = false; - unsigned int end_col_count = end_col_idx.size(); - for (unsigned int i = 0; i < end_col_count; ++i) - { - // approximate land mask boundaries for the western coast of the US, - T lat_val = p_lat[con_comp_r[end_col_idx[i]]]; - if ((lat_val >= river_end_lat_low) && (lat_val <= river_end_lat_high)) - { - T lon_val = p_lon[con_comp_c[end_col_idx[i]]]; - end_criteria = true; - if (!top_touch) - { - top_touch = true; - top_lat = lat_val; - top_lon = lon_val; - } - bot_lat = lat_val; - bot_lon = lon_val; - } - } - - ar.end_top_lat = top_lat; - ar.end_top_lon = top_lon; - ar.end_bot_lat = bot_lat; - ar.end_bot_lon = bot_lon; - - return end_criteria; -} - -/* -* Calculate geodesic distance between two lat, long pairs -* CODE borrowed from: from http://www.geodatasource.com/developers/c -* from http://osiris.tuwien.ac.at/~wgarn/gis-gps/latlong.html -* from http://www.codeproject.com/KB/cpp/Distancecplusplus.aspx -*/ -template -T geodesic_distance(T lat1, T lon1, T lat2, T lon2) -{ - T deg_to_rad = T(M_PI/180.0); - - T dlat1 = lat1*deg_to_rad; - T dlon1 = lon1*deg_to_rad; - T dlat2 = lat2*deg_to_rad; - T dlon2 = lon2*deg_to_rad; - - T dLon = dlon1 - dlon2; - T dLat = dlat1 - dlat2; - - T sin_dLat_half_sq = sin(dLat/T(2.0)); - sin_dLat_half_sq *= sin_dLat_half_sq; - - T sin_dLon_half_sq = sin(dLon/T(2.0)); - sin_dLon_half_sq *= sin_dLon_half_sq; - - T aHarv = sin_dLat_half_sq - + cos(dlat1)*cos(dlat2)*sin_dLon_half_sq; - - T cHarv = T(2.0)*atan2(sqrt(aHarv), sqrt(T(1.0) - aHarv)); - - T R = T(6371.0); - T distance = R*cHarv; - - return distance; -} - -/* -* This function calculates the average geodesic width -* As each pixel represents certain area, the total area -* is the product of the number of pixels and the area of -* one pixel. The average width is: the total area divided -* by the medial axis length -* We are calculating the average width, since we are not -* determining where exactly to cut off the tropical region -* to calculate the real width of an atmospheric river -*/ -template -T avg_width( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - T ar_len, - const T *p_lat, - const T *p_lon) -{ -/* - // TODO -- need bounds checking when doing things like - // p_lat[con_comp_r[0] + 1]. also because it's potentially - // a stretched cartesian mesh need to compute area of - // individual cells - - // length of cell in lat direction - T lat_val[2] = {p_lat[con_comp_r[0]], p_lat[con_comp_r[0] + 1]}; - T lon_val[2] = {p_lon[con_comp_c[0]], p_lon[con_comp_c[0]]}; - T dlat = geodesic_distance(lat_val[0], lon_val[0], lat_val[1], lon_val[1]); - - // length of cell in lon direction - lat_val[1] = lat_val[0]; - lon_val[1] = p_lon[con_comp_c[0] + 1]; - T dlon = geodesic_distance(lat_val[0], lon_val[0], lat_val[1], lon_val[1]); -*/ - (void)con_comp_c; - // compute area of the first cell in the input mesh - // length of cell in lat direction - T lat_val[2] = {p_lat[0], p_lat[1]}; - T lon_val[2] = {p_lon[0], p_lon[0]}; - T dlat = geodesic_distance(lat_val[0], lon_val[0], lat_val[1], lon_val[1]); - - // length of cell in lon direction - lat_val[1] = lat_val[0]; - lon_val[1] = p_lon[1]; - T dlon = geodesic_distance(lat_val[0], lon_val[0], lat_val[1], lon_val[1]); - - // area - T pixel_area = dlat*dlon; - T total_area = pixel_area*con_comp_r.size(); - - // avg width - T avg_width = total_area/ar_len; - return avg_width; -} - -/* -* Find the middle point between two pairs of lat and lon values -* http://stackoverflow.com/questions/4164830/geographic-midpoint-between-two-coordinates -*/ -template -void geodesic_midpoint(T lat1, T lon1, T lat2, T lon2, T &mid_lat, T &mid_lon) -{ - T deg_to_rad = T(M_PI/180.0); - T dLon = (lon2 - lon1) * deg_to_rad; - T dLat1 = lat1 * deg_to_rad; - T dLat2 = lat2 * deg_to_rad; - T dLon1 = lon1 * deg_to_rad; - - T Bx = cos(dLat2) * cos(dLon); - T By = cos(dLat2) * sin(dLon); - - mid_lat = atan2(sin(dLat1)+sin(dLat2), - sqrt((cos(dLat1)+Bx)*(cos(dLat1)+Bx)+By*By)); - - mid_lon = dLon1 + atan2(By, (cos(dLat1)+Bx)); - - T rad_to_deg = T(180.0/M_PI); - mid_lat *= rad_to_deg; - mid_lon *= rad_to_deg; -} - -/* -* Find the length along the medial axis of a connected component -* Medial length is the sum of the distances between the medial -* points in the connected component -*/ -template -T medial_length( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon) -{ - std::vector jb_r1; - std::vector jb_c1; - std::vector jb_c2; - - long row_track = -1; - - unsigned long count = con_comp_r.size(); - for (unsigned long i = 0; i < count; ++i) - { - if (row_track != con_comp_r[i]) - { - jb_r1.push_back(con_comp_r[i]); - jb_c1.push_back(con_comp_c[i]); - - jb_c2.push_back(con_comp_c[i]); - - row_track = con_comp_r[i]; - } - else - { - jb_c2.back() = con_comp_c[i]; - } - } - - T total_dist = T(); - - long b_count = jb_r1.size() - 1; - for (long i = 0; i < b_count; ++i) - { - T lat_val[2]; - T lon_val[2]; - - lat_val[0] = p_lat[jb_r1[i]]; - lat_val[1] = p_lat[jb_r1[i]]; - - lon_val[0] = p_lon[jb_c1[i]]; - lon_val[1] = p_lon[jb_c2[i]]; - - T mid_lat1; - T mid_lon1; - - geodesic_midpoint( - lat_val[0], lon_val[0], lat_val[1], lon_val[1], - mid_lat1, mid_lon1); - - lat_val[0] = p_lat[jb_r1[i+1]]; - lat_val[1] = p_lat[jb_r1[i+1]]; - - lon_val[0] = p_lon[jb_c1[i+1]]; - lon_val[1] = p_lon[jb_c2[i+1]]; - - T mid_lat2; - T mid_lon2; - - geodesic_midpoint( - lat_val[0], lon_val[0], lat_val[1], lon_val[1], - mid_lat2, mid_lon2); - - total_dist - += geodesic_distance(mid_lat1, mid_lon1, mid_lat2, mid_lon2); - } - - return total_dist; -} - -/* -// Suren's function -// helper return true if the geometric conditions -// on an ar are satisfied. also stores the length -// and width of the river. -template -bool river_geometric_criteria( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon, - double river_length, - double river_width, - atmospheric_river &ar) -{ - ar.length = medial_length(con_comp_r, con_comp_c, p_lat, p_lon); - - ar.width = avg_width(con_comp_r, con_comp_c, - static_cast(ar.length), p_lat, p_lon); - - return (ar.length >= river_length) && (ar.width <= river_width); -} -*/ - -// Junmin's function for height of a triangle -template -T triangle_height(T base, T s1, T s2) -{ - // area from Heron's fomula - T p = (base + s1 + s2)/T(2); - T area = p*(p - base)*(p - s1)*(p - s2); - // detect impossible triangle - if (area < T()) - return std::min(s1, s2); - // height from A = 1/2 b h - return T(2)*sqrt(area)/base; -} - -// TDataProcessor::check_geodesic_width_top_down -// Junmin's function for detecting river based on -// it's geometric properties -template -bool river_geometric_criteria( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon, - double river_length, - double river_width, - atmospheric_river &ar) -{ - std::vector distinct_rows; - std::vector leftmost_col; - std::vector rightmost_col; - - int row_track = -1; - size_t count = con_comp_r.size(); - for (size_t i = 0; i < count; ++i) - { - if (row_track != con_comp_r[i]) - { - row_track = con_comp_r[i]; - - distinct_rows.push_back(con_comp_r[i]); - leftmost_col.push_back(con_comp_c[i]); - rightmost_col.push_back(con_comp_c[i]); - } - else - { - rightmost_col.back() = con_comp_c[i]; - } - } - - // river metrics - T length_from_top = T(); - T min_width = std::numeric_limits::max(); - T max_width = std::numeric_limits::lowest(); - - for (long i = distinct_rows.size() - 2; i >= 0; --i) - { - // for each row w respect to row above it. triangulate - // a quadrilateral composed of left and right most points - // in this and the above rows. ccw ordering from lower - // left corner is A,B,D,C. - - // low left-right distance - T AB = geodesic_distance( - p_lat[distinct_rows[i]], p_lon[leftmost_col[i]], - p_lat[distinct_rows[i]], p_lon[rightmost_col[i]]); - - // left side bottom-top distance - T AC = geodesic_distance( - p_lat[distinct_rows[i]], p_lon[leftmost_col[i]], - p_lat[distinct_rows[i+1]], p_lon[leftmost_col[i]]); - - // distance from top left to bottom right, across - T BC = geodesic_distance( - p_lat[distinct_rows[i]], p_lon[rightmost_col[i]], - p_lat[distinct_rows[i+1]], p_lon[leftmost_col[i+1]]); - - // high left-right distance - T CD = geodesic_distance( - p_lat[distinct_rows[i+1]], p_lon[leftmost_col[i+1]], - p_lat[distinct_rows[i+1]], p_lon[rightmost_col[i+1]]); - - // right side bottom-top distance - T BD = geodesic_distance( - p_lat[distinct_rows[i]], p_lon[rightmost_col[i]], - p_lat[distinct_rows[i+1]], p_lon[rightmost_col[i+1]]); - - T height_from_b = triangle_height(AC, AB, BC); - T height_from_c = triangle_height(BD, BC, CD); - - T curr_min = std::min(height_from_b, height_from_c); - - // test width criteria - if (curr_min > river_width) - { - // TODO -- first time through the loop length == 0. is it intentional - // to discard the detection or should length calc take place before this test? - // note: first time through loop none of the event details have been recoreded - if (length_from_top <= river_length) - { - // too short to be a river - return false; - } - else - { - // part of a connected region is AR - ar.min_width = static_cast(min_width); - ar.max_width = static_cast(max_width); - ar.length = static_cast(length_from_top); - return true; - } - } - - // update width - min_width = std::min(min_width, curr_min); - max_width = std::max(max_width, curr_min); - - // update length - T mid_bot_lat; - T mid_bot_lon; - geodesic_midpoint( - p_lat[distinct_rows[i]], p_lon[leftmost_col[i]], - p_lat[distinct_rows[i]], p_lon[rightmost_col[i]], - mid_bot_lat, mid_bot_lon); - - T mid_top_lat; - T mid_top_lon; - geodesic_midpoint( - p_lat[distinct_rows[i+1]], p_lon[leftmost_col[i+1]], - p_lat[distinct_rows[i+1]], p_lon[rightmost_col[i+1]], - mid_top_lat, mid_top_lon); - - length_from_top += geodesic_distance( - mid_bot_lat, mid_bot_lon, mid_top_lat, mid_top_lon); - } - - // check the length criteria. - // TODO: if we are here the widtrh critera was not met - // so the following detection is based solely on the length? - if (length_from_top > river_length) - { - // AR - ar.min_width = static_cast(min_width); - ar.max_width = static_cast(max_width); - ar.length = static_cast(length_from_top); - return true; - } - - return false; -} - - - -// Junmin's function checkRightBoundary -// note: if land sea mask is not available land array -// must all be true. -template -bool river_end_criteria( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const std::vector &land, - const T *p_lat, - const T *p_lon, - T river_end_lat_low, - T river_end_lon_low, - T river_end_lat_high, - T river_end_lon_high, - atmospheric_river &ar) -{ - // locate component points within shoreline - // box - bool first_crossing = false; - bool event_detected = false; - - T top_lat = T(); - T top_lon = T(); - T bot_lat = T(); - T bot_lon = T(); - - std::vector right_bound_col_idx; - size_t count = con_comp_c.size(); - for (size_t i = 0; i < count; ++i) - { - T lat = p_lat[con_comp_r[i]]; - T lon = p_lon[con_comp_c[i]]; - - if ((lat >= river_end_lat_low) && (lat <= river_end_lat_high) - && (lon >= river_end_lon_low) && (lon <= river_end_lon_high)) - { - if (!event_detected) - event_detected = land[i]; - - if (!first_crossing) - { - first_crossing = true; - top_lat = lat; - top_lon = lon; - } - bot_lat = lat; - bot_lon = lon; - } - } - - ar.end_top_lat = top_lat; - ar.end_top_lon = top_lon; - ar.end_bot_lat = bot_lat; - ar.end_bot_lon = bot_lon; - - return event_detected; -} - -// Junmin's function -template -void classify_event( - const std::vector &con_comp_r, - const std::vector &con_comp_c, - const T *p_lat, - const T *p_lon, - T start_lat, - T start_lon, - atmospheric_river &ar) -{ - // classification determined by first detected point in event - // is closer to left or to bottom - T lat = p_lat[con_comp_r[0]]; - T lon = p_lon[con_comp_c[0]]; - - ar.pe = false; - if ((lat - start_lat) < (lon - start_lon)) - ar.pe = true; // PE -} - -/* -* The main function that checks whether an AR event exists in -* given sub-plane of data. This currently applies only to the Western -* coast of the USA. return true if an ar is found. -*/ -bool ar_detect( - const_p_teca_variant_array lat, - const_p_teca_variant_array lon, - const_p_teca_variant_array land_sea_mask, - p_teca_unsigned_int_array con_comp, - unsigned long n_comp, - double river_start_lat, - double river_start_lon, - double river_end_lat_low, - double river_end_lon_low, - double river_end_lat_high, - double river_end_lon_high, - double percent_in_mesh, - double river_width, - double river_length, - double land_threshold_low, - double land_threshold_high, - atmospheric_river &ar) -{ - NESTED_TEMPLATE_DISPATCH_FP( - const teca_variant_array_impl, - lat.get(), - 1, - - NESTED_TEMPLATE_DISPATCH( - const teca_variant_array_impl, - land_sea_mask.get(), - 2, - - const NT1 *p_lat = dynamic_cast(lat.get())->get(); - const NT1 *p_lon = dynamic_cast(lon.get())->get(); - - const NT2 *p_land_sea_mask - = dynamic_cast(land_sea_mask.get())->get(); - - NT1 start_lat = static_cast(river_start_lat); - NT1 start_lon = static_cast(river_start_lon); - NT1 end_lat_low = static_cast(river_end_lat_low); - NT1 end_lon_low = static_cast(river_end_lon_low); - NT1 end_lat_high = static_cast(river_end_lat_high); - NT1 end_lon_high = static_cast(river_end_lon_high); - - unsigned long num_rows = lat->size(); - unsigned long num_cols = lon->size(); - - // # in PE is % of points in regious mesh - unsigned long num_rc = num_rows*num_cols; - unsigned long thr_count = num_rc*percent_in_mesh/100.0; - - unsigned int *p_labels = con_comp->get(); - for (unsigned int i = 1; i <= n_comp; ++i) - { - // for all discrete connected component labels - // verify if there exists an AR - std::vector con_comp_r; - std::vector con_comp_c; - std::vector land; - - for (unsigned long r = 0, q = 0; r < num_rows; ++r) - { - for (unsigned long c = 0; c < num_cols; ++c, ++q) - { - if (p_labels[q] == i) - { - // gather points of this connected component - con_comp_r.push_back(r); - con_comp_c.push_back(c); - - // identify them as land or not - land.push_back( - (p_land_sea_mask[q] >= land_threshold_low) - && (p_land_sea_mask[q] < land_threshold_high)); - } - } - } - - // check for ar criteria - unsigned long count = con_comp_r.size(); - if ((count > thr_count) - && river_end_criteria( - con_comp_r, con_comp_c, land, - p_lat, p_lon, - end_lat_low, end_lon_low, - end_lat_high, end_lon_high, - ar) - && river_geometric_criteria( - con_comp_r, con_comp_c, p_lat, p_lon, - river_length, river_width, ar)) - { - // determine if PE or AR - classify_event( - con_comp_r, con_comp_c, p_lat, p_lon, - start_lat, start_lon, ar); - return true; - } - } - ) - ) - return false; -} - -#if TECA_DEBUG > 0 -// helper to dump a dataset for debugging -int write_mesh( - const const_p_teca_cartesian_mesh &mesh, - const const_p_teca_variant_array &vapor, - const const_p_teca_variant_array &thresh, - const const_p_teca_variant_array &ccomp, - const const_p_teca_variant_array &lsmask, - const std::string &file_name) -{ - p_teca_cartesian_mesh m = teca_cartesian_mesh::New(); - m->copy_metadata(mesh); - - p_teca_array_collection pac = m->get_point_arrays(); - pac->append("vapor", std::const_pointer_cast(vapor)); - pac->append("thresh", std::const_pointer_cast(thresh)); - pac->append("ccomp", std::const_pointer_cast(ccomp)); - pac->append("lsmask", std::const_pointer_cast(lsmask)); - - p_teca_programmable_algorithm s = teca_programmable_algorithm::New(); - s->set_name("serve_mesh"); - s->set_number_of_input_connections(0); - s->set_number_of_output_ports(1); - s->set_execute_callback( - [m] (unsigned int, const std::vector &, - const teca_metadata &) -> const_p_teca_dataset { return m; } - ); - - p_teca_cartesian_mesh_writer w - = teca_cartesian_mesh_writer::New(); - - w->set_file_name(file_name); - w->set_input_connection(s->get_output_port()); - w->update(); - - return 0; -} -#endif diff --git a/alg/teca_ar_detect.h b/alg/teca_ar_detect.h deleted file mode 100644 index 11ef0f199..000000000 --- a/alg/teca_ar_detect.h +++ /dev/null @@ -1,136 +0,0 @@ -#ifndef teca_ar_detect_h -#define teca_ar_detect_h - -#include "teca_shared_object.h" -#include "teca_algorithm.h" -#include "teca_metadata.h" - -#include -#include - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_ar_detect) - -/** -Suren and Junmin's atmospheric river detector. - -The algorithm searches for atmospheric rivers that -end on the California coast in water vapor data over -a specific subset of the input data. A river is detected -based on it's length, width, and percent area of the -search space. The algorithm can optionally use a -land-sea mask to increase accuracy of the California -coast. Without the land-sea mask a box is used. -*/ -class teca_ar_detect : public teca_algorithm -{ -public: - TECA_ALGORITHM_STATIC_NEW(teca_ar_detect) - TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_ar_detect) - TECA_ALGORITHM_CLASS_NAME(teca_ar_detect) - ~teca_ar_detect(); - - // report/initialize to/from Boost program options - // objects. - TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() - TECA_SET_ALGORITHM_PROPERTIES() - - // set/get the name of the integrated water vapor variable - TECA_ALGORITHM_PROPERTY(std::string, water_vapor_variable) - - // set/get threshold on water vapor variable used - // to segment the data - TECA_ALGORITHM_PROPERTY(double, low_water_vapor_threshold) - TECA_ALGORITHM_PROPERTY(double, high_water_vapor_threshold) - - // set/get the region of interest in lat lon coordinate system - // defaults are 19 56 180 250 - TECA_ALGORITHM_PROPERTY(double, search_lat_low) - TECA_ALGORITHM_PROPERTY(double, search_lon_low) - TECA_ALGORITHM_PROPERTY(double, search_lat_high) - TECA_ALGORITHM_PROPERTY(double, search_lon_high) - - // set/get the river source region in lat lon coordinate system - // defaults are 18 180 - TECA_ALGORITHM_PROPERTY(double, river_start_lat_low) - TECA_ALGORITHM_PROPERTY(double, river_start_lon_low) - - // set/get the river ladfall region in lat lon coordinate system - // defaults are 29 233 56 238 - TECA_ALGORITHM_PROPERTY(double, river_end_lat_low) - TECA_ALGORITHM_PROPERTY(double, river_end_lon_low) - TECA_ALGORITHM_PROPERTY(double, river_end_lat_high) - TECA_ALGORITHM_PROPERTY(double, river_end_lon_high) - - // set/get the area as a percent of the search space that - // a potential river must occupy - TECA_ALGORITHM_PROPERTY(double, percent_in_mesh) - - // set/get the minimum river width and length. defaults - // are 1250 2000 - TECA_ALGORITHM_PROPERTY(double, river_width) - TECA_ALGORITHM_PROPERTY(double, river_length) - - // set/get the land-sea mask variable. this array - // will be used to identify land from ocean using - // land_threshold properties. - TECA_ALGORITHM_PROPERTY(std::string, land_sea_mask_variable) - - // set/get the land classification range [low high). defaults - // are [1.0 DOUBLE_MAX) - TECA_ALGORITHM_PROPERTY(double, land_threshold_low) - TECA_ALGORITHM_PROPERTY(double, land_threshold_high) - - // send humand readable representation to the - // stream - virtual void to_stream(std::ostream &os) const override; - -protected: - teca_ar_detect(); - - // helper that computes the output extent - int get_active_extent( - p_teca_variant_array lat, - p_teca_variant_array lon, - std::vector &extent) const; - -private: - virtual - teca_metadata get_output_metadata( - unsigned int port, - const std::vector &input_md) override; - - virtual - std::vector get_upstream_request( - unsigned int port, - const std::vector &input_md, - const teca_metadata &request) override; - - virtual - const_p_teca_dataset execute( - unsigned int port, - const std::vector &input_data, - const teca_metadata &request) override; - -private: - std::string water_vapor_variable; - std::string land_sea_mask_variable; - double low_water_vapor_threshold; - double high_water_vapor_threshold; - double search_lat_low; - double search_lon_low; - double search_lat_high; - double search_lon_high; - double river_start_lat_low; - double river_start_lon_low; - double river_end_lat_low; - double river_end_lon_low; - double river_end_lat_high; - double river_end_lon_high; - double percent_in_mesh; - double river_width; - double river_length; - double land_threshold_low; - double land_threshold_high; -}; - -#endif diff --git a/alg/teca_bayesian_ar_detect.cxx b/alg/teca_bayesian_ar_detect.cxx index 8a7705a09..c14dd6f1f 100644 --- a/alg/teca_bayesian_ar_detect.cxx +++ b/alg/teca_bayesian_ar_detect.cxx @@ -608,8 +608,10 @@ void teca_bayesian_ar_detect::internals_t::clear() teca_bayesian_ar_detect::teca_bayesian_ar_detect() : min_component_area_variable("min_component_area"), min_ivt_variable("min_water_vapor"), - hwhm_latitude_variable("hwhm_latitude"), thread_pool_size(1), - verbose(0), internals(new internals_t) + hwhm_latitude_variable("hwhm_latitude"), + ar_probability_variable("ar_probability"), + thread_pool_size(1), + internals(new internals_t) { this->set_number_of_input_connections(1); this->set_number_of_output_ports(1); @@ -631,23 +633,25 @@ void teca_bayesian_ar_detect::get_properties_description( opts.add_options() TECA_POPTS_GET(std::string, prefix, ivt_variable, - "name of the water vapor variable (\"\")") + "Set the name of the integrated vaopr transport(IVT) variable to" + " compute AR probability from.") TECA_POPTS_GET(std::string, prefix, min_component_area_variable, - "name of the column in the parameter table containing the " - "component area threshold (\"min_component_area\")") + "Set the name of the column in the parameter table containing the " + "minimum feature area threshold.") TECA_POPTS_GET(std::string, prefix, min_ivt_variable, - "name of the column in the parameter table containing the " - "water vapor threshold (\"min_water_vapor\")") + "Set the name of the column in the parameter table containing the " + "minimum percentile IVT threshold.") TECA_POPTS_GET(std::string, prefix, hwhm_latitude_variable, - "name of the column in the parameter table containing the " - "half width at half max latitude (\"hwhm_latitude\")") + "Set the name of the column in the parameter table containing the " + "half width at half max latitude mask value.") + TECA_POPTS_GET(std::string, prefix, ar_probability_variable, + "Set the name of the variable to store the computed AR probability in.") TECA_POPTS_GET(int, prefix, thread_pool_size, - "number of threads to parallelize execution over (1)") - TECA_POPTS_GET(int, prefix, verbose, - "flag indicating diagnostic info should be displayed in " - "the terminal (0)") + "Set the number of threads to parallelize execution over.") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -655,10 +659,13 @@ void teca_bayesian_ar_detect::get_properties_description( void teca_bayesian_ar_detect::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, ivt_variable) TECA_POPTS_SET(opts, std::string, prefix, min_component_area_variable) TECA_POPTS_SET(opts, std::string, prefix, min_ivt_variable) TECA_POPTS_SET(opts, std::string, prefix, hwhm_latitude_variable) + TECA_POPTS_SET(opts, std::string, prefix, ar_probability_variable) TECA_POPTS_SET(opts, int, prefix, thread_pool_size) TECA_POPTS_SET(opts, int, prefix, verbose) } @@ -693,7 +700,10 @@ void teca_bayesian_ar_detect::set_thread_pool_size(int n) // -------------------------------------------------------------------------- unsigned int teca_bayesian_ar_detect::get_thread_pool_size() const noexcept { - return this->internals->queue->size(); + unsigned int n_threads = 0; + if (this->internals->queue) + n_threads = this->internals->queue->size(); + return n_threads; } // -------------------------------------------------------------------------- @@ -805,7 +815,7 @@ teca_metadata teca_bayesian_ar_detect::get_output_metadata( // report the variable that we compute, for each timestep from the // parameter tables. teca_metadata md(input_md[0]); - md.append("variables", std::string("ar_probability")); + md.append("variables", std::string(this->ar_probability_variable)); // add attributes to enable CF I/O teca_metadata atts; @@ -816,7 +826,7 @@ teca_metadata teca_bayesian_ar_detect::get_output_metadata( 0, "unitless", "posterior AR flag", "the posterior probability of the presence of an atmospheric river"); - atts.set("ar_probability", (teca_metadata)prob_atts); + atts.set(this->ar_probability_variable, (teca_metadata)prob_atts); unsigned long num_params = this->internals->parameter_table->get_number_of_rows(); @@ -875,7 +885,7 @@ std::vector teca_bayesian_ar_detect::get_upstream_request( arrays.insert(this->ivt_variable); // remove what we produce - arrays.erase("ar_probability"); + arrays.erase(this->ar_probability_variable); arrays.erase("ar_count"); arrays.erase("parameter_table_row"); @@ -1039,7 +1049,7 @@ const_p_teca_dataset teca_bayesian_ar_detect::execute( // set up the reduction which computes the average over runs of all control // parameter combinations provided in the parameter table ::parameter_table_reduction reduce(parameter_table_size, - "wv_cc", "ar_probability"); + "wv_cc", this->ar_probability_variable); p_teca_programmable_reduce pr = teca_programmable_reduce::New(); pr->set_name("parameter_table_reduce"); diff --git a/alg/teca_bayesian_ar_detect.h b/alg/teca_bayesian_ar_detect.h index c2805cf99..aecbd9411 100644 --- a/alg/teca_bayesian_ar_detect.h +++ b/alg/teca_bayesian_ar_detect.h @@ -10,31 +10,35 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_bayesian_ar_detect) -/// CASCADE BARD atmospheric river detector +/// The TECA BARD atmospheric river detector. /** -Given a point wise IVT (integrated vapor transport) field and a training -parameter table computes the point wise probability of an atmospheric river -using the CASCADE BARD algorithm. - -Required inputs: - - 1. IVT (integrated vapor transport) array on a Cartesian nesh. - 2. a compatible parameter table. columns of which are : min IVT, - component area, HWHM lattitude - -The names of the input varibale and columns can be specified at run time -through algorithm properties. - -Produces: - - A Cartesian mesh with probability of an AR stored in the point centered - array named "ar_probability". The diagnostic quantites "ar_count" amd - "parameter_table_row" are stored in information arrays. - -For more information see: - -Detection of Atmospheric Rivers with Inline Uncertainty Quantification: TECA-BARD v1.0 -O'Brien, T. A et al. Geoscientific Model Development, 2020 + * Given a point wise IVT (integrated vapor transport) field and a training + * parameter table computes the point wise probability of an atmospheric river + * using the TECA BARD algorithm. + * + * Required inputs: + * + * 1. IVT (integrated vapor transport) array on a Cartesian nesh. + * 2. a compatible parameter table. columns of which are : min IVT, + * component area, HWHM lattitude + * + * The names of the input varibale and columns can be specified at run time + * through algorithm properties. + * + * Produces: + * + * A Cartesian mesh with probability of an AR stored in the point centered + * array named "ar_probability". The diagnostic quantites "ar_count" amd + * "parameter_table_row" are stored in information arrays. + * + * For more information see: + * + * O’Brien, T. A., Risser, M. D., Loring, B., Elbashandy, A. A., Krishnan, H., + * Johnson, J., Patricola, C. M., O’Brien, J. P., Mahesh, A., Arriaga Ramirez, + * S., Rhoades, A. M., Charn, A., Inda Díaz, H., & Collins, W. D. (2020). + * Detection of atmospheric rivers with inline uncertainty quantification: + * TECA-BARD v1.0.1. Geoscientific Model Development, 13(12), 6131–6148. + * https://doi.org/10.5194/gmd-13-6131-2020 */ class teca_bayesian_ar_detect : public teca_algorithm { @@ -49,28 +53,54 @@ class teca_bayesian_ar_detect : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the name of the input array + /** @name ivt_variable + * Sets the name of the array containing the IVT field to detect ARs in. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, ivt_variable) + ///@} - // set the names of columns in the parameter table. + /** @name min_ivt_variable + * Set the names of the minimum IVT column in the parameter table. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, min_ivt_variable) - TECA_ALGORITHM_PROPERTY(std::string, min_component_area_variable) - TECA_ALGORITHM_PROPERTY(std::string, hwhm_latitude_variable) + ///@} - // flag indicating verbose terminal output is desired. - // default is 0 - TECA_ALGORITHM_PROPERTY(int, verbose) + /** @name min_component_area_variable + * Set the names of the minimum area column in the parameter table. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, min_component_area_variable) + ///@} - // set/get the number of threads in the pool. setting - // to -1 results in a thread per core factoring in all MPI - // ranks running on the node. the default is -1. + /** @name hwhm_latitude_variable + * Set the names of the HWHM column in the parameter table. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, hwhm_latitude_variable) + ///@} + + /** @name probability variable + * Set the name of the variable to store output probability as. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, ar_probability_variable) + ///@} + + /** Set the number of threads in the pool. Setting to -1 results in a + * thread per core factoring in all MPI ranks running on the node. the + * default is -1. + */ void set_thread_pool_size(int n_threads); + + /// Get the number of threads in the pool. unsigned int get_thread_pool_size() const noexcept; - // override the input connections because we are going to - // take the first input and use it to generate metadata. - // the second input then becomes the only one the pipeline - // knows about. + /** override the input connections because we are going to take the first + * input and use it to generate metadata. the second input then becomes + * the only one the pipeline knows about. + */ void set_input_connection(unsigned int id, const teca_algorithm_output_port &port) override; @@ -98,8 +128,8 @@ class teca_bayesian_ar_detect : public teca_algorithm std::string min_component_area_variable; std::string min_ivt_variable; std::string hwhm_latitude_variable; + std::string ar_probability_variable; int thread_pool_size; - int verbose; struct internals_t; internals_t *internals; diff --git a/alg/teca_bayesian_ar_detect_parameters.cxx b/alg/teca_bayesian_ar_detect_parameters.cxx index ca58d5c85..23511a112 100644 --- a/alg/teca_bayesian_ar_detect_parameters.cxx +++ b/alg/teca_bayesian_ar_detect_parameters.cxx @@ -973,9 +973,11 @@ void teca_bayesian_ar_detect_parameters::get_properties_description( opts.add_options() TECA_POPTS_GET(long, prefix, number_of_rows, - "the number of parameter table rows to serve (-1)") + "the number of parameter table rows to serve") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -983,6 +985,8 @@ void teca_bayesian_ar_detect_parameters::get_properties_description( void teca_bayesian_ar_detect_parameters::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, long, prefix, number_of_rows) } #endif diff --git a/alg/teca_bayesian_ar_detect_parameters.h b/alg/teca_bayesian_ar_detect_parameters.h index 7f5b1dc56..518305301 100644 --- a/alg/teca_bayesian_ar_detect_parameters.h +++ b/alg/teca_bayesian_ar_detect_parameters.h @@ -5,10 +5,10 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_bayesian_ar_detect_parameters) -/** -An algorithm that constructs and serves up the parameter -table needed to run the Bayesain AR detector -*/ +/** @brief + * An algorithm that constructs and serves up the parameter + * table needed to run the Bayesian AR detector. + */ class teca_bayesian_ar_detect_parameters : public teca_algorithm { public: @@ -22,12 +22,16 @@ class teca_bayesian_ar_detect_parameters : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // control the number of rows coppied into the table. The rows are - // copppied in sequential order starting from row zero. The default value - // of -1 is used to serve all rows. See also get_parameter_table_size. + /** @name number_of_rows + * control the number of rows copied into the table. The rows are copied + * in sequential order starting from row zero. The default value of -1 is + * used to serve all rows. See also get_parameter_table_size. + */ + ///@{ TECA_ALGORITHM_PROPERTY(long, number_of_rows) + ///@} - // return the number of rows in the internal parameter table. + /// return the number of rows in the internal parameter table. unsigned long get_parameter_table_size(); protected: diff --git a/alg/teca_binary_segmentation.h b/alg/teca_binary_segmentation.h index 983a497fb..138a28b03 100644 --- a/alg/teca_binary_segmentation.h +++ b/alg/teca_binary_segmentation.h @@ -10,17 +10,17 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_binary_segmentation) -/// an algorithm that computes a binary segmentation +/// An algorithm that computes a binary segmentation. /** -an algorithm that computes a binary segmentation for 1D, 2D, and 3D data. The -segmentation is computed using threshold operation where values in a range -(low, high] are in the segmentation (assigned 1). Values outside the range -are outside of the segmentation (assigned 0). - -The algorithm has 2 modes, BY_VALUE and BY_PERCENTILE. In the BY_VALUE mode, -the test for inclusion is applied on the raw data. In the BY_PERCENTILE mode -the range is given in percentiles and each data point is converted to a -percentile before applying the test for inclusion. + * an algorithm that computes a binary segmentation for 1D, 2D, and 3D data. + * The segmentation is computed using threshold operation where values in a + * range (low, high] are in the segmentation (assigned 1). Values outside the + * range are outside of the segmentation (assigned 0). + * + * The algorithm has 2 modes, BY_VALUE and BY_PERCENTILE. In the BY_VALUE mode, + * the test for inclusion is applied on the raw data. In the BY_PERCENTILE mode + * the range is given in percentiles and each data point is converted to a + * percentile before applying the test for inclusion. */ class teca_binary_segmentation : public teca_algorithm { @@ -31,26 +31,59 @@ class teca_binary_segmentation : public teca_algorithm ~teca_binary_segmentation(); // set the name of the output array to store the resulting segmentation in + /** @name segmentation_variable + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, segmentation_variable) + ///@} + // set extra metadata for the segmentation variable + /** @name segmentation_variable_attributes + */ + ///@{ TECA_ALGORITHM_PROPERTY(teca_metadata, segmentation_variable_attributes) + ///@} + - // set the name of the input array to segment + /** @name threshold_variable + * set the name of the input array to segment + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, threshold_variable) + ///@} - // Set the threshold range. The defaults are (-infinity, infinity]. + /** @name low_threshold_value + * Set the threshold range. The defaults are (-infinity, infinity]. + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, low_threshold_value) - TECA_ALGORITHM_PROPERTY(double, high_threshold_value) + ///@} - // Set the threshold mode. In BY_PERCENTILE mode low and high thresholds - // define the percentiles (0 to 100) between which data is in the - // segmentation. default is BY_VALUE. + /** @name high_threshold_value + * Set the threshold range. The defaults are (-infinity, infinity]. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(double, high_threshold_value) + ///@} + + /** @name threshold_mode + * Set the threshold mode. In BY_PERCENTILE mode low and high thresholds + * define the percentiles (0 to 100) between which data is in the + * segmentation. default is BY_VALUE. + */ + ///@{ + /// The threshold_mode modes enum {BY_VALUE=0, BY_PERCENTILE=1}; - TECA_ALGORITHM_PROPERTY(int, threshold_mode); + TECA_ALGORITHM_PROPERTY(int, threshold_mode) + + /// Set the threshold_mode to percentile. void set_threshold_by_percentile() { set_threshold_mode(BY_PERCENTILE); } + + /// Set the threshold_mode to value. void set_threshold_by_value() { set_threshold_mode(BY_VALUE); } + ///@} protected: teca_binary_segmentation(); diff --git a/alg/teca_cartesian_mesh_regrid.cxx b/alg/teca_cartesian_mesh_regrid.cxx index 632738efa..b8f90aa35 100644 --- a/alg/teca_cartesian_mesh_regrid.cxx +++ b/alg/teca_cartesian_mesh_regrid.cxx @@ -18,6 +18,28 @@ using std::endl; //#define TECA_DEBUG +// always use nearest neighbor interpolation for integers +// to avoid truncation errors. an alternative would be to +// implement rounding in the interpolator for integer types +template +int get_interpolation_mode(int desired_mode, + typename std::enable_if::value>::type* = 0) +{ + (void)desired_mode; + return teca_cartesian_mesh_regrid::nearest; +} + +// use the requested interpolation mode for floating point +// data +template +int get_interpolation_mode(int desired_mode, + typename std::enable_if::value>::type* = 0) +{ + return desired_mode; +} + + +// 3D template int interpolate(unsigned long target_nx, unsigned long target_ny, unsigned long target_nz, const NT1 *p_target_xc, const NT1 *p_target_yc, @@ -53,31 +75,100 @@ int interpolate(unsigned long target_nx, unsigned long target_ny, return 0; } +// 2D - x-y +template +int interpolate(unsigned long target_nx, unsigned long target_ny, + const NT1 *p_target_xc, const NT1 *p_target_yc, + NT3 *p_target_a, const NT2 *p_source_xc, + const NT2 *p_source_yc, const NT3 *p_source_a, + unsigned long source_ihi, unsigned long source_jhi, + unsigned long source_nx) +{ + interp_t f; + unsigned long q = 0; + for (unsigned long j = 0; j < target_ny; ++j) + { + NT2 ty = static_cast(p_target_yc[j]); + for (unsigned long i = 0; i < target_nx; ++i, ++q) + { + NT2 tx = static_cast(p_target_xc[i]); + if (f(tx,ty, + p_source_xc, p_source_yc, + p_source_a, source_ihi, source_jhi, + source_nx, p_target_a[q])) + { + TECA_ERROR("failed to interpolate i=(" << i << ", " << j + << ") x=(" << tx << ", " << ty << ", " << ")") + return -1; + } + } + } + return 0; +} + template int interpolate(int mode, unsigned long target_nx, unsigned long target_ny, - unsigned long target_nz, const taget_coord_t *p_target_xc, const taget_coord_t *p_target_yc, - const taget_coord_t *p_target_zc, array_t *p_target_a, const source_coord_t *p_source_xc, - const source_coord_t *p_source_yc, const source_coord_t *p_source_zc, const array_t *p_source_a, - unsigned long source_ihi, unsigned long source_jhi, unsigned long source_khi, - unsigned long source_nx, unsigned long source_nxy) + unsigned long target_nz, const taget_coord_t *p_target_xc, + const taget_coord_t *p_target_yc, const taget_coord_t *p_target_zc, + array_t *p_target_a, const source_coord_t *p_source_xc, + const source_coord_t *p_source_yc, const source_coord_t *p_source_zc, + const array_t *p_source_a, unsigned long source_ihi, unsigned long source_jhi, + unsigned long source_khi, unsigned long source_nx, unsigned long source_ny, + unsigned long source_nz) { using nearest_interp_t = teca_coordinate_util::interpolate_t<0>; using linear_interp_t = teca_coordinate_util::interpolate_t<1>; - switch (mode) + unsigned long source_nxy = source_nx*source_ny; + + switch (get_interpolation_mode(mode)) { case teca_cartesian_mesh_regrid::nearest: - return interpolate( - target_nx, target_ny, target_nz, p_target_xc, p_target_yc, p_target_zc, - p_target_a, p_source_xc, p_source_yc, p_source_zc, p_source_a, - source_ihi, source_jhi, source_khi, source_nx, source_nxy); + { + if ((target_nz == 1) && (source_nz == 1)) + { + // 2D in the x-y plane + return interpolate( + target_nx, target_ny, p_target_xc, p_target_yc, + p_target_a, p_source_xc, p_source_yc, p_source_a, + source_ihi, source_jhi, source_nx); + } + else + { + // 3D + return interpolate( + target_nx, target_ny, target_nz, p_target_xc, + p_target_yc, p_target_zc, p_target_a, p_source_xc, + p_source_yc, p_source_zc, p_source_a, source_ihi, + source_jhi, source_khi, source_nx, source_nxy); + } break; + } case teca_cartesian_mesh_regrid::linear: - return interpolate( - target_nx, target_ny, target_nz, p_target_xc, p_target_yc, p_target_zc, - p_target_a, p_source_xc, p_source_yc, p_source_zc, p_source_a, - source_ihi, source_jhi, source_khi, source_nx, source_nxy); + { + if ((target_nz == 1) && (source_nz == 1)) + { + // 2D in the x-y plane + return interpolate( + target_nx, target_ny, p_target_xc, p_target_yc, + p_target_a, p_source_xc, p_source_yc, p_source_a, + source_ihi, source_jhi, source_nx); + } + else + { + // 3D + return interpolate( + target_nx, target_ny, target_nz, p_target_xc, + p_target_yc, p_target_zc, p_target_a, p_source_xc, + p_source_yc, p_source_zc, p_source_a, source_ihi, + source_jhi, source_khi, source_nx, source_nxy); + } break; + } } TECA_ERROR("invalid interpolation mode \"" << mode << "\"") @@ -86,6 +177,7 @@ int interpolate(int mode, unsigned long target_nx, unsigned long target_ny, + // -------------------------------------------------------------------------- teca_cartesian_mesh_regrid::teca_cartesian_mesh_regrid() : target_input(0), interpolation_mode(nearest) @@ -109,13 +201,15 @@ void teca_cartesian_mesh_regrid::get_properties_description( opts.add_options() TECA_POPTS_GET(int, prefix, target_input, - "select input connection that contains metadata (0)") - TECA_POPTS_GET(std::vector, prefix, arrays, - "list of arrays to move from source to target mesh ("")") + "select input connection that contains metadata") + TECA_POPTS_MULTI_GET(std::vector, prefix, arrays, + "list of arrays to move from source to target mesh") TECA_POPTS_GET(int, prefix, interpolation_mode, - "linear or nearest interpolation (1)") + "linear or nearest interpolation") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -123,6 +217,8 @@ void teca_cartesian_mesh_regrid::get_properties_description( void teca_cartesian_mesh_regrid::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, int, prefix, target_input) TECA_POPTS_SET(opts, std::vector, prefix, arrays) TECA_POPTS_SET(opts, int, prefix, interpolation_mode) @@ -276,7 +372,9 @@ std::vector teca_cartesian_mesh_regrid::get_upstream_request( else { if (teca_coordinate_util::bounds_to_extent(request_bounds, - target_x, target_y, target_z, target_extent)) + target_x, target_y, target_z, target_extent) || + teca_coordinate_util::validate_extent(target_x->size(), + target_y->size(), target_z->size(), target_extent, true)) { TECA_ERROR("invalid bounds requested [" << request_bounds[0] << ", " << request_bounds[1] << ", " << request_bounds[2] << ", " @@ -294,6 +392,24 @@ std::vector teca_cartesian_mesh_regrid::get_upstream_request( target_z->get(target_extent[4], target_bounds[4]); target_z->get(target_extent[5], target_bounds[5]); + // if the source is 2D, the cf_reader may have faked the vertical dimension. + // in that case, use the source's vertical coordinate in the requested bounds + teca_metadata source_coords; + p_teca_variant_array source_z; + + if (input_md[md_src].get("coordinates", source_coords) + || !(source_z = source_coords.get("z"))) + { + TECA_ERROR("failed to locate source mesh coordinates") + return up_reqs; + } + + if (source_z->size() == 1) + { + source_z->get(0, target_bounds[4]); + source_z->get(0, target_bounds[5]); + } + // send the target bounds to the source as well source_req.set("bounds", target_bounds, 6); @@ -301,10 +417,19 @@ std::vector teca_cartesian_mesh_regrid::get_upstream_request( up_reqs[md_tgt] = target_req; up_reqs[md_src] = source_req; +#ifdef TECA_DEBUG + std::cerr << "source request = "; + source_req.to_stream(std::cerr); + std::cerr << std::endl; + + std::cerr << "target request = "; + target_req.to_stream(std::cerr); + std::cerr << std::endl; +#endif + return up_reqs; } - // -------------------------------------------------------------------------- const_p_teca_dataset teca_cartesian_mesh_regrid::execute( unsigned int port, const std::vector &input_data, @@ -347,7 +472,7 @@ const_p_teca_dataset teca_cartesian_mesh_regrid::execute( // get the list of arrays to move std::vector req_arrays; - request.get("regrid_arrays", req_arrays); + request.get("arrays", req_arrays); // add any explicitly named std::copy(this->arrays.begin(), this->arrays.end(), @@ -362,7 +487,25 @@ const_p_teca_dataset teca_cartesian_mesh_regrid::execute( for (; it != end; ++it) { if (!target->get_point_arrays()->has(*it)) - source_arrays.push_back(*it); + { + if (source->get_point_arrays()->has(*it)) + { + source_arrays.push_back(*it); + } + else + { + TECA_ERROR("Array \"" << *it + << "\" is neither present in source or target mesh") + return nullptr; + } + } + } + + // catch a user error + if (!source_arrays.size() && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_WARNING("No arrays will be interpolated") } // move the arrays @@ -384,7 +527,6 @@ const_p_teca_dataset teca_cartesian_mesh_regrid::execute( unsigned long source_nx = source_xc->size(); unsigned long source_ny = source_yc->size(); unsigned long source_nz = source_zc->size(); - unsigned long source_nxy = source_nx*source_ny; unsigned long source_ihi = source_nx - 1; unsigned long source_jhi = source_ny - 1; unsigned long source_khi = source_nz - 1; @@ -392,20 +534,20 @@ const_p_teca_dataset teca_cartesian_mesh_regrid::execute( NESTED_TEMPLATE_DISPATCH_FP( const teca_variant_array_impl, target_xc.get(), - 1, + _TGT, - const NT1 *p_target_xc = std::dynamic_pointer_cast(target_xc)->get(); - const NT1 *p_target_yc = std::dynamic_pointer_cast(target_yc)->get(); - const NT1 *p_target_zc = std::dynamic_pointer_cast(target_zc)->get(); + const NT_TGT *p_target_xc = std::dynamic_pointer_cast(target_xc)->get(); + const NT_TGT *p_target_yc = std::dynamic_pointer_cast(target_yc)->get(); + const NT_TGT *p_target_zc = std::dynamic_pointer_cast(target_zc)->get(); NESTED_TEMPLATE_DISPATCH_FP( const teca_variant_array_impl, source_xc.get(), - 2, + _SRC, - const NT2 *p_source_xc = std::dynamic_pointer_cast(source_xc)->get(); - const NT2 *p_source_yc = std::dynamic_pointer_cast(source_yc)->get(); - const NT2 *p_source_zc = std::dynamic_pointer_cast(source_zc)->get(); + const NT_SRC *p_source_xc = std::dynamic_pointer_cast(source_xc)->get(); + const NT_SRC *p_source_yc = std::dynamic_pointer_cast(source_yc)->get(); + const NT_SRC *p_source_zc = std::dynamic_pointer_cast(source_zc)->get(); size_t n_arrays = source_arrays.size(); for (size_t i = 0; i < n_arrays; ++i) @@ -417,25 +559,37 @@ const_p_teca_dataset teca_cartesian_mesh_regrid::execute( NESTED_TEMPLATE_DISPATCH( teca_variant_array_impl, target_a.get(), - 3, + _DATA, - const NT3 *p_source_a = std::static_pointer_cast(source_a)->get(); - NT3 *p_target_a = std::static_pointer_cast(target_a)->get(); + const NT_DATA *p_source_a = std::static_pointer_cast(source_a)->get(); + NT_DATA *p_target_a = std::static_pointer_cast(target_a)->get(); if (interpolate(this->interpolation_mode, target_nx, target_ny, target_nz, p_target_xc, p_target_yc, p_target_zc, p_target_a, p_source_xc, p_source_yc, p_source_zc, p_source_a, source_ihi, source_jhi, - source_khi, source_nx, source_nxy)) + source_khi, source_nx, source_ny, source_nz)) { TECA_ERROR("Failed to move \"" << source_arrays[i] << "\"") return nullptr; } - - target_ac->set(source_arrays[i], target_a); ) + else + { + TECA_ERROR("Unsupported array type " << source_a->get_class_name()) } - ) + + target_ac->set(source_arrays[i], target_a); + } ) + else + { + TECA_ERROR("Unupported coordinate type " << source_xc->get_class_name()) + } + ) + else + { + TECA_ERROR("Unupported coordinate type " << target_xc->get_class_name()) + } return target; } diff --git a/alg/teca_cartesian_mesh_regrid.h b/alg/teca_cartesian_mesh_regrid.h index f9ae85da2..d67c5f7f0 100644 --- a/alg/teca_cartesian_mesh_regrid.h +++ b/alg/teca_cartesian_mesh_regrid.h @@ -4,26 +4,29 @@ #include "teca_shared_object.h" #include "teca_algorithm.h" #include "teca_metadata.h" -#include "teca_variant_array_fwd.h" +#include "teca_variant_array.h" #include #include TECA_SHARED_OBJECT_FORWARD_DECL(teca_cartesian_mesh_regrid) -/// transfer data between overlapping meshes of potentially different resolution -/** -an algorithm that transfers data between cartesian meshes defined in the same -world coordinate system but potentially different resolutions. nearest or -linear interpolation are supported. - -By default the first input is the target mesh. the second input is the source -mesh. This can be changed by setting the target_input property. - -the arrays to move from source to target can be selected using add_array api or -in the request key regrid_source_arrays. this is a spatial regriding operation -for temporal regriding see teca_mesh_temporal_regrid. -*/ +/** @brief + * Transfers data between spatially overlapping meshes of potentially different + * resolutions. + * + * @details + * an algorithm that transfers data between cartesian meshes defined in the + * same world coordinate system but potentially different resolutions. nearest + * or linear interpolation are supported. + * + * By default the first input is the target mesh. the second input is the + * source mesh. This can be changed by setting the target_input property. + * + * the arrays to move from source to target can be selected using add_array api + * or in the request key "arrays". this is a spatial regriding operation for + * temporal regriding see teca_mesh_temporal_regrid. + */ class teca_cartesian_mesh_regrid : public teca_algorithm { public: @@ -37,23 +40,33 @@ class teca_cartesian_mesh_regrid : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the list of arrays to move from the source - // to the target + /** @name array + * set the list of arrays to move from the source to the target + */ + //@{ TECA_ALGORITHM_VECTOR_PROPERTY(std::string, array) + //@} - // set the input connection from which metadata such as arrays - // and time steps are taken from. + /** @name target_input + * set the input connection which provides the output geometry. + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, target_input) - - // set the interpolation mode used in transfering - // data between meshes of differing resolution. - // in nearest mode value at the nearest grid point - // is used, in linear mode bi/tri linear interpolation - // is used. + ///@} + + /** @name interpolation_mode + * set the interpolation mode used in transfering data between meshes of + * differing resolution. in nearest mode value at the nearest grid point + * is used, in linear mode bi/tri linear interpolation is used. + */ + //@{ enum {nearest=0, linear=1}; TECA_ALGORITHM_PROPERTY(int, interpolation_mode) void set_interpolation_mode_nearest(){ interpolation_mode = nearest; } void set_interpolation_mode_linear(){ interpolation_mode = linear; } + //@} + + protected: teca_cartesian_mesh_regrid(); diff --git a/alg/teca_cartesian_mesh_source.cxx b/alg/teca_cartesian_mesh_source.cxx index 7946d188f..2139a9864 100644 --- a/alg/teca_cartesian_mesh_source.cxx +++ b/alg/teca_cartesian_mesh_source.cxx @@ -26,12 +26,19 @@ struct teca_cartesian_mesh_source::internals_t // the world space [x0 x1 y0 y1 z0 z1 t0 t1] generate // equally spaced coordinate axes x,y,z,t static - void initialize_axes(int type_code, unsigned long *extent, - double *bounds, p_teca_variant_array &x_axis, + void initialize_axes(int type_code, const unsigned long *extent, + const double *bounds, p_teca_variant_array &x_axis, p_teca_variant_array &y_axis, p_teca_variant_array &z_axis, p_teca_variant_array &t_axis); + static + void initialize_axes(int type_code, const unsigned long *extent, + const double *bounds, p_teca_variant_array &x_axis, + p_teca_variant_array &y_axis, p_teca_variant_array &z_axis); + + // cached metadata teca_metadata metadata; + p_teca_variant_array t_axis; }; @@ -45,21 +52,25 @@ void teca_cartesian_mesh_source::internals_t::initialize_axis( unsigned long nx = i1 - i0 + 1; x->resize(nx); + num_t *px = x->get(); + // avoid divide by zero if (nx < 2) + { + px[0] = x0; return; + } num_t dx = (x1 - x0)/(nx - 1l); num_t xx = x0 + i0*dx; - num_t *px = x->get(); for (unsigned long i = 0; i < nx; ++i) px[i] = xx + dx*i; } // -------------------------------------------------------------------------- void teca_cartesian_mesh_source::internals_t::initialize_axes(int type_code, - unsigned long *extent, double *bounds, p_teca_variant_array &x_axis, + const unsigned long *extent, const double *bounds, p_teca_variant_array &x_axis, p_teca_variant_array &y_axis, p_teca_variant_array &z_axis, p_teca_variant_array &t_axis) { @@ -86,6 +97,30 @@ void teca_cartesian_mesh_source::internals_t::initialize_axes(int type_code, ) } +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::internals_t::initialize_axes(int type_code, + const unsigned long *extent, const double *bounds, p_teca_variant_array &x_axis, + p_teca_variant_array &y_axis, p_teca_variant_array &z_axis) +{ + // gernate equally spaced coordinate axes x,y,z,t + x_axis = teca_variant_array_factory::New(type_code); + y_axis = x_axis->new_instance(); + z_axis = x_axis->new_instance(); + + TEMPLATE_DISPATCH(teca_variant_array_impl, + x_axis.get(), + + internals_t::initialize_axis(std::static_pointer_cast(x_axis), + extent[0], extent[1], bounds[0], bounds[1]); + + internals_t::initialize_axis(std::static_pointer_cast(y_axis), + extent[2], extent[3], bounds[2], bounds[3]); + + internals_t::initialize_axis(std::static_pointer_cast(z_axis), + extent[4], extent[5], bounds[4], bounds[5]); + ) +} + // -------------------------------------------------------------------------- @@ -93,9 +128,7 @@ teca_cartesian_mesh_source::teca_cartesian_mesh_source() : coordinate_type_code(teca_variant_array_code::get()), field_type_code(teca_variant_array_code::get()), x_axis_variable("lon"), y_axis_variable("lat"), z_axis_variable("plev"), - t_axis_variable("time"), x_axis_units("degrees_east"), - y_axis_units("degrees_north"), z_axis_units("pascals"), - calendar("Gregorian"), time_units("seconds since 1970-01-01 00:00:00"), + t_axis_variable("time"), whole_extents{0l, 359l, 0l, 179l, 0l, 0l, 0l, 0l}, bounds{0., 360, -90., 90., 0., 0., 0., 0.}, internals(new internals_t) @@ -129,7 +162,6 @@ void teca_cartesian_mesh_source::set_properties(const std::string &prefix, } #endif - // -------------------------------------------------------------------------- void teca_cartesian_mesh_source::set_modified() { @@ -143,11 +175,311 @@ void teca_cartesian_mesh_source::set_modified() void teca_cartesian_mesh_source::clear_cached_metadata() { this->internals->metadata.clear(); + teca_algorithm::set_modified(); +} + + + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_x_axis_variable(const std::string &name) +{ + this->x_axis_variable = name; + this->x_axis_attributes.clear(); + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_x_axis_variable(const std::string &name, + const teca_metadata &atts) +{ + this->x_axis_variable = name; + this->x_axis_attributes = atts; + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_x_axis_variable(const teca_metadata &md) +{ + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + if (coords.get("x_variable", this->x_axis_variable)) + return -1; + + teca_metadata atts; + if (md.get("attributes", atts)) + return -1; + + if (atts.get(this->x_axis_variable, this->x_axis_attributes)) + return -1; + + return 0; +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_y_axis_variable(const std::string &name) +{ + this->y_axis_variable = name; + this->y_axis_attributes.clear(); + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_y_axis_variable(const std::string &name, + const teca_metadata &atts) +{ + this->y_axis_variable = name; + this->y_axis_attributes = atts; + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_y_axis_variable(const teca_metadata &md) +{ + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + if (coords.get("y_variable", this->y_axis_variable)) + return -1; + + teca_metadata atts; + if (md.get("attributes", atts)) + return -1; + + if (atts.get(this->y_axis_variable, this->y_axis_attributes)) + return -1; + + return 0; +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_z_axis_variable(const std::string &name) +{ + this->z_axis_variable = name; + this->z_axis_attributes.clear(); + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_z_axis_variable(const std::string &name, + const teca_metadata &atts) +{ + this->z_axis_variable = name; + this->z_axis_attributes = atts; + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_z_axis_variable(const teca_metadata &md) +{ + // get coordinates and attributes, fail if either are missing + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + if (coords.get("x_variable", this->z_axis_variable)) + return -1; + + teca_metadata atts; + if (md.get("attributes", atts)) + return -1; + + if (atts.get(this->z_axis_variable, this->z_axis_attributes)) + return -1; + + return 0; +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_t_axis_variable(const std::string &name) +{ + this->t_axis_variable = name; + this->t_axis_attributes.clear(); + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_calendar( + const std::string &calendar, const std::string &units) +{ + this->t_axis_attributes.clear(); + this->t_axis_attributes.set("calendar", calendar); + this->t_axis_attributes.set("units", units); + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_t_axis_variable(const std::string &name, + const teca_metadata &atts) +{ + this->t_axis_variable = name; + this->t_axis_attributes = atts; + teca_algorithm::set_modified(); +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_t_axis_variable(const teca_metadata &md) +{ + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + if (coords.get("t_variable", this->t_axis_variable)) + return -1; + + teca_metadata atts; + if (md.get("attributes", atts)) + return -1; + + if (atts.get(this->t_axis_variable, this->t_axis_attributes)) + return -1; + + return 0; +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_t_axis(const teca_metadata &md) +{ + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + this->internals->t_axis = coords.get("t"); + + return 0; +} + +// -------------------------------------------------------------------------- +void teca_cartesian_mesh_source::set_t_axis(const p_teca_variant_array &t) +{ + this->internals->t_axis = t; +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_output_metadata(const teca_metadata &md) +{ + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + teca_metadata atts; + if (md.get("attributes", atts)) + return -1; + + // get the coordinate axes. + const_p_teca_variant_array x = coords.get("x"); + const_p_teca_variant_array y = coords.get("y"); + const_p_teca_variant_array z = coords.get("z"); + const_p_teca_variant_array t = coords.get("t"); + + // because of assumptions made in execute, all must be provided + if (!x || !y || !z || !t) + return -1; + + unsigned long nx = x->size(); + unsigned long ny = y->size(); + unsigned long nz = z->size(); + unsigned long nxyz = nx*ny*nz; + + // clear out any variables, and replace with those that we provide. + std::vector vars; + std::vector::iterator it = this->field_generators.begin(); + std::vector::iterator end = this->field_generators.end(); + for (; it != end; ++it) + { + vars.push_back(it->name); + + // correct size + teca_metadata var_atts = it->attributes; + var_atts.set("size", nxyz); + + atts.set(it->name, var_atts); + } + + // copy the metadata + this->set_modified(); + + this->internals->metadata = md; + this->internals->metadata.set("variables", vars); + + return 0; +} + +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_spatial_extents(const teca_metadata &md, + bool three_d) +{ + // get coordinates and attributes, fail if either are missing + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + teca_metadata attributes; + if (md.get("attributes", attributes)) + return -1; + + // get the coordinate axes + p_teca_variant_array x = coords.get("x"); + p_teca_variant_array y = coords.get("y"); + p_teca_variant_array z = coords.get("z"); + + // verify + if (!x || !y || (three_d && !z)) + return -1; + + // set the extents + this->whole_extents[0] = 0; + this->whole_extents[1] = x->size() - 1; + this->whole_extents[2] = 0; + this->whole_extents[3] = y->size() - 1; + this->whole_extents[4] = 0; + this->whole_extents[5] = three_d ? z->size() - 1 : 0; + + return 0; +} +// -------------------------------------------------------------------------- +int teca_cartesian_mesh_source::set_spatial_bounds(const teca_metadata &md, + bool three_d) +{ + // get coordinates and attributes, fail if either are missing + teca_metadata coords; + if (md.get("coordinates", coords)) + return -1; + + teca_metadata attributes; + if (md.get("attributes", attributes)) + return -1; + + // get the coordinate axes + p_teca_variant_array x = coords.get("x"); + p_teca_variant_array y = coords.get("y"); + p_teca_variant_array z = coords.get("z"); + + // verify + if (!x || !y || (three_d && !z)) + return -1; + + // get the bounds + x->get(0lu, this->bounds[0]); + x->get(x->size() - 1lu, this->bounds[1]); + y->get(0lu, this->bounds[2]); + y->get(y->size() - 1lu, this->bounds[3]); + z->get(0lu, this->bounds[4]); + + unsigned long khi = three_d ? z->size() - 1lu : 0lu; + z->get(khi, this->bounds[5]); + + // set the coordinate type + this->set_coordinate_type_code(x->type_code()); + + return 0; } // -------------------------------------------------------------------------- void teca_cartesian_mesh_source::append_field_generator( - const std::string &name, const teca_array_attributes &atts, + const std::string &name, const teca_metadata &atts, field_generator_callback &callback) { this->append_field_generator({name, atts, callback}); @@ -181,59 +513,59 @@ teca_metadata teca_cartesian_mesh_source::get_output_metadata( // generate cooridnate axes p_teca_variant_array x_axis, y_axis, z_axis, t_axis; - internals_t::initialize_axes(this->coordinate_type_code, - this->whole_extents.data(), this->bounds.data(), x_axis, - y_axis, z_axis, t_axis); + if (this->internals->t_axis) + { + // generate x,y,z axes but use cached time axis + internals_t::initialize_axes(this->coordinate_type_code, + this->whole_extents.data(), this->bounds.data(), x_axis, + y_axis, z_axis); - size_t nx = this->whole_extents[1] - this->whole_extents[0] + 1; - size_t ny = this->whole_extents[3] - this->whole_extents[2] + 1; - size_t nz = this->whole_extents[5] - this->whole_extents[4] + 1; - size_t nt = this->whole_extents[7] - this->whole_extents[6] + 1; - size_t nxyz = nx*ny*nz; + t_axis = this->internals->t_axis; + } + else + { + // generate x,y,z and t axes + internals_t::initialize_axes(this->coordinate_type_code, + this->whole_extents.data(), this->bounds.data(), x_axis, + y_axis, z_axis, t_axis); + } - std::string x_ax_var_name = (this->x_axis_variable.empty() ? "x" : this->x_axis_variable); - std::string y_ax_var_name = (this->y_axis_variable.empty() ? "y" : this->y_axis_variable); - std::string z_ax_var_name = (this->z_axis_variable.empty() ? "z" : this->z_axis_variable); - std::string t_ax_var_name = (this->t_axis_variable.empty() ? "t" : this->t_axis_variable); + size_t nx = x_axis->size(); + size_t ny = y_axis->size(); + size_t nz = z_axis->size(); + size_t nt = t_axis->size(); // construct attributes - teca_metadata x_atts; - x_atts.set("units", (this->x_axis_units.empty() ? "meters" : this->x_axis_units)); - x_atts.set("type_code", this->coordinate_type_code); + teca_metadata x_atts = this->x_axis_attributes; + x_atts.set("type_code", x_axis->type_code()); x_atts.set("size", nx); - teca_metadata y_atts; - y_atts.set("units", (this->y_axis_units.empty() ? "meters" : this->y_axis_units)); - y_atts.set("type_code", this->coordinate_type_code); + teca_metadata y_atts = this->y_axis_attributes; + y_atts.set("type_code", y_axis->type_code()); y_atts.set("size", ny); - teca_metadata z_atts; - z_atts.set("units", (this->z_axis_units.empty() ? "meters" : this->z_axis_units)); - z_atts.set("type_code", this->coordinate_type_code); + teca_metadata z_atts = this->z_axis_attributes; + z_atts.set("type_code", z_axis->type_code()); z_atts.set("size", nz); - teca_metadata t_atts; - t_atts.set("units", (this->time_units.empty() ? - "seconds since 1970-01-01 00:00:00" : this->time_units)); - - t_atts.set("calendar", (this->calendar.empty() ? - "standard" : this->calendar)); - - t_atts.set("type_code", this->coordinate_type_code); + teca_metadata t_atts = this->t_axis_attributes; + t_atts.set("type_code", t_axis->type_code()); t_atts.set("size", nt); teca_metadata atts; - atts.set(x_ax_var_name, x_atts); - atts.set(y_ax_var_name, y_atts); - atts.set(z_ax_var_name, z_atts); - atts.set(t_ax_var_name, t_atts); + atts.set(this->x_axis_variable, x_atts); + atts.set(this->y_axis_variable, y_atts); + atts.set(this->z_axis_variable, z_atts); + + if (!this->t_axis_variable.empty()) + atts.set(this->t_axis_variable, t_atts); // construct dataset metadata teca_metadata coords; - coords.set("x_variable", x_ax_var_name); - coords.set("y_variable", y_ax_var_name); - coords.set("z_variable", z_ax_var_name); - coords.set("t_variable", t_ax_var_name); + coords.set("x_variable", this->x_axis_variable); + coords.set("y_variable", this->y_axis_variable); + coords.set("z_variable", this->z_axis_variable); + coords.set("t_variable", this->t_axis_variable); coords.set("x", x_axis); coords.set("y", y_axis); @@ -243,6 +575,7 @@ teca_metadata teca_cartesian_mesh_source::get_output_metadata( this->internals->metadata.set("whole_extent", this->whole_extents); this->internals->metadata.set("coordinates", coords); + size_t nxyz = nx*ny*nz; std::vector vars; std::vector::iterator it = this->field_generators.begin(); std::vector::iterator end = this->field_generators.end(); @@ -251,18 +584,24 @@ teca_metadata teca_cartesian_mesh_source::get_output_metadata( vars.push_back(it->name); // correct size - teca_array_attributes var_atts = it->attributes; - var_atts.size = nxyz; + teca_metadata var_atts = it->attributes; + var_atts.set("size", nxyz); - atts.set(it->name, teca_metadata(var_atts)); + atts.set(it->name, var_atts); } this->internals->metadata.set("variables", vars); this->internals->metadata.set("attributes", atts); - this->internals->metadata.set("number_of_time_steps", t_axis->size()); - this->internals->metadata.set("index_initializer_key", std::string("number_of_time_steps")); - this->internals->metadata.set("index_request_key", std::string("time_step")); + // setup the execution control keys + this->internals->metadata.set("number_of_time_steps", + t_axis->size()); + + this->internals->metadata.set("index_initializer_key", + std::string("number_of_time_steps")); + + this->internals->metadata.set("index_request_key", + std::string("time_step")); return this->internals->metadata; } @@ -320,25 +659,44 @@ const_p_teca_dataset teca_cartesian_mesh_source::execute(unsigned int port, { // bounds key was present, convert the bounds to an // an extent that covers them. - if (teca_coordinate_util::bounds_to_extent( - req_bounds, in_x, in_y, in_z, req_extent)) + if (teca_coordinate_util::bounds_to_extent(req_bounds, + in_x, in_y, in_z, req_extent) || + teca_coordinate_util::validate_extent(in_x->size(), + in_y->size(), in_z->size(), req_extent, true)) { TECA_ERROR("invalid bounds requested.") return nullptr; } } - // get the timestep - unsigned long time_step = 0; - if (request.get("time_step", time_step)) + // get the timestep, no matter what the key is named we treat it as + // a time step. this is to support metadata provided by another source + // eg. a different reader. + std::string request_key; + if (request.get("index_request_key", request_key)) + { + TECA_ERROR("Request is missing the \"index_request_key\"") + return nullptr; + } + + unsigned long req_index = 0; + if (request.get(request_key, req_index)) + { + TECA_ERROR("Request is missing \"" << request_key << "\"") + return nullptr; + } + + // check that the we have a time value for the requested index. + if (req_index >= in_t->size()) { - TECA_ERROR("Request is missing time_step") + TECA_ERROR("The requested index " << req_index + << " is out of bounds [0, " << in_t->size() << "]") return nullptr; } // get the time double t = 0.; - in_t->get(time_step, t); + in_t->get(req_index, t); // slice axes on the requested extent p_teca_variant_array out_x = in_x->new_copy(req_extent[0], req_extent[1]); @@ -351,22 +709,29 @@ const_p_teca_dataset teca_cartesian_mesh_source::execute(unsigned int port, std::string x_variable = this->x_axis_variable.empty() ? "x" : this->x_axis_variable; std::string y_variable = this->y_axis_variable.empty() ? "y" : this->y_axis_variable; std::string z_variable = this->z_axis_variable.empty() ? "z" : this->z_axis_variable; - std::string t_variable = this->t_axis_variable.empty() ? "t" : this->t_axis_variable; mesh->set_x_coordinates(x_variable, out_x); mesh->set_y_coordinates(y_variable, out_y); mesh->set_z_coordinates(z_variable, out_z); + // get the calendar + std::string calendar; + std::string units; + teca_metadata atts; + this->internals->metadata.get("attributes", atts); + atts.get("calendar", calendar); + atts.get("units", units); + // set metadata mesh->set_whole_extent(md_whole_extent); mesh->set_extent(req_extent); - mesh->set_time_step(time_step); + mesh->set_time_step(req_index); mesh->set_time(t); - mesh->set_calendar(this->calendar); - mesh->set_time_units(this->time_units); + mesh->set_calendar(calendar); + mesh->set_time_units(units); teca_metadata &mesh_md = mesh->get_metadata(); - mesh_md.set("index_request_key", std::string("time_step")); + mesh_md.set("index_request_key", request_key); // generate fields over the requested subset std::vector::iterator it = this->field_generators.begin(); @@ -378,8 +743,6 @@ const_p_teca_dataset teca_cartesian_mesh_source::execute(unsigned int port, } // pass the attributes - teca_metadata atts; - this->internals->metadata.get("attributes", atts); mesh_md.set("attributes", atts); return mesh; diff --git a/alg/teca_cartesian_mesh_source.h b/alg/teca_cartesian_mesh_source.h index be1e5039a..d357ad0ba 100644 --- a/alg/teca_cartesian_mesh_source.h +++ b/alg/teca_cartesian_mesh_source.h @@ -2,7 +2,7 @@ #define teca_cartesian_mesh_source_h #include "teca_algorithm.h" -#include "teca_array_attributes.h" +#include "teca_metadata.h" #include #include @@ -10,19 +10,27 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_cartesian_mesh_source) -// f(x, y, z, t) -// given spatial coordinate axes x,y,z and the time t, return the field +/** The signature of the callback used to specify user defined fields. + * f(x, y, z, t) -> w + * Given spatial coordinate axes x,y,z and the time t, return the + * 3D field w. + */ using field_generator_callback = std::function; +/** An object that bundles field name, the metadata attributes needed for I/O, + * and a field generator callback. Use this with append_field_generator + */ struct field_generator { std::string name; - teca_array_attributes attributes; + teca_metadata attributes; field_generator_callback generator; }; +using field_generator_t = field_generator; + inline bool operator==(const field_generator &l, const field_generator &r) { @@ -35,12 +43,25 @@ bool operator!=(const field_generator &l, const field_generator &r) return l.name != r.name; } -using field_generator_t = field_generator; - -/** -An algorithm that constructs and serves up a Cartesian mesh -of the specified dimensions. -*/ +/** @brief + * An algorithm that generates a teca_cartesian_mesh of the requested + * spatial and temporal dimensions with optional user defined fields. + * + * @details + * User defined fields are specified by passing callbacks and metadata + * via field_generator and append_field_generator + * + * The spatial and temporal dimensions are set by the combination of + * whole_extent and bounds. + * + * The names of coordinate axes are set by the combination + * of x_axis_variable, y_axis_variable, z_axis_variable, + * and t_axis_variable + * + * The units of the coordinate axes are set by the combination of + * x_axis_units, y_axis_units, z_axis_units, calendar, + * and time_units. + */ class teca_cartesian_mesh_source : public teca_algorithm { public: @@ -54,65 +75,211 @@ class teca_cartesian_mesh_source : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set/get the type code for generated coordinates. - // default is a 64 bit floating point type. Use - // teca_variant_array_type::get() to get specific type - // codes for C++ POD types NT. + /** @name coordinate_type_code + * set/get the type code for generated coordinates. The default is a 64 bit + * floating point type. Use teca_variant_array_code::get() to get + * specific type codes for C++ POD types NT. + */ + ///@{ TECA_ALGORITHM_PROPERTY(unsigned int, coordinate_type_code) + ///@} + + /** @name field_type_code + * set/get the type code for generated fields. The default is a 64 bit + * floating point type. Use teca_variant_array_code::get() to get + * specific type codes for C++ POD types NT. + */ + ///@{ TECA_ALGORITHM_PROPERTY(unsigned int, field_type_code) + ///@} - // set/get the global index space extent of the data. the extents are - // given by 8 values, 6 spatial plus 2 temporal, in the following order - // [i0 i1 j0 j1 k0 k1 q0 q1] - // this should be the same on all ranks elements. + /** @name whole_extent + * set/get the global index space extent of the data. the extents are + * given by 8 values, 6 spatial plus 2 temporal, in the following order + * [i0 i1 j0 j1 k0 k1 q0 q1] This should be the same on all ranks + */ + ///@{ TECA_ALGORITHM_VECTOR_PROPERTY(unsigned long, whole_extent) - // set/get the global bounds of the data. the bounds are 8 values 6 spatial - // plus 2 temporal in the following order. - // [x0 x1 y0 y1 z0 z1 t0 t1] - // this should be the same on all ranks elements. + /** Set the spatial extents from a metadata object following the + * conventions defined by the teca_cf_reader. If three_d is true the + * extents in the z-direction are copied, otherwise they are set to 0. + * Returns zero if successful and non-zero if the supplied metadata is + * missing any of the requisite information. + **/ + int set_spatial_extents(const teca_metadata &md, bool three_d = true); + ///@} + + /** @name bounds + * set/get the global bounds of the data. the bounds are 8 values 6 spatial + * plus 2 temporal in the following order. [x0 x1 y0 y1 z0 z1 t0 t1] + * this should be the same on all ranks. + */ + ///@{ TECA_ALGORITHM_VECTOR_PROPERTY(double, bound) - // set the variable to use for the coordinate axes. - // the defaults are: x => lon, y => lat, z = plev, - // t => time - TECA_ALGORITHM_PROPERTY(std::string, x_axis_variable) - TECA_ALGORITHM_PROPERTY(std::string, y_axis_variable) - TECA_ALGORITHM_PROPERTY(std::string, z_axis_variable) - TECA_ALGORITHM_PROPERTY(std::string, t_axis_variable) - - // set the units of spatial axes. The defaults are: - // degrees_east, degrees_north, and pressure_level - TECA_ALGORITHM_PROPERTY(std::string, x_axis_units) - TECA_ALGORITHM_PROPERTY(std::string, y_axis_units) - TECA_ALGORITHM_PROPERTY(std::string, z_axis_units) - - // number of time steps to generate - TECA_ALGORITHM_PROPERTY(std::string, calendar) - TECA_ALGORITHM_PROPERTY(std::string, time_units) - - // set the named callbacks to generate fields on the mesh. A callback - // function must have the signature f(x,y,z,t). - TECA_ALGORITHM_VECTOR_PROPERTY(field_generator_t, field_generator); - - // set a callback function f(x,y,z,t) that generates a field named name - // x,y,z are coordinate axes in variant arrays, t is the double precision - // time value. + /** Set the spatial bounds from a metadata object following the conventions + * defined by the teca_cf_reader. Returns zero if successful and + * non-zero if the supplied metadata is missing any of the requisite + * information. + */ + int set_spatial_bounds(const teca_metadata &md, bool three_d = true); + + ///@} + + /** @name x_axis_variable + * set the name of the variable to use for the coordinate axes and + * optionally associated attributes. + */ + ///@{ + /** set the name of the t_axis_variable */ + void set_x_axis_variable(const std::string &name); + + /** Set the name of the variable and its attributes. See + * teca_array_attributes for more information. + */ + void set_x_axis_variable(const std::string &name, const teca_metadata &atts); + + /** Set the name of the variable and its attributes using conventions + * defined by the teca_cf_reader. Returns zero if successful and + * non-zero if the supplied metadata is missing any of the requisite + * information. + */ + int set_x_axis_variable(const teca_metadata &md); + ///@} + + /** @name y_axis_variable + * set the name of the variable to use for the coordinate axes and + * optionally associated attributes. + */ + ///@{ + /** set the name of the y_axis_variable */ + void set_y_axis_variable(const std::string &name); + + /** Set the name of the variable and its attributes. See + * teca_array_attributes for more information. + */ + void set_y_axis_variable(const std::string &name, const teca_metadata &atts); + + /** Set the name of the variable and its attributes using conventions + * defined by the teca_cf_reader. Returns zero if successful and + * non-zero if the supplied metadata is missing any of the requisite + * information. + */ + int set_y_axis_variable(const teca_metadata &md); + ///@} + + /** @name z_axis_variable + * set the name of the variable to use for the coordinate axes and + * optionally associated attributes. + */ + ///@{ + /** set the name of the z_axis_variable */ + void set_z_axis_variable(const std::string &name); + + /** Set the name of the variable and its attributes. See + * teca_array_attributes for more information. + */ + void set_z_axis_variable(const std::string &name, const teca_metadata &atts); + + /** Set the name of the variable and its attributes using conventions + * defined by the teca_cf_reader. Returns zero if successful and + * non-zero if the supplied metadata is missing any of the requisite + * information. + */ + int set_z_axis_variable(const teca_metadata &md); + ///@} + + /** @name t_axis_variable + * set the name of the variable to use for the coordinate axes and + * optionally associated attributes. + */ + ///@{ + /** set the name of the t_axis_variable */ + void set_t_axis_variable(const std::string &name); + + /** Set the calendar, and time units of the t_axis_variable */ + void set_calendar(const std::string &calendar, const std::string &units); + + /** Set the name of the variable and its attributes. See + * teca_array_attributes for more information. + */ + void set_t_axis_variable(const std::string &name, + const teca_metadata &atts); + + /** Set the name of the variable and its attributes using conventions + * defined by the teca_cf_reader. Returns zero if successful and + * non-zero if the supplied metadata is missing any of the requisite + * information. + */ + int set_t_axis_variable(const teca_metadata &md); + + /** Set the time axis using coordinate conventions defined by the + * teca_cf_reader. When a time axis is provided values are served up from + * the array rather than being generated. Execution control keys are also + * made use of if present. Returns zero if successful and non-zero if the + * supplied metadata is missing any of the requisite information. + */ + int set_t_axis(const teca_metadata &md); + + /** Set the time axis directly. When a time axis is provided values are + * served up from the array rather than being generated. Execution control + * keys are also made use of if present. + */ + void set_t_axis(const p_teca_variant_array &t); + ///@} + + /** @name output_metadata + * Set the output metadata directly. The provided metadata must contain + * "coordinates" as defined by the teca_cf_reader because these are + * required for mesh generation. Pipeline execution control keys as defined + * by teca_index_executive are also required. Calendaring metadata is + * recommended. A copy of the passed object is made but "variables" are + * replaced with those generated by this class, if any. As a result be sure + * to specifiy field generators before calling this method. Returns 0 if + * successful, and non-zero if the supplied metadata doesn't contain the + * expected information. No error messages are sent to the terminal. + */ + ///@{ + int set_output_metadata(const teca_metadata &md); + ///@} + + /** @name append_field_generator + * set a callback function f(x,y,z,t) that generates a field named name + * x,y,z are coordinate axes in variant arrays, t is the double precision + * time value. + */ + ///@{ void append_field_generator(const std::string &name, - const teca_array_attributes &atts, field_generator_callback &callback); + const teca_metadata &atts, field_generator_callback &callback); + ///@} + + /** @name field_generator + * Set/get the named callbacks that generate fields on the mesh. These + * should be packaged in the field_generator struct so that field name + * and attributes for I/O are provided together with the callback. + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(field_generator_t, field_generator) + ///@} protected: teca_cartesian_mesh_source(); private: + /// implements the report phase of pipeline execution teca_metadata get_output_metadata(unsigned int port, const std::vector &input_md) override; + /// implements the execute phase of pipeline execution const_p_teca_dataset execute(unsigned int port, const std::vector &input_data, const teca_metadata &request) override; + /// updates the modification state void set_modified() override; + + /// clears cached metadata in response to modification of algorithm properties void clear_cached_metadata(); private: @@ -122,11 +289,10 @@ class teca_cartesian_mesh_source : public teca_algorithm std::string y_axis_variable; std::string z_axis_variable; std::string t_axis_variable; - std::string x_axis_units; - std::string y_axis_units; - std::string z_axis_units; - std::string calendar; - std::string time_units; + teca_metadata x_axis_attributes; + teca_metadata y_axis_attributes; + teca_metadata z_axis_attributes; + teca_metadata t_axis_attributes; std::vector whole_extents; std::vector bounds; diff --git a/alg/teca_cartesian_mesh_subset.cxx b/alg/teca_cartesian_mesh_subset.cxx index e9b21991d..81837966d 100644 --- a/alg/teca_cartesian_mesh_subset.cxx +++ b/alg/teca_cartesian_mesh_subset.cxx @@ -41,13 +41,15 @@ void teca_cartesian_mesh_subset::get_properties_description( + (prefix.empty()?"teca_cartesian_mesh_subset":prefix)); opts.add_options() - TECA_POPTS_GET(vector, prefix, bounds, + TECA_POPTS_MULTI_GET(std::vector, prefix, bounds, "bounding box given by x0,x1,y0,y1,z0,z1") TECA_POPTS_GET(bool, prefix, cover_bounds, "(T)use smallest subset covering or (F)largest " "subset contained by bounds") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -55,7 +57,9 @@ void teca_cartesian_mesh_subset::get_properties_description( void teca_cartesian_mesh_subset::set_properties( const string &prefix, variables_map &opts) { - TECA_POPTS_SET(opts, vector, prefix, bounds) + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, std::vector, prefix, bounds) TECA_POPTS_SET(opts, bool, prefix, cover_bounds) } #endif @@ -85,8 +89,10 @@ teca_metadata teca_cartesian_mesh_subset::get_output_metadata( } this->extent.resize(6, 0UL); - if (teca_coordinate_util::bounds_to_extent( - this->bounds.data(), x, y, z, this->extent.data())) + if (teca_coordinate_util::bounds_to_extent(this->bounds.data(), + x, y, z, this->extent.data()) || + teca_coordinate_util::validate_extent(x->size(), + y->size(), z->size(), this->extent.data(), true)) { TECA_ERROR("Failed to convert bounds to extent") return teca_metadata(); diff --git a/alg/teca_cartesian_mesh_subset.h b/alg/teca_cartesian_mesh_subset.h index 3b2b2bf56..b4d9761ae 100644 --- a/alg/teca_cartesian_mesh_subset.h +++ b/alg/teca_cartesian_mesh_subset.h @@ -4,7 +4,7 @@ #include "teca_shared_object.h" #include "teca_algorithm.h" #include "teca_metadata.h" -#include "teca_variant_array_fwd.h" +#include "teca_variant_array.h" #include #include @@ -13,13 +13,13 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_cartesian_mesh_subset) /// applies a subset given in world coordinates to the upstream request /** -an algorithm that applies a subset specified in -world coordinates to upstream requests. the subset -is specified as bounding box of the form [x_low to x_high, -y_low to y_high, z_low to z_high]. The subset can be either -the smallest subset containing the bounding box or the -largest set contained by the bounding box, and is controled -by the cover_bounds property. + * an algorithm that applies a subset specified in + * world coordinates to upstream requests. the subset + * is specified as bounding box of the form [x_low to x_high, + * y_low to y_high, z_low to z_high]. The subset can be either + * the smallest subset containing the bounding box or the + * largest set contained by the bounding box, and is controled + * by the cover_bounds property. */ class teca_cartesian_mesh_subset : public teca_algorithm { @@ -34,20 +34,26 @@ class teca_cartesian_mesh_subset : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // define the bounding box of the subset - // this algorithm converts this into an - // extent into the upstream dataset. - TECA_ALGORITHM_PROPERTY(std::vector, bounds); + /** @name bounds + * define the bounding box of the subset this algorithm converts this into + * an extent into the upstream dataset. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::vector, bounds) void set_bounds(double low_x, double high_x, double low_y, double high_y, double low_z, double high_z) { this->set_bounds({low_x, high_x, low_y, high_y, low_z, high_z}); } + ///@} - // control how bounds are converted. if true - // smallest subset covering the bounding box is - // used. if false the largest subset contained - // by the bounding box is used. + /** @name cover_bounds + * control how bounds are converted. if true smallest subset covering the + * bounding box is used. if false the largest subset contained by the + * bounding box is used. + */ + ///@{ TECA_ALGORITHM_PROPERTY(bool, cover_bounds) + ///@} protected: teca_cartesian_mesh_subset(); diff --git a/alg/teca_component_area_filter.cxx b/alg/teca_component_area_filter.cxx index 2441e8d73..62773c3ed 100644 --- a/alg/teca_component_area_filter.cxx +++ b/alg/teca_component_area_filter.cxx @@ -3,7 +3,7 @@ #include "teca_variant_array.h" #include "teca_metadata.h" #include "teca_cartesian_mesh.h" -#include "teca_metadata_util.h" +#include "teca_string_util.h" #include #include @@ -82,31 +82,30 @@ void teca_component_area_filter::get_properties_description( TECA_POPTS_GET(std::string, prefix, component_variable, "name of the varibale containing connected component labeling") TECA_POPTS_GET(std::string, prefix, number_of_components_key, - "name of the key that contains the number of components" - "\"number_of_components\")") + "name of the key that contains the number of components") TECA_POPTS_GET(std::string, prefix, component_ids_key, - "name of the key that contains the list of component ids " - "\"component_ids\")") + "name of the key that contains the list of component ids") TECA_POPTS_GET(std::string, prefix, component_area_key, - "name of the key that contains the list of component areas " - "(\"component_area\")") + "name of the key that contains the list of component areas") TECA_POPTS_GET(int, prefix, mask_value, "components with area outside of the range will be replaced " - "by this label value (-1)") + "by this label value") TECA_POPTS_GET(double, prefix, low_area_threshold, "set the lower end of the range of areas to pass through. " - "components smaller than this are masked out. (-inf)") + "components smaller than this are masked out.") TECA_POPTS_GET(double, prefix, high_area_threshold, "set the higher end of the range of areas to pass through. " - "components larger than this are masked out. (+inf)") + "components larger than this are masked out.") TECA_POPTS_GET(std::string, prefix, variable_post_fix, "set a string that will be appended to variable names and " - "metadata keys in the filter's output (\"\")") + "metadata keys in the filter's output") TECA_POPTS_GET(int, prefix, contiguous_component_ids, "when the region label ids start at 0 and are consecutive " - "this flag enables use of an optimization (0)") + "this flag enables use of an optimization") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -114,6 +113,8 @@ void teca_component_area_filter::get_properties_description( void teca_component_area_filter::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, component_variable) TECA_POPTS_SET(opts, std::string, prefix, number_of_components_key) TECA_POPTS_SET(opts, std::string, prefix, component_ids_key) @@ -184,7 +185,7 @@ std::vector teca_component_area_filter::get_upstream_request( const std::string &var_post_fix = this->variable_post_fix; if (!var_post_fix.empty()) { - teca_metadata_util::remove_post_fix(arrays, var_post_fix); + teca_string_util::remove_post_fix(arrays, var_post_fix); } req.set("arrays", arrays); diff --git a/alg/teca_component_area_filter.h b/alg/teca_component_area_filter.h index f6c8e5a22..6ef2669c0 100644 --- a/alg/teca_component_area_filter.h +++ b/alg/teca_component_area_filter.h @@ -12,29 +12,29 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_component_area_filter) /// An algorithm that applies a mask based on connected component area /** -The filter masks the regions identified by an integer label that are outside -the range bracketed by the 'low_area_threshold' and 'high_area_threshold' -properties. These default to -inf and +inf, hence by default no regions are -masked. The mask value may be set by the 'mask_value' property which defaults -to '0'. - -The filter expects an integer field containing connected component labels. -This field is named by the 'component_variable' property. Additionally a list -of label ids and coresponding areas is expected in the dataset metadata. The -properties 'component_ids_key' and 'component_area_key' identify the latter -metadata. These default to the names used by the 'teca_2d_component_area' -algotihm, 'component_ids' and 'component_area'. - -Applying the 'teca_connected_component' algorithm followed by the -'teca_2d_component_area' algorithm is the easiest way to get valid inputs for -the 'component_area_filter'. - -The filtered coomponent ids are put in the output dataset along with the -updated lists of valid component ids and component area metadata keys. By -default the filtered data replaces the input data in the output. However, the -input data can be retained by setting the 'variable_post_fix' property, a -string that will be appended to the names of the filtered component array and -metadata keys. + * The filter masks the regions identified by an integer label that are outside + * the range bracketed by the 'low_area_threshold' and 'high_area_threshold' + * properties. These default to -inf and +inf, hence by default no regions are + * masked. The mask value may be set by the 'mask_value' property which + * defaults to '0'. + * + * The filter expects an integer field containing connected component labels. + * This field is named by the 'component_variable' property. Additionally a + * list of label ids and coresponding areas is expected in the dataset + * metadata. The properties 'component_ids_key' and 'component_area_key' + * identify the latter metadata. These default to the names used by the + * 'teca_2d_component_area' algotihm, 'component_ids' and 'component_area'. + * + * Applying the 'teca_connected_component' algorithm followed by the + * 'teca_2d_component_area' algorithm is the easiest way to get valid inputs + * for the 'component_area_filter'. + * + * The filtered coomponent ids are put in the output dataset along with the + * updated lists of valid component ids and component area metadata keys. By + * default the filtered data replaces the input data in the output. However, + * the input data can be retained by setting the 'variable_post_fix' property, + * a string that will be appended to the names of the filtered component array + * and metadata keys. */ class teca_component_area_filter : public teca_algorithm { @@ -51,43 +51,78 @@ class teca_component_area_filter : public teca_algorithm // set the name of the input array containing connected // component labels + /** @name component_variable + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_variable) + ///@} - // set the name of the dataset metadata key holding the number of - // components left after the filter is applied + /** @name number_of_components_key + * set the name of the dataset metadata key holding the number of + * components left after the filter is applied + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, number_of_components_key) + ///@} - // set the name of the dataset metadata key holding connected component - // label ids + /** @name component_ids_key + * set the name of the dataset metadata key holding connected component + * label ids + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_ids_key) + ///@} - // set the name of the dataset metadata key holding connected component - // areas + /** @name component_area_key + * set the name of the dataset metadata key holding connected component + * areas + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_area_key) - - // set this to be the default label id for the filtered out component - // areas. This will typically correspond to the label used for cells - // outside of the segmentation (i.e. in the background). One can use this - // property to override the mask value. The default mask value is '-1' - // which results in aquiring the mask value from input metadata key - // `background_id`. Use -2 to specify no background label. + ///@} + + /** @name mask_value + * set this to be the default label id for the filtered out component + * areas. This will typically correspond to the label used for cells + * outside of the segmentation (i.e. in the background). One can use this + * property to override the mask value. The default mask value is '-1' + * which results in aquiring the mask value from input metadata key + * `background_id`. Use -2 to specify no background label. + */ + ///@{ TECA_ALGORITHM_PROPERTY(long, mask_value) + ///@} - // set the range identifying values to area filter. - // The defaults are (-infinity, infinity]. + /** @name low_area_threshold + * set the range identifying values to area filter. The defaults are + * (-infinity, infinity]. + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, low_area_threshold) - TECA_ALGORITHM_PROPERTY(double, high_area_threshold) + ///@} - // a string to be appended to the name of the output variable. - // setting this to an empty string will result in the masked array - // replacing the input array in the output. default is an empty - // string "" + /** @name high_area_threshold + */ + ///@{ + TECA_ALGORITHM_PROPERTY(double, high_area_threshold) + ///@} + + /** @name variable_post_fix + * a string to be appended to the name of the output variable. setting + * this to an empty string will result in the masked array replacing the + * input array in the output. default is an empty string "" + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, variable_post_fix) + ///@} - // set this only if you know for certain that label ids - // are contiguous and start at 0. this enables use of a - // faster implementation. + /** @name contiguous_component_ids + * set this only if you know for certain that label ids are contiguous and + * start at 0. this enables use of a faster implementation. + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, contiguous_component_ids) + ///@} protected: teca_component_area_filter(); diff --git a/alg/teca_component_statistics.cxx b/alg/teca_component_statistics.cxx index a5de6f0ee..773e2f74f 100644 --- a/alg/teca_component_statistics.cxx +++ b/alg/teca_component_statistics.cxx @@ -47,6 +47,8 @@ void teca_component_statistics::get_properties_description( "list of arrays to compute statistics for") ;*/ + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -54,8 +56,7 @@ void teca_component_statistics::get_properties_description( void teca_component_statistics::set_properties( const std::string &prefix, variables_map &opts) { - (void) prefix; - (void) opts; + this->teca_algorithm::set_properties(prefix, opts); //TECA_POPTS_SET(opts, std::vector, prefix, dependent_variables) } diff --git a/alg/teca_connected_components.h b/alg/teca_connected_components.h index 0f188f7e0..693ffa1b4 100644 --- a/alg/teca_connected_components.h +++ b/alg/teca_connected_components.h @@ -12,35 +12,35 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_connected_components) /// an algorithm that computes connected component labeling /** -an algorithm that computes connected component labeling for 1D, 2D, and 3D -data. The components are computed from a binary segmentation provided on the -input. - -the input binary segmentation is labeled and stored in a variable named by the -component_variable property. the component ids are added to the output -dataset metadata in an key named 'component_ids', and the number of components -is stored in a key named 'number_of_components'. These keys facilitate further -processing as one need not scan the labeled data to get the list of label ids. - -The cells outside of the segmentation (i.e. the background) are always assigned -the label 0. The cells belonging to connected regions inside the segmentation -are labeled starting from 1 up to number_of_components - 1. - -output keys: - - number_of_components - number of component ids found. this will always be - at least 1 long as the cells outside the segmentation - are assigned the label 0. - - component_ids - a vector containing the label of each component. This is - always starts with 0, where the label 0 identifies cells - out side of the segmentation, and ranges up to - number_of_components - 1, where the labels from 1 up to - number_of_components - 1 identify connected regions of - cells inside the segmentation. - - background_id - the label used for cells outside of the segmentation, - i.e. the background. always 0. + * an algorithm that computes connected component labeling for 1D, 2D, and 3D + * data. The components are computed from a binary segmentation provided on the + * input. + * + * the input binary segmentation is labeled and stored in a variable named by the + * component_variable property. the component ids are added to the output + * dataset metadata in an key named 'component_ids', and the number of components + * is stored in a key named 'number_of_components'. These keys facilitate further + * processing as one need not scan the labeled data to get the list of label ids. + * + * The cells outside of the segmentation (i.e. the background) are always assigned + * the label 0. The cells belonging to connected regions inside the segmentation + * are labeled starting from 1 up to number_of_components - 1. + * + * output keys: + * + * | name | description | + * | ---- | ----------- | + * | number_of_components | number of component ids found. this will always be | + * | | at least 1 long as the cells outside the segmentation | + * | | are assigned the label 0. | + * | component_ids | a vector containing the label of each component. This is | + * | | always starts with 0, where the label 0 identifies cells | + * | | out side of the segmentation, and ranges up to | + * | | number_of_components - 1, where the labels from 1 up to | + * | | number_of_components - 1 identify connected regions of | + * | | cells inside the segmentation. | + * | background_id | the label used for cells outside of the segmentation, | + * | | i.e. the background. always 0. | */ class teca_connected_components : public teca_algorithm { diff --git a/alg/teca_dataset_diff.cxx b/alg/teca_dataset_diff.cxx index 3905871cb..ffa008ae5 100644 --- a/alg/teca_dataset_diff.cxx +++ b/alg/teca_dataset_diff.cxx @@ -33,7 +33,7 @@ // -------------------------------------------------------------------------- teca_dataset_diff::teca_dataset_diff() - : relative_tolerance(1.0e-6), absolute_tolerance(-1.0), verbose(1) + : relative_tolerance(1.0e-6), absolute_tolerance(-1.0) { this->set_number_of_input_connections(2); this->set_number_of_output_ports(1); @@ -54,15 +54,18 @@ void teca_dataset_diff::get_properties_description( opts.add_options() TECA_POPTS_GET(double, prefix, relative_tolerance, "relative test tolerance") TECA_POPTS_GET(double, prefix, absolute_tolerance, "absolute test tolerance") - TECA_POPTS_GET(int, prefix, verbose, "print status messages as the diff runs") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } // -------------------------------------------------------------------------- void teca_dataset_diff::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, double, prefix, relative_tolerance) TECA_POPTS_SET(opts, double, prefix, absolute_tolerance) TECA_POPTS_SET(opts, int, prefix, verbose) diff --git a/alg/teca_dataset_diff.h b/alg/teca_dataset_diff.h index 57bfffbba..881b928f6 100644 --- a/alg/teca_dataset_diff.h +++ b/alg/teca_dataset_diff.h @@ -18,16 +18,14 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_dataset_diff) /// compute the element wise difference between to datasets /** -a two input algorithm that compares datasets by examining each -element of their contained arrays. a threshold is used to detect -when an element is different. a report containing the string FAIL -is issued to stderr stream when a difference is detected. this -algorithm is the core of TECA's regression test suite. - -by convention the first input produces the reference dataset, -and the second input produces the dataset to validate. this is -primarilly to support map-reduce implementation where after -the reduction only rank 0 has data. + * a two input algorithm that compares datasets by examining each element of their + * contained arrays. a threshold is used to detect when an element is different. a + * report containing the string FAIL is issued to stderr stream when a difference + * is detected. this algorithm is the core of TECA's regression test suite. + * + * by convention the first input produces the reference dataset, and the second + * input produces the dataset to validate. this is primarilly to support + * map-reduce implementation where after the reduction only rank 0 has data. */ class teca_dataset_diff : public teca_algorithm { @@ -41,19 +39,24 @@ class teca_dataset_diff : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // Relative tolerance below which two floating-point numbers a and b are - // considered equal. if |a - b| <= max(|a|,|b|)*tol then a is equal to b. - // the relative tolerance is used with numbers not close to zero. + /** @name relative_tolerance + * Relative tolerance below which two floating-point numbers a and b are + * considered equal. if |a - b| <= max(|a|,|b|)*tol then a is equal to b. + * the relative tolerance is used with numbers not close to zero. + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, relative_tolerance) - - // The absolute tolerance below which two floating point numbers a and b are - // considered equal. if |a - b| <= tol then a is equal to b. The absolute - // tolerance is used with numbers close to zero. + ///@} + + /** @name absolute_tolerance + * The absolute tolerance below which two floating point numbers a and b + * are considered equal. if |a - b| <= tol then a is equal to b. The + * absolute tolerance is used with numbers close to zero. + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, absolute_tolerance) + ///@} - // if set infromation about the test progress is displayed during - // the test. - TECA_ALGORITHM_PROPERTY(int, verbose) protected: teca_dataset_diff(); @@ -107,7 +110,6 @@ class teca_dataset_diff : public teca_algorithm private: double relative_tolerance; double absolute_tolerance; - int verbose; }; #endif diff --git a/alg/teca_derived_quantity.cxx b/alg/teca_derived_quantity.cxx index 46345bf8f..4d7f68b6e 100644 --- a/alg/teca_derived_quantity.cxx +++ b/alg/teca_derived_quantity.cxx @@ -31,12 +31,14 @@ void teca_derived_quantity::get_properties_description( + (prefix.empty()?"teca_derived_quantity":prefix)); opts.add_options() - TECA_POPTS_GET(std::vector, prefix, dependent_variables, + TECA_POPTS_MULTI_GET(std::vector, prefix, dependent_variables, "list of arrays needed to compute the derived quantity") TECA_POPTS_GET(std::string, prefix, derived_variable, "name of the derived quantity") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -44,6 +46,8 @@ void teca_derived_quantity::get_properties_description( void teca_derived_quantity::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::vector, prefix, dependent_variables) TECA_POPTS_SET(opts, std::string, prefix, derived_variable) } diff --git a/alg/teca_derived_quantity.h b/alg/teca_derived_quantity.h index 922a87691..40b7510b5 100644 --- a/alg/teca_derived_quantity.h +++ b/alg/teca_derived_quantity.h @@ -3,7 +3,7 @@ #include "teca_programmable_algorithm.h" #include "teca_metadata.h" -#include "teca_dataset_fwd.h" +#include "teca_dataset.h" #include "teca_shared_object.h" #include @@ -13,14 +13,13 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_derived_quantity) /// a programmable algorithm specialized for simple array based computations /** -A programmable algorithm specialized for simple array based -computations. A user provided callable(see set execute_callback) -which operates on one or more arrays(the dependent variables) to -produce a new array (the derived quantity). The purpose of this -class is to implement the request and report phases of the pipeline -consistently for this common use case. An implementation specific -context(operation_name) differentiates between multiple instances -in the same pipeline. + * A programmable algorithm specialized for simple array based computations. A + * user provided callable(see set execute_callback) which operates on one or more + * arrays(the dependent variables) to produce a new array (the derived quantity). + * The purpose of this class is to implement the request and report phases of the + * pipeline consistently for this common use case. An implementation specific + * context(operation_name) differentiates between multiple instances in the same +pipeline. */ class teca_derived_quantity : public teca_programmable_algorithm { diff --git a/alg/teca_derived_quantity_numerics.h b/alg/teca_derived_quantity_numerics.h index a755714d2..d5f375fae 100644 --- a/alg/teca_derived_quantity_numerics.h +++ b/alg/teca_derived_quantity_numerics.h @@ -1,21 +1,22 @@ #ifndef teca_numerics_h #define teca_numerics_h +/// @file + #include "teca_mesh.h" #include #include -// this namespace contains numeric code that could be reused -// by teca_derived_quantity +/// Numeric code that could be reused by teca_derived_quantity namespace teca_derived_quantity_numerics { -// an execute function designed for use with teca_derived_quantity -// on a teca_mesh. shallow copies the input and computes the -// point-wise average of the named variables. -// -// for every i -// avg[i] = (v0[i] + v1[i])/2 -// +/** an execute function designed for use with teca_derived_quantity + * on a teca_mesh. shallow copies the input and computes the + * point-wise average of the named variables. + * + * for every i + * avg[i] = (v0[i] + v1[i])/2 + */ struct point_wise_average { // construct the class with two input array names, v0,v1 @@ -70,12 +71,12 @@ struct point_wise_average std::string m_avg; // output variable name }; -// an execute function designed for use with teca_derived_quantity -// on a teca_mesh. compute the point-wise difference of two variables -// -// for every i -// diff[i] = v1[i] - v0[i] -// +/** an execute function designed for use with teca_derived_quantity + * on a teca_mesh. compute the point-wise difference of two variables + * + * for every i + * diff[i] = v1[i] - v0[i] + */ struct point_wise_difference { // construct the class with two input array names, v0,v1 diff --git a/alg/teca_descriptive_statistics.cxx b/alg/teca_descriptive_statistics.cxx index 2c7794f0a..042684228 100644 --- a/alg/teca_descriptive_statistics.cxx +++ b/alg/teca_descriptive_statistics.cxx @@ -128,7 +128,7 @@ void teca_descriptive_statistics::get_properties_description( + (prefix.empty()?"teca_descriptive_statistics":prefix)); opts.add_options() - TECA_POPTS_GET(std::vector, prefix, dependent_variables, + TECA_POPTS_MULTI_GET(std::vector, prefix, dependent_variables, "list of arrays to compute statistics for") ; diff --git a/alg/teca_descriptive_statistics.h b/alg/teca_descriptive_statistics.h index b426dca64..416476c1e 100644 --- a/alg/teca_descriptive_statistics.h +++ b/alg/teca_descriptive_statistics.h @@ -12,8 +12,8 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_descriptive_statistics) /// compute descriptive statistics over a set of arrays. /** -compute the min, max, avg, median, standard deviation of a -set of named arrays. the results are returned in a table. + * compute the min, max, avg, median, standard deviation of a set of named + * arrays. the results are returned in a table. */ class teca_descriptive_statistics : public teca_algorithm { diff --git a/alg/teca_elevation_mask.cxx b/alg/teca_elevation_mask.cxx new file mode 100644 index 000000000..bc662a313 --- /dev/null +++ b/alg/teca_elevation_mask.cxx @@ -0,0 +1,384 @@ +#include "teca_elevation_mask.h" + +#include "teca_cartesian_mesh.h" +#include "teca_array_collection.h" +#include "teca_variant_array.h" +#include "teca_metadata.h" +#include "teca_array_attributes.h" + +#include "teca_dataset_source.h" +#include "teca_dataset_capture.h" +#include "teca_cartesian_mesh_regrid.h" +#include "teca_index_executive.h" + +#include +#include +#include +#include +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +//#define TECA_DEBUG + +struct teca_elevation_mask::internals_t +{ + // compute the valid value mask such that for each point the mask + // is 1 where the mesh point is above the surface of the Earth and + // 0 otherwise + template + static void mask_by_surface_elevation( + size_t nx, size_t ny, size_t nz, + mask_t * __restrict__ mask, + const elev_num_t * __restrict__ surface_elev, + const mesh_num_t * __restrict__ mesh_height) + { + size_t nxy = nx*ny; + for (size_t k = 0; k < nz; ++k) + { + const mesh_num_t * __restrict__ mesh_height_k = mesh_height + k*nxy; + mask_t * __restrict__ mask_k = mask + k*nxy; + for (size_t q = 0; q < nxy; ++q) + { + mask_k[q] = mesh_height_k[q] >= (mesh_num_t)surface_elev[q] ? mask_t(1) : mask_t(0); + } + } + } +}; + + +// -------------------------------------------------------------------------- +teca_elevation_mask::teca_elevation_mask() : + mesh_height_variable("zg"), surface_elevation_variable("z") +{ + this->set_number_of_input_connections(2); + this->set_number_of_output_ports(1); +} + +// -------------------------------------------------------------------------- +teca_elevation_mask::~teca_elevation_mask() +{ +} + +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_elevation_mask::get_properties_description( + const std::string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_elevation_mask":prefix)); + + opts.add_options() + + TECA_POPTS_GET(std::string, prefix, surface_elevation_variable, + "Set the name of the variable containing surface elevation" + " values in meters above mean sea level") + + TECA_POPTS_GET(std::string, prefix, mesh_height_variable, + "Set the name of the variable containing point wise mesh height" + " values in meters above mean sea level") + + TECA_POPTS_MULTI_GET(std::vector, prefix, mask_variables, + "Set the names of the variables to store the generated mask in." + " Each name is assigned a reference to the mask.") + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_elevation_mask::set_properties( + const std::string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, std::string, prefix, surface_elevation_variable) + TECA_POPTS_SET(opts, std::string, prefix, mesh_height_variable) + TECA_POPTS_SET(opts, std::vector, prefix, mask_variables) +} +#endif + +// -------------------------------------------------------------------------- +teca_metadata teca_elevation_mask::get_output_metadata( + unsigned int port, + const std::vector &input_md) +{ +#ifdef TECA_DEBUG + cerr << teca_parallel_id() + << "teca_elevation_mask::get_output_metadata" << endl; +#endif + (void)port; + + // validate runtime provided settings + unsigned int n_mask_vars = this->mask_variables.size(); + if (n_mask_vars == 0) + { + TECA_ERROR("The names of the mask_variables were not provided") + return teca_metadata(); + } + + // pass metadata from the input mesh through. + const teca_metadata &mesh_md = input_md[0]; + teca_metadata out_md(mesh_md); + + // add the mask arrays we will generate + for (unsigned int i = 0; i < n_mask_vars; ++i) + out_md.append("variables", this->mask_variables[i]); + + // insert attributes to enable this to be written by the CF writer + teca_metadata attributes; + out_md.get("attributes", attributes); + + teca_metadata mesh_height_atts; + if (attributes.get(this->mesh_height_variable, mesh_height_atts)) + { + TECA_WARNING("Failed to get mesh_height_variable \"" + << this->mesh_height_variable << "\" attrbibutes." + " Writing the result will not be possible") + } + else + { + // get the centering and size from the array + unsigned int centering = 0; + mesh_height_atts.get("centering", centering); + + unsigned long size = 0; + mesh_height_atts.get("size", size); + + // construct output attributes + teca_array_attributes mask_atts( + teca_variant_array_code::get(), + centering, size, "none", "", "elevation mask"); + + // add one for each output + for (unsigned int i = 0; i < n_mask_vars; ++i) + attributes.set(this->mask_variables[i], (teca_metadata)mask_atts); + + // update the attributes collection + out_md.set("attributes", attributes); + } + + return out_md; +} + +// -------------------------------------------------------------------------- +std::vector teca_elevation_mask::get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) +{ + (void)port; + (void)input_md; + + std::vector up_reqs; + + // get the names of the arrays we need to request + if (this->mesh_height_variable.empty()) + { + TECA_ERROR("The mesh_height_variable was not specified") + return up_reqs; + } + + if (this->surface_elevation_variable.empty()) + { + TECA_ERROR("The surface_elevation_variable was not specified") + return up_reqs; + } + + // need to make the request for the surface elevation field using bounds + double req_bounds[6] = {0.0}; + if (request.get("bounds", req_bounds, 6)) + { + // bounds not specified, try to get an extent and convert to a bounds + unsigned long req_extent[6]; + if (request.get("extent", req_extent, 6)) + { + TECA_ERROR("Neither bounds nor extent were specified in the request") + return up_reqs; + } + + const teca_metadata &md = input_md[0]; + + teca_metadata coords; + p_teca_variant_array x,y; + + if (md.get("coordinates", coords) || + !(x = coords.get("x")) || !(y = coords.get("y"))) + { + TECA_ERROR("Failed to get mesh coordinates") + return up_reqs; + } + + x->get(req_extent[0], req_bounds[0]); + x->get(req_extent[1], req_bounds[1]); + y->get(req_extent[2], req_bounds[2]); + y->get(req_extent[3], req_bounds[3]); + } + + // input port 0 will source the mesh height field, and any other data + // requested by the down stream. copy the incoming request to preserve the + // downstream requirements and add the mesh height variable + teca_metadata req_0(request); + + std::set mesh_arrays; + if (req_0.has("arrays")) + req_0.get("arrays", mesh_arrays); + + mesh_arrays.insert(this->mesh_height_variable); + + // intercept request for our output + int n_mask_vars = this->mask_variables.size(); + for (int i = 0; i < n_mask_vars; ++i) + mesh_arrays.erase(this->mask_variables[i]); + + req_0.set("arrays", mesh_arrays); + + + // input port 1 provides the surface elevation field, request it + // preserve bounds etc + const teca_metadata &elev_md = input_md[1]; + + std::string req_key; + if (elev_md.get("index_request_key", req_key)) + { + TECA_ERROR("Metadata is missing \"index_request_key\"") + return up_reqs; + } + + // surface elevations don't change over the timescale of concern + // always request index 0 + teca_metadata req_1; + req_1.set(req_key, 0ul); + req_1.set("index_request_key", req_key); + + // request the surface elevation + std::vector elev_arrays(1, this->surface_elevation_variable); + req_1.set("arrays", elev_arrays); + + // at the bounds of interest + req_1.set("bounds", req_bounds, 6); + + // package the requests and send them up + up_reqs.push_back(req_0); + up_reqs.push_back(req_1); + + + return up_reqs; +} + +// -------------------------------------------------------------------------- +const_p_teca_dataset teca_elevation_mask::execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + cerr << teca_parallel_id() << "teca_elevation_mask::execute" << endl; +#endif + (void)port; + (void)request; + + // check for an error upstream + if ((input_data.size() != 2) || !input_data[0] || !input_data[1]) + { + TECA_ERROR("Invalid inputs detected") + return nullptr; + } + + // get the input 3D mesh + const_p_teca_cartesian_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); + + if (!in_mesh) + { + TECA_ERROR("Data to mask on input port 0 is not a" + " teca_cartesian_mesh. Got " << input_data[0]->get_class_name()) + return nullptr; + } + + // get the mesh dimensions + unsigned long extent[6]; + in_mesh->get_extent(extent); + + unsigned long nx = extent[1] - extent[0] + 1; + unsigned long ny = extent[3] - extent[2] + 1; + unsigned long nz = extent[5] - extent[4] + 1; + + // get the mesh height, this is a 3d field with the altitude for + // each mesh point + const_p_teca_variant_array mesh_height = + in_mesh->get_point_arrays()->get(this->mesh_height_variable); + + if (!mesh_height) + { + TECA_ERROR("Mesh to mask is missing the height field \"" + << this->mesh_height_variable << "\"") + return nullptr; + } + + // get the surface elevations + const_p_teca_cartesian_mesh in_elev + = std::dynamic_pointer_cast(input_data[1]); + + if (!in_elev) + { + TECA_ERROR("Data to mask on input port 0 is not a" + " teca_cartesian_mesh. Got " << input_data[0]->get_class_name()) + return nullptr; + } + + // get the surface elevation, this is a 2d field with surface altitude + // + // at each mesh point. regridding has been performed so that the horizontal + // coordinates are the same as the 3d mesh for which masks will be generated + const_p_teca_variant_array surface_elev = + in_elev->get_point_arrays()->get(this->surface_elevation_variable); + + if (!surface_elev) + { + TECA_ERROR("Surface elevation data has no array \"" + << this->surface_elevation_variable << "\"") + return nullptr; + } + + // compute the mask + p_teca_char_array mask = teca_char_array::New(mesh_height->size()); + char *p_mask = mask->get(); + + NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, + surface_elev.get(), + _SURF, + + const NT_SURF *p_surface_elev = + static_cast(surface_elev.get())->get(); + + NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, + mesh_height.get(), + _MESH, + + const NT_MESH *p_mesh_height = + static_cast(mesh_height.get())->get(); + + internals_t::mask_by_surface_elevation(nx, ny, nz, + p_mask, p_surface_elev, p_mesh_height); + + ) + ) + + // allocate the output mesh + p_teca_cartesian_mesh out_mesh = std::dynamic_pointer_cast + (std::const_pointer_cast(in_mesh)->new_shallow_copy()); + + // store the results under the requested names + int n_mask_vars = this->mask_variables.size(); + for (int i = 0; i < n_mask_vars; ++i) + { + out_mesh->get_point_arrays()->set(this->mask_variables[i], mask); + } + + return out_mesh; +} diff --git a/alg/teca_elevation_mask.h b/alg/teca_elevation_mask.h new file mode 100644 index 000000000..fea8a7fbf --- /dev/null +++ b/alg/teca_elevation_mask.h @@ -0,0 +1,115 @@ +#ifndef teca_elevation_mask_h +#define teca_elevation_mask_h + +#include "teca_shared_object.h" +#include "teca_algorithm.h" +#include "teca_metadata.h" + +#include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_elevation_mask) + +/** @brief + * Generates a mask indicating where mesh points with a vertical pressure + * coordinate lie above the surface of the Earth. The mask is set to 1 where + * data is above the Earth's surface and 0 otherwise. + * + * @details + * Given a 3D height field containing the altitude of each point in meters + * above mean sea level, and a 2D height field corresponding to height in + * meters above mean sea level of the surface of the Earth, generate a mask + * that is 1 where the 3D point is on or above the surface of the Earth and 0 + * where it is below. + * + * The name of the 3D height field is specified by the mesh_height_variable + * property. The name of the 2D height field conntaining elveation of the + * Earth's surface is specified by the surface_elevation_variable property. + * + * The 3D mesh height field must be provided on input 0, and the 2D surface + * height field on input 1. Use the mask_names property to name the output + * mask. If more than one name is provided each name will reference a pointer + * to the mask. Consider using names of the form X_valid in which case the + * output is compatible with the teca_valid_value_mask and will be treated + * as missing values by down stream algorithms. + * + * If the simulation does not provide the 3D height field, for simulations + * where the acceleration due to the Earth's gravity is assumed constant, + * teca_geopotential_height can generate the 3D height field. + * + * The primary use case of this algorithm is when dealing with calculations on + * 3D meshes with a vertical pressure coordinate and there is a need to + * identify and treat specially the mesh points that are below the surface of + * the Earth. There are a number of alternatives available depending on the + * data. If your data has a _FillValue where data is below the surface then + * use teca_valid_value_mask instead of this algorithm. If your data has + * surface pressure field use teca_pressure_level_mask instead of this + * algorithm. If your dataset has surface temperature, and mean sea level + * pressure fields then use teca_surface_pressure to generate the surface + * pressure field and use teca_pressure_level_mask instead of this algorithm. + */ +class teca_elevation_mask : public teca_algorithm +{ +public: + TECA_ALGORITHM_STATIC_NEW(teca_elevation_mask) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_elevation_mask) + TECA_ALGORITHM_CLASS_NAME(teca_elevation_mask) + ~teca_elevation_mask(); + + /** @name program_options + * report/initialize to/from Boost program options objects. + */ + ///@{ + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + ///@} + + /** @name mesh_height_variable + * Set the name of the 3D height field + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, mesh_height_variable) + ///@} + + /** @name surface_elevation_variable + * Set the name of the variable containing the elevation of the Earth's + * surface. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, surface_elevation_variable) + ///@} + + /** @name mask_variables + * set the names of the variables to store the generated mask in + * each variable will contain a reference to the mask + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(std::string, mask_variable) + ///@} + +protected: + teca_elevation_mask(); + +private: + teca_metadata get_output_metadata( + unsigned int port, + const std::vector &input_md) override; + + std::vector get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) override; + + const_p_teca_dataset execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) override; + +private: + std::string mesh_height_variable; + std::string surface_elevation_variable; + std::vector mask_variables; + struct internals_t; +}; + +#endif diff --git a/alg/teca_evaluate_expression.cxx b/alg/teca_evaluate_expression.cxx index 69f485a30..4a1948321 100644 --- a/alg/teca_evaluate_expression.cxx +++ b/alg/teca_evaluate_expression.cxx @@ -18,7 +18,7 @@ #include #endif #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #if defined(TECA_HAS_MPI) #include @@ -59,6 +59,8 @@ void teca_evaluate_expression::get_properties_description( "when set columns used in the calculation are removed from output") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -66,6 +68,8 @@ void teca_evaluate_expression::get_properties_description( void teca_evaluate_expression::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, expression) TECA_POPTS_SET(opts, std::string, prefix, result_variable) TECA_POPTS_SET(opts, int, prefix, remove_dependent_variables) diff --git a/alg/teca_evaluate_expression.h b/alg/teca_evaluate_expression.h index 91ce9b9f1..afbe546b6 100644 --- a/alg/teca_evaluate_expression.h +++ b/alg/teca_evaluate_expression.h @@ -10,23 +10,24 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_evaluate_expression) -/** -An algorithm that evaluates an expression stores the -result in a new variable. - -the expression parser supports the following operations: - +,-,*,/,%,<.<=,>,>=,==,!=,&&,||.!,? - -grouping in the expression is denoted in the usual -way: () - -constants in the expression are expanded to full length -arrays and can be typed. The supported types are: - d,f,L,l,i,s,c -coresponding to double,float,long long, long, int, -short and char repsectively. integer types can be -unsigned by including u after the code. -*/ +/** @brief + * An algorithm that evaluates an expression stores the + * result in a new variable. + * + * @details + * The expression parser supports the following operations: + * +,-,*,/,%,<.<=,>,>=,==,!=,&&,||.!,? + * + * Grouping in the expression is denoted in the usual + * way: () + * + * Constants in the expression are expanded to full length + * arrays and can be typed. The supported types are: + * d,f,L,l,i,s,c + * Corresponding to double,float, long long, long, int, + * short and char respectively. Integer types can be + * unsigned by including u after the code. + */ class teca_evaluate_expression : public teca_algorithm { public: @@ -40,18 +41,32 @@ class teca_evaluate_expression : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set/get the expression to evaluate + /** @name expression + * Set the expression to evaluate. + */ + ///@{ + /// Set the expression. void set_expression(const std::string &expr); + /// Get the expression. std::string get_expression() { return this->expression; } - - // set the name of the variable to store the result in - TECA_ALGORITHM_PROPERTY(std::string, result_variable); - - // when set columns used in the calculation are removed - // from the output. deault off. + ///@} + + /** @name result_variable + * set the name of the variable to store the result in + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, result_variable) + ///@} + + /** @name remove_dependent_variables + * when set columns used in the calculation are removed from the output. + * default off. + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, remove_dependent_variables) + ///@} protected: teca_evaluate_expression(); diff --git a/alg/teca_face_to_cell_centering.cxx b/alg/teca_face_to_cell_centering.cxx index 993df96c4..6e6f5c939 100644 --- a/alg/teca_face_to_cell_centering.cxx +++ b/alg/teca_face_to_cell_centering.cxx @@ -128,26 +128,26 @@ teca_face_to_cell_centering::~teca_face_to_cell_centering() void teca_face_to_cell_centering::get_properties_description( const string &prefix, options_description &global_opts) { - (void)prefix; - (void)global_opts; - /*options_description opts("Options for " + options_description opts("Options for " + (prefix.empty()?"teca_face_to_cell_centering":prefix)); - opts.add_options() + /*opts.add_options() TECA_POPTS_GET(int, prefix, mode, - "transform mode (mode_wrf_v3)") - ; + "Set the coordinate transform mode. The valid modes" + " are: mode_wrf_v3)") + ;*/ - global_opts.add(opts);*/ + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); } // -------------------------------------------------------------------------- void teca_face_to_cell_centering::set_properties( const string &prefix, variables_map &opts) { - (void)prefix; - (void)opts; //TECA_POPTS_SET(opts, int, prefix, mode) + this->teca_algorithm::set_properties(prefix, opts); } #endif diff --git a/alg/teca_face_to_cell_centering.h b/alg/teca_face_to_cell_centering.h index 563c9ae1f..458b7f9f0 100644 --- a/alg/teca_face_to_cell_centering.h +++ b/alg/teca_face_to_cell_centering.h @@ -10,10 +10,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_face_to_cell_centering) -/// an algorithm that transforms vertical cooridinates -/** -An algorithm that transforms vertical coordinates of a mesh. -*/ +/// An algorithm that transforms from face to cell centering class teca_face_to_cell_centering : public teca_algorithm { public: diff --git a/alg/teca_geography.h b/alg/teca_geography.h index 1e1d35401..59592018b 100644 --- a/alg/teca_geography.h +++ b/alg/teca_geography.h @@ -1,48 +1,48 @@ #ifndef teca_geography_h #define teca_geography_h +/// @file + #include #include +/// Codes for dealing with geography namespace teca_geography { -/** -get the number of cyclone basins. cyclone basin ids are -in the range 0 to number of basins - 1. +/** get the number of cyclone basins. cyclone basin ids are + * in the range 0 to number of basins - 1. */ unsigned long get_number_of_cyclone_basins(); /** -get the unique list of names describing available cyclone basins. -the list can be indexed by the ids returned by the -get_cyclone_basin/s functions. + * get the unique list of names describing available cyclone basins. + * the list can be indexed by the ids returned by the + * get_cyclone_basin/s functions. */ void get_cyclone_basin_names(std::vector &names, std::vector &long_names); -/** -load polygons describing the cyclone basins used by TECA - -upon return: - - sizes array has been appended with the size of each basin - starts array has been appended with the starting index of - each basin's coordinates - x/y_coordinates have been appended with the coordinates - ids array has been appended with the basin id - -some basins are comprised of multiple polygons because -they split over the periodic boundary. hence the ids array -is used to identify a basin. +/** load polygons describing the cyclone basins used by TECA + * + * upon return: + * + * sizes array has been appended with the size of each basin + * starts array has been appended with the starting index of + * each basin's coordinates + * x/y_coordinates have been appended with the coordinates + * ids array has been appended with the basin id + * + * some basins are comprised of multiple polygons because + * they split over the periodic boundary. hence the ids array + * is used to identify a basin. */ void get_cyclone_basins(std::vector &sizes, std::vector &starts, std::vector &x_coordinates, std::vector &y_coordinates, std::vector &ids, std::vector &names, std::vector &long_names); -/** -load a cylcone basin by name. Either the short or long name -can be used. see get_cyclone_basin_names. +/** load a cyclone basin by name. Either the short or long name + * can be used. see get_cyclone_basin_names. */ int get_cyclone_basin(const std::string &rname, std::vector &sizes, std::vector &starts, @@ -50,9 +50,8 @@ int get_cyclone_basin(const std::string &rname, std::vector &ids, std::vector &names, std::vector &long_names); -/** -load a cyclone basin by it's region id. region ids must be in the range -of 0 to get_number_of_cylone_basins() - 1. +/** load a cyclone basin by it's region id. region ids must be in the range + * of 0 to get_number_of_cyclone_basins() - 1. */ int get_cyclone_basin(unsigned int rid, std::vector &sizes, std::vector &starts, diff --git a/alg/teca_geometry.h b/alg/teca_geometry.h index bb7306948..d096faff9 100644 --- a/alg/teca_geometry.h +++ b/alg/teca_geometry.h @@ -1,9 +1,12 @@ #ifndef teca_geometry_h #define teca_geometry_h +/// @file + +/// Codes dealing with computational geometry namespace teca_geometry { -// tests if a point is Left|On|Right of an infinite line. +/// tests if a point is Left|On|Right of an infinite line. template bool left(n_t e0x, n_t e0y, n_t e1x, n_t e1y, n_t px, n_t py) { @@ -14,8 +17,9 @@ bool left(n_t e0x, n_t e0y, n_t e1x, n_t e1y, n_t px, n_t py) ((e1x - e0x)*(py - e0y) - (px - e0x)*(e1y - e0y)) >= n_t(); } -// winding number test for a point in a polygon -// winding number is 0 when the point is outside. +/** Winding number test for a point in a polygon. The winding number is 0 when + * the point is outside. + */ template bool point_in_poly(n_t px, n_t py, n_t *vx, n_t *vy, unsigned long nppts) diff --git a/alg/teca_indexed_dataset_cache.cxx b/alg/teca_indexed_dataset_cache.cxx new file mode 100644 index 000000000..f907edc00 --- /dev/null +++ b/alg/teca_indexed_dataset_cache.cxx @@ -0,0 +1,351 @@ +#include "teca_indexed_dataset_cache.h" + +#include "teca_metadata.h" +#include "teca_priority_queue.h" + +#include +#include +#include +#include + +#include +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +//#define TECA_DEBUG + +struct cache_entry +{ + cache_entry() : m_data(nullptr), m_keep(1) {} + + std::mutex m_mutex; // for access to the cache and time + std::condition_variable m_cond; // use to wait for another thread to provide the data + const_p_teca_dataset m_data; // the dataset + unsigned long m_keep; // when 0 safe to delete the element +}; + +using p_cache_entry = std::shared_ptr; + +using index_t = unsigned long; +using priority_t = unsigned long; + +using data_map_t = std::map; +using use_map_t = std::map; + +using heap_t = teca_priority_queue, // to look up priorities + std::less<>, // heapify by smallest + mapped_key_t>; // location tracking container + +using p_heap_t = std::shared_ptr; + +struct teca_indexed_dataset_cache::internals_t +{ + internals_t() : m_current_time(0) + { + mapped_key_priority priority_lookup(m_time_used); + m_heap = heap_t::New(priority_lookup); + } + + std::mutex m_mutex; // for access to the following + p_heap_t m_heap; // heap with least recently used dataset at the top + use_map_t m_time_used; // the use time of each cached dataset + data_map_t m_data; // cached data + priority_t m_current_time; // the current time of use +}; + + +// -------------------------------------------------------------------------- +teca_indexed_dataset_cache::teca_indexed_dataset_cache() : + max_cache_size(0), internals(new internals_t) +{ + this->set_number_of_input_connections(1); + this->set_number_of_output_ports(1); +} + +// -------------------------------------------------------------------------- +teca_indexed_dataset_cache::~teca_indexed_dataset_cache() +{ + delete this->internals; +} + +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_indexed_dataset_cache::get_properties_description( + const std::string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_indexed_dataset_cache":prefix)); + + opts.add_options() + + TECA_POPTS_GET(unsigned long, prefix, max_cache_size, + "Sets the maximum number of datasets to cache.") + + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_indexed_dataset_cache::set_properties( + const std::string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, unsigned long, prefix, max_cache_size) +} +#endif + +// -------------------------------------------------------------------------- +void teca_indexed_dataset_cache::clear_cache() +{ + { + std::lock_guard lock(this->internals->m_mutex); + this->internals->m_heap->clear(); + this->internals->m_time_used.clear(); + this->internals->m_data.clear(); + this->internals->m_current_time = 0; + } +} + +// -------------------------------------------------------------------------- +std::vector teca_indexed_dataset_cache::get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_indexed_dataset_cache::get_upstream_request" << std::endl; +#endif + (void)port; + (void)input_md; + + std::vector up_reqs; + + // force the user to set the cache size + if (this->max_cache_size == 0) + { + TECA_ERROR("max_cache_size is 0, you must set the" + " cache size before use.") + return up_reqs; + } + + // get the requested index + std::string request_key; + if (request.get("index_request_key", request_key)) + { + TECA_ERROR("Failed to locate the index_request_key") + return up_reqs; + } + + index_t index = 0; + if (request.get(request_key, index)) + { + TECA_ERROR("Failed to get the requested index using the" + " index_request_key \"" << request_key << "\"") + return up_reqs; + } + + { + std::lock_guard lock(this->internals->m_mutex); + + // is this index in the cache? + if (this->internals->m_time_used.count(index)) + { + // yes, update the use time + this->internals->m_time_used[index] = ++this->internals->m_current_time; + this->internals->m_heap->modified(index); + + // make a note that it needs to be served one more time before + // it can be removed + p_cache_entry elem = this->internals->m_data[index]; + + { + std::lock_guard elock(elem->m_mutex); + ++elem->m_keep; + } + +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "update entry " << index + << " keep=" << elem->m_keep << std::endl; +#endif + return up_reqs; + } + + // no, not in cache + // set the use time and put in the heap + this->internals->m_time_used[index] = ++this->internals->m_current_time; + this->internals->m_heap->push(index); + + // add an empty cache enrty + this->internals->m_data[index] = std::make_shared(); + +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "add entry " << index << " " + << this->internals->m_current_time << std::endl; +#endif + } + + // generate the request for this index + up_reqs.push_back(request); + return up_reqs; +} + +// -------------------------------------------------------------------------- +const_p_teca_dataset teca_indexed_dataset_cache::execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_indexed_dataset_cache::execute" << std::endl; +#endif + (void)port; + + // get the requested index + std::string request_key; + if (request.get("index_request_key", request_key)) + { + TECA_ERROR("Failed to locate the index_request_key") + return nullptr; + } + + index_t index = 0; + if (request.get(request_key, index)) + { + TECA_ERROR("Failed to get the requested index using the" + " index_request_key \"" << request_key << "\"") + return nullptr; + } + + const_p_teca_dataset data_out; + + // get the cache element associated with the requested index + p_cache_entry elem; + { + std::lock_guard lock(this->internals->m_mutex);; + data_map_t::iterator it = this->internals->m_data.find(index); + if (it == this->internals->m_data.end()) + { + TECA_ERROR("The cache is in an invalid state") + return nullptr; + } + elem = it->second; + } + + if (input_data.size()) + { + // add new data to the cache + { + std::lock_guard elock(elem->m_mutex); + elem->m_data = input_data[0]; + --elem->m_keep; + } + // notify other threads that may be waiting for this data + elem->m_cond.notify_all(); +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "add data " << index + << " keep=" << elem->m_keep << std::endl; +#endif + } + else + { + // fetch existing data from the cache + if (!elem->m_data) + { + // data is not yet ready, it will be provided by another thread + std::unique_lock elock(elem->m_mutex); + if (!elem->m_data) + { + // data is not ready wait for another thread to provide + elem->m_cond.wait(elock, [&]{ return bool(elem->m_data); }); + --elem->m_keep; + } + } + else + { + // data is ready + std::lock_guard elock(elem->m_mutex); + --elem->m_keep; + } +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "use data " << index + << " keep=" << elem->m_keep << std::endl; +#endif + } + + // return the dataset + data_out = elem->m_data; + + // enforce the max cache size + { + std::lock_guard lock(this->internals->m_mutex); + unsigned long n_cached = this->internals->m_time_used.size(); + if (n_cached > this->max_cache_size) + { +#ifdef TECA_DEBUG + std::cerr << "cache too large " << n_cached << std::endl; + this->internals->m_heap->to_stream(std::cerr, false); +#endif + // might have to save some elements if they haven't been served yet + std::vector save; + save.reserve(n_cached); + + unsigned long n_to_rm = n_cached - this->max_cache_size; + + // make one pass over the cache in lru order, or stop if we find + // enough elements that can be deleted + for (unsigned long i = 0; n_to_rm && (i < n_cached); ++i) + { + index_t idx = this->internals->m_heap->pop(); + + p_cache_entry elem = this->internals->m_data[idx]; + + // have all requests for the data been served? + unsigned long keep = 0; + { + std::lock_guard elock(elem->m_mutex); + keep = elem->m_keep; + } + if (keep) + { + // no, delete later + save.push_back(idx); +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "save " + << idx << " keep=" << keep << std::endl; +#endif + } + else + { + // yes, delete now + this->internals->m_data.erase(idx); + this->internals->m_time_used.erase(idx); + --n_to_rm; +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "evict " + << idx << std::endl; +#endif + } + } + + // put elements we couldn't remove because they haven't been + // served yet back on the heap + unsigned long n = save.size(); + for (unsigned long i = 0; i < n; ++i) + { + this->internals->m_heap->push(save[i]); + } + } + } + + return data_out; +} diff --git a/alg/teca_indexed_dataset_cache.h b/alg/teca_indexed_dataset_cache.h new file mode 100644 index 000000000..d7863c7d6 --- /dev/null +++ b/alg/teca_indexed_dataset_cache.h @@ -0,0 +1,65 @@ +#ifndef teca_indexed_dataset_cache_h +#define teca_indexed_dataset_cache_h + +#include "teca_shared_object.h" +#include "teca_algorithm.h" +#include "teca_metadata.h" + +#include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_indexed_dataset_cache) + +/// Caches N datasets such that repeated requests for the same dataset are served from the cache +/** + * A cache storing up to N datasets. Datasets are identified using their + * request index. Repeated requests for the same dataset (ie same index) are + * served from the cache. When more than N unique datasets have been requested + * the cache is modified such that the least recently used dataset is replaced. + */ +class teca_indexed_dataset_cache : public teca_algorithm +{ +public: + TECA_ALGORITHM_STATIC_NEW(teca_indexed_dataset_cache) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_indexed_dataset_cache) + TECA_ALGORITHM_CLASS_NAME(teca_indexed_dataset_cache) + ~teca_indexed_dataset_cache(); + + // report/initialize to/from Boost program options + // objects. + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + + /** @name max_cache_size + * Set the max number of datasets to cache. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(unsigned long, max_cache_size) + ///@} + + /// clear any cached data. + void clear_cache(); + +protected: + teca_indexed_dataset_cache(); + +private: + + std::vector get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) override; + + const_p_teca_dataset execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) override; + +private: + unsigned long max_cache_size; + + struct internals_t; + internals_t *internals; +}; + +#endif diff --git a/alg/teca_integrated_vapor_transport.cxx b/alg/teca_integrated_vapor_transport.cxx index 39f2fd5f2..01405eaa1 100644 --- a/alg/teca_integrated_vapor_transport.cxx +++ b/alg/teca_integrated_vapor_transport.cxx @@ -156,15 +156,17 @@ void teca_integrated_vapor_transport::get_properties_description( opts.add_options() TECA_POPTS_GET(std::string, prefix, wind_u_variable, - "name of the variable containg the lon component of the wind vector (ua)") + "name of the variable containg the lon component of the wind vector") TECA_POPTS_GET(std::string, prefix, wind_v_variable, - "name of the variable containg the lat component of the wind vector (va)") - TECA_POPTS_GET(std::string, prefix, specific_humidty_variable, - "name of the variable containg the specific humidity (hus)") + "name of the variable containg the lat component of the wind vector") + TECA_POPTS_GET(std::string, prefix, specific_humidity_variable, + "name of the variable containg the specific humidity") TECA_POPTS_GET(double, prefix, fill_value, - "the value of the NetCDF _FillValue attribute (1e20)") + "the value of the NetCDF _FillValue attribute") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -172,6 +174,8 @@ void teca_integrated_vapor_transport::get_properties_description( void teca_integrated_vapor_transport::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, wind_u_variable) TECA_POPTS_SET(opts, std::string, prefix, wind_v_variable) TECA_POPTS_SET(opts, std::string, prefix, specific_humidity_variable) @@ -419,5 +423,22 @@ const_p_teca_dataset teca_integrated_vapor_transport::execute( ) ) + // pass 2D arrays through. + p_teca_array_collection in_arrays = + std::const_pointer_cast(in_mesh->get_point_arrays()); + + p_teca_array_collection out_arrays = out_mesh->get_point_arrays(); + + int n_arrays = in_arrays->size(); + for (int i = 0; i < n_arrays; ++i) + { + p_teca_variant_array array = in_arrays->get(i); + if (array->size() == nxy) + { + // pass the array. + out_arrays->append(in_arrays->get_name(i), array); + } + } + return out_mesh; } diff --git a/alg/teca_integrated_vapor_transport.h b/alg/teca_integrated_vapor_transport.h index 44bd896a4..4ce7649a7 100644 --- a/alg/teca_integrated_vapor_transport.h +++ b/alg/teca_integrated_vapor_transport.h @@ -10,19 +10,21 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_integrated_vapor_transport) -/// an algorithm that computes integrated vapor transport (IVT) +/// An algorithm that computes integrated vapor transport (IVT) /** -Compute integrated vaport transport (IVT) from wind vector and -specific humidity. - -IVT = - \frac{1}{g} \int_{p_0}^{p_1} \vec{v} q dp - -where q is the specific humidity, and \vec{v} = (u, v) are the -longitudinal and latitudinal components of wind. - -This calculation is an instance of a vertical reduction where -a 3D mesh is transformed into a 2D one. -*/ + * Compute integrated vapor transport (IVT) from wind vector and + * specific humidity. + * + * \f[ + * IVT = \frac{1}{g} \int_{p_{sfc}}^{p_{top}} \vec{v} q dp + * \f] + * + * where q is the specific humidity, and \f$\vec{v} = (u, v)\f$ are the + * longitudinal and latitudinal components of wind. + * + * This calculation is an instance of a vertical reduction where + * a 3D mesh is transformed into a 2D one. + */ class teca_integrated_vapor_transport : public teca_vertical_reduction { public: @@ -31,35 +33,59 @@ class teca_integrated_vapor_transport : public teca_vertical_reduction TECA_ALGORITHM_CLASS_NAME(teca_integrated_vapor_transport) ~teca_integrated_vapor_transport(); - // report/initialize to/from Boost program options - // objects. + /** @name program_options + * report/initialize to/from Boost program options objects. + */ + ///@{ TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() + ///@} - // set the name of the varaiable that contains the longitudinal - // component of the wind vector ("ua") + /** @name wind_u_variable + * set the name of the varaiable that contains the longitudinal component + * of the wind vector ("ua") + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, wind_u_variable) + ///@} - // set the name of the varaiable that contains the latitudinal - // component of the wind vector ("va") + /** @name wind_v_variable + * set the name of the varaiable that contains the latitudinal component of + * the wind vector ("va") + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, wind_v_variable) - - // set the name of the variable that contains the specific - // humidity ("hus") - TECA_ALGORITHM_PROPERTY(std::string, - specific_humidity_variable) - - // set the name of the varaiable that contains the longitudinal - // component of the ivt vector ("ivt_u") + ///@} + + /** @name specific_humidity_variable + * set the name of the variable that contains the specific humidity ("hus") + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, specific_humidity_variable) + ///@} + + /** @name ivt_u_variable + * set the name of the varaiable that contains the longitudinal component + * of the ivt vector ("ivt_u") + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, ivt_u_variable) + ///@} - // set the name of the varaiable that contains the latitudinal - // component of the ivt vector ("ivt_v") + /** @name ivt_v_variable + * set the name of the varaiable that contains the latitudinal component of + * the ivt vector ("ivt_v") + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, ivt_v_variable) + ///@} - // set the _fillValue attribute for the output data. - // default 1.0e20 + /** @name fill_value + * set the _fillValue attribute for the output data. default 1.0e20 + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, fill_value) + ///@} protected: teca_integrated_vapor_transport(); diff --git a/alg/teca_integrated_water_vapor.cxx b/alg/teca_integrated_water_vapor.cxx new file mode 100644 index 000000000..481dfc690 --- /dev/null +++ b/alg/teca_integrated_water_vapor.cxx @@ -0,0 +1,362 @@ +#include "teca_integrated_water_vapor.h" + +#include "teca_cartesian_mesh.h" +#include "teca_array_collection.h" +#include "teca_variant_array.h" +#include "teca_metadata.h" +#include "teca_coordinate_util.h" + +#include +#include +#include +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +using std::string; +using std::vector; +using std::cerr; +using std::endl; +using std::cos; + +//#define TECA_DEBUG + +namespace { +template +void cartesian_iwv(unsigned long nx, unsigned long ny, unsigned long nz, + const coord_t *plev, const num_t *q, num_t *iwv) +{ + unsigned long nxy = nx*ny; + + // initialize the result + memset(iwv, 0, nxy*sizeof(num_t)); + + // work an x-y slice at a time + unsigned long nzm1 = nz - 1; + for (unsigned long k = 0; k < nzm1; ++k) + { + // dp over the slice + num_t h2 = num_t(0.5) * (plev[k+1] - plev[k]); + + // the current two x-y-planes of data + unsigned long knxy = k*nxy; + const num_t *q_k0 = q + knxy; + const num_t *q_k1 = q_k0 + nxy; + + // accumulate this plane of data using trapazoid rule + for (unsigned long i = 0; i < nxy; ++i) + { + iwv[i] += h2 * (q_k0[i] + q_k1[i]); + } + } + + // check the sign, in this way we can handle both increasing and decreasing + // pressure coordinates + num_t s = plev[1] - plev[0] < num_t(0) ? num_t(-1) : num_t(1); + + // scale by -1/g + num_t m1g = s/num_t(9.80665); + for (unsigned long i = 0; i < nxy; ++i) + iwv[i] *= m1g; +} + +template +void cartesian_iwv(unsigned long nx, unsigned long ny, unsigned long nz, + const coord_t *plev, const num_t *q, const char *q_valid, num_t *iwv) +{ + unsigned long nxy = nx*ny; + + // initialize the result + memset(iwv, 0, nxy*sizeof(num_t)); + + // work an x-y slice at a time + unsigned long nzm1 = nz - 1; + for (unsigned long k = 0; k < nzm1; ++k) + { + // dp over the slice + num_t h2 = num_t(0.5) * (plev[k+1] - plev[k]); + + // the current two x-y-planes of data + unsigned long knxy = k*nxy; + const num_t *q_k0 = q + knxy; + const num_t *q_k1 = q_k0 + nxy; + + const char *q_valid_k0 = q_valid + knxy; + const char *q_valid_k1 = q_valid_k0 + nxy; + + // accumulate this plane of data using trapazoid rule + for (unsigned long i = 0; i < nxy; ++i) + { + iwv[i] += ((q_valid_k0[i] && q_valid_k1[i]) ? + h2 * (q_k0[i] + q_k1[i]) : num_t(0)); + } + } + + // check the sign, in this way we can handle both increasing and decreasing + // pressure coordinates + num_t s = plev[1] - plev[0] < num_t(0) ? num_t(-1) : num_t(1); + + // scale by -1/g + num_t m1g = s/num_t(9.80665); + for (unsigned long i = 0; i < nxy; ++i) + iwv[i] *= m1g; +} +} + +// -------------------------------------------------------------------------- +teca_integrated_water_vapor::teca_integrated_water_vapor() : + specific_humidity_variable("Q"), iwv_variable("IWV"), + fill_value(1.0e20) +{ + this->set_number_of_input_connections(1); + this->set_number_of_output_ports(1); +} + +// -------------------------------------------------------------------------- +teca_integrated_water_vapor::~teca_integrated_water_vapor() +{} + +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_integrated_water_vapor::get_properties_description( + const string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_integrated_water_vapor":prefix)); + + opts.add_options() + TECA_POPTS_GET(std::string, prefix, specific_humidity_variable, + "name of the variable containg the specific humidity") + TECA_POPTS_GET(double, prefix, fill_value, + "the value of the NetCDF _FillValue attribute") + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_integrated_water_vapor::set_properties( + const string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, std::string, prefix, specific_humidity_variable) + TECA_POPTS_SET(opts, double, prefix, fill_value) +} +#endif + +// -------------------------------------------------------------------------- +teca_metadata teca_integrated_water_vapor::get_output_metadata( + unsigned int port, + const std::vector &input_md) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_integrated_water_vapor::get_output_metadata" << std::endl; +#endif + (void)port; + + // set things up in the first pass, and don't modify in subsequent passes + // due to threading concerns + + if (this->get_number_of_derived_variables() == 0) + { + // the base class will handle dealing with the transformation of + // mesh dimensions and reporting the array we produce, but we have + // to determine the data type and tell the name of the produced array. + const teca_metadata &md = input_md[0]; + + teca_metadata attributes; + if (md.get("attributes", attributes)) + { + TECA_ERROR("Failed to determine output data type " + "because attributes are misisng") + return teca_metadata(); + } + + teca_metadata hus_atts; + if (attributes.get(this->specific_humidity_variable, hus_atts)) + { + TECA_ERROR("Failed to determine output data type " + "because attributes for \"" << this->specific_humidity_variable + << "\" are misisng") + return teca_metadata(); + } + + int type_code = 0; + if (hus_atts.get("type_code", type_code)) + { + TECA_ERROR("Failed to determine output data type " + "because attributes for \"" << this->specific_humidity_variable + << "\" is misisng a \"type_code\"") + return teca_metadata(); + } + + teca_array_attributes iwv_atts( + type_code, teca_array_attributes::point_centering, + 0, "kg m^{-1} s^{-1}", "longitudinal integrated vapor transport", + "the longitudinal component of integrated vapor transport", + 1, this->fill_value); + + + // install name and attributes of the output variables in the base classs + this->append_derived_variable(this->iwv_variable); + this->append_derived_variable_attribute(iwv_atts); + } + + if (this->get_number_of_dependent_variables() == 0) + { + // install the names of the input variables in the base class + this->append_dependent_variable(this->specific_humidity_variable); + } + + // invoke the base class method, which does the work of transforming + // the mesh and reporting the variables and their attributes. + return teca_vertical_reduction::get_output_metadata(port, input_md); +} + +// -------------------------------------------------------------------------- +std::vector teca_integrated_water_vapor::get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) +{ + // invoke the base class method + return teca_vertical_reduction::get_upstream_request(port, input_md, request); +} + +// -------------------------------------------------------------------------- +const_p_teca_dataset teca_integrated_water_vapor::execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_integrated_water_vapor::execute" << std::endl; +#endif + (void)port; + + // get the input mesh + const_p_teca_cartesian_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); + + if (!in_mesh) + { + TECA_ERROR("Failed to compute IWV because a cartesian mesh is required.") + return nullptr; + } + + // get the input dimensions + unsigned long extent[6] = {0}; + if (in_mesh->get_extent(extent)) + { + TECA_ERROR("Failed to compute IWV because mesh extent is missing.") + return nullptr; + } + + unsigned long nx = extent[1] - extent[0] + 1; + unsigned long ny = extent[3] - extent[2] + 1; + unsigned long nz = extent[5] - extent[4] + 1; + + // get the pressure coordinates + const_p_teca_variant_array p = in_mesh->get_z_coordinates(); + if (!p) + { + TECA_ERROR("Failed to compute IWV because pressure coordinates are missing") + return nullptr; + } + + if (p->size() < 2) + { + TECA_ERROR("Failed to compute IWV because z dimensions " + << p->size() << " < 2 as required by the integration method") + return nullptr; + } + + // gather the input arrays + const_p_teca_variant_array q = + in_mesh->get_point_arrays()->get(this->specific_humidity_variable); + + if (!q) + { + TECA_ERROR("Failed to compute IWV because specific humidity \"" + << this->specific_humidity_variable << "\" is missing") + return nullptr; + } + + const_p_teca_variant_array q_valid = + in_mesh->get_point_arrays()->get(this->specific_humidity_variable + "_valid"); + + // the base class will construct the output mesh + p_teca_cartesian_mesh out_mesh + = std::dynamic_pointer_cast( + std::const_pointer_cast( + teca_vertical_reduction::execute(port, input_data, request))); + + if (!out_mesh) + { + TECA_ERROR("Failed to compute IWV because the output mesh was " + "not constructed") + return nullptr; + } + + // allocate the output arrays + unsigned long nxy = nx*ny; + p_teca_variant_array iwv = q->new_instance(nxy); + + // store the result + out_mesh->get_point_arrays()->set(this->iwv_variable, iwv); + + // calculate IWV + NESTED_TEMPLATE_DISPATCH_FP(const teca_variant_array_impl, + p.get(), _COORDS, + + const NT_COORDS *p_p = static_cast(p.get())->get(); + + NESTED_TEMPLATE_DISPATCH_FP(teca_variant_array_impl, + iwv.get(), _DATA, + + NT_DATA *p_iwv = static_cast(iwv.get())->get(); + + const NT_DATA *p_q = static_cast(q.get())->get(); + + const char *p_q_valid = nullptr; + if (q_valid) + { + using TT_MASK = teca_char_array; + + p_q_valid = dynamic_cast(q_valid.get())->get(); + + ::cartesian_iwv(nx, ny, nz, p_p, p_q, p_q_valid, p_iwv); + } + else + { + ::cartesian_iwv(nx, ny, nz, p_p, p_q, p_iwv); + } + ) + ) + + // pass 2D arrays through. + p_teca_array_collection in_arrays = + std::const_pointer_cast(in_mesh->get_point_arrays()); + + p_teca_array_collection out_arrays = out_mesh->get_point_arrays(); + + int n_arrays = in_arrays->size(); + for (int i = 0; i < n_arrays; ++i) + { + p_teca_variant_array array = in_arrays->get(i); + if (array->size() == nxy) + { + // pass the array. + out_arrays->append(in_arrays->get_name(i), array); + } + } + + return out_mesh; +} diff --git a/alg/teca_integrated_water_vapor.h b/alg/teca_integrated_water_vapor.h new file mode 100644 index 000000000..3070e6a49 --- /dev/null +++ b/alg/teca_integrated_water_vapor.h @@ -0,0 +1,85 @@ +#ifndef teca_integrated_water_vapor_h +#define teca_integrated_water_vapor_h + +#include "teca_shared_object.h" +#include "teca_vertical_reduction.h" +#include "teca_metadata.h" + +#include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_integrated_water_vapor) + +/// An algorithm that computes integrated water vapor (IWV) +/** + * Compute integrated vaport transport (IWV) from the specific humidity. + * + * \f[ + * IWV = \frac{1}{g} \int_{p_{sfc}}^{p_{top}} q dp + * \f] + * + * where q is the specific humidity. + * + * This calculation is an instance of a vertical reduction where + * a 3D mesh is transformed into a 2D one. + */ +class teca_integrated_water_vapor : public teca_vertical_reduction +{ +public: + TECA_ALGORITHM_STATIC_NEW(teca_integrated_water_vapor) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_integrated_water_vapor) + TECA_ALGORITHM_CLASS_NAME(teca_integrated_water_vapor) + ~teca_integrated_water_vapor(); + + // report/initialize to/from Boost program options + // objects. + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + + /** @name specific_humidity_variable + * set the name of the variable that contains the specific humidity ("hus") + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, specific_humidity_variable) + ///@} + + /** @name iwv_variable + * set the name of the varaiable that contains the integrated water vapor + * ("iwv"). + */ + ///@{ + TECA_ALGORITHM_PROPERTY(std::string, iwv_variable) + ///@} + + /** @name fill_value + * set the _fillValue attribute for the output data. default 1.0e20 + */ + ///@{ + TECA_ALGORITHM_PROPERTY(double, fill_value) + ///@} + +protected: + teca_integrated_water_vapor(); + +private: + teca_metadata get_output_metadata( + unsigned int port, + const std::vector &input_md) override; + + std::vector get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) override; + + const_p_teca_dataset execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) override; + +private: + std::string specific_humidity_variable; + std::string iwv_variable; + double fill_value; +}; + +#endif diff --git a/alg/teca_l2_norm.cxx b/alg/teca_l2_norm.cxx index 9d66dbdd0..df2aed92f 100644 --- a/alg/teca_l2_norm.cxx +++ b/alg/teca_l2_norm.cxx @@ -1,6 +1,6 @@ #include "teca_l2_norm.h" -#include "teca_cartesian_mesh.h" +#include "teca_mesh.h" #include "teca_array_collection.h" #include "teca_variant_array.h" #include "teca_metadata.h" @@ -92,6 +92,8 @@ void teca_l2_norm::get_properties_description( "array to store the computed norm in") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -99,6 +101,8 @@ void teca_l2_norm::get_properties_description( void teca_l2_norm::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, component_0_variable) TECA_POPTS_SET(opts, std::string, prefix, component_1_variable) TECA_POPTS_SET(opts, std::string, prefix, component_2_variable) @@ -279,12 +283,12 @@ const_p_teca_dataset teca_l2_norm::execute( (void)port; // get the input mesh - const_p_teca_cartesian_mesh in_mesh - = std::dynamic_pointer_cast(input_data[0]); + const_p_teca_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); if (!in_mesh) { - TECA_ERROR("Failed to compute l2 norm. dataset is not a teca_cartesian_mesh") + TECA_ERROR("Failed to compute l2 norm. dataset is not a teca_mesh") return nullptr; } @@ -355,11 +359,10 @@ const_p_teca_dataset teca_l2_norm::execute( // create the output mesh, pass everything through, and // add the l2 norm array - p_teca_cartesian_mesh out_mesh = teca_cartesian_mesh::New(); - - out_mesh->shallow_copy(std::const_pointer_cast(in_mesh)); + p_teca_mesh out_mesh = std::static_pointer_cast + (std::const_pointer_cast(in_mesh)->new_shallow_copy()); - out_mesh->get_point_arrays()->append( + out_mesh->get_point_arrays()->set( this->get_l2_norm_variable(request), l2_norm); return out_mesh; diff --git a/alg/teca_l2_norm.h b/alg/teca_l2_norm.h index f67e23581..5a67eceb0 100644 --- a/alg/teca_l2_norm.h +++ b/alg/teca_l2_norm.h @@ -10,10 +10,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_l2_norm) -/// an algorithm that computes L2 norm -/** -Compute L2 norm -*/ +/// An algorithm that computes L2 norm class teca_l2_norm : public teca_algorithm { public: @@ -27,19 +24,43 @@ class teca_l2_norm : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the arrays that contain the vector components - // to compute norm from + /** @name component_0_variable + * Set the arrays that contain the vector components to compute the norm + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_0_variable) + ///@} + + /** @name component_1_variable + * Set the arrays that contain the vector components to compute the norm + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_1_variable) + ///@} + + /** @name component_2_variable + * Set the arrays that contain the vector components to compute the norm + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_2_variable) + ///@} - // set the name of the array to store the result in. - // the default is "l2_norm" + /** @name l2_norm_variable + * set the name of the array to store the result in. the default is + * "l2_norm" + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, l2_norm_variable) + ///@} protected: teca_l2_norm(); + // helpers to get the variable names from either the incoming + // request or the class member variable. std::string get_component_0_variable(const teca_metadata &request); std::string get_component_1_variable(const teca_metadata &request); std::string get_component_2_variable(const teca_metadata &request); diff --git a/alg/teca_laplacian.cxx b/alg/teca_laplacian.cxx index 43c481e44..9d9966dd3 100644 --- a/alg/teca_laplacian.cxx +++ b/alg/teca_laplacian.cxx @@ -44,7 +44,7 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, // delta lon squared as a function of latitude num_t d_lon = (lon[1] - lon[0]) * deg_to_rad() * earth_radius(); // tan(lat) - num_t *tan_lat = static_cast(malloc(n_bytes)); + num_t *tan_lat = static_cast(malloc(n_bytes)); for (unsigned long j = 0; j < n_lat; ++j) { delta_lon_sq[j] = pow(d_lon * cos(lat[j] * deg_to_rad()),2); @@ -65,7 +65,7 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, { // set the current row in the u/v/w arrays unsigned long jj = j*n_lon; - /* + /* * The following f_* variables describe the field * f in a grid oriented fashion: * @@ -74,12 +74,12 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, * f_ijm f_ji f_ijp * * f_imjm f_imj f_imjp - * + * * The 'j' direction represents longitude, the - * 'i' direciton represents latitude. + * 'i' direciton represents latitude. * * Note: The laplacian represented here uses the chain - * rule to separate the (1/cos(lat)*d(cos(lat)*df/dlat)/dlat + * rule to separate the (1/cos(lat)*d(cos(lat)*df/dlat)/dlat * term into two terms. * */ @@ -94,15 +94,15 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, // set the pointer index for the output field w // ... this is index i,j num_t *ww = w + jj; - // create a dummy variable for u**2 + // create a dummy variable for u**2 num_t dlon_sq = delta_lon_sq[j]; for (unsigned long i = 1; i < max_i; ++i) { // calculate the laplacian in spherical coordinates, assuming // constant radius R. - ww[i] = (f_imj[i] - num_t(2)*f_ij[i] + f_ipj[i])/dlat_sq - - tan_lat[j]*(f_ipj[i]-f_imj[i])/dlat + + ww[i] = (f_imj[i] - num_t(2)*f_ij[i] + f_ipj[i])/dlat_sq - + tan_lat[j]*(f_ipj[i]-f_imj[i])/dlat + (f_ijm[i] - num_t(2)*f_ij[i] + f_ijp[i])/dlon_sq; } } @@ -125,13 +125,13 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, // set the pointer index for the output field w // ... this is index i,j num_t *ww = w + jj; - // create a dummy variable for u**2 + // create a dummy variable for u**2 num_t dlon_sq = delta_lon_sq[j]; // calculate the laplacian in spherical coordinates, assuming // constant radius R. - ww[0] = (f_imj[0] - num_t(2)*f_ij[0] + f_ipj[0])/dlat_sq - - tan_lat[j]*(f_ipj[0]-f_imj[0])/dlat + + ww[0] = (f_imj[0] - num_t(2)*f_ij[0] + f_ipj[0])/dlat_sq - + tan_lat[j]*(f_ipj[0]-f_imj[0])/dlat + (f_ijm[0] - num_t(2)*f_ij[0] + f_ijp[0])/dlon_sq; } @@ -152,13 +152,13 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, // set the pointer index for the output field w // ... this is index i,j num_t *ww = w + jj + max_i; - // create a dummy variable for u**2 + // create a dummy variable for u**2 num_t dlon_sq = delta_lon_sq[j]; // calculate the laplacian in spherical coordinates, assuming // constant radius R. - ww[0] = (f_imj[0] - num_t(2)*f_ij[0] + f_ipj[0])/dlat_sq - - tan_lat[j]*(f_ipj[0]-f_imj[0])/dlat + + ww[0] = (f_imj[0] - num_t(2)*f_ij[0] + f_ipj[0])/dlat_sq - + tan_lat[j]*(f_ipj[0]-f_imj[0])/dlat + (f_ijm[0] - num_t(2)*f_ij[0] + f_ijp[0])/dlon_sq; } } @@ -193,7 +193,7 @@ void laplacian(num_t *w, const pt_t *lon, const pt_t *lat, // -------------------------------------------------------------------------- teca_laplacian::teca_laplacian() : - component_0_variable(), + component_0_variable(), laplacian_variable("laplacian") { this->set_number_of_input_connections(1); @@ -219,6 +219,8 @@ void teca_laplacian::get_properties_description( "array to store the computed laplacian in") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -226,6 +228,8 @@ void teca_laplacian::get_properties_description( void teca_laplacian::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, component_0_variable) TECA_POPTS_SET(opts, std::string, prefix, laplacian_variable) } diff --git a/alg/teca_laplacian.h b/alg/teca_laplacian.h index 247e00bd4..35e623d39 100644 --- a/alg/teca_laplacian.h +++ b/alg/teca_laplacian.h @@ -10,10 +10,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_laplacian) -/// an algorithm that computes laplacian -/** -Compute laplacian from a vector field. -*/ +/// An algorithm that computes the Laplacian from a vector field. class teca_laplacian : public teca_algorithm { public: @@ -27,14 +24,29 @@ class teca_laplacian : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the arrays that contain the vector components - // to compute laplacian from + /** @name component_0_variable + * Set the arrays that contain the vector components to compute laplacian + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_0_variable) + ///@} + + /** @name component_1_variable + * Set the arrays that contain the vector components to compute laplacian + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_1_variable) + ///@} - // set the name of the array to store the result in. - // the default is "laplacian" + /** @name laplacian_variable + * Set the name of the array to store the result in. the default is + * "laplacian". + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, laplacian_variable) + ///@} protected: teca_laplacian(); diff --git a/alg/teca_latitude_damper.cxx b/alg/teca_latitude_damper.cxx index 34478dcfd..0a4804a1e 100644 --- a/alg/teca_latitude_damper.cxx +++ b/alg/teca_latitude_damper.cxx @@ -3,7 +3,7 @@ #include "teca_variant_array.h" #include "teca_metadata.h" #include "teca_cartesian_mesh.h" -#include "teca_metadata_util.h" +#include "teca_string_util.h" #include #include @@ -74,27 +74,31 @@ void teca_latitude_damper::get_properties_description( { options_description opts("Options for " + (prefix.empty()?"teca_latitude_damper":prefix)); - + opts.add_options() TECA_POPTS_GET(double, prefix, center, "set the center (mu) for the gaussian filter") TECA_POPTS_GET(double, prefix, half_width_at_half_max, "set the value of the half width at half maximum (HWHM) " "to calculate sigma from: sigma = HWHM/std::sqrt(2.0*std::log(2.0))") - TECA_POPTS_GET(std::vector, prefix, damped_variables, + TECA_POPTS_MULTI_GET(std::vector, prefix, damped_variables, "set the variables that will be damped by the inverted " "gaussian filter") TECA_POPTS_GET(std::string, prefix, variable_post_fix, "set the post-fix that will be attached to the variables " "that will be saved in the output") ; - + + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } // -------------------------------------------------------------------------- void teca_latitude_damper::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, double, prefix, center) TECA_POPTS_SET(opts, double, prefix, half_width_at_half_max) TECA_POPTS_SET(opts, std::vector, prefix, damped_variables) @@ -213,14 +217,14 @@ std::vector teca_latitude_damper::get_upstream_request( arrays.insert(damped_vars.begin(), damped_vars.end()); - // Cleaning off the postfix for arrays passed in the pipeline. + // Cleaning off the postfix for arrays passed in the pipeline. // For ex a down stream could request "foo_damped" then we'd // need to request "foo". also remove "foo_damped" from the // request. const std::string &var_post_fix = this->variable_post_fix; if (!var_post_fix.empty()) { - teca_metadata_util::remove_post_fix(arrays, var_post_fix); + teca_string_util::remove_post_fix(arrays, var_post_fix); } req.set("arrays", arrays); diff --git a/alg/teca_latitude_damper.h b/alg/teca_latitude_damper.h index 9cab2b7ff..908b6cc22 100644 --- a/alg/teca_latitude_damper.h +++ b/alg/teca_latitude_damper.h @@ -10,21 +10,24 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_latitude_damper) +/// Inverted Gaussian damper for scalar fields. /** -damps the specified scalar field(s) using an inverted Gaussian centered on a -given latitude with a half width specified in degrees latitude. The paramters -defining the Gaussian (center, half width at half max) can be specified by the -user directly or by down stream algorithm via the following keys in the request. - -request keys: - - teca_latitude_damper::damped_variables - teca_latitude_damper::half_width_at_half_max - teca_latitude_damper::center - -note that user specified values take precedence over request keys. When using -request keys be sure to include the variable post-fix. -*/ + * Damps the specified scalar field(s) using an inverted Gaussian centered on a + * given latitude with a half width specified in degrees latitude. The + * parameters defining the Gaussian (center, half width at half max) can be + * specified by the user directly or by down stream algorithm via the + * following keys in the request. + * + * request keys: + * + * teca_latitude_damper::damped_variables + * teca_latitude_damper::half_width_at_half_max + * teca_latitude_damper::center + * + * @note + * User specified values take precedence over request keys. When + * using request keys be sure to include the variable post-fix. + */ class teca_latitude_damper : public teca_algorithm { public: @@ -38,7 +41,7 @@ class teca_latitude_damper : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the center of the Gaussian in units of degress latitude. + // set the center of the Gaussian in units of degrees latitude. // default is 0.0 deg lat TECA_ALGORITHM_PROPERTY(double, center) diff --git a/alg/teca_mask.h b/alg/teca_mask.h index 6c7dcb84d..02b6072e8 100644 --- a/alg/teca_mask.h +++ b/alg/teca_mask.h @@ -1,6 +1,8 @@ #ifndef teca_mask_h #define teca_mask_h +/// @file + #include "teca_shared_object.h" #include "teca_algorithm.h" #include "teca_metadata.h" @@ -11,9 +13,8 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_mask) /// an algorithm that masks a range of values -/** -An algorithm to mask a range of values in an array. Values -in the range are replaced with the mask value. +/** An algorithm to mask a range of values in an array. Values + * in the range are replaced with the mask value. */ class teca_mask : public teca_algorithm { diff --git a/alg/teca_normalize_coordinates.cxx b/alg/teca_normalize_coordinates.cxx index a9544c7ae..a4f946816 100644 --- a/alg/teca_normalize_coordinates.cxx +++ b/alg/teca_normalize_coordinates.cxx @@ -5,6 +5,7 @@ #include "teca_variant_array.h" #include "teca_metadata.h" #include "teca_coordinate_util.h" +#include "teca_metadata_util.h" #include #include @@ -16,35 +17,80 @@ #include #endif -using std::cerr; -using std::endl; - //#define TECA_DEBUG -struct teca_normalize_coordinates::internals_t +struct internals { - internals_t() {} - ~internals_t() {} - - static - p_teca_variant_array normalize_axis(const const_p_teca_variant_array &x); - - static - void normalize_extent(p_teca_variant_array out_x, - p_teca_variant_array out_y, p_teca_variant_array out_z, - unsigned long *whole_extent, unsigned long *extent_in, - unsigned long *extent_out); - - static - void normalize_variables(bool normalize_x, - bool normalize_y, bool normalize_z, unsigned long *extent, - p_teca_array_collection data); + // check and flip the axis. templated on comparison type std::less puts in + // ascending order, std::greater puts in descending order. + template typename compare_t> + static int reorder(p_teca_variant_array &x_out, + const const_p_teca_variant_array &x, bool &do_reorder); + + // transforms coordinates from [-180, 180] to [0, 360]. + template + static data_t periodic_shift_x(data_t x); + + // transforms coordinates from [0, 360] to [-180, 180]. + template + static data_t inv_periodic_shift_x(data_t x); + + template + static void periodic_shift_x(coord_t *pxo, + unsigned long *pmap, const coord_t *px, unsigned long nx); + + template + static void inv_periodic_shift_x(unsigned long *pmap, + const coord_t *px, unsigned long nx); + + static int inv_periodic_shift_x( + p_teca_unsigned_long_array &map, + const const_p_teca_variant_array &x); + + static int periodic_shift_x(p_teca_variant_array &out_x, + p_teca_unsigned_long_array &shift_map, + const const_p_teca_variant_array &in_x, + bool &shifted_x); + + static int periodic_shift_x(p_teca_array_collection data, + const teca_metadata &attributes, + const const_p_teca_unsigned_long_array &shift_map, + const unsigned long *extent_in, + const unsigned long *extent_out); + + // put the y-axis in ascending order if it is not. if a transformation was + // applied reordered_y is set. + static int ascending_order_y( + p_teca_variant_array &out_y, const const_p_teca_variant_array &in_y, + bool &reorder_y); + + // apply corresponding transformation that put the -y-axis in ascending + // order to all data arrays + static int ascending_order_y(p_teca_array_collection data, + const teca_metadata &attributes, const unsigned long *extent); + + + template + struct indirect_less; }; +template +struct internals::indirect_less +{ + indirect_less(data_t *data) : m_data(data) {} + + bool operator()(const unsigned long &l, const unsigned long &r) + { + return m_data[l] < m_data[r]; + } + + data_t *m_data; +}; // -------------------------------------------------------------------------- -p_teca_variant_array teca_normalize_coordinates::internals_t::normalize_axis( - const const_p_teca_variant_array &x) +template typename compare_t> +int internals::reorder(p_teca_variant_array &x_out, + const const_p_teca_variant_array &x, bool &do_reorder) { unsigned long nx = x->size(); unsigned long x1 = nx - 1; @@ -52,190 +98,343 @@ p_teca_variant_array teca_normalize_coordinates::internals_t::normalize_axis( NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, x.get(), _C, - // detect coordinate axis in descending order, reorder it in ascending - // order. for instance an // input of x = (90 30 -30 -90) is transformed - // to x = (-90 -30 30 90) const NT_C *px = dynamic_cast(x.get())->get(); - if (px[x1] < px[0]) - { - p_teca_variant_array xo = x->new_instance(nx); - NT_C *pxo = static_cast*>(xo.get())->get(); - pxo += x1; - for (unsigned long i = 0; i < nx; ++i) - pxo[-i] = px[i]; + // if comp(x0, x1) reverse the axis. + // when comp is less than the output will be ascending + // when comp is greater than the output will be descending + compare_t compare; + do_reorder = compare(px[x1], px[0]); - return xo; - } + if (!do_reorder) + return 0; + + p_teca_variant_array xo = x->new_instance(nx); + NT_C *pxo = static_cast*>(xo.get())->get(); + + pxo += x1; + for (unsigned long i = 0; i < nx; ++i) + pxo[-i] = px[i]; + + x_out = xo; + + return 0; ) - return nullptr; + + TECA_ERROR("Unsupported coordinate type " << x->get_class_name()) + return -1; } // -------------------------------------------------------------------------- -void teca_normalize_coordinates::internals_t::normalize_extent( - p_teca_variant_array out_x, p_teca_variant_array out_y, - p_teca_variant_array out_z, unsigned long *whole_extent, - unsigned long *extent_in, unsigned long *extent_out) +template +data_t internals::periodic_shift_x(data_t x) { -#if defined(TECA_DEBUG) - cerr << "out=[" << out_x << ", " << out_y << ", " << out_z << "]" << endl - << "whole_extent=[" << whole_extent[0] << ", " << whole_extent[1] << ", " - << whole_extent[2] << ", " << whole_extent[3] << ", " << whole_extent[4] - << ", " << whole_extent[5] << "]" << endl << "extent_in=[" << extent_in[0] - << ", " << extent_in[1] << ", " << extent_in[2] << ", " << extent_in[3] - << ", " << extent_in[4] << ", " << extent_in[5] << "]" << endl; -#endif + if (x < data_t(0)) + return x + data_t(360); + return x; +} - memcpy(extent_out, extent_in, 6*sizeof(unsigned long)); +// -------------------------------------------------------------------------- +template +data_t internals::inv_periodic_shift_x(data_t x) +{ + if (x > data_t(180)) + return x - data_t(360); + return x; +} - // detect coordinate axes in descending order, transform the incoming - // extents from ascending order coordinates back to original descending - // order coordinate system so the upstream gets the correct extent - if (out_x) - { - unsigned long wnx = whole_extent[1] - whole_extent[0]; - extent_out[0] = wnx - extent_in[1]; - extent_out[1] = wnx - extent_in[0]; - } +// -------------------------------------------------------------------------- +template +void internals::periodic_shift_x(coord_t *pxo, + unsigned long *pmap, const coord_t *px, unsigned long nx) +{ + coord_t *tmp = (coord_t*)malloc(nx*sizeof(coord_t)); - if (out_y) - { - unsigned long wny = whole_extent[3] - whole_extent[2]; - extent_out[2] = wny - extent_in[3]; - extent_out[3] = wny - extent_in[2]; - } + // apply the periodic shift, this will leave the axis out of order + for (unsigned long i = 0; i < nx; ++i) + tmp[i] = periodic_shift_x(px[i]); - if (out_z) - { - unsigned long wnz = whole_extent[5] - whole_extent[4]; - extent_out[4] = wnz - extent_in[5]; - extent_out[5] = wnz - extent_in[4]; - } + // construct the map the puts the axis back into order. + for (unsigned long i = 0; i < nx; ++i) + pmap[i] = i; -#if defined(TECA_DEBUG) - cerr << "extent_out=[" << extent_out[0] << ", " << extent_out[1] << ", " - << extent_out[2] << ", " << extent_out[3] << ", " << extent_out[4] - << ", " << extent_out[5] << "]" << endl; -#endif + indirect_less comp(tmp); + std::sort(pmap, pmap + nx, comp); + + // reoder the periodic shifted values + for (unsigned long i = 0; i < nx; ++i) + pxo[i] = tmp[pmap[i]]; + + free(tmp); } +// -------------------------------------------------------------------------- +template +void internals::inv_periodic_shift_x(unsigned long *pmap, + const coord_t *px, unsigned long nx) +{ + coord_t *tmp = (coord_t*)malloc(nx*sizeof(coord_t)); + + // apply the periodic shift, this will leave the axis out of order + for (unsigned long i = 0; i < nx; ++i) + tmp[i] = inv_periodic_shift_x(px[i]); + + // construct the map that the puts the axis back into order. + for (unsigned long i = 0; i < nx; ++i) + pmap[i] = i; + + indirect_less comp(tmp); + std::sort(pmap, pmap + nx, comp); + + free(tmp); +} // -------------------------------------------------------------------------- -void teca_normalize_coordinates::internals_t::normalize_variables( - bool normalize_x, bool normalize_y, bool normalize_z, - unsigned long *extent, p_teca_array_collection data) +int internals::inv_periodic_shift_x(p_teca_unsigned_long_array &map, + const const_p_teca_variant_array &x) { - unsigned long nx = extent[1] - extent[0] + 1; - unsigned long ny = extent[3] - extent[2] + 1; - unsigned long nz = extent[5] - extent[4] + 1; + unsigned long nx = x->size(); + + NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, + x.get(), _C, + + const NT_C *px = dynamic_cast(x.get())->get(); + + map = teca_unsigned_long_array::New(nx); + unsigned long *pmap = map->get(); - unsigned long nxy = nx*ny; - unsigned long nxyz = nxy*nz; + inv_periodic_shift_x(pmap, px, nx); + return 0; + ) + + TECA_ERROR("Unsupported coordinate type " << x->get_class_name()) + return -1; +} + +// -------------------------------------------------------------------------- +int internals::periodic_shift_x(p_teca_variant_array &x_out, + p_teca_unsigned_long_array &map, const const_p_teca_variant_array &x, + bool &shifted_x) +{ +// ignore warning about unsigned integer types. these will never need +// the periodic shift and the code will ignore them. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wtype-limits" + unsigned long nx = x->size(); unsigned long x1 = nx - 1; - unsigned long y1 = ny - 1; - unsigned long z1 = nz - 1; - unsigned int n_arrays = data->size(); + NESTED_TEMPLATE_DISPATCH(const teca_variant_array_impl, + x.get(), _C, - // for any coodinate axes that have been transformed from descending order - // into ascending order, apply the same transform to the scalar data arrays - if (normalize_x) - { - for (unsigned int l = 0; l < n_arrays; ++l) + const NT_C *px = dynamic_cast(x.get())->get(); + + // check that the shift is needed. + shifted_x = (px[0] < NT_C(0)); + + if (!shifted_x) + return 0; + + // this approach requires that coordinates are in ascending + // order + if (px[x1] < px[0]) { - p_teca_variant_array a = data->get(l); - p_teca_variant_array ao = a->new_instance(nxyz); - NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, - a.get(), _A, - NT_A *pa = static_cast(a.get())->get(); - NT_A *pao = static_cast(ao.get())->get(); - for (unsigned long k = 0; k < nz; ++k) - { - unsigned long kk = k*nxy; - for (unsigned long j = 0; j < ny; ++j) - { - unsigned long jj = kk + j*nx; + TECA_ERROR("A periodic shift can only be apllied to" + " coordinates in ascending order") + return -1; + } - NT_A *par = pa + jj; - NT_A *paor = pao + jj + x1; + // in its current form this approach handles coordinates in + // -180 to 180. + if ((px[0] < NT_C(-180)) || (px[x1] > NT_C(180))) + { + TECA_ERROR("Invalid x-axis coordinate range [" + << px[0] << " , " << px[x1] << "] coordinates in the" + " range [-180.0, 180] are required") + return -1; + } - for (unsigned long i = 0; i < nx; ++i) - paor[-i] = par[i]; - } - } - ) - data->set(l, ao); + // if 2 coordinate points touch the periodic boundary remnove 1 so the + // output does not have an extranious data point at the boundary. + if (teca_coordinate_util::equal(px[0], NT_C(-180)) && + teca_coordinate_util::equal(px[nx-1], NT_C(180))) + { + nx -= 1; } - } - if (normalize_y) + p_teca_variant_array xo = x->new_instance(nx); + NT_C *pxo = static_cast*> + (xo.get())->get(); + + map = teca_unsigned_long_array::New(nx); + unsigned long *pmap = map->get(); + + periodic_shift_x(pxo, pmap, px, nx); + + x_out = xo; + + return 0; + ) + + TECA_ERROR("Unsupported coordinate type " << x->get_class_name()) + return -1; +#pragma GCC diagnostic pop +} + +// -------------------------------------------------------------------------- +int internals::ascending_order_y( + p_teca_variant_array &out_y, const const_p_teca_variant_array &in_y, + bool &reorder_y) +{ + if (reorder(out_y, in_y, reorder_y)) + return -1; + + return 0; +} + +// -------------------------------------------------------------------------- +int internals::periodic_shift_x(p_teca_array_collection data, + const teca_metadata &attributes, + const const_p_teca_unsigned_long_array &shift_map, + const unsigned long *extent_in, + const unsigned long *extent_out) +{ + // apply periodic shift in the x-direction + const unsigned long *pmap = shift_map->get(); + + unsigned int n_arrays = data->size(); + for (unsigned int l = 0; l < n_arrays; ++l) { - unsigned int n_arrays = data->size(); - for (unsigned int l = 0; l < n_arrays; ++l) + // get the extent of the input/output array + const std::string &array_name = data->get_name(l); + teca_metadata array_attributes; + if (attributes.get(array_name, array_attributes)) { - p_teca_variant_array a = data->get(l); - p_teca_variant_array ao = a->new_instance(nxyz); - NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, - a.get(), _A, - NT_A *pa = static_cast(a.get())->get(); - NT_A *pao = static_cast(ao.get())->get(); - for (unsigned long k = 0; k < nz; ++k) + TECA_ERROR("Failed to get the attributes for \"" + << array_name << "\"") + return -1; + } + + unsigned long array_extent_in[6] = {0ul}; + teca_metadata_util::get_array_extent(array_attributes, + extent_in, array_extent_in); + + unsigned long array_extent_out[6] = {0ul}; + teca_metadata_util::get_array_extent(array_attributes, + extent_out, array_extent_out); + + // input and output arrays may be different size if there was a duplicated + // coordinate point on the periodic boundary + unsigned long nxi = array_extent_in[1] - array_extent_in[0] + 1; + unsigned long nyi = array_extent_in[3] - array_extent_in[2] + 1; + unsigned long nxyi = nxi*nyi; + + unsigned long nxo = array_extent_out[1] - array_extent_out[0] + 1; + unsigned long nyo = array_extent_out[3] - array_extent_out[2] + 1; + unsigned long nzo = array_extent_out[5] - array_extent_out[4] + 1; + unsigned long nxyo = nxo*nyo; + unsigned long nxyzo = nxyo*nzo; + + p_teca_variant_array a = data->get(l); + + p_teca_variant_array ao = a->new_instance(nxyzo); + NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, + a.get(), _A, + NT_A *pa = static_cast(a.get())->get(); + NT_A *pao = static_cast(ao.get())->get(); + for (unsigned long k = 0; k < nzo; ++k) + { + unsigned long kki = k*nxyi; + unsigned long kko = k*nxyo; + for (unsigned long j = 0; j < nyo; ++j) { - unsigned long kk = k*nxy; - for (unsigned long j = 0; j < ny; ++j) + unsigned long jji = kki + j*nxi; + unsigned long jjo = kko + j*nxo; + + NT_A *par = pa + jji; + NT_A *paor = pao + jjo; + + for (unsigned long i = 0; i < nxo; ++i) { - unsigned long jj = kk + j*nx; - unsigned long jjo = kk + (y1 - j)*nx; - NT_A *par = pa + jj; - NT_A *paor = pao + jjo; - for (unsigned long i = 0; i < nx; ++i) - paor[i] = par[i]; + paor[i] = par[pmap[i]]; } } - ) - data->set(l, ao); - } + } + ) + + data->set(l, ao); } - if (normalize_z) + return 0; +} + +// -------------------------------------------------------------------------- +int internals::ascending_order_y(p_teca_array_collection data, + const teca_metadata &attributes, const unsigned long *mesh_extent) +{ + // for any coodinate axes that have been transformed from descending order + // into ascending order, apply the same transform to the scalar data arrays + unsigned int n_arrays = data->size(); + for (unsigned int l = 0; l < n_arrays; ++l) { - for (unsigned int l = 0; l < n_arrays; ++l) - { - p_teca_variant_array a = data->get(l); - p_teca_variant_array ao = a->new_instance(nxyz); - NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, - a.get(), _A, - NT_A *pa = static_cast(a.get())->get(); - NT_A *pao = static_cast(ao.get())->get(); - for (unsigned long k = 0; k < nz; ++k) - { - unsigned long kk = k*nxy; - unsigned long kko = (z1 - k)*nxy; - for (unsigned long j = 0; j < ny; ++j) - { - unsigned long jnx = j*nx; - unsigned long jj = kk + jnx; - unsigned long jjo = kko + jnx; + const std::string &array_name = data->get_name(l); - NT_A *par = pa + jj; - NT_A *paor = pao + jjo; + // get the extent of the array + unsigned long array_extent[6] = {0ul}; - for (unsigned long i = 0; i < nx; ++i) - paor[i] = par[i]; - } - } - ) - data->set(l, ao); + teca_metadata array_attributes; + if (attributes.get(array_name, array_attributes)) + { + TECA_ERROR("Failed to get attributes for array \"" + << array_name << "\"") + return -1; } + + teca_metadata_util::get_array_extent(array_attributes, + mesh_extent, array_extent); + + unsigned long nx = array_extent[1] - array_extent[0] + 1; + unsigned long ny = array_extent[3] - array_extent[2] + 1; + unsigned long nz = array_extent[5] - array_extent[4] + 1; + + unsigned long nxy = nx*ny; + unsigned long nxyz = nxy*nz; + + unsigned long y1 = ny - 1; + + // apply the transform + p_teca_variant_array a = data->get(l); + p_teca_variant_array ao = a->new_instance(nxyz); + NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, + a.get(), _A, + NT_A *pa = static_cast(a.get())->get(); + NT_A *pao = static_cast(ao.get())->get(); + for (unsigned long k = 0; k < nz; ++k) + { + unsigned long kk = k*nxy; + for (unsigned long j = 0; j < ny; ++j) + { + unsigned long jj = kk + j*nx; + unsigned long jjo = kk + (y1 - j)*nx; + NT_A *par = pa + jj; + NT_A *paor = pao + jjo; + for (unsigned long i = 0; i < nx; ++i) + paor[i] = par[i]; + } + } + ) + data->set(l, ao); } + + return 0; } + + // -------------------------------------------------------------------------- -teca_normalize_coordinates::teca_normalize_coordinates() : internals(nullptr) +teca_normalize_coordinates::teca_normalize_coordinates() : + enable_periodic_shift_x(0), enable_y_axis_ascending(1) { - this->internals = new teca_normalize_coordinates::internals_t; - this->set_number_of_input_connections(1); this->set_number_of_output_ports(1); } @@ -243,20 +442,37 @@ teca_normalize_coordinates::teca_normalize_coordinates() : internals(nullptr) // -------------------------------------------------------------------------- teca_normalize_coordinates::~teca_normalize_coordinates() { - delete this->internals; } #if defined(TECA_HAS_BOOST) // -------------------------------------------------------------------------- void teca_normalize_coordinates::get_properties_description( - const std::string &/*prefix*/, options_description &/*global_opts*/) + const std::string &prefix, options_description &global_opts) { + options_description opts("Options for " + + (prefix.empty()?"teca_normalize_coordinates":prefix)); + + opts.add_options() + TECA_POPTS_GET(int, prefix, enable_periodic_shift_x, + "Enables application of periodic shift in the x-direction.") + TECA_POPTS_GET(int, prefix, enable_y_axis_ascending, + "Enables transformtion the ensures the y-axis is in" + " ascending order.") + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); } // -------------------------------------------------------------------------- void teca_normalize_coordinates::set_properties( - const std::string &/*prefix*/, variables_map &/*opts*/) + const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, int, prefix, enable_periodic_shift_x) + TECA_POPTS_SET(opts, int, prefix, enable_y_axis_ascending) } #endif @@ -265,8 +481,8 @@ teca_metadata teca_normalize_coordinates::get_output_metadata( unsigned int port, const std::vector &input_md) { #ifdef TECA_DEBUG - cerr << teca_parallel_id() - << "teca_normalize_coordinates::get_output_metadata" << endl; + std::cerr << teca_parallel_id() + << "teca_normalize_coordinates::get_output_metadata" << std::endl; #endif (void)port; @@ -287,20 +503,68 @@ teca_metadata teca_normalize_coordinates::get_output_metadata( return teca_metadata(); } - // check for and transform coordinate axes from descending order - // to ascending order - p_teca_variant_array out_x, out_y, out_z; - if ((out_x = this->internals->normalize_axis(in_x))) + // ensure x-axis is in 0 to 360 + bool shifted_x = false; + p_teca_variant_array out_x; + p_teca_unsigned_long_array shift_map; + if (this->enable_periodic_shift_x && + internals::periodic_shift_x(out_x, shift_map, in_x, shifted_x)) + { + TECA_ERROR("Failed to apply periodic shift to the x-axis") + return teca_metadata(); + } + + // ensure y-axis ascending + bool reordered_y = false; + p_teca_variant_array out_y; + if (this->enable_y_axis_ascending && + internals::ascending_order_y(out_y, in_y, reordered_y)) + { + TECA_ERROR("Failed to put the y-axis in ascending order") + return teca_metadata(); + } + + // pass normalized coordinates + if (out_x) + { coords.set("x", out_x); - if ((out_y = this->internals->normalize_axis(in_y))) - coords.set("y", out_y); + // update the whole extent in case the coordinate axis touches the periodic boundary + if (out_x->size() != in_x->size()) + { + unsigned long whole_extent[6]; + if (out_md.get("whole_extent", whole_extent, 6)) + { + TECA_ERROR("Failed to get the input whole_extent") + return teca_metadata(); + } + whole_extent[1] -= 1; + out_md.set("whole_extent", whole_extent); + } + } - if ((out_z = this->internals->normalize_axis(in_z))) - coords.set("z", out_z); + if (out_y) + coords.set("y", out_y); - if (out_x || out_y || out_z) + if (out_x || out_y) + { + double bounds[6] = {0.0}; + teca_coordinate_util::get_cartesian_mesh_bounds( + out_x ? out_x : in_x, out_y ? out_y : in_y, in_z, + bounds); out_md.set("coordinates", coords); + out_md.set("bounds", bounds); + } + + if ((this->verbose > 1) && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + if (reordered_y) + TECA_STATUS("The y-axis will be transformed to be in ascending order.") + + if (shifted_x) + TECA_STATUS("The x-axis will be transformed from [-180, 180] to [0, 360].") + } return out_md; } @@ -310,6 +574,10 @@ std::vector teca_normalize_coordinates::get_upstream_request( unsigned int port, const std::vector &input_md, const teca_metadata &request) { +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_normalize_coordinates::get_upstream_request" << std::endl; +#endif (void)port; std::vector up_reqs; @@ -327,28 +595,50 @@ std::vector teca_normalize_coordinates::get_upstream_request( return up_reqs; } - p_teca_variant_array in_x, in_y, in_z; + p_teca_variant_array in_x, in_y, z; if (!(in_x = coords.get("x")) || !(in_y = coords.get("y")) - || !(in_z = coords.get("z"))) + || !(z = coords.get("z"))) { TECA_ERROR("metadata is missing coordinate arrays") return up_reqs; } - // now convert the original coordinate axes into the - // normalized system. this isn't cached for thread safety - p_teca_variant_array out_x, out_y, out_z; - out_x = this->internals->normalize_axis(in_x); - out_y = this->internals->normalize_axis(in_y); - out_z = this->internals->normalize_axis(in_z); + // ensure x-axis is in 0 to 360 + bool shifted_x = false; + p_teca_variant_array out_x; + p_teca_unsigned_long_array shift_map; + if (this->enable_periodic_shift_x && + internals::periodic_shift_x(out_x, shift_map, in_x, shifted_x)) + { + TECA_ERROR("Failed to apply periodic shift to the x-axis") + return up_reqs; + } + + // compute the inverse map + p_teca_unsigned_long_array inv_shift_map; + if (shifted_x && internals::inv_periodic_shift_x(inv_shift_map, out_x)) + { + TECA_ERROR("Failed to compute the inverse shifty map") + return up_reqs; + } - // normalized system is the same as the original, pass the request up - if (!out_x && !out_y && !out_z) + // ensure y-axis ascending + bool reordered_y = false; + p_teca_variant_array out_y; + if (this->enable_y_axis_ascending && + internals::ascending_order_y(out_y, in_y, reordered_y)) { - up_reqs.push_back(request); + TECA_ERROR("Failed to put the y-axis in ascending order") return up_reqs; } + // get the transformed bounds + const_p_teca_variant_array x = out_x ? out_x : in_x; + const_p_teca_variant_array y = out_y ? out_y : in_y; + + double bounds[6] = {0.0}; + teca_coordinate_util::get_cartesian_mesh_bounds(x, y, z, bounds); + // get the original extent unsigned long whole_extent[6] = {0}; if (input_md[0].get("whole_extent", whole_extent, 6)) @@ -359,35 +649,122 @@ std::vector teca_normalize_coordinates::get_upstream_request( // get the extent that is being requested unsigned long extent_in[6] = {0}; - unsigned long extent_out[6] = {0}; - double bounds[6] = {0.0}; - if (req.get("bounds", bounds, 6)) + double req_bounds[6] = {0.0}; + if (req.get("bounds", req_bounds, 6)) { - // bounds key not present, check for extent key - // if not present use whole_extent + // bounds key not present, check for extent key if not present use + // whole_extent if (request.get("extent", extent_in, 6)) + { + // correct in case we removed a duplicated point at the periodic + // boundary + if (out_x && (in_x->size() != out_x->size())) + whole_extent[1] -= 1; + memcpy(extent_in, whole_extent, 6*sizeof(unsigned long)); + } + + // convert extent to bounds + x->get(extent_in[0], req_bounds[0]); + x->get(extent_in[1], req_bounds[1]); + y->get(extent_in[2], req_bounds[2]); + y->get(extent_in[3], req_bounds[3]); + z->get(extent_in[4], req_bounds[4]); + z->get(extent_in[5], req_bounds[5]); } else { - // bounds key was present, convert the bounds to an - // an extent that covers them. - if (teca_coordinate_util::bounds_to_extent(bounds, - (out_x ? out_x : in_x), (out_y ? out_y : in_y), - (out_z ? out_z : in_z), extent_in)) - { - TECA_ERROR("invalid bounds requested.") - return up_reqs; - } - // remove the bounds request, which will force the reader to // use the given extent req.remove("bounds"); } - // apply the trsnaform if needed - this->internals->normalize_extent(out_x, out_y, out_z, - whole_extent, extent_in, extent_out); + // validate the requested bounds + if (!teca_coordinate_util::same_orientation(bounds, req_bounds) || + !teca_coordinate_util::covers(bounds, req_bounds)) + { + TECA_ERROR("Invalid request. The requested bounds [" + << req_bounds[0] << ", " << req_bounds[1] << ", " + << req_bounds[2] << ", " << req_bounds[3] << ", " + << req_bounds[4] << ", " << req_bounds[5] + << "] is not covered by the available bounds [" + << bounds[0] << ", " << bounds[1] << ", " + << bounds[2] << ", " << bounds[3] << ", " + << bounds[4] << ", " << bounds[5] << "]") + return up_reqs; + } + + // transform the bounds + double tfm_bounds[6]; + memcpy(tfm_bounds, req_bounds, 6*sizeof(double)); + + if (shifted_x) + { + // if a bounds request crosses the periodic boundary + // then it needs to be split into 2 requests. Eg: a request + // for [90, 270] becomes [-180, -90], [90, 180]. + // If this comes up a lot we could implement as follows: + // issue both requests here and merge the data in execute. + + if (((req_bounds[0] < 180.0) && (req_bounds[1] > 180.0)) || + teca_coordinate_util::equal(req_bounds[0], 180.0) || + teca_coordinate_util::equal(req_bounds[1], 180.0)) + { + // crosses periodic boundary (TODO handle subseting) + unsigned long x1 = in_x->size() - 1; + in_x->get(0, tfm_bounds[0]); + in_x->get(x1, tfm_bounds[1]); + + TECA_WARNING("The requested x-axis bounds" + " [" << req_bounds[0] << ", " << req_bounds[1] << "] cross a" + " periodic boundary. Subsetting across a periodic boundary is" + " not fully implemented. Requesting the entire x-axis [" + << tfm_bounds[0] << ", " << tfm_bounds[1] << "]") + } + else + { + tfm_bounds[0] = internals::inv_periodic_shift_x(tfm_bounds[0]); + tfm_bounds[1] = internals::inv_periodic_shift_x(tfm_bounds[1]); + } + } + + if (reordered_y) + { + std::swap(tfm_bounds[2], tfm_bounds[3]); + } + + // convert the transformed bounds to an + // an extent that covers them in the upstream coordinate system + unsigned long extent_out[6]; + memcpy(extent_out, extent_in, 6*sizeof(unsigned long)); + + if (teca_coordinate_util::bounds_to_extent(tfm_bounds, + in_x, in_y, z, extent_out) || + teca_coordinate_util::validate_extent(in_x->size(), + in_y->size(), z->size(), extent_out, true)) + { + TECA_ERROR("invalid bounds requested.") + return up_reqs; + } + +#ifdef TECA_DEBUG + std::cerr << "req_bounds = [" << req_bounds[0] + << ", " << req_bounds[1] << ", " << req_bounds[2] + << ", " << req_bounds[3] << ", " << req_bounds[4] + << ", " << req_bounds[5] << "]" << std::endl + << "tfm_bounds = [" << tfm_bounds[0] + << ", " << tfm_bounds[1] << ", " << tfm_bounds[2] + << ", " << tfm_bounds[3] << ", " << tfm_bounds[4] + << ", " << tfm_bounds[5] << "]" << std::endl + << "extent_in = [" << extent_in[0] + << ", " << extent_in[1] << ", " << extent_in[2] + << ", " << extent_in[3] << ", " << extent_in[4] + << ", " << extent_in[5] << "]" << std::endl + << "extent_out = [" << extent_out[0] + << ", " << extent_out[1] << ", " << extent_out[2] + << ", " << extent_out[3] << ", " << extent_out[4] + << ", " << extent_out[5] << "]" << std::endl; +#endif // send the request up req.set("extent", extent_out, 6); @@ -402,7 +779,8 @@ const_p_teca_dataset teca_normalize_coordinates::execute(unsigned int port, const teca_metadata &request) { #ifdef TECA_DEBUG - cerr << teca_parallel_id() << "teca_normalize_coordinates::execute" << endl; + std::cerr << teca_parallel_id() + << "teca_normalize_coordinates::execute" << std::endl; #endif (void)port; (void)request; @@ -413,7 +791,7 @@ const_p_teca_dataset teca_normalize_coordinates::execute(unsigned int port, if (!in_mesh) { - TECA_ERROR("Failed to compute l2 norm. dataset is not a teca_cartesian_mesh") + TECA_ERROR("The input dataset is not a teca_cartesian_mesh") return nullptr; } @@ -426,37 +804,98 @@ const_p_teca_dataset teca_normalize_coordinates::execute(unsigned int port, const_p_teca_variant_array in_y = in_mesh->get_y_coordinates(); const_p_teca_variant_array in_z = in_mesh->get_z_coordinates(); - // transform the axes to ascending order if needed - p_teca_variant_array out_x, out_y, out_z; - if ((out_x = this->internals->normalize_axis(in_x))) + // get the extent + unsigned long extent_in[6]; + in_mesh->get_extent(extent_in); + + unsigned long extent_out[6]; + memcpy(extent_out, extent_in, 6*sizeof(unsigned long)); + + // ensure x-axis is in 0 to 360 + bool shifted_x = false; + p_teca_variant_array out_x; + p_teca_unsigned_long_array shift_map; + if (this->enable_periodic_shift_x && + internals::periodic_shift_x(out_x, shift_map, in_x, shifted_x)) { - std::string var; - in_mesh->get_x_coordinate_variable(var); - out_mesh->set_x_coordinates(var, out_x); + TECA_ERROR("Failed to apply periodic shift to the x-axis") + return nullptr; } - if ((out_y = this->internals->normalize_axis(in_y))) + teca_metadata attributes; + + if (shifted_x) { + in_mesh->get_attributes(attributes); + + if (this->verbose && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_STATUS("The x-axis will be transformed from [-180, 180] to [0, 360].") + } + std::string var; - in_mesh->get_y_coordinate_variable(var); - out_mesh->set_y_coordinates(var, out_y); + in_mesh->get_x_coordinate_variable(var); + out_mesh->set_x_coordinates(var, out_x); + + // correct extent in case the coordinate axis touches the periodic boundary + if (out_x && (out_x->size() != in_x->size())) + { + if (teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_WARNING("The coordinate and data on the periodic boundary" + " at x = +/- 180 is duplicated.") + } + + extent_out[1] -= 1; + out_mesh->set_extent(extent_out); + + unsigned long whole_extent[6]; + in_mesh->get_whole_extent(whole_extent); + whole_extent[1] -= 1; + + out_mesh->set_whole_extent(whole_extent); + } + + if (internals::periodic_shift_x(out_mesh->get_point_arrays(), + attributes, shift_map, extent_in, extent_out)) + { + TECA_ERROR("Failed to apply periodic shift in the x direction") + return nullptr; + } } - if ((out_z = this->internals->normalize_axis(in_z))) + // ensure y-axis ascending + bool reordered_y = false; + p_teca_variant_array out_y; + if (this->enable_y_axis_ascending && + internals::ascending_order_y(out_y, in_y, reordered_y)) { - std::string var; - in_mesh->get_z_coordinate_variable(var); - out_mesh->set_z_coordinates(var, out_z); + TECA_ERROR("Failed to put the y-axis in ascending order") + return nullptr; } - // apply the same set of transforms to the data - if (out_x || out_y || out_z) + if (reordered_y) { - unsigned long extent[6]; - in_mesh->get_extent(extent); + if (attributes.empty()) + in_mesh->get_attributes(attributes); + + if (this->verbose && + teca_mpi_util::mpi_rank_0(this->get_communicator())) + { + TECA_STATUS("The y-axis will be transformed to be in ascending order.") + } - this->internals->normalize_variables(out_x.get(), - out_y.get(), out_z.get(), extent, out_mesh->get_point_arrays()); + std::string var; + in_mesh->get_y_coordinate_variable(var); + out_mesh->set_y_coordinates(var, out_y); + + if (internals::ascending_order_y(out_mesh->get_point_arrays(), + attributes, extent_out)) + { + TECA_ERROR("Failed to put point arrays into ascending order") + return nullptr; + } } return out_mesh; diff --git a/alg/teca_normalize_coordinates.h b/alg/teca_normalize_coordinates.h index 87d071603..265a269ab 100644 --- a/alg/teca_normalize_coordinates.h +++ b/alg/teca_normalize_coordinates.h @@ -10,12 +10,27 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_normalize_coordinates) -/// an algorithm to ensure that coordinates are in ascending order -/** -Transformations of coordinates and data to/from ascending order -are made as data and information pass up and down stream through -the algorithm. -*/ +/** @brief + * An algorithm to ensure that Cartesian mesh coordinates follow conventions + * + * @details + * When enabled, transformations of coordinates and data are applied such that + * Cartesian meshes are follow the conventions: + * + * 1. the x-axis coordinates are in the range of 0 to 360. + * 2. the y-axis coordinate are in ascending order. + * + * These transformations are automatically applied and can be enabled or + * disbaled as needed. The properties enable_periodic_shift and + * enable_y_axis_ascending provide a way to enable/disable the transforms. + * + * Subset requests are not implemented when the periodic shift is enabled. When + * a request is made for data that crosses the periodic boundary, the request + * is modified to request the entire x-axis. + * + * If data point opn the periodic boundary is duplicated, the data at 180 is + * dropped and a warning is issued.g + */ class teca_normalize_coordinates : public teca_algorithm { public: @@ -29,6 +44,30 @@ class teca_normalize_coordinates : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() + /** @name enable_periodic_shift_x + * If set, this enables an automatic transformation of the x-axis + * coordinates and data from [-180, 180] to [0, 360]. When enabled, the + * transformation is applied if the lowest x coordinate is less than 0 and + * skipped otherwise. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, enable_periodic_shift_x) + ///@} + + /** @name enable_y_axis_ascending + * If set, this enables an automatic transformation of the y-axis + * coordinates and data from descending to ascending order. The + * transformation is applied if the lowest y coordinate is greater than the + * highest y coordinate skipped otherwise. Many TECA algorithms are written + * to process data with y-axis coordinates in ascending order, thus the + * transform is enabled by default. Setting this to 0 disables the + * transform for cases where it is desirable to pass data through + * unmodified. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, enable_y_axis_ascending) + ///@} + protected: teca_normalize_coordinates(); @@ -45,8 +84,8 @@ class teca_normalize_coordinates : public teca_algorithm const teca_metadata &request) override; private: - struct internals_t; - internals_t *internals; + int enable_periodic_shift_x; + int enable_y_axis_ascending; }; #endif diff --git a/alg/teca_parser.h b/alg/teca_parser.h index 540d8db28..7e1703b0e 100644 --- a/alg/teca_parser.h +++ b/alg/teca_parser.h @@ -1,6 +1,8 @@ #ifndef teca_parser_h #define teca_parser_h +/// @file + #include "teca_common.h" #include @@ -42,14 +44,14 @@ << "\" requires " << _nreq << " operands, given " \ << _ngive << ". ", _expr, _pos) +/// Codes dealing with expression parsing. namespace teca_parser { -/** -class that recognizes and extracts tokens during parsing. -given a pointer (first argument) the methods return the -number of chars in the token, or 0 when the pointer doesn't -point to a valid token, and copies the token into the buffer -(second argument). +/** Recognizes and extracts tokens during parsing. + * given a pointer (first argument) the methods return the + * number of chars in the token, or 0 when the pointer doesn't + * point to a valid token, and copies the token into the buffer + * (second argument). */ class tokenizer { @@ -64,13 +66,12 @@ class tokenizer static unsigned int get_operator_precedence(char *op); }; -/** -convert infix expression to postfix. returns the postfix form -of the expression in a string allocated with malloc. caller to -free the string. return nullptr if there is an error. - -template types implement detection of classes of syntactical -tokens. groups, constants, variables, and operators. +/** Convert infix expression to postfix. returns the postfix form + * of the expression in a string allocated with malloc. caller to + * free the string. return nullptr if there is an error. + * + * template types implement detection of classes of syntactical + * tokens. groups, constants, variables, and operators. */ template char *infix_to_postfix(const char *iexpr, std::set *variables) @@ -191,15 +192,14 @@ char *infix_to_postfix(const char *iexpr, std::set *variables) return rpnexpr; } -/** -evaluate a postfix expression. returns non zero if an error occurred. -the result of the evaluted expression is returned in iexpr_result. - -template types define the intermediate types used in the calculation. -arg_t would likely be the const form of work_t. resolvers for constants, -variables, and operators are passed. The purpose of the resolvers is -to identify token class and implement variable lookup, and operator -evaluation. +/** evaluate a postfix expression. returns non zero if an error occurred. + * the result of the evaluated expression is returned in iexpr_result. + * + * template types define the intermediate types used in the calculation. + * arg_t would likely be the const form of work_t. resolvers for constants, + * variables, and operators are passed. The purpose of the resolvers is + * to identify token class and implement variable lookup, and operator + * evaluation. */ template diff --git a/alg/teca_rename_variables.cxx b/alg/teca_rename_variables.cxx new file mode 100644 index 000000000..16c95d9eb --- /dev/null +++ b/alg/teca_rename_variables.cxx @@ -0,0 +1,225 @@ +#include "teca_rename_variables.h" + +#include "teca_mesh.h" +#include "teca_array_collection.h" +#include "teca_variant_array.h" +#include "teca_metadata.h" +#include "teca_array_attributes.h" + +#include +#include +#include +#include +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +using std::string; +using std::vector; +using std::set; +using std::cerr; +using std::endl; + +//#define TECA_DEBUG + +// -------------------------------------------------------------------------- +teca_rename_variables::teca_rename_variables() : + original_variable_names(), new_variable_names() +{ + this->set_number_of_input_connections(1); + this->set_number_of_output_ports(1); +} + +// -------------------------------------------------------------------------- +teca_rename_variables::~teca_rename_variables() +{} + +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_rename_variables::get_properties_description( + const string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_rename_variables":prefix)); + + opts.add_options() + TECA_POPTS_MULTI_GET(std::vector, prefix, original_variable_names, + "Sets the list of original_variable_names to rename.") + TECA_POPTS_MULTI_GET(std::vector, prefix, new_variable_names, + "Sets the list of new names, one for each variable to rename.") + ; + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_rename_variables::set_properties( + const string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, std::vector, prefix, original_variable_names) + TECA_POPTS_SET(opts, std::vector, prefix, new_variable_names) + +} +#endif + +// -------------------------------------------------------------------------- +teca_metadata teca_rename_variables::get_output_metadata( + unsigned int port, + const std::vector &input_md) +{ +#ifdef TECA_DEBUG + cerr << teca_parallel_id() + << "teca_rename_variables::get_output_metadata" << endl; +#endif + (void)port; + + // validate the user provided values. + if (this->original_variable_names.size() != this->new_variable_names.size()) + { + TECA_ERROR("Each variable to rename must have a " + " corresponding output_variable_name.") + return teca_metadata(); + } + + teca_metadata out_md(input_md[0]); + + // update the list of original_variable_names to reflect the new names + std::set out_vars; + if (out_md.get("variables", out_vars)) + { + TECA_ERROR("Failed to get the list of variables") + return teca_metadata(); + } + + unsigned long n_vars = this->original_variable_names.size(); + for (unsigned long i = 0; i < n_vars; ++i) + { + std::set::iterator it = out_vars.find(this->original_variable_names[i]); + if (it == out_vars.end()) + { + TECA_ERROR("No such variable \"" << this->original_variable_names[i] + << "\" to rename") + return teca_metadata(); + } + + out_vars.erase(it); + out_vars.insert(this->new_variable_names[i]); + } + + // update the list of attributes to reflect the new names + teca_metadata attributes; + if (out_md.get("attributes", attributes)) + { + TECA_ERROR("Failed to get attributes") + return teca_metadata(); + } + + for (unsigned long i = 0; i < n_vars; ++i) + { + const std::string &var_name = this->original_variable_names[i]; + + teca_metadata atts; + if (attributes.get(var_name, atts)) + { + TECA_ERROR("Failed to get attributes for \"" << var_name << "\"") + return teca_metadata(); + } + + attributes.remove(var_name); + + attributes.set(this->new_variable_names[i], atts); + } + + out_md.set("attributes", attributes); + + return out_md; +} + +// -------------------------------------------------------------------------- +std::vector teca_rename_variables::get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) +{ + (void)port; + (void)input_md; + + vector up_reqs; + + // copy the incoming request to preserve the downstream requirements. + // replace renamed original_variable_names with their original name + teca_metadata req(request); + + std::set arrays; + if (req.has("arrays")) + req.get("arrays", arrays); + + unsigned long n_vars = this->new_variable_names.size(); + for (unsigned long i = 0; i < n_vars; ++i) + { + std::set::iterator it = arrays.find(this->new_variable_names[i]); + if (it != arrays.end()) + { + arrays.erase(it); + arrays.insert(this->original_variable_names[i]); + } + + } + + req.set("arrays", arrays); + up_reqs.push_back(req); + + return up_reqs; +} + +// -------------------------------------------------------------------------- +const_p_teca_dataset teca_rename_variables::execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + cerr << teca_parallel_id() << "teca_rename_variables::execute" << endl; +#endif + (void)port; + (void)request; + + // get the input mesh + const_p_teca_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); + + if (!in_mesh) + { + TECA_ERROR("The input dataset is not a teca_mesh") + return nullptr; + } + + // create the output mesh, pass everything through. + p_teca_mesh out_mesh = std::static_pointer_cast + (std::const_pointer_cast(in_mesh)->new_shallow_copy()); + + + // rename the arrays if they are found + p_teca_array_collection arrays = out_mesh->get_point_arrays(); + + unsigned long n_vars = this->original_variable_names.size(); + for (unsigned long i = 0; i < n_vars; ++i) + { + const std::string var_name = this->original_variable_names[i]; + + p_teca_variant_array array = arrays->get(var_name); + if (array) + { + arrays->remove(var_name); + arrays->set(this->new_variable_names[i], array); + } + } + + return out_mesh; +} diff --git a/alg/teca_rename_variables.h b/alg/teca_rename_variables.h new file mode 100644 index 000000000..24ebb7103 --- /dev/null +++ b/alg/teca_rename_variables.h @@ -0,0 +1,68 @@ +#ifndef teca_rename_variables_h +#define teca_rename_variables_h + +#include "teca_shared_object.h" +#include "teca_algorithm.h" +#include "teca_metadata.h" + +#include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_rename_variables) + +/// An algorithm that renames variables. +class teca_rename_variables : public teca_algorithm +{ +public: + TECA_ALGORITHM_STATIC_NEW(teca_rename_variables) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_rename_variables) + TECA_ALGORITHM_CLASS_NAME(teca_rename_variables) + ~teca_rename_variables(); + + // report/initialize to/from Boost program options + // objects. + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + + /** @name original_variable_names + * Set the list of variables to rename. For each variable to rename a new + * name must be specified at the same index in the new_variable_names + * list. The two lists must be the same length. + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(std::string, original_variable_name) + ///@} + + /** @name new_variable_names + * Set the names of the renamed variables. The new names are applied to the + * list of variables to rename in the same order and the two lists must be + * the same length. + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(std::string, new_variable_name) + ///@} + +protected: + teca_rename_variables(); + +private: + teca_metadata get_output_metadata( + unsigned int port, + const std::vector &input_md) override; + + std::vector get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) override; + + const_p_teca_dataset execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) override; + +private: + std::vector original_variable_names; + std::vector new_variable_names; +}; + +#endif diff --git a/alg/teca_saffir_simpson.h b/alg/teca_saffir_simpson.h index 89a73c168..618f44827 100644 --- a/alg/teca_saffir_simpson.h +++ b/alg/teca_saffir_simpson.h @@ -1,44 +1,51 @@ #ifndef teca_saffir_simpson #define teca_saffir_simpson +/// @file + #include +/// Codes dealing with the Saffir-Simpson scale namespace teca_saffir_simpson { -// Saffir-Simpson scale prescribes the following limits: -// CAT wind km/h -// -1: 0- 63 : Tropical depression -// 0: 63-119 : Tropical storm -// 1: 119-153 km/h -// 2: 154-177 km/h -// 3: 178-208 km/h -// 4: 209-251 km/h -// 5: 252 km/h or higher +/** Saffir-Simpson scale prescribes the following limits: + * CAT wind km/h + * -1: 0- 63 : Tropical depression + * 0: 63-119 : Tropical storm + * 1: 119-153 km/h + * 2: 154-177 km/h + * 3: 178-208 km/h + * 4: 209-251 km/h + * 5: 252 km/h or higher + */ constexpr double low_wind_bound_kmph[] = {0.0, 63.0, 119.0, 154.0, 178.0, 209.0, 252.0}; -// get the high bound for the given class of storm +/// get the high bound for the given class of storm constexpr double high_wind_bound_kmph[] = {63.0, 119.0, 154.0, 178.0, 209.0, 252.0, std::numeric_limits::max()}; +/// get the lower bound for the given class of storm template constexpr n_t get_lower_bound_kmph(int c) { return low_wind_bound_kmph[++c]; } +/// get the higher bound for the given class of storm template constexpr n_t get_upper_bound_kmph(int c) { return high_wind_bound_kmph[++c]; } -// given wind speed in km/hr return Saffir-Simpson category -// NOTE: there is some ambiguity in the above as -// it's defined using integers. we are not converting -// to integer here. -// get the low bound for the given class of storm +/** given wind speed in km/hr return Saffir-Simpson category + * NOTE: there is some ambiguity in the above as + * it's defined using integers. we are not converting + * to integer here. + * get the low bound for the given class of storm + */ template int classify_kmph(n_t w) { @@ -62,25 +69,26 @@ int classify_kmph(n_t w) return 5; } -// get the low bound for the given class of storm +/// get the low bound for the given class of storm template constexpr n_t get_lower_bound_mps(int c) { return get_lower_bound_kmph(c)/n_t(3.6); } -// get the high bound for the given class of storm +/// get the high bound for the given class of storm template constexpr n_t get_upper_bound_mps(int c) { return get_upper_bound_kmph(c)/n_t(3.6); } -// given wind speed in km/hr return Saffir-Simpson category -// NOTE: there is some ambiguity in the above as -// it's defined using integers. we are not converting -// to integer here. -// get the low bound for the given class of storm +/** given wind speed in km/hr return Saffir-Simpson category + * NOTE: there is some ambiguity in the above as + * it's defined using integers. we are not converting + * to integer here. + * get the low bound for the given class of storm + */ template int classify_mps(n_t w) { diff --git a/alg/teca_temporal_average.cxx b/alg/teca_simple_moving_average.cxx similarity index 84% rename from alg/teca_temporal_average.cxx rename to alg/teca_simple_moving_average.cxx index 3e19f32c9..3e4e4eb8f 100644 --- a/alg/teca_temporal_average.cxx +++ b/alg/teca_simple_moving_average.cxx @@ -1,4 +1,4 @@ -#include "teca_temporal_average.h" +#include "teca_simple_moving_average.h" #include "teca_mesh.h" #include "teca_array_collection.h" @@ -21,7 +21,7 @@ using std::endl; //#define TECA_DEBUG // -------------------------------------------------------------------------- -teca_temporal_average::teca_temporal_average() +teca_simple_moving_average::teca_simple_moving_average() : filter_width(3), filter_type(backward) { this->set_number_of_input_connections(1); @@ -29,16 +29,16 @@ teca_temporal_average::teca_temporal_average() } // -------------------------------------------------------------------------- -teca_temporal_average::~teca_temporal_average() +teca_simple_moving_average::~teca_simple_moving_average() {} #if defined(TECA_HAS_BOOST) // -------------------------------------------------------------------------- -void teca_temporal_average::get_properties_description( +void teca_simple_moving_average::get_properties_description( const string &prefix, options_description &global_opts) { options_description opts("Options for " - + (prefix.empty()?"teca_temporal_average":prefix)); + + (prefix.empty()?"teca_simple_moving_average":prefix)); opts.add_options() TECA_POPTS_GET(unsigned int, prefix, filter_width, @@ -47,20 +47,24 @@ void teca_temporal_average::get_properties_description( "use a backward(0), forward(1) or centered(2) stencil") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } // -------------------------------------------------------------------------- -void teca_temporal_average::set_properties( +void teca_simple_moving_average::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, unsigned int, prefix, filter_width) TECA_POPTS_SET(opts, int, prefix, filter_type) } #endif // -------------------------------------------------------------------------- -std::vector teca_temporal_average::get_upstream_request( +std::vector teca_simple_moving_average::get_upstream_request( unsigned int port, const std::vector &input_md, const teca_metadata &request) @@ -80,7 +84,7 @@ std::vector teca_temporal_average::get_upstream_request( break; } cerr << teca_parallel_id() - << "teca_temporal_average::get_upstream_request filter_type=" + << "teca_simple_moving_average::get_upstream_request filter_type=" << type << endl; #endif (void) port; @@ -128,38 +132,42 @@ std::vector teca_temporal_average::get_upstream_request( TECA_ERROR("Invalid \"filter_type\" " << this->filter_type) return up_reqs; } + first = std::max(0l, first); + last = std::min(num_steps - 1, last); + // make a request for each time that will be used in the + // average for (long i = first; i <= last; ++i) { - // make a request for each time that will be used in the - // average - if ((i >= 0) && (i < num_steps)) - { + teca_metadata up_req(request); + up_req.set("time_step", i); + up_reqs.push_back(up_req); + } + #ifdef TECA_DEBUG - cerr << teca_parallel_id() - << "request time_step " << i << endl; + cerr << teca_parallel_id() << "processing " << active_step + << " request " << first << " - " << last << endl; #endif - teca_metadata up_req(request); - up_req.set("time_step", i); - up_reqs.push_back(up_req); - } - } return up_reqs; } // -------------------------------------------------------------------------- -const_p_teca_dataset teca_temporal_average::execute( +const_p_teca_dataset teca_simple_moving_average::execute( unsigned int port, const std::vector &input_data, const teca_metadata &request) { #ifdef TECA_DEBUG cerr << teca_parallel_id() - << "teca_temporal_average::execute" << endl; + << "teca_simple_moving_average::execute" << endl; #endif (void)port; + // nothing to do + if ((input_data.size() < 1) || !input_data[0]) + return nullptr; + // create output and copy metadata, coordinates, etc p_teca_mesh out_mesh = std::dynamic_pointer_cast(input_data[0]->new_instance()); diff --git a/alg/teca_temporal_average.h b/alg/teca_simple_moving_average.h similarity index 62% rename from alg/teca_temporal_average.h rename to alg/teca_simple_moving_average.h index ca00c5c40..5b2f82d63 100644 --- a/alg/teca_temporal_average.h +++ b/alg/teca_simple_moving_average.h @@ -1,5 +1,5 @@ -#ifndef teca_temporal_average_h -#define teca_temporal_average_h +#ifndef teca_simple_moving_average_h +#define teca_simple_moving_average_h #include "teca_shared_object.h" #include "teca_algorithm.h" @@ -8,7 +8,7 @@ #include #include -TECA_SHARED_OBJECT_FORWARD_DECL(teca_temporal_average) +TECA_SHARED_OBJECT_FORWARD_DECL(teca_simple_moving_average) /// an algorithm that averages data in time /** @@ -16,33 +16,41 @@ an algorithm that averages data in time. filter_width controls the number of time steps to average over. all arrays in the input data are processed. */ -class teca_temporal_average : public teca_algorithm +class teca_simple_moving_average : public teca_algorithm { public: - TECA_ALGORITHM_STATIC_NEW(teca_temporal_average) - TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_temporal_average) - TECA_ALGORITHM_CLASS_NAME(teca_temporal_average) - ~teca_temporal_average(); + TECA_ALGORITHM_STATIC_NEW(teca_simple_moving_average) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_simple_moving_average) + TECA_ALGORITHM_CLASS_NAME(teca_simple_moving_average) + ~teca_simple_moving_average(); // report/initialize to/from Boost program options // objects. TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the number of steps to average. should be odd. + /** @name filter_width + * set the number of steps to average. should be odd. + */ + ///@{ TECA_ALGORITHM_PROPERTY(unsigned int, filter_width) + ///@} - // select the filter stencil, default is backward + /** @name filter_type + * select the filter stencil, default is backward + */ + ///@{ enum { backward, centered, forward }; TECA_ALGORITHM_PROPERTY(int, filter_type) + ///@} protected: - teca_temporal_average(); + teca_simple_moving_average(); private: std::vector get_upstream_request( diff --git a/alg/teca_table_calendar.cxx b/alg/teca_table_calendar.cxx index 79768f323..57f5ef945 100644 --- a/alg/teca_table_calendar.cxx +++ b/alg/teca_table_calendar.cxx @@ -16,7 +16,7 @@ #include #endif #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #if defined(TECA_HAS_MPI) #include @@ -73,6 +73,8 @@ void teca_table_calendar::get_properties_description( "prepended to all output column names") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -80,6 +82,8 @@ void teca_table_calendar::get_properties_description( void teca_table_calendar::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, units) TECA_POPTS_SET(opts, std::string, prefix, calendar) TECA_POPTS_SET(opts, std::string, prefix, time_column) @@ -270,7 +274,7 @@ const_p_teca_dataset teca_table_calendar::execute( int curr_minute = 0; double curr_second = 0; - if (calcalcs::date(curr_time[i], &curr_year, &curr_month, + if (teca_calcalcs::date(curr_time[i], &curr_year, &curr_month, &curr_day, &curr_hour, &curr_minute, &curr_second, units.c_str(), calendar.c_str())) { diff --git a/alg/teca_table_calendar.h b/alg/teca_table_calendar.h index 3f58ec4c5..fbc0cd494 100644 --- a/alg/teca_table_calendar.h +++ b/alg/teca_table_calendar.h @@ -10,21 +10,24 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_calendar) -/// an algorithm that transforms NetCDF CF-2 time -/// variable into an absolute date. -/** -Transform NetCDF CF-2 time variable into an absolute -date. By default the "time" column is used, but this -can be over road by set_active_column methods. the -table must have temporal metadata containing base date -and calendar units following the CF-2 convention. -the output table will contain year,month,day,hours, -minutes columns. - -NOTE: this should be used in serial, as the udunits -package loads an xml file in each instance. The -CalCalcs package also has thread safety issues. -*/ +/** @brief + * An algorithm that transforms NetCDF CF-2 time + * variable into an absolute date. + * + * @details + * Transform NetCDF CF-2 time variable into an absolute + * date. By default the "time" column is used, but this + * can be over road by set_active_column methods. the + * table must have temporal metadata containing base date + * and calendar units following the CF-2 convention. + * the output table will contain year,month,day,hours, + * minutes columns. + * + * @note + * This should be used in serial, as the UDUNITS + * package loads an XML file in each instance. The + * CalCalcs package also has thread safety issues. + */ class teca_table_calendar : public teca_algorithm { public: diff --git a/alg/teca_table_reduce.h b/alg/teca_table_reduce.h index 2740494d6..6add595c6 100644 --- a/alg/teca_table_reduce.h +++ b/alg/teca_table_reduce.h @@ -2,7 +2,7 @@ #define teca_table_reduce_h #include "teca_shared_object.h" -#include "teca_dataset_fwd.h" +#include "teca_dataset.h" #include "teca_metadata.h" #include "teca_index_reduce.h" @@ -11,12 +11,11 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_reduce) -// a reduction on tabular data over time steps +/// A reduction on tabular data over time steps. /** -a reduction on tabular data over time steps. -tabular data from each time step is collected and -concatenated into a big table. -*/ + * Tabular data from each time step is collected and + * concatenated into a big table. + */ class teca_table_reduce : public teca_index_reduce { public: diff --git a/alg/teca_table_region_mask.cxx b/alg/teca_table_region_mask.cxx index 1b5b9bed9..75bbc37a8 100644 --- a/alg/teca_table_region_mask.cxx +++ b/alg/teca_table_region_mask.cxx @@ -17,7 +17,7 @@ #include #endif #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #if defined(TECA_HAS_MPI) #include @@ -49,11 +49,11 @@ void teca_table_region_mask::get_properties_description( opts.add_options() TECA_POPTS_GET(std::string, prefix, x_coordinate_column, - "name of the column containing x cooridnates. default \"lon\"") + "name of the column containing x cooridnates.") TECA_POPTS_GET(std::string, prefix, y_coordinate_column, - "name of the column containing y cooridnates. default \"lat\"") + "name of the column containing y cooridnates.") TECA_POPTS_GET(std::string, prefix, result_column, - "name of the column to store the mask in. default \"region_mask\"") + "name of the column to store the mask in.") TECA_POPTS_MULTI_GET(std::vector, prefix, region_sizes, "the number of points in each region") TECA_POPTS_MULTI_GET(std::vector, prefix, region_x_coordinates, @@ -62,6 +62,8 @@ void teca_table_region_mask::get_properties_description( "list of y coordinates describing the regions") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -69,6 +71,8 @@ void teca_table_region_mask::get_properties_description( void teca_table_region_mask::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, x_coordinate_column) TECA_POPTS_SET(opts, std::string, prefix, y_coordinate_column) TECA_POPTS_SET(opts, std::string, prefix, result_column) diff --git a/alg/teca_table_region_mask.h b/alg/teca_table_region_mask.h index 3b3f7dafc..434bfdce6 100644 --- a/alg/teca_table_region_mask.h +++ b/alg/teca_table_region_mask.h @@ -10,13 +10,16 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_region_mask) -/** - an algorithm that identifies rows in the table that are -inside the list of regions provided. a new column, called -the mask column is created. It has 1's if the row is in -the set of regions, otherwise 0's. The invert property -can be used to invert the result. -*/ +/** @brief + * An algorithm that identifies rows in the table that are + * inside the list of regions provided. + * + * @details + * A new column, called the mask column is created. + * It has 1's if the row is in the set of regions, + * otherwise 0's. The invert property + * can be used to invert the result. + */ class teca_table_region_mask : public teca_algorithm { public: @@ -37,7 +40,7 @@ class teca_table_region_mask : public teca_algorithm // set the name of the column to store the mask in // the mask is a column of 1 and 0 indicating if the - // row satsifies the criteria or not. the default is + // row satisfies the criteria or not. the default is // "region_mask" TECA_ALGORITHM_PROPERTY(std::string, result_column); diff --git a/alg/teca_table_remove_rows.cxx b/alg/teca_table_remove_rows.cxx index 986023cc4..6840f8641 100644 --- a/alg/teca_table_remove_rows.cxx +++ b/alg/teca_table_remove_rows.cxx @@ -17,7 +17,7 @@ #include #endif #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #if defined(TECA_HAS_MPI) #include @@ -56,6 +56,8 @@ void teca_table_remove_rows::get_properties_description( "when set columns used in the calculation are removed from output") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -63,6 +65,8 @@ void teca_table_remove_rows::get_properties_description( void teca_table_remove_rows::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, mask_expression) TECA_POPTS_SET(opts, int, prefix, remove_dependent_variables) } diff --git a/alg/teca_table_remove_rows.h b/alg/teca_table_remove_rows.h index 543b9eada..882f3b259 100644 --- a/alg/teca_table_remove_rows.h +++ b/alg/teca_table_remove_rows.h @@ -10,23 +10,24 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_remove_rows) -/** -an algorithm that removes rows from a table where -a given expression evaluates to true. - -the expression parser supports the following operations: - +,-,*,/,%,<.<=,>,>=,==,!=,&&,||.!,? - -grouping in the expression is denoted in the usual -way: () - -constants in the expression are expanded to full length -arrays and can be typed. The supported types are: - d,f,L,l,i,s,c -coresponding to double,float,long long, long, int, -short and char repsectively. integer types can be -unsigned by including u after the code. -*/ +/** @brief + * An algorithm that removes rows from a table where + * a given expression evaluates to true. + * + * @details + * The expression parser supports the following operations: + * +,-,*,/,%,<.<=,>,>=,==,!=,&&,||.!,? + * + * Grouping in the expression is denoted in the usual + * way: () + * + * Constants in the expression are expanded to full length + * arrays and can be typed. The supported types are: + * d,f,L,l,i,s,c + * Corresponding to double,float, long long, long, int, + * short and char respectively. Integer types can be + * unsigned by including u after the code. + */ class teca_table_remove_rows : public teca_algorithm { public: @@ -40,17 +41,27 @@ class teca_table_remove_rows : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the expression to use to determine which rows - // are removed. rows are removed where the expression - // evaluates true. + /** @name mask_expression + * set the expression to use to determine which rows are removed. rows are + * removed where the expression evaluates true. + */ + ///@{ + /// Set the mask expression void set_mask_expression(const std::string &expr); + /// Get the mask expression std::string get_mask_expression() { return this->mask_expression; } + ///@} - // when set columns used in the calculation are removed - // from the output. deault off. + /** @name remove_dependent_variables + * when set columns used in the calculation are removed from the output. + * default off. + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, remove_dependent_variables) + ///@} + protected: teca_table_remove_rows(); diff --git a/alg/teca_table_sort.cxx b/alg/teca_table_sort.cxx index e85daabbb..ef8fae195 100644 --- a/alg/teca_table_sort.cxx +++ b/alg/teca_table_sort.cxx @@ -76,6 +76,8 @@ void teca_table_sort::get_properties_description( "if set a stable sort will be used") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -83,6 +85,8 @@ void teca_table_sort::get_properties_description( void teca_table_sort::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, index_column) TECA_POPTS_SET(opts, int, prefix, index_column_id) TECA_POPTS_SET(opts, int, prefix, stable_sort) diff --git a/alg/teca_table_to_stream.cxx b/alg/teca_table_to_stream.cxx index 58b15a108..f9ee2cc84 100644 --- a/alg/teca_table_to_stream.cxx +++ b/alg/teca_table_to_stream.cxx @@ -54,6 +54,9 @@ void teca_table_to_stream::get_properties_description( TECA_POPTS_GET(std::string, prefix, stream, "name of stream to send output to. stderr, stdout") ; + + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -61,6 +64,8 @@ void teca_table_to_stream::get_properties_description( void teca_table_to_stream::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, header) TECA_POPTS_SET(opts, std::string, prefix, footer) TECA_POPTS_SET(opts, std::string, prefix, stream) @@ -91,6 +96,25 @@ void teca_table_to_stream::set_stream(const std::string &s) } } +// -------------------------------------------------------------------------- +std::string teca_table_to_stream::get_stream() +{ + if (this->stream == &std::cerr) + { + return "stderr"; + } + else if (this->stream == &std::cout) + { + return "stdout"; + } + else if (!this->stream) + { + return "null"; + } + + return "unknown"; +} + // -------------------------------------------------------------------------- void teca_table_to_stream::set_stream_to_stderr() { diff --git a/alg/teca_table_to_stream.h b/alg/teca_table_to_stream.h index c2fcd631c..c88c6d747 100644 --- a/alg/teca_table_to_stream.h +++ b/alg/teca_table_to_stream.h @@ -11,8 +11,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_to_stream) -/// an algorithm that serializes a table to a c++ stream object. -/// This is primarilly useful for debugging. +/// An algorithm that serializes a table to a C++ stream object. class teca_table_to_stream : public teca_algorithm { public: @@ -40,6 +39,9 @@ class teca_table_to_stream : public teca_algorithm void set_stream_to_stderr(); void set_stream_to_stdout(); + // get the stream name + std::string get_stream(); + protected: teca_table_to_stream(); diff --git a/alg/teca_tc_candidates.cxx b/alg/teca_tc_candidates.cxx index 76ac71657..b08eede71 100644 --- a/alg/teca_tc_candidates.cxx +++ b/alg/teca_tc_candidates.cxx @@ -4,7 +4,6 @@ #include "teca_variant_array.h" #include "teca_table.h" #include "teca_database.h" -#include "teca_calendar.h" #include "teca_coordinate_util.h" #include "gfdl_tc_candidates.h" @@ -71,34 +70,36 @@ void teca_tc_candidates::get_properties_description( "name of core temperature variable") TECA_POPTS_GET(double, prefix, max_core_radius, "maximum number of degrees latitude separation between " - "vorticity max and pressure min defining a storm (2.0)") + "vorticity max and pressure min defining a storm") TECA_POPTS_GET(double, prefix, min_vorticity_850mb, - "minimum vorticty to be considered a tropical storm (1.6e-4)") + "minimum vorticty to be considered a tropical storm") TECA_POPTS_GET(double, prefix, vorticity_850mb_window, "size of the search window in degrees. storms core must have a " - "local vorticity max centered on this window (7.74446)") + "local vorticity max centered on this window") TECA_POPTS_GET(double, prefix, max_pressure_delta, - "maximum pressure change within specified radius (400.0)") + "maximum pressure change within specified radius") TECA_POPTS_GET(double, prefix, max_pressure_radius, - "radius in degrees over which max pressure change is computed (5.0)") + "radius in degrees over which max pressure change is computed") TECA_POPTS_GET(double, prefix, max_core_temperature_delta, - "maximum core temperature change over the specified radius (0.8)") + "maximum core temperature change over the specified radius") TECA_POPTS_GET(double, prefix, max_core_temperature_radius, - "radius in degrees over which max core temperature change is computed (5.0)") + "radius in degrees over which max core temperature change is computed") TECA_POPTS_GET(double, prefix, max_thickness_delta, - "maximum thickness change over the specified radius (50.0)") + "maximum thickness change over the specified radius") TECA_POPTS_GET(double, prefix, max_thickness_radius, - "radius in degrees over with max thickness change is comuted (4.0)") + "radius in degrees over with max thickness change is comuted") TECA_POPTS_GET(double, prefix, search_lat_low, - "lowest latitude in degrees to search for storms (-80.0)") + "lowest latitude in degrees to search for storms") TECA_POPTS_GET(double, prefix, search_lat_high, - "highest latitude in degrees to search for storms (80.0)") + "highest latitude in degrees to search for storms") TECA_POPTS_GET(double, prefix, search_lon_low, - "lowest longitude in degrees to search for stroms (1)") + "lowest longitude in degrees to search for stroms") TECA_POPTS_GET(double, prefix, search_lon_high, - "highest longitude in degrees to search for storms (0)") + "highest longitude in degrees to search for storms") ; + this->teca_algorithm::get_properties_description(prefix, opts); + opts.add(ard_opts); } @@ -106,6 +107,8 @@ void teca_tc_candidates::get_properties_description( void teca_tc_candidates::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, surface_wind_speed_variable) TECA_POPTS_SET(opts, std::string, prefix, vorticity_850mb_variable) TECA_POPTS_SET(opts, std::string, prefix, sea_level_pressure_variable) @@ -453,8 +456,9 @@ const_p_teca_dataset teca_tc_candidates::execute(unsigned int port, cerr << std::endl; #endif seconds_t dt(t1 - t0); - TECA_STATUS("teca_tc_candidates step=" << time_step - << " t=" << time_offset << ", dt=" << dt.count() << " sec") + + TECA_STATUS("At step " << time_step << " time " << time_offset << " " + << n_candidates << " candidates detected in " << dt.count() << " seconds") return out_table; } diff --git a/alg/teca_tc_candidates.h b/alg/teca_tc_candidates.h index 85b1dcdcb..7c45c50a5 100644 --- a/alg/teca_tc_candidates.h +++ b/alg/teca_tc_candidates.h @@ -10,41 +10,42 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_tc_candidates) +/// GFDL tropical storms detection algorithm. /** -GFDL tropical storms detection algorithm -for more information see -"Seasonal forecasting of tropical storms using coupled GCM integrations" - - --- INPUT - Gwind - wind speed at 850 mb - Gvort - vorticity_850mb at 850 mb - Gtbar - mean core_temperature for warm core layer - Gpsl - sea level sea_level_pressure - Gthick - thickness of 200 to 1000 mb layer - Grlon - longitudes - Grlat - latitudes - iyear - year - imon - month - iday - day of month - ihour - hour - iucy - unit for output - - --- OUTPUT - --- record # 1 - num0 - day - imon0 - month - iyear - year - number - number of cyclones found - --- records # 2...number+1 - idex, jdex - (i,j) index of cyclone - svort_max - max vorticity_850mb - swind_max - max wind - spsl_min - min sea level sea_level_pressure - svort_lon, svort_lat - longitude & latitude of max vorticity_850mb - spsl_lon, spsl_lat - longitude & latitude of min slp - stemperature_lon, stemperature_lat - longitude & latitude of warm core - sthick_lon, sthick_lat - longitude & latitude of max thickness -*/ + * For more information see + * "Seasonal forecasting of tropical storms using coupled GCM integrations" + * + * --- INPUT + * Gwind - wind speed at 850 mb + * Gvort - vorticity_850mb at 850 mb + * Gtbar - mean core_temperature for warm core layer + * Gpsl - sea level sea_level_pressure + * Gthick - thickness of 200 to 1000 mb layer + * Grlon - longitudes + * Grlat - latitudes + * iyear - year + * imon - month + * iday - day of month + * ihour - hour + * iucy - unit for output + * + * --- OUTPUT + * --- record # 1 + * num0 - day + * imon0 - month + * iyear - year + * number - number of cyclones found + * --- records # 2...number+1 + * idex, jdex - (i,j) index of cyclone + * svort_max - max vorticity_850mb + * swind_max - max wind + * spsl_min - min sea level sea_level_pressure + * svort_lon, svort_lat - longitude & latitude of max vorticity_850mb + * spsl_lon, spsl_lat - longitude & latitude of min slp + * stemperature_lon, stemperature_lat - longitude & latitude of warm + * core + * sthick_lon, sthick_lat - longitude & latitude of max thickness + */ class teca_tc_candidates : public teca_algorithm { public: @@ -85,19 +86,19 @@ class teca_tc_candidates : public teca_algorithm TECA_ALGORITHM_PROPERTY(double, max_thickness_radius) // set/get the bounding box to search for storms - // in units of degreees lat,lon + // in units of degrees lat,lon TECA_ALGORITHM_PROPERTY(double, search_lat_low) TECA_ALGORITHM_PROPERTY(double, search_lat_high) TECA_ALGORITHM_PROPERTY(double, search_lon_low) TECA_ALGORITHM_PROPERTY(double, search_lon_high) // set/get the number of iterations to search for the - // storm local minimum. raising this paramter might increase - // detections but the detector will run slowerd. default is + // storm local minimum. raising this parameter might increase + // detections but the detector will run slower. default is // 50. TECA_ALGORITHM_PROPERTY(int, minimizer_iterations) - // send humand readable representation to the + // send human readable representation to the // stream virtual void to_stream(std::ostream &os) const override; diff --git a/alg/teca_tc_classify.cxx b/alg/teca_tc_classify.cxx index 42ceca00e..503c58100 100644 --- a/alg/teca_tc_classify.cxx +++ b/alg/teca_tc_classify.cxx @@ -17,7 +17,7 @@ #include #endif #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #if defined(TECA_HAS_MPI) #include @@ -81,6 +81,8 @@ void teca_tc_classify::get_properties_description( "each region. if not provided names are generated from ids") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -88,6 +90,8 @@ void teca_tc_classify::get_properties_description( void teca_tc_classify::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, track_id_column) TECA_POPTS_SET(opts, std::string, prefix, time_column) TECA_POPTS_SET(opts, std::string, prefix, surface_wind_column) diff --git a/alg/teca_tc_classify.h b/alg/teca_tc_classify.h index efec9e852..1f2ae884a 100644 --- a/alg/teca_tc_classify.h +++ b/alg/teca_tc_classify.h @@ -12,12 +12,12 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_tc_classify) /// an algorithm that classifies storms using Saphire-Simpson scale /** -An algorithm that classifies storms using Saphire-Simpson scale -a column containing the classification is added to the output - -An algorithm that sorts the storms by geographic region -and category. tyhe output is a table where regions is -mapped to columns and category is mapped to rows. + * An algorithm that classifies storms using Saphire-Simpson scale a column + * containing the classification is added to the output + * + * An algorithm that sorts the storms by geographic region and category. tyhe + * output is a table where regions is mapped to columns and category is mapped to +rows. */ class teca_tc_classify : public teca_algorithm { diff --git a/alg/teca_tc_trajectory.cxx b/alg/teca_tc_trajectory.cxx index 61889b7a3..a6b274c11 100644 --- a/alg/teca_tc_trajectory.cxx +++ b/alg/teca_tc_trajectory.cxx @@ -38,10 +38,12 @@ int teca_tc_trajectory(var_t r_crit, var_t wind_crit, double n_wind_crit, const int *storm_uid, const coord_t *d_lon, const coord_t *d_lat, const var_t *wind_max, const var_t *vort_max, const var_t *psl, const int *have_twc, const int *have_thick, const var_t *twc_max, - const var_t *thick_max, unsigned long n_rows, p_teca_table track_table) + const var_t *thick_max, unsigned long n_rows, p_teca_table track_table, + unsigned long &n_tracks) { const coord_t DEG_TO_RAD = M_PI/180.0; unsigned long track_id = 0; + n_tracks = 0; // convert from dsegrees to radians unsigned long nbytes = n_rows*sizeof(coord_t); @@ -218,6 +220,8 @@ int teca_tc_trajectory(var_t r_crit, var_t wind_crit, double n_wind_crit, free(r_lat); free(available); + n_tracks = track_id + 1; + return 0; } }; @@ -248,16 +252,18 @@ void teca_tc_trajectory::get_properties_description( opts.add_options() TECA_POPTS_GET(double, prefix, max_daily_distance, - "max distance a storm can move on the same track in single day (1600 km)") + "max distance a storm can move on the same track in single day") TECA_POPTS_GET(double, prefix, min_wind_speed, - "minimum wind speed to be worthy of tracking (17.0 ms^-1)") + "minimum wind speed to be worthy of tracking") TECA_POPTS_GET(double, prefix, min_wind_duration, "minimum number of, not necessarily consecutive, days thickness, " - "core temp, and wind speed criteria must be satisfied (2.0 days)") + "core temp, and wind speed criteria must be satisfied") TECA_POPTS_GET(unsigned long, prefix, step_interval, - "number of time steps between valid candidate data. (1 step)") + "number of time steps between valid candidate data.") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -265,6 +271,8 @@ void teca_tc_trajectory::get_properties_description( void teca_tc_trajectory::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, double, prefix, max_daily_distance) TECA_POPTS_SET(opts, double, prefix, min_wind_speed) TECA_POPTS_SET(opts, double, prefix, min_wind_duration) @@ -405,7 +413,18 @@ const_p_teca_dataset teca_tc_trajectory::execute( } unsigned long n_rows = candidates->get_number_of_rows(); + + // check that there are some candidates to work with. + if (n_rows < 1) + { + TECA_ERROR("Failed to form TC tracks because there were no candiates") + return nullptr; + } + + unsigned long n_tracks = 0; + std::chrono::high_resolution_clock::time_point t0, t1; + t0 = std::chrono::high_resolution_clock::now(); NESTED_TEMPLATE_DISPATCH_FP(const teca_variant_array_impl, lon.get(), _COORD, @@ -445,24 +464,25 @@ const_p_teca_dataset teca_tc_trajectory::execute( dynamic_cast(thick_max.get())->get(); // invoke the track finder - t0 = std::chrono::high_resolution_clock::now(); if (internal::teca_tc_trajectory( static_cast(this->max_daily_distance), static_cast(this->min_wind_speed), this->min_wind_duration, this->step_interval, p_step, p_time, p_storm_id, p_lon, p_lat, p_wind_max, p_vort_max, p_psl_min, p_have_twc, p_have_thick, - p_twc_max, p_thick_max, n_rows, storm_tracks)) + p_twc_max, p_thick_max, n_rows, storm_tracks, n_tracks)) { - TECA_ERROR("GFDL TC trajectory analysis encountered an error") + TECA_ERROR("Failed to form tracks") return nullptr; } - t1 = std::chrono::high_resolution_clock::now(); ) ) + t1 = std::chrono::high_resolution_clock::now(); seconds_t dt(t1 - t0); - TECA_STATUS("teca_tc_trajectory n_candidates=" << n_rows << ", n_tracks=" - << storm_tracks->get_number_of_rows() << ", dt=" << dt.count() << " sec") + + TECA_STATUS("Formed " << n_tracks << " tracks comprised of " + << storm_tracks->get_number_of_rows() << " of the " << n_rows + << " avilable candidates in " << dt.count() << " seconds") return storm_tracks; } diff --git a/alg/teca_tc_trajectory.h b/alg/teca_tc_trajectory.h index 2154cc881..26c487e6a 100644 --- a/alg/teca_tc_trajectory.h +++ b/alg/teca_tc_trajectory.h @@ -11,28 +11,25 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_tc_trajectory) /// GFDL tropical storms trajectory tracking algorithm -/** -GFDL tropical storms trajectory tracking algorithm - -for more information see -"Seasonal forecasting of tropical storms using coupled GCM integrations" - -computes trajectories from table of detections. trajectories -are stored in a table. - -the trajectory computation makes use of the following paramteters: - -max_daily_distance (1600 km) - event must be within this distance on the - following day to be considered as part of the trajectory. - -min_wind_speed (17 m/s) - 850 mb wind sped must be above this value. - -min_wind_duration (2 days) - criteria must be satisfied for this many days to be - a candidate -*/ +/** for more information see + * "Seasonal forecasting of tropical storms using coupled GCM integrations" + * + * computes trajectories from table of detections. trajectories + * are stored in a table. + * + * the trajectory computation makes use of the following paramteters: + * + * max_daily_distance (1600 km) + * event must be within this distance on the + * following day to be considered as part of the trajectory. + * + * min_wind_speed (17 m/s) + * 850 mb wind sped must be above this value. + * + * min_wind_duration (2 days) + * criteria must be satisfied for this many days to be + * a candidate + */ class teca_tc_trajectory : public teca_algorithm { public: diff --git a/alg/teca_tc_wind_radii.cxx b/alg/teca_tc_wind_radii.cxx index 20ba6713c..de14e7c00 100644 --- a/alg/teca_tc_wind_radii.cxx +++ b/alg/teca_tc_wind_radii.cxx @@ -596,6 +596,8 @@ void teca_tc_wind_radii::get_properties_description(const std::string &prefix, "the average wind speed over the interval is used.") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -603,6 +605,8 @@ void teca_tc_wind_radii::get_properties_description(const std::string &prefix, void teca_tc_wind_radii::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, storm_id_column) TECA_POPTS_SET(opts, std::string, prefix, storm_x_coordinate_column) TECA_POPTS_SET(opts, std::string, prefix, storm_y_coordinate_column) diff --git a/alg/teca_tc_wind_radii.h b/alg/teca_tc_wind_radii.h index 6b9befc38..d44b0282d 100644 --- a/alg/teca_tc_wind_radii.h +++ b/alg/teca_tc_wind_radii.h @@ -12,15 +12,14 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_tc_wind_radii) /// computes wind radius at the specified coordinates /** -Compute storm size and adds it to the table. There are two inputs, -the first serves up tables of storms to compute the storm radius -for. One must set the names of the columns that contain storm ids, -x-coordnates, y-coordinates, and time coordinate. For each event -the wind radius is computed. Computations are parallelized over -storm id. The second input serves up wind velocity data most likely -this will be from a NetCDF CF2 simulation dataset. By default -radius is computed at the transitions on the Saffir-Simpson -scale. + * Compute storm size and adds it to the table. There are two inputs, the first + * serves up tables of storms to compute the storm radius for. One must set the + * names of the columns that contain storm ids, x-coordnates, y-coordinates, + * and time coordinate. For each event the wind radius is computed. + * Computations are parallelized over storm id. The second input serves up wind + * velocity data most likely this will be from a NetCDF CF2 simulation dataset. + * By default radius is computed at the transitions on the Saffir-Simpson + * scale. */ class teca_tc_wind_radii : public teca_algorithm { @@ -35,59 +34,110 @@ class teca_tc_wind_radii : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the name of the column that defines the track ids - // if set the specified column is coppied into the output - // metadata and accessed with the key event_id + /** @name storm_id_column + * set the name of the column that defines the track ids if set the + * specified column is coppied into the output metadata and accessed with + * the key event_id + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, storm_id_column) - - // set the name of the columns that define the event position - // if set the columns are coppied into the output metadata - // and accessed with the keys storm_x_coordinate, storm_y_coordinate + ///@} + + /** @name storm_x_coordinate_column + * set the name of the columns that define the event position if set the + * columns are coppied into the output metadata and accessed with the keys + * storm_x_coordinate, storm_y_coordinate + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, storm_x_coordinate_column) + ///@} + + /** @name storm_y_coordinate_column + * set the name of the columns that define the event position if set the + * columns are coppied into the output metadata and accessed with the keys + * storm_x_coordinate, storm_y_coordinate + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, storm_y_coordinate_column) + ///@} - // set the name of the column containing peak instantanious - // surface wind speed + /** @name storm_wind_speed_column + * set the name of the column containing peak instantanious surface wind + * speed + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, storm_wind_speed_column) - - // set the name of the column that defines the event time - // if set the specified column is coppied into the output - // metadata and accessed with the key event_time + ///@} + + /** @name storm_time_column + * set the name of the column that defines the event time if set the + * specified column is coppied into the output metadata and accessed with + * the key event_time + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, storm_time_column) + ///@} // set the name of the wind variable components + /** @name wind_u_variable + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, wind_u_variable) + ///@} + + /** @name wind_v_variable + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, wind_v_variable) + ///@} - // set the radius in degrees of latitude to sample the wind - // field + /** @name search_radius + * set the radius in degrees of latitude to sample the wind field + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, search_radius) - - // set the radius in degrees of latitude beyond which to - // terminate the search for peak wind speed. if the peak - // lies beyond this distance search is terminated and a - // warning is displayed. + ///@} + + /** @name core_radius + * set the radius in degrees of latitude beyond which to terminate the + * search for peak wind speed. if the peak lies beyond this distance search + * is terminated and a warning is displayed. + */ + ///@{ TECA_ALGORITHM_PROPERTY(double, core_radius) + ///@} - // number of bins to discetize by in the radial direction + /** @name number_of_radial_bins + * number of bins to discetize by in the radial direction + */ + ///@{ TECA_ALGORITHM_PROPERTY(int, number_of_radial_bins) + ///@} - // set the wind speeds (in m/s) to find the radius of. the - // defualt values are the transition speeds of the Saffir-Simpson - // scale. + /** @name critical_wind_speed + * set the wind speeds (in m/s) to find the radius of. the defualt values + * are the transition speeds of the Saffir-Simpson scale. + */ + ///@{ TECA_ALGORITHM_VECTOR_PROPERTY(double, critical_wind_speed) - - // set the profile type. PROFILE_MAX uses the maximum - // wind speed on each interval of the discretization, while - // PROFILE_AVERAGE uses the average on each interval + ///@} + + /** @name profile_type + * Set the profile type. PROFILE_MAX uses the maximum wind speed on each + * interval of the discretization, while PROFILE_AVERAGE uses the average + * on each interval + */ + ///@{ + /// Profile types enum {PROFILE_MAX = 0, PROFILE_AVERAGE = 1}; - TECA_ALGORITHM_PROPERTY(int, profile_type); + TECA_ALGORITHM_PROPERTY(int, profile_type) + ///@} - // override the input connections because we are going to - // take the first input and use it to generate metadata. - // the second input then becomes the only one the pipeline - // knows about. + /** override the input connections because we are going to take the first + * input and use it to generate metadata. the second input then becomes + * the only one the pipeline knows about. + */ void set_input_connection(unsigned int id, const teca_algorithm_output_port &port) override; diff --git a/alg/teca_temporal_reduction.py b/alg/teca_temporal_reduction.py index 034d551f2..005f9125f 100644 --- a/alg/teca_temporal_reduction.py +++ b/alg/teca_temporal_reduction.py @@ -1,7 +1,7 @@ import sys import numpy as np - +# @cond class teca_temporal_reduction_internals: class time_point: """ @@ -181,7 +181,7 @@ def __next__(self): sm = self.month t0 = '%04d-%02d-01 00:00:00' % (sy, sm) - i0 = coordinate_util.time_step_of(self.t, True, True, + i0 = coordinate_util.time_step_of(self.t, False, True, self.calendar, self.units, t0) @@ -259,7 +259,7 @@ def __next__(self): month = self.month t0 = '%04d-%02d-01 00:00:00' % (self.year, self.month) - i0 = coordinate_util.time_step_of(self.t, True, True, + i0 = coordinate_util.time_step_of(self.t, False, True, self.calendar, self.units, t0) @@ -350,7 +350,7 @@ def __next__(self): t0 = '%04d-%02d-%02d 00:00:00' % \ (self.year, self.month, self.day) - i0 = coordinate_util.time_step_of(self.t, True, True, + i0 = coordinate_util.time_step_of(self.t, False, True, self.calendar, self.units, t0) @@ -531,6 +531,7 @@ def New(op_name): reduction_operator.maximum() raise RuntimeError('Invalid operator %s' % (op_name)) +# @endcond class teca_temporal_reduction(teca_threaded_python_algorithm): diff --git a/alg/teca_unpack_data.cxx b/alg/teca_unpack_data.cxx new file mode 100644 index 000000000..5972070f0 --- /dev/null +++ b/alg/teca_unpack_data.cxx @@ -0,0 +1,363 @@ +#include "teca_unpack_data.h" + +#include "teca_cartesian_mesh.h" +#include "teca_array_collection.h" +#include "teca_variant_array.h" +#include "teca_metadata.h" +#include "teca_array_attributes.h" + +#include +#include +#include +#include +#include + +#if defined(TECA_HAS_BOOST) +#include +#endif + +//#define TECA_DEBUG + +namespace +{ +template +void transform(output_t * __restrict__ p_out, input_t * __restrict__ p_in, + size_t n, output_t scale, output_t offset) +{ + for (size_t i = 0; i < n; ++i) + p_out[i] = p_in[i] * scale + offset; +} + +template +void transform(output_t * __restrict__ p_out, input_t * __restrict__ p_in, + mask_t * __restrict__ p_mask, size_t n, output_t scale, output_t offset, + output_t fill) +{ + for (size_t i = 0; i < n; ++i) + p_out[i] = (p_mask[i] ? p_in[i] * scale + offset : fill); +} +} + + +// -------------------------------------------------------------------------- +teca_unpack_data::teca_unpack_data() : + output_data_type(teca_variant_array_code::get()) +{ + this->set_number_of_input_connections(1); + this->set_number_of_output_ports(1); +} + +// -------------------------------------------------------------------------- +teca_unpack_data::~teca_unpack_data() +{} + +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_unpack_data::get_properties_description( + const std::string &prefix, options_description &global_opts) +{ + options_description opts("Options for " + + (prefix.empty()?"teca_unpack_data":prefix)); + + opts.add_options() + TECA_POPTS_GET(int, prefix, output_data_type, + "Sets the type of the transformed data to either single or double" + " precision floating point. Use 11 for single precision and 12 for" + " double precision.") + TECA_POPTS_GET(int, prefix, verbose, "Enables verbose output") + ; + + + this->teca_algorithm::get_properties_description(prefix, opts); + + global_opts.add(opts); +} + +// -------------------------------------------------------------------------- +void teca_unpack_data::set_properties( + const std::string &prefix, variables_map &opts) +{ + this->teca_algorithm::set_properties(prefix, opts); + + TECA_POPTS_SET(opts, int, prefix, output_data_type) + TECA_POPTS_SET(opts, int, prefix, verbose) +} +#endif + +// -------------------------------------------------------------------------- +int teca_unpack_data::validate_output_data_type(int val) +{ + // validate the output type + if ((val != ((int)teca_variant_array_code::get())) && + (val != ((int)teca_variant_array_code::get()))) + { + TECA_ERROR("Invlaid output data type " << val << ". Use " + << teca_variant_array_code::get() + << " to select double precision output and " + << teca_variant_array_code::get() + << " to select single precision output") + return -1; + } + return 0; +} + +// -------------------------------------------------------------------------- +teca_metadata teca_unpack_data::get_output_metadata( + unsigned int port, + const std::vector &input_md) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() + << "teca_unpack_data::get_output_metadata" << endl; +#endif + (void)port; + + // for each array on the input look for the presence of scale_factor and + // add_offset if both attributes are present then modify the output data + // type. + teca_metadata out_md(input_md[0]); + + std::vector variables; + if (out_md.get("variables", variables)) + { + TECA_ERROR("Failed to get the list of variables") + return teca_metadata(); + } + + teca_metadata attributes; + if (out_md.get("attributes", attributes)) + { + TECA_ERROR("Failed to get the array attributes") + return teca_metadata(); + } + + size_t n_vars = variables.size(); + for (size_t i = 0; i < n_vars; ++i) + { + const std::string &array_name = variables[i]; + + teca_metadata array_atts; + if (attributes.get(array_name, array_atts)) + { + // this could be reported as an error or a warning but unless this + // becomes problematic quietly ignore it + continue; + } + + // if both scale_factor and add_offset attributes are present then + // the data will be transformed. Update the output type. + if (array_atts.has("scale_factor") && array_atts.has("add_offset")) + { + array_atts.set("type_code", this->output_data_type); + + array_atts.remove("scale_factor"); + array_atts.remove("add_offset"); + + if (array_atts.has("_FillValue") || array_atts.has("missing_value")) + { + array_atts.remove("_FillValue"); + array_atts.remove("missing_value"); + + if (this->output_data_type == ((int)teca_variant_array_code::get())) + array_atts.set("_FillValue", 1e20); + else if (this->output_data_type == ((int)teca_variant_array_code::get())) + array_atts.set("_FillValue", 1e20f); + } + + attributes.set(array_name, array_atts); + } + } + + out_md.set("attributes", attributes); + return out_md; +} + +// -------------------------------------------------------------------------- +std::vector teca_unpack_data::get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) +{ + (void)port; + + std::vector up_reqs; + + // copy the incoming request to preserve the downstream + // requirements and add the arrays we need + teca_metadata req(request); + + // get the list of variable available. we need to see if + // the valid value mask is available and if so request it + const teca_metadata &md = input_md[0]; + + std::set variables; + if (md.get("variables", variables)) + { + TECA_ERROR("Metadata issue. variables is missing") + return up_reqs; + } + + teca_metadata attributes; + if (md.get("attributes", attributes)) + { + TECA_ERROR("Failed to get the array attributes") + return up_reqs; + } + + // add the dependent variables into the requested arrays + std::set arrays_up; + if (req.has("arrays")) + req.get("arrays", arrays_up); + + std::vector arrays_in(arrays_up.begin(), arrays_up.end()); + int n_arrays = arrays_in.size(); + for (int i = 0; i < n_arrays; ++i) + { + const std::string &array_name = arrays_in[i]; + + teca_metadata array_atts; + if (attributes.get(array_name, array_atts)) + { + // this could be reported as an error or a warning but unless this + // becomes problematic quietly ignore it + continue; + } + + // if both scale_factor and add_offset attributes are present then + // the data will be transformed. Update the output type. + if (array_atts.has("scale_factor") && array_atts.has("add_offset") && + (array_atts.has("_FillValue") || array_atts.has("missing_value"))) + { + // request the valid value mask if they are available. + std::string mask_var = array_name + "_valid"; + if (variables.count(mask_var)) + arrays_up.insert(mask_var); + } + } + + // update the request + req.set("arrays", arrays_up); + + // send it up + up_reqs.push_back(req); + return up_reqs; +} + +// -------------------------------------------------------------------------- +const_p_teca_dataset teca_unpack_data::execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) +{ +#ifdef TECA_DEBUG + std::cerr << teca_parallel_id() << "teca_unpack_data::execute" << endl; +#endif + (void)port; + (void)request; + + // get the input mesh + const_p_teca_mesh in_mesh + = std::dynamic_pointer_cast(input_data[0]); + + if (!in_mesh) + { + TECA_ERROR("Input dataset is not a teca_mesh") + return nullptr; + } + + p_teca_mesh out_mesh = + std::static_pointer_cast(in_mesh->new_instance()); + + out_mesh->shallow_copy(std::const_pointer_cast(in_mesh)); + + teca_metadata attributes; + if (out_mesh->get_attributes(attributes)) + { + TECA_ERROR("Failed to get attributes") + return nullptr; + } + + // for each array + p_teca_array_collection point_arrays = out_mesh->get_point_arrays(); + int n_arrays = point_arrays->size(); + for (int i = 0; i < n_arrays; ++i) + { + const std::string &array_name = point_arrays->get_name(i); + + // skip valid value masks + size_t len = array_name.size(); + if ((len > 6) && (strcmp("_valid", array_name.c_str() + len - 6) == 0)) + continue; + + // check if this array is to be transformed + teca_metadata array_atts; + double scale = 0.0; + double offset = 0.0; + if (attributes.get(array_name, array_atts) || + array_atts.get("scale_factor", scale) || + array_atts.get("add_offset", offset)) + continue; + + // check for valid value mask + std::string mask_name = array_name + "_valid"; + p_teca_variant_array mask = point_arrays->get(mask_name); + + // get the input + p_teca_variant_array in_array = point_arrays->get(i); + + // allocate the output + p_teca_variant_array out_array = + teca_variant_array_factory::New(this->output_data_type); + if (!out_array) + { + TECA_ERROR("Failed to allocate the output array") + return nullptr; + } + + unsigned long n_elem = in_array->size(); + out_array->resize(n_elem); + + // transform arrays + NESTED_TEMPLATE_DISPATCH(teca_variant_array_impl, + in_array.get(), + _IN, + NT_IN *p_in = dynamic_cast(in_array.get())->get(); + NESTED_TEMPLATE_DISPATCH_FP(teca_variant_array_impl, + out_array.get(), + _OUT, + NT_OUT *p_out = dynamic_cast(out_array.get())->get(); + + if (mask) + { + NESTED_TEMPLATE_DISPATCH_I(teca_variant_array_impl, + mask.get(), + _MASK, + NT_MASK *p_mask = dynamic_cast(mask.get())->get(); + ::transform(p_out, p_in, p_mask, + n_elem, NT_OUT(scale), NT_OUT(offset), NT_OUT(1e20)); + ) + + } + else + { + ::transform(p_out, p_in, n_elem, NT_OUT(scale), NT_OUT(offset)); + } + ) + ) + + // poass to the output + point_arrays->set(i, out_array); + + // update the metadata + array_atts.set("type_code", this->output_data_type); + attributes.set(array_name, array_atts); + + if (this->verbose) + { + TECA_STATUS("Unpacked \"" << array_name << "\" scale_factor = " + << scale << " add_offset = " << offset) + } + } + + return out_mesh; +} diff --git a/alg/teca_unpack_data.h b/alg/teca_unpack_data.h new file mode 100644 index 000000000..373535f2a --- /dev/null +++ b/alg/teca_unpack_data.h @@ -0,0 +1,84 @@ +#ifndef teca_unpack_data_h +#define teca_unpack_data_h + +#include "teca_shared_object.h" +#include "teca_algorithm.h" +#include "teca_metadata.h" +#include "teca_variant_array.h" + +#include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_unpack_data) + +/// an algorithm that unpacks NetCDF packed values +/** + * Applies a data transform according to the NetCDF attribute conventions for + * packed data values. + * https://www.unidata.ucar.edu/software/netcdf/docs/attribute_conventions.html + * + * Variables in the input dataset are scanned for the presence + * of the `scale_factor` and `add_offset` attributes. When both are present + * an element wise transformation is applied such that + * + * out[i] = scale_factor * in[i] + add_offset + * + * The input array is expected to be an integer type while the type of the output + * array may be either float or double. Valid value masks may be necessary for + * correct results, see `teca_valid_value_mask`. +*/ +class teca_unpack_data : public teca_algorithm +{ +public: + TECA_ALGORITHM_STATIC_NEW(teca_unpack_data) + TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_unpack_data) + TECA_ALGORITHM_CLASS_NAME(teca_unpack_data) + ~teca_unpack_data(); + + // report/initialize to/from Boost program options + // objects. + TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() + TECA_SET_ALGORITHM_PROPERTIES() + + /** @name output_data_type + * set the output data type. use teca_variant_array_code::get() to get + * the numeric code corresponding to the data type T. The default output + * data type is single precision floating point. + */ + ///@{ + TECA_ALGORITHM_PROPERTY_V(int, output_data_type) + + /// set the output data type to double precision floating point + void set_output_data_type_to_float() + { this->set_output_data_type(teca_variant_array_code::get()); } + + /// set the output data type to single precision floating point + void set_output_data_type_to_double() + { this->set_output_data_type(teca_variant_array_code::get()); } + ///@} + +protected: + teca_unpack_data(); + +private: + teca_metadata get_output_metadata( + unsigned int port, + const std::vector &input_md) override; + + std::vector get_upstream_request( + unsigned int port, + const std::vector &input_md, + const teca_metadata &request) override; + + const_p_teca_dataset execute( + unsigned int port, + const std::vector &input_data, + const teca_metadata &request) override; + + int validate_output_data_type(int val); + +private: + int output_data_type; +}; + +#endif diff --git a/alg/teca_valid_value_mask.cxx b/alg/teca_valid_value_mask.cxx index c9240efec..90cbddbfe 100644 --- a/alg/teca_valid_value_mask.cxx +++ b/alg/teca_valid_value_mask.cxx @@ -37,7 +37,7 @@ bool is_mask_array(const std::string &array) // -------------------------------------------------------------------------- teca_valid_value_mask::teca_valid_value_mask() : - mask_arrays(), enable_valid_range(0), verbose(0) + mask_arrays(), enable_valid_range(0) { this->set_number_of_input_connections(1); this->set_number_of_output_ports(1); @@ -62,10 +62,10 @@ void teca_valid_value_mask::get_properties_description( TECA_POPTS_GET(int, prefix, enable_valid_range, "If set non-zero vald_range, valid_min, and valid_max attributes" " would be used if there is no _FillValue attribute.") - TECA_POPTS_GET(int, prefix, verbose, - "If set then status messages are sent to the terminal.") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -73,9 +73,10 @@ void teca_valid_value_mask::get_properties_description( void teca_valid_value_mask::set_properties( const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::vector, prefix, mask_arrays) TECA_POPTS_SET(opts, int, prefix, enable_valid_range) - TECA_POPTS_SET(opts, int, prefix, verbose) } #endif diff --git a/alg/teca_valid_value_mask.h b/alg/teca_valid_value_mask.h index ebea5f21a..dfa90b9a3 100644 --- a/alg/teca_valid_value_mask.h +++ b/alg/teca_valid_value_mask.h @@ -27,7 +27,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_valid_value_mask) * a variable named "V" it will be named "V_valid". * * Masks are requested for specific arrays in one of two ways. One may use the - * @ref mask_arrays algorithm property to explicitly name the list of variables to + * mask_arrays algorithm property to explicitly name the list of variables to * compute masks for. Alternatively, a heuristic applied to incoming requests * determines if masks should be generated. Specifically the string "_valid" is * looked for at the end of each requested array. If it is found then the mask @@ -48,8 +48,7 @@ class teca_valid_value_mask : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - /** @anchor mask_arrays - * @name mask_arrays + /** @name mask_arrays * explicitly set a list of input arrays to process. By default * all arrays are processed. Use this property to compute masks * for a subset of the arrays, @@ -58,8 +57,7 @@ class teca_valid_value_mask : public teca_algorithm TECA_ALGORITHM_VECTOR_PROPERTY(std::string, mask_array) ///@} - /** @anchor enable_valid_range - * @name enable_valid_range + /** @name enable_valid_range * enable the use of valid_range, valid_min, valid_max attributes. * by default this is off. */ @@ -67,15 +65,6 @@ class teca_valid_value_mask : public teca_algorithm TECA_ALGORITHM_PROPERTY(int, enable_valid_range) ///@} - /** @anchor verbose - * @name verbose - * if set to a non-zero value, rank 0 will send status information to the - * terminal. The default setting of zero results in minimal output. - */ - ///@{ - TECA_ALGORITHM_PROPERTY(int, verbose) - ///@} - protected: teca_valid_value_mask(); @@ -94,7 +83,6 @@ class teca_valid_value_mask : public teca_algorithm private: std::vector mask_arrays; int enable_valid_range; - int verbose; }; #endif diff --git a/alg/teca_variant_array_operand.h b/alg/teca_variant_array_operand.h index b74b030d3..785a61f64 100644 --- a/alg/teca_variant_array_operand.h +++ b/alg/teca_variant_array_operand.h @@ -1,36 +1,43 @@ #ifndef teca_variant_array_operand_h #define teca_variant_array_operand_h +/// @file + #include "teca_variant_array.h" #include "teca_array_collection.h" +/// Codes dealing with the operands of run time specified operations on teca_variant_array namespace teca_variant_array_operand { -// class that handles conversion of literals to varaint_arrays -// and name resolution of variables. +/** @brief + * Handles conversion of literals to variant_arrays and the name resolution of + * variables. + */ class resolver { public: - // given a text representation of a numeric value in s - // convert and return a variant_array filled with the - // numeric value. the type is determined by the last - // 1 or 2 characters in the string, Valid type codes - // are: - // d -- double - // f -- float - // L -- long long - // l -- long - // i -- int - // s -- short - // c -- char - // u -- unsigned, augments any of the integer types - // the return is non-zero if an error occured, zero - // otherwise. + /** given a text representation of a numeric value in s convert and + * return a variant_array filled with the numeric value. the type is + * determined by the last 1 or 2 characters in the string, Valid type + * codes are: + * + * d -- double + * f -- float + * L -- long long + * l -- long + * i -- int + * s -- short + * c -- char + * u -- unsigned, augments any of the integer types + * + * the return is non-zero if an error occurred, zero otherwise. + */ int get_constant(const char *s, p_teca_variant_array &c); - // given the name of a variable in var_name, set var to point - // to the array of coresponding name. see set/get_variables. + /** given the name of a variable in var_name, set var to point to the + * array of corresponding name. see set/get_variables. + */ int get_variable(const char *var_name, const_p_teca_variant_array &var) { @@ -39,11 +46,11 @@ class resolver return 0; } - // set/get the set of arrays used for variable name - // resolution. + /// get the set of arrays used for variable name resolution. const_p_teca_array_collection get_variables() { return m_variables; } + /// set the set of arrays used for variable name resolution. void set_variables(const_p_teca_array_collection v) { m_variables = v; } diff --git a/alg/teca_variant_array_operator.h b/alg/teca_variant_array_operator.h index e1c0decaf..f886af2d3 100644 --- a/alg/teca_variant_array_operator.h +++ b/alg/teca_variant_array_operator.h @@ -1,11 +1,15 @@ #ifndef teca_variant_array_operator #define teca_variant_array_operator +/// @file + #include "teca_variant_array.h" #include "teca_type_select.h" +/// Codes dealing with run time specified operations on teca_variant_arrays namespace teca_variant_array_operator { +/// @cond namespace internal { // -------------------------------------------------------------------------- @@ -64,6 +68,7 @@ p_teca_variant_array apply(unsigned long n, return out; } }; +/// @endcond // -------------------------------------------------------------------------- template diff --git a/alg/teca_vertical_coordinate_transform.cxx b/alg/teca_vertical_coordinate_transform.cxx index 6e9b25d72..6fba697a9 100644 --- a/alg/teca_vertical_coordinate_transform.cxx +++ b/alg/teca_vertical_coordinate_transform.cxx @@ -92,9 +92,11 @@ void teca_vertical_coordinate_transform::get_properties_description( opts.add_options() TECA_POPTS_GET(int, prefix, mode, - "transform mode (mode_wrf_v3)") + "Sets the coordinate transform mode. The modes are: mode_wrf_v3") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -102,6 +104,8 @@ void teca_vertical_coordinate_transform::get_properties_description( void teca_vertical_coordinate_transform::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, int, prefix, mode) } #endif diff --git a/alg/teca_vertical_coordinate_transform.h b/alg/teca_vertical_coordinate_transform.h index 1e3b27ba1..da5073bfb 100644 --- a/alg/teca_vertical_coordinate_transform.h +++ b/alg/teca_vertical_coordinate_transform.h @@ -10,10 +10,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_vertical_coordinate_transform) -/// an algorithm that transforms vertical cooridinates -/** -An algorithm that transforms vertical coordinates of a mesh. -*/ +/// An algorithm that transforms the vertical cooridinates of a mesh class teca_vertical_coordinate_transform : public teca_algorithm { public: diff --git a/alg/teca_vertical_reduction.cxx b/alg/teca_vertical_reduction.cxx index be806acdd..f823cfba7 100644 --- a/alg/teca_vertical_reduction.cxx +++ b/alg/teca_vertical_reduction.cxx @@ -35,9 +35,9 @@ void teca_vertical_reduction::get_properties_description( + (prefix.empty()?"teca_vertical_reduction":prefix)); opts.add_options() - TECA_POPTS_GET(std::vector, prefix, dependent_variables, + TECA_POPTS_MULTI_GET(std::vector, prefix, dependent_variables, "list of arrays needed to compute the derived quantity") - TECA_POPTS_GET(std::vector, prefix, derived_variables, + TECA_POPTS_MULTI_GET(std::vector, prefix, derived_variables, "name of the derived quantity") ; diff --git a/alg/teca_vertical_reduction.h b/alg/teca_vertical_reduction.h index a47362408..bbeb72487 100644 --- a/alg/teca_vertical_reduction.h +++ b/alg/teca_vertical_reduction.h @@ -11,11 +11,11 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_vertical_reduction) -/// base class for vertical reducitons +/// The base class for vertical reducitons. /** -implements common operations associated with computing a vertical -reduction where a 3D dataset is transformed into a 2D dataset -by a reduction along the 3rd spatial dimension. + * implements common operations associated with computing a vertical + * reduction where a 3D dataset is transformed into a 2D dataset + * by a reduction along the 3rd spatial dimension. */ class teca_vertical_reduction : public teca_algorithm { @@ -30,16 +30,27 @@ class teca_vertical_reduction : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set/get the list of variables that are needed to produce - // the derived quantity + /** @name dependent_variable + * set/get the list of variables that are needed to produce the derived + * quantity + */ + ///@{ TECA_ALGORITHM_VECTOR_PROPERTY(std::string, dependent_variable) + ///@} - // set/get the name of the variable that is produced + /** @name derived_variable + * set/get the name of the variable that is produced + */ + ///@{ TECA_ALGORITHM_VECTOR_PROPERTY(std::string, derived_variable) + ///@} - // set/get the attributes of the variable that is produced - TECA_ALGORITHM_VECTOR_PROPERTY(teca_array_attributes, - derived_variable_attribute) + /** @name derived_variable_attribute + * Set the attributes of the variable that is produced. + */ + ///@{ + TECA_ALGORITHM_VECTOR_PROPERTY(teca_array_attributes, derived_variable_attribute) + ///@} protected: teca_vertical_reduction(); diff --git a/alg/teca_vorticity.cxx b/alg/teca_vorticity.cxx index 66fcf21d7..b6635686e 100644 --- a/alg/teca_vorticity.cxx +++ b/alg/teca_vorticity.cxx @@ -161,6 +161,8 @@ void teca_vorticity::get_properties_description( "array to store the computed vorticity in") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -168,6 +170,8 @@ void teca_vorticity::get_properties_description( void teca_vorticity::set_properties( const string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, std::string, prefix, component_0_variable) TECA_POPTS_SET(opts, std::string, prefix, component_1_variable) TECA_POPTS_SET(opts, std::string, prefix, vorticity_variable) diff --git a/alg/teca_vorticity.h b/alg/teca_vorticity.h index 7ac3226f6..b42cd0d6c 100644 --- a/alg/teca_vorticity.h +++ b/alg/teca_vorticity.h @@ -10,10 +10,7 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_vorticity) -/// an algorithm that computes vorticity -/** -Compute vorticity from a vector field. -*/ +/// An algorithm that computes vorticity from a vector field. class teca_vorticity : public teca_algorithm { public: @@ -27,14 +24,29 @@ class teca_vorticity : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set the arrays that contain the vector components - // to compute vorticity from + /** @name component_0_variable + * set the arrays that contain the vector components to compute vorticity + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_0_variable) + ///@} + + /** @name component_1_variable + * set the arrays that contain the vector components to compute vorticity + * from. + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, component_1_variable) + ///@} - // set the name of the array to store the result in. - // the default is "vorticity" + /** @name vorticity_variable + * set the name of the array to store the result in. the default is + * "vorticity" + */ + ///@{ TECA_ALGORITHM_PROPERTY(std::string, vorticity_variable) + ///@} protected: teca_vorticity(); diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 2d8a1be85..67b18de57 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -22,6 +22,9 @@ teca_add_app(teca_bayesian_ar_detect LIBS ${teca_app_link} teca_add_app(teca_integrated_vapor_transport LIBS ${teca_app_link} FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) +teca_add_app(teca_integrated_water_vapor LIBS ${teca_app_link} + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) + teca_add_app(teca_tc_detect LIBS ${teca_app_link} FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) diff --git a/apps/teca_app_util.h b/apps/teca_app_util.h index 9269c9896..5b4f4967e 100644 --- a/apps/teca_app_util.h +++ b/apps/teca_app_util.h @@ -1,25 +1,30 @@ #ifndef teca_app_util_h #define teca_app_util_h +/// @file + #include "teca_config.h" #include #include +/// Codes shared among the command line applications namespace teca_app_util { -// check for flag and if found print the help message -// and the option definitions. return non-zero if the flag -// was found. +/** Check for flag and if found print the help message + * and the option definitions. return non-zero if the flag + * was found. + */ int process_command_line_help(int rank, const std::string &flag, boost::program_options::options_description &opt_defs, boost::program_options::variables_map &opt_vals); -// parses the command line options and checks for --help, --advanced_help, and -// --full_help flags. if any are found prints the associated option -// defintions. if any of the help flags were found 1 is returned. If there is -// an error -1 is returned. Otherwise 0 is returned. +/** parses the command line options and checks for --help, --advanced_help, and + * --full_help flags. if any are found prints the associated option + * defintions. if any of the help flags were found 1 is returned. If there is + * an error -1 is returned. Otherwise 0 is returned. + */ int process_command_line_help(int rank, int argc, char **argv, boost::program_options::options_description &basic_opt_defs, boost::program_options::options_description &advanced_opt_defs, diff --git a/apps/teca_bayesian_ar_detect.cpp b/apps/teca_bayesian_ar_detect.cpp index dedfb99bc..9d11971ce 100644 --- a/apps/teca_bayesian_ar_detect.cpp +++ b/apps/teca_bayesian_ar_detect.cpp @@ -8,15 +8,21 @@ #include "teca_bayesian_ar_detect_parameters.h" #include "teca_binary_segmentation.h" #include "teca_l2_norm.h" +#include "teca_apply_binary_mask.h" #include "teca_multi_cf_reader.h" #include "teca_integrated_vapor_transport.h" #include "teca_valid_value_mask.h" +#include "teca_cartesian_mesh_source.h" +#include "teca_cartesian_mesh_regrid.h" +#include "teca_indexed_dataset_cache.h" +#include "teca_elevation_mask.h" +#include "teca_unpack_data.h" #include "teca_mpi_manager.h" #include "teca_coordinate_util.h" #include "teca_table.h" #include "teca_dataset_source.h" #include "teca_app_util.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" #include #include @@ -82,6 +88,22 @@ int main(int argc, char **argv) ("write_ivt", "\nwhen this flag is present IVT vector is written to disk with" " the result\n") + ("dem", value(), "\nA teca_cf_reader regex identifying the" + " file containing surface elevation field or DEM.\n") + ("dem_variable", value()->default_value("Z"), + "\nSets the name of the variable containing the surface elevation field\n") + ("mesh_height", value()->default_value("Zg"), + "\nSets the name of the variable containing the point wise vertical height" + " in meters above mean sea level\n") + + ("ar_probability", value()->default_value("ar_probability"), + "\nSets the name of the variable to store the computed AR probability mask in.\n") + + ("ar_weighted_variables", value>()->multitoken(), + "\nAn optional list of variables to weight with the computed AR probability." + " Each such variable will be multiplied by the computed AR probability, and" + " written to disk as \"NAME_ar_wgtd\".\n") + ("x_axis_variable", value()->default_value("lon"), "\nname of x coordinate variable\n") ("y_axis_variable", value()->default_value("lat"), @@ -92,16 +114,29 @@ int main(int argc, char **argv) ("periodic_in_x", value()->default_value(1), "\nFlags whether the x dimension (typically longitude) is periodic.\n") - ("binary_ar_threshold", value()->default_value(2.0/3.0,"0.667"), - "\nprobability threshold for segmenting ar_probability to produce ar_binary_tag\n") - - ("output_file", value()->default_value(std::string("CASCADE_BARD_%t%.nc")), + ("segment_ar_probability", "\nA flag that enables a binary segmentation of AR" + " probability to be produced. --segment_threshold controls the segmentation." + " threshold and --segment_variable to set the name of the variable to store" + " the result in.\n") + ("segment_threshold", value()->default_value(2.0/3.0,"0.667"), + "\nSets the threshold value that is used when segmenting ar_probability." + " See also --segment_ar_probability\n") + ("segment_variable", value()->default_value("ar_binary_tag"), + "\nSet the name of the variable to store the result of a binary" + " segmentation of AR probabilty. See also --segment_ar_probability.") + + ("output_file", value()->default_value(std::string("TECA_BARD_%t%.nc")), "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a" " human readable date and time corresponding to the time of the first time step in" " the file. Use --cf_writer::date_format to change the formatting\n") - + ("file_layout", value()->default_value("monthly"), + "\nSelects the size and layout of the set of output files. May be one of" + " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured" + " such that each file contains one of the selected interval. For the number_of_steps" + " option use --steps_per_file.\n") ("steps_per_file", value()->default_value(128), - "\nnumber of time steps per output file\n") + "\nThe number of time steps per output file when --file_layout number_of_steps is" + " specified.\n") ("first_step", value()->default_value(0), "\nfirst time step to process\n") ("last_step", value()->default_value(-1), "\nlast time step to process\n") @@ -142,6 +177,38 @@ int main(int argc, char **argv) p_teca_multi_cf_reader mcf_reader = teca_multi_cf_reader::New(); mcf_reader->get_properties_description("mcf_reader", advanced_opt_defs); + p_teca_valid_value_mask vv_mask = teca_valid_value_mask::New(); + vv_mask->get_properties_description("vv_mask", advanced_opt_defs); + + p_teca_unpack_data unpack = teca_unpack_data::New(); + unpack->get_properties_description("unpack", advanced_opt_defs); + + p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); + norm_coords->get_properties_description("norm_coords", advanced_opt_defs); + + p_teca_cf_reader elev_reader = teca_cf_reader::New(); + elev_reader->get_properties_description("elev_reader", advanced_opt_defs); + elev_reader->set_t_axis_variable(""); + + p_teca_normalize_coordinates elev_coords = teca_normalize_coordinates::New(); + elev_coords->get_properties_description("elev_coords", advanced_opt_defs); + elev_coords->set_enable_periodic_shift_x(1); + + p_teca_indexed_dataset_cache elev_cache = teca_indexed_dataset_cache::New(); + elev_cache->get_properties_description("elev_cache", advanced_opt_defs); + elev_cache->set_max_cache_size(1); + + p_teca_cartesian_mesh_source elev_mesh = teca_cartesian_mesh_source::New(); + elev_mesh->get_properties_description("elev_mesh", advanced_opt_defs); + + p_teca_cartesian_mesh_regrid elev_regrid = teca_cartesian_mesh_regrid::New(); + elev_regrid->get_properties_description("elev_regrid", advanced_opt_defs); + + p_teca_elevation_mask elev_mask = teca_elevation_mask::New(); + elev_mask->get_properties_description("elev_mask", advanced_opt_defs); + elev_mask->set_surface_elevation_variable("Z"); + elev_mask->set_mesh_height_variable("ZG"); + p_teca_l2_norm l2_norm = teca_l2_norm::New(); l2_norm->get_properties_description("ivt_magnitude", advanced_opt_defs); l2_norm->set_component_0_variable("IVT_U"); @@ -156,12 +223,6 @@ int main(int argc, char **argv) ivt_int->set_ivt_u_variable("IVT_U"); ivt_int->set_ivt_v_variable("IVT_V"); - p_teca_valid_value_mask vv_mask = teca_valid_value_mask::New(); - vv_mask->get_properties_description("vv_mask", advanced_opt_defs); - - p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); - norm_coords->get_properties_description("norm_coords", advanced_opt_defs); - // parameter source p_teca_bayesian_ar_detect_parameters params = teca_bayesian_ar_detect_parameters::New(); @@ -172,7 +233,7 @@ int main(int argc, char **argv) p_teca_bayesian_ar_detect ar_detect = teca_bayesian_ar_detect::New(); ar_detect->get_properties_description("ar_detect", advanced_opt_defs); ar_detect->set_ivt_variable("IVT"); - + ar_detect->set_ar_probability_variable("ar_probability"); // segment the ar probability field p_teca_binary_segmentation ar_tag = teca_binary_segmentation::New(); @@ -180,6 +241,12 @@ int main(int argc, char **argv) ar_tag->set_threshold_variable("ar_probability"); ar_tag->set_segmentation_variable("ar_binary_tag"); + // mask any requested variables by the AR probability + p_teca_apply_binary_mask ar_mask = teca_apply_binary_mask::New(); + ar_mask->get_properties_description("ar_mask", advanced_opt_defs); + ar_mask->set_mask_variable("ar_probability"); + ar_mask->set_output_variable_prefix("ar_wgtd_"); + // Add an executive for the writer p_teca_index_executive exec = teca_index_executive::New(); @@ -189,16 +256,21 @@ int main(int argc, char **argv) cf_writer->set_verbose(0); cf_writer->set_thread_pool_size(1); cf_writer->set_steps_per_file(128); + cf_writer->set_layout(teca_cf_writer::monthly); // package basic and advanced options for display options_description all_opt_defs(help_width, help_width - 4); all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -207,34 +279,60 @@ int main(int argc, char **argv) // options will override them cf_reader->set_properties("cf_reader", opt_vals); mcf_reader->set_properties("mcf_reader", opt_vals); - l2_norm->set_properties("ivt_magnitude", opt_vals); - ivt_int->set_properties("ivt_integral", opt_vals); vv_mask->set_properties("vv_mask", opt_vals); norm_coords->set_properties("norm_coords", opt_vals); + unpack->set_properties("unpack", opt_vals); + elev_reader->set_properties("elev_reader", opt_vals); + elev_coords->set_properties("elev_coords", opt_vals); + elev_mesh->set_properties("elev_mesh", opt_vals); + elev_cache->set_properties("elev_cache", opt_vals); + elev_regrid->set_properties("elev_regrid", opt_vals); + elev_mask->set_properties("elev_mask", opt_vals); + l2_norm->set_properties("ivt_magnitude", opt_vals); + ivt_int->set_properties("ivt_integral", opt_vals); params->set_properties("parameter_table", opt_vals); ar_detect->set_properties("ar_detect", opt_vals); + ar_mask->set_properties("ar_mask", opt_vals); cf_writer->set_properties("cf_writer", opt_vals); // now pass in the basic options, these are processed // last so that they will take precedence // configure the pipeline from the command line options. - p_teca_algorithm head; // configure the reader bool have_file = opt_vals.count("input_file"); bool have_regex = opt_vals.count("input_regex"); + if ((have_file && have_regex) || !(have_file || have_regex)) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " + "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " + "and --input_regex to activate the cf_reader (CAM like datasets)") + } + return -1; + } + + + p_teca_algorithm reader; if (have_file) { mcf_reader->set_input_file(opt_vals["input_file"].as()); - head = mcf_reader; + reader = mcf_reader; } else if (have_regex) { cf_reader->set_files_regex(opt_vals["input_regex"].as()); - head = cf_reader; + reader = cf_reader; } - p_teca_algorithm reader = head; + + // add transformation stages to the pipeline + norm_coords->set_input_connection(reader->get_output_port()); + vv_mask->set_input_connection(norm_coords->get_output_port()); + unpack->set_input_connection(vv_mask->get_output_port()); + + p_teca_algorithm head = unpack; if (!opt_vals["periodic_in_x"].defaulted()) { @@ -290,10 +388,21 @@ int main(int argc, char **argv) ar_detect->set_ivt_variable(opt_vals["ivt"].as()); } - // add the ivt caluation stages if needed + // add the ivt calculation stages if needed bool do_ivt = opt_vals.count("compute_ivt"); bool do_ivt_magnitude = opt_vals.count("compute_ivt_magnitude"); + if (do_ivt && do_ivt_magnitude) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("Only one of --compute_ivt and compute_ivt_magnitude can " + "be specified. --compute_ivt implies --compute_ivt_magnitude") + } + return -1; + } + + teca_metadata md; if (do_ivt) { std::string z_var = "plev"; @@ -303,8 +412,51 @@ int main(int argc, char **argv) cf_reader->set_z_axis_variable(z_var); mcf_reader->set_z_axis_variable(z_var); - vv_mask->set_input_connection(head->get_output_port()); - ivt_int->set_input_connection(vv_mask->get_output_port()); + // add the elevation mask stages + if (opt_vals.count("dem")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Generating elevation mask") + + elev_reader->set_files_regex(opt_vals["dem"].as()); + + elev_coords->set_input_connection(elev_reader->get_output_port()); + + md = head->update_metadata(); + + elev_mesh->set_spatial_bounds(md, false); + elev_mesh->set_spatial_extents(md, false); + elev_mesh->set_x_axis_variable(md); + elev_mesh->set_y_axis_variable(md); + elev_mesh->set_z_axis_variable(md); + elev_mesh->set_t_axis_variable(md); + elev_mesh->set_t_axis(md); + + elev_regrid->set_input_connection(0, elev_mesh->get_output_port()); + elev_regrid->set_input_connection(1, elev_coords->get_output_port()); + + elev_cache->set_input_connection(elev_regrid->get_output_port()); + + elev_mask->set_input_connection(0, head->get_output_port()); + elev_mask->set_input_connection(1, elev_cache->get_output_port()); + + if (!opt_vals["dem_variable"].defaulted()) + elev_mask->set_surface_elevation_variable( + opt_vals["dem_variable"].as()); + + if (!opt_vals["mesh_height"].defaulted()) + elev_mask->set_mesh_height_variable( + opt_vals["mesh_height"].as()); + + elev_mask->set_mask_variables({ + ivt_int->get_specific_humidity_variable() + "_valid", + ivt_int->get_wind_u_variable() + "_valid", + ivt_int->get_wind_v_variable() + "_valid"}); + + head = elev_mask; + } + + ivt_int->set_input_connection(head->get_output_port()); l2_norm->set_input_connection(ivt_int->get_output_port()); head = l2_norm; @@ -315,8 +467,71 @@ int main(int argc, char **argv) head = l2_norm; } + // connect the detector and post detector operations + ar_detect->set_input_connection(0, params->get_output_port()); + ar_detect->set_input_connection(1, head->get_output_port()); + + if (!opt_vals["ar_probability"].defaulted()) + ar_detect->set_ar_probability_variable( + opt_vals["ar_probability"].as()); + + head = ar_detect; + + // configure binary segmentation + bool do_segment = opt_vals.count("segment_ar_probability"); + if (do_segment) + { + ar_tag->set_input_connection(0, ar_detect->get_output_port()); + + // set input and output variable names + ar_tag->set_threshold_variable( + ar_detect->get_ar_probability_variable()); + + if (!opt_vals["segement_variable"].defaulted()) + ar_tag->set_segmentation_variable( + opt_vals["segment_variable"].as()); + + // set threshold value + double ar_tag_threshold = opt_vals["segment_threshold"].as(); + ar_tag->set_low_threshold_value(ar_tag_threshold); + + // add I/O metadata + teca_metadata seg_atts; + seg_atts.set("long_name", std::string("binary indicator of atmospheric river")); + seg_atts.set("description", std::string("binary indicator of atmospheric river")); + seg_atts.set("scheme", std::string("TECA_BARD")); + seg_atts.set("version", std::string("1.0")); + seg_atts.set("note", + std::string("derived by thresholding ar_probability >= ") + + std::to_string(ar_tag_threshold)); + + ar_tag->set_segmentation_variable_attributes(seg_atts); + + head = ar_tag; + } + + // configure weight by AR probability + std::vector weighted_vars; + int do_weighted = opt_vals.count("ar_weighted_variables"); + if (opt_vals.count("ar_weighted_variables")) + { + weighted_vars = + opt_vals["ar_weighted_variables"].as>(); + + ar_mask->set_input_connection(0, head->get_output_port()); + ar_mask->set_masked_variables(weighted_vars); + ar_mask->set_mask_variable(ar_detect->get_ar_probability_variable()); + + head = ar_mask; + } + + // connect and configure the writer + cf_writer->set_input_connection(head->get_output_port()); + // tell the writer to write ivt if needed - std::vector point_arrays({"ar_probability", "ar_binary_tag"}); + std::vector point_arrays( + {ar_detect->get_ar_probability_variable()}); + if ((do_ivt || do_ivt_magnitude) && opt_vals.count("write_ivt_magnitude")) { point_arrays.push_back(l2_norm->get_l2_norm_variable()); @@ -328,12 +543,23 @@ int main(int argc, char **argv) point_arrays.push_back(ivt_int->get_ivt_v_variable()); } - cf_writer->set_information_arrays({"ar_count", "parameter_table_row"}); - cf_writer->set_point_arrays(point_arrays); + if (do_segment) + { + point_arrays.push_back(ar_tag->get_segmentation_variable()); + } + + // tell the writer to write ar weighted variables if needed + if (do_weighted) + { + point_arrays.insert(point_arrays.end(), + weighted_vars.begin(), weighted_vars.end()); + ar_mask->get_output_variable_names(point_arrays); + } - if (!opt_vals["output_file"].defaulted()) - cf_writer->set_file_name(opt_vals["output_file"].as()); + cf_writer->set_file_name(opt_vals["output_file"].as()); + cf_writer->set_information_arrays({"ar_count", "parameter_table_row"}); + cf_writer->set_point_arrays(point_arrays); if (!opt_vals["steps_per_file"].defaulted()) cf_writer->set_steps_per_file(opt_vals["steps_per_file"].as()); @@ -344,6 +570,14 @@ int main(int argc, char **argv) if (!opt_vals["last_step"].defaulted()) cf_writer->set_last_step(opt_vals["last_step"].as()); + if (!opt_vals["file_layout"].defaulted() && + cf_writer->set_layout(opt_vals["file_layout"].as())) + { + TECA_ERROR("An invalid file layout was provided \"" + << opt_vals["file_layout"].as() << "\"") + return -1; + } + if (opt_vals.count("verbose")) { ar_detect->set_verbose(1); @@ -356,29 +590,6 @@ int main(int argc, char **argv) else ar_detect->set_thread_pool_size(-1); - - // some minimal check for missing options - if ((have_file && have_regex) || !(have_file || have_regex)) - { - if (mpi_man.get_comm_rank() == 0) - { - TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " - "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " - "and --input_regex to activate the cf_reader (CAM like datasets)") - } - return -1; - } - - if (do_ivt && do_ivt_magnitude) - { - if (mpi_man.get_comm_rank() == 0) - { - TECA_ERROR("Only one of --compute_ivt and compute_ivt_magnitude can " - "be specified. --compute_ivt implies --compute_ivt_magnitude") - } - return -1; - } - if (cf_writer->get_file_name().empty()) { if (mpi_man.get_comm_rank() == 0) @@ -389,20 +600,14 @@ int main(int argc, char **argv) return -1; } - // connect the fixed stages of the pipeline - norm_coords->set_input_connection(head->get_output_port()); - ar_detect->set_input_connection(0, params->get_output_port()); - ar_detect->set_input_connection(1, norm_coords->get_output_port()); - ar_tag->set_input_connection(0, ar_detect->get_output_port()); - cf_writer->set_input_connection(ar_tag->get_output_port()); - // look for requested time step range, start bool parse_start_date = opt_vals.count("start_date"); bool parse_end_date = opt_vals.count("end_date"); if (parse_start_date || parse_end_date) { // run the reporting phase of the pipeline - teca_metadata md = reader->update_metadata(); + if (md.empty()) + md = reader->update_metadata(); teca_metadata atrs; if (md.get("attributes", atrs)) @@ -423,10 +628,8 @@ int main(int argc, char **argv) } teca_metadata coords; - p_teca_double_array time; - if (md.get("coordinates", coords) || - !(time = std::dynamic_pointer_cast( - coords.get("t")))) + p_teca_variant_array time; + if (md.get("coordinates", coords) || !(time = coords.get("t"))) { TECA_ERROR("failed to determine time coordinate") return -1; @@ -463,20 +666,6 @@ int main(int argc, char **argv) } } - // set the threshold for calculating ar_binary_tag - double ar_tag_threshold = opt_vals["binary_ar_threshold"].as(); - ar_tag->set_low_threshold_value(ar_tag_threshold); - - // add metadata for ar_binary_tag - teca_metadata seg_atts; - seg_atts.set("long_name", std::string("binary indicator of atmospheric river")); - seg_atts.set("description", std::string("binary indicator of atmospheric river")); - seg_atts.set("scheme", std::string("cascade_bard")); - seg_atts.set("version", std::string("1.0")); - seg_atts.set("note", - std::string("derived by thresholding ar_probability >= ") + - std::to_string(ar_tag_threshold)); - ar_tag->set_segmentation_variable_attributes(seg_atts); // run the pipeline cf_writer->set_executive(exec); diff --git a/apps/teca_cartesian_mesh_diff.cpp b/apps/teca_cartesian_mesh_diff.cpp index 26be1d2e2..d950b814e 100644 --- a/apps/teca_cartesian_mesh_diff.cpp +++ b/apps/teca_cartesian_mesh_diff.cpp @@ -66,9 +66,10 @@ int main(int argc, char **argv) // initialize command line options description // set up some common options to simplify use for most // common scenarios + int help_width = 100; options_description basic_opt_defs( "teca_cartesian_mesh_diff an application that compares two datasets.\n\n" - "Command line options", 120, -1 + "Command line options", help_width, help_width - 4 ); basic_opt_defs.add_options() ("reference_dataset", value()->required(), @@ -105,7 +106,7 @@ int main(int argc, char **argv) "control over all runtime modifiable parameters. The basic options\n" "(see" "--help) map to these, and will override them if both are\n" "specified.\n\n" - "Advanced command line options", -1, 1 + "Advanced command line options", help_width, help_width - 4 ); // create the pipeline stages here, they contain the @@ -125,7 +126,7 @@ int main(int argc, char **argv) } // package basic and advanced options for display - options_description all_opt_defs(-1, -1); + options_description all_opt_defs(help_width, help_width - 4); all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line @@ -149,7 +150,7 @@ int main(int argc, char **argv) << std::endl << basic_opt_defs << std::endl << std::endl; - return -1; + return 0; } if (opt_vals.count("advanced_help")) @@ -159,7 +160,7 @@ int main(int argc, char **argv) << std::endl << advanced_opt_defs << std::endl << std::endl; - return -1; + return 0; } if (opt_vals.count("full_help")) @@ -169,7 +170,7 @@ int main(int argc, char **argv) << std::endl << all_opt_defs << std::endl << std::endl; - return -1; + return 0; } } diff --git a/apps/teca_cf_restripe.cpp b/apps/teca_cf_restripe.cpp index bbf7584bd..ae2496a9e 100644 --- a/apps/teca_cf_restripe.cpp +++ b/apps/teca_cf_restripe.cpp @@ -4,6 +4,10 @@ #include "teca_variant_array.h" #include "teca_cf_reader.h" #include "teca_multi_cf_reader.h" +#include "teca_normalize_coordinates.h" +#include "teca_cartesian_mesh_regrid.h" +#include "teca_cartesian_mesh_source.h" +#include "teca_rename_variables.h" #include "teca_cf_writer.h" #include "teca_dataset_diff.h" #include "teca_index_executive.h" @@ -13,7 +17,7 @@ #include "teca_mpi_manager.h" #include "teca_mpi.h" #include "teca_app_util.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" #include #include @@ -56,19 +60,37 @@ int main(int argc, char **argv) ("point_arrays", value>()->multitoken(), "\nA list of point centered arrays to write\n") - ("information_arrays", value>()->multitoken(), "\nA list of non-geometric arrays to write\n") - - ("output_file", value()->default_value(std::string("IVT_%t%.nc")), + ("output_file", value(), "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a" " human readable date and time corresponding to the time of the first time step in" " the file. Use --cf_writer::date_format to change the formatting\n") - - ("steps_per_file", value(), "\nnumber of time steps per output file\n") - + ("file_layout", value()->default_value("monthly"), + "\nSelects the size and layout of the set of output files. May be one of" + " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured" + " such that each file contains one of the selected interval. For the number_of_steps" + " option use --steps_per_file.\n") + ("steps_per_file", value(), "\nThe number of time steps per output file when " + " --file_layout number_of_steps is specified.\n") + + ("normalize_coordinates", "\nEnable coordinate normalization pipeline stage\n") + + ("regrid", "\nEnable mesh regridding pipeline stage. When enabled requires --dims" + " to be provided\n") + ("dims", value>()->multitoken(), + "\nA 3-tuple of values specifying the mesh size of the output dataset in the x, y," + " and z dimensions. The accepted format for dimensions is: nx ny nz\n") ("bounds", value>()->multitoken(), - "\nlat lon lev bounding box to subset with\n") + "\nA hex-tuple of low and high values specifying lon lat lev bounding box to subset" + " the input dataset with. The accepted format for bounds is: x0 x1 y0 y1 z0 z1\n") + + ("rename", "\nEnable variable renaming stage\n") + ("original_name", value>()->multitoken(), + "\nA list of variables to rename. Use --new_name to set the new names\n") + ("new_name", value>()->multitoken(), + "\nThe new names to use when renaming variables. Use --original_name to set the" + " list of variables to rename\n") ("first_step", value(), "\nfirst time step to process\n") ("last_step", value(), "\nlast time step to process\n") @@ -97,15 +119,29 @@ int main(int argc, char **argv) // documentation and parse command line. // objects report all of their properties directly // set default options here so that command line options override - // them. while we are at it connect the pipeline + // them. p_teca_cf_reader cf_reader = teca_cf_reader::New(); cf_reader->get_properties_description("cf_reader", advanced_opt_defs); p_teca_multi_cf_reader mcf_reader = teca_multi_cf_reader::New(); mcf_reader->get_properties_description("mcf_reader", advanced_opt_defs); + p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); + norm_coords->get_properties_description("norm_coords", advanced_opt_defs); + + p_teca_cartesian_mesh_regrid regrid = teca_cartesian_mesh_regrid::New(); + regrid->set_interpolation_mode_linear(); + regrid->get_properties_description("regrid", advanced_opt_defs); + + p_teca_cartesian_mesh_source regrid_src = teca_cartesian_mesh_source::New(); + regrid_src->get_properties_description("regrid_source", advanced_opt_defs); + + p_teca_rename_variables rename = teca_rename_variables::New(); + rename->get_properties_description("rename", advanced_opt_defs); + p_teca_cf_writer cf_writer = teca_cf_writer::New(); cf_writer->get_properties_description("cf_writer", advanced_opt_defs); + cf_writer->set_layout(teca_cf_writer::monthly); // Add an executive for the writer p_teca_index_executive exec = teca_index_executive::New(); @@ -115,10 +151,14 @@ int main(int argc, char **argv) all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -127,6 +167,10 @@ int main(int argc, char **argv) // options will override them cf_reader->set_properties("cf_reader", opt_vals); mcf_reader->set_properties("mcf_reader", opt_vals); + norm_coords->set_properties("norm_coords", opt_vals); + regrid->set_properties("regrid", opt_vals); + regrid_src->set_properties("regrid_source", opt_vals); + rename->set_properties("rename", opt_vals); cf_writer->set_properties("cf_writer", opt_vals); // now pass in the basic options, these are processed @@ -175,6 +219,14 @@ int main(int argc, char **argv) cf_writer->set_information_arrays( opt_vals["information_arrays"].as>()); + if (!opt_vals["file_layout"].defaulted() && + cf_writer->set_layout(opt_vals["file_layout"].as())) + { + TECA_ERROR("An invalid file layout was provided \"" + << opt_vals["file_layout"].as() << "\"") + return -1; + } + if (opt_vals.count("steps_per_file")) cf_writer->set_steps_per_file( opt_vals["steps_per_file"].as()); @@ -185,9 +237,48 @@ int main(int argc, char **argv) if (opt_vals.count("last_step")) cf_writer->set_last_step(opt_vals["last_step"].as()); - if (opt_vals.count("bounds")) - exec->set_bounds( - opt_vals["bounds"].as>()); + std::vector bounds; + bool have_bounds = opt_vals.count("bounds"); + if (have_bounds) + { + bounds = opt_vals["bounds"].as>(); + if (bounds.size() != 6) + { + TECA_ERROR("An invlaid bounds specification was provided in" + " --bounds, size != 6. Use: --bounds x0 x1 y0 y1 z0 z1") + return -1; + } + } + + bool do_regrid = opt_vals.count("regrid"); + + // when not regriding let the executive subset. when regriding + // the regrid algorithm handles subsetting and the executive should + // request the entire domain. + if (have_bounds && !do_regrid) + exec->set_bounds(bounds); + + // when regriding target mesh dimensions must be provided + std::vector dims; + if (do_regrid) + { + if (opt_vals.count("dims")) + { + dims = opt_vals["dims"].as>(); + if (dims.size() != 3) + { + TECA_ERROR("An invlaid dimension specification was provided in" + " --dims, size != 3. Use: --dims nx ny nz") + return -1; + } + } + else + { + TECA_ERROR("The --regrid option requires that --dims" + " also be specified") + return -1; + } + } if (opt_vals.count("verbose")) { @@ -222,10 +313,24 @@ int main(int argc, char **argv) return -1; } + // add the normalize coordinates stage before accessing metadata + p_teca_algorithm head = reader; + if (opt_vals.count("normalize_coordinates")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Added cooridnate normalization stage"); + + norm_coords->set_input_connection(reader->get_output_port()); + head = norm_coords; + } + // if no point arrays were specified on the command line by default // write all point arrays teca_metadata md; teca_metadata atts; + teca_metadata time_atts; + std::string calendar; + std::string units; // TODO -- this will need some more work in the reader as currently // all arrays are marked as being point centered, but here we need // to identify only the arrays on the mesh. @@ -233,7 +338,7 @@ int main(int argc, char **argv) { // run the reporting phase of the pipeline if (md.empty()) - md = cf_reader->update_metadata(); + md = head->update_metadata(); // if array attributes are present, use them to locate the set of // point centered arrrays @@ -271,32 +376,30 @@ int main(int argc, char **argv) { // run the reporting phase of the pipeline if (md.empty()) - md = cf_reader->update_metadata(); + md = head->update_metadata(); if (atts.empty() && md.get("attributes", atts)) { - TECA_ERROR("metadata missing attributes") + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("metadata missing attributes") return -1; } - teca_metadata time_atts; - std::string calendar; - std::string units; if (atts.get("time", time_atts) || time_atts.get("calendar", calendar) || time_atts.get("units", units)) { - TECA_ERROR("failed to determine the calendaring parameters") + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("failed to determine the calendaring parameters") return -1; } teca_metadata coords; - p_teca_double_array time; - if (md.get("coordinates", coords) || - !(time = std::dynamic_pointer_cast( - coords.get("t")))) + p_teca_variant_array time; + if (md.get("coordinates", coords) || !(time = coords.get("t"))) { - TECA_ERROR("failed to determine time coordinate") + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("failed to determine time coordinate") return -1; } @@ -308,8 +411,9 @@ int main(int argc, char **argv) if (teca_coordinate_util::time_step_of(time, true, true, calendar, units, start_date, first_step)) { - TECA_ERROR("Failed to locate time step for start date \"" - << start_date << "\"") + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("Failed to locate time step for start date \"" + << start_date << "\"") return -1; } cf_writer->set_first_step(first_step); @@ -323,16 +427,114 @@ int main(int argc, char **argv) if (teca_coordinate_util::time_step_of(time, false, true, calendar, units, end_date, last_step)) { - TECA_ERROR("Failed to locate time step for end date \"" - << end_date << "\"") + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("Failed to locate time step for end date \"" + << end_date << "\"") return -1; } cf_writer->set_last_step(last_step); } } - // connect the pipeline - cf_writer->set_input_connection(reader->get_output_port()); + // set up regriding + if (do_regrid) + { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Added regrid stage"); + + // run the reporting phase of the pipeline, the resulting metadata + // can be used to automatically determine the calendaring parameters + // and spatial bounds + if (md.empty()) + md = head->update_metadata(); + + // use the calendar and time axis of the input dataset + if (regrid_src->set_t_axis_variable(md) || regrid_src->set_t_axis(md)) + { + if (mpi_man.get_comm_rank() == 0) + TECA_WARNING("Failed to determine the time axis, assuming a" + " single time step") + + p_teca_double_array t = teca_double_array::New(1); + regrid_src->set_t_axis_variable(""); + regrid_src->set_t_axis(t); + } + + // to construct the target mesh we need bounds. if no bounds are + // specified on the command line use those of the input dataset and + // error out if that fails + if (have_bounds) + { + // extend to include time + bounds.resize(8, 0.0); + regrid_src->set_bounds(bounds); + } + else + { + // try to determine the bounds from the input mesh metadata + if (regrid_src->set_spatial_bounds(md)) + { + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("Failed to determine target mesh bounds from the" + " input metadata. Use --bounds to specify them manually.") + return -1; + } + } + + // set the target mesh dimensions + regrid_src->set_whole_extents({0lu, dims[0] - 1lu, + 0lu, dims[1] - 1lu, 0lu, dims[2] - 1lu, 0lu, 0lu}); + + // connect to the pipeline + regrid->set_input_connection(0, regrid_src->get_output_port()); + regrid->set_input_connection(1, head->get_output_port()); + head = regrid; + } + + // add rename stage + if (opt_vals.count("rename")) + { + if (!opt_vals.count("original_name")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("--original_name is required when renaming variables") + return -1; + } + + std::vector original_name = + opt_vals["original_name"].as>(); + + if (!opt_vals.count("new_name")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("--new_name is required when renaming variables") + return -1; + } + + std::vector new_name = + opt_vals["new_name"].as>(); + + if (original_name.size() != new_name.size()) + { + if (mpi_man.get_comm_rank() == 0) + TECA_ERROR("--original_name and --new_name must have the same" + " number of values") + return -1; + + } + + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Added rename stage"); + + rename->set_input_connection(head->get_output_port()); + rename->set_original_variable_names(original_name); + rename->set_new_variable_names(new_name); + + head = rename; + } + + // add the writer last + cf_writer->set_input_connection(head->get_output_port()); // run the pipeline cf_writer->set_executive(exec); diff --git a/apps/teca_deeplab_ar_detect.in b/apps/teca_deeplab_ar_detect.in index 6d07f5c3d..e124de1e6 100755 --- a/apps/teca_deeplab_ar_detect.in +++ b/apps/teca_deeplab_ar_detect.in @@ -87,6 +87,14 @@ parser.add_argument('--output_file', type=str, required=True, ' time of the first time step in the file. Use --date_format to change' ' the formatting') +parser.add_argument('--file_layout', type=str, default='monthly', + help='Selects the size and layout of the set of output' + ' files. May be one of number_of_steps, daily,' + ' monthly, seasonal, or yearly. Files are structured' + ' such that each file contains one of the selected' + ' interval. For the number_of_steps option use' + ' --steps_per_file.') + parser.add_argument('--steps_per_file', type=int, required=False, default=128, help='number of time steps per output file') @@ -112,7 +120,7 @@ parser.add_argument('--pytorch_model', type=str, required=False, parser.add_argument('--t_axis_variable', type=str, required=False, help='time dimension name') -parser.add_argument('--t_calendar', type=str, required=False, +parser.add_argument('--calendar', type=str, required=False, help='time calendar') parser.add_argument('--t_units', type=str, required=False, @@ -165,8 +173,8 @@ reader.set_y_axis_variable(args.y_axis_variable) if args.t_axis_variable is not None: reader.set_t_axis_variable(args.t_axis_variable) -if args.t_calendar: - reader.set_t_calendar(args.t_calendar) +if args.calendar: + reader.set_calendar(args.calendar) if args.t_units: reader.set_t_units(args.t_units) @@ -241,6 +249,7 @@ writer.set_input_connection(ar_tag.get_output_port()) writer.set_executive(exe) writer.set_thread_pool_size(1) writer.set_file_name(args.output_file) +writer.set_layout(args.file_layout) writer.set_steps_per_file(args.steps_per_file) writer.set_first_step(args.first_step) writer.set_last_step(args.last_step) diff --git a/apps/teca_integrated_vapor_transport.cpp b/apps/teca_integrated_vapor_transport.cpp index d078f6b31..b8b0d89eb 100644 --- a/apps/teca_integrated_vapor_transport.cpp +++ b/apps/teca_integrated_vapor_transport.cpp @@ -10,18 +10,25 @@ #include "teca_multi_cf_reader.h" #include "teca_integrated_vapor_transport.h" #include "teca_valid_value_mask.h" +#include "teca_unpack_data.h" +#include "teca_cartesian_mesh_source.h" +#include "teca_cartesian_mesh_regrid.h" +#include "teca_elevation_mask.h" +#include "teca_indexed_dataset_cache.h" #include "teca_mpi_manager.h" #include "teca_coordinate_util.h" #include "teca_table.h" #include "teca_dataset_source.h" #include "teca_app_util.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" #include #include #include #include +#include "teca_cartesian_mesh_writer.h" + using namespace std; using boost::program_options::value; @@ -57,13 +64,11 @@ int main(int argc, char **argv) ("wind_u", value()->default_value("U"), "\nname of variable with the 3D longitudinal component of the wind vector.\n") - ("wind_v", value()->default_value("V"), "\nname of variable with the 3D latitudinal component of the wind vector.\n") ("ivt_u", value()->default_value("IVT_U"), "\nname to use for the longitudinal component of the integrated vapor transport vector.\n") - ("ivt_v", value()->default_value("IVT_V"), "\nname to use for the latitudinal component of the integrated vapor transport vector.\n") @@ -84,9 +89,14 @@ int main(int argc, char **argv) "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a" " human readable date and time corresponding to the time of the first time step in" " the file. Use --cf_writer::date_format to change the formatting\n") - + ("file_layout", value()->default_value("monthly"), + "\nSelects the size and layout of the set of output files. May be one of" + " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured" + " such that each file contains one of the selected interval. For the number_of_steps" + " option use --steps_per_file.\n") ("steps_per_file", value()->default_value(128), - "\nnumber of time steps per output file\n") + "\nThe number of time steps per output file when --file_layout number_of_steps is" + " specified.\n") ("x_axis_variable", value()->default_value("lon"), "\nname of x coordinate variable\n") @@ -95,6 +105,16 @@ int main(int argc, char **argv) ("z_axis_variable", value()->default_value("plev"), "\nname of z coordinate variable\n") + ("dem", value(), "\nA teca_cf_reader regex identifying the" + " file containing surface elevation field or DEM.\n") + + ("dem_variable", value()->default_value("Z"), + "\nSets the name of the variable containing the surface elevation field\n") + + ("mesh_height", value()->default_value("Zg"), + "\nSets the name of the variable containing the point wise vertical height" + " in meters above mean sea level\n") + ("first_step", value()->default_value(0), "\nfirst time step to process\n") ("last_step", value()->default_value(-1), "\nlast time step to process\n") @@ -102,9 +122,9 @@ int main(int argc, char **argv) " format. Note: There must be a space between the date and time specification\n") ("end_date", value(), "\nThe last time to process in 'Y-M-D h:m:s' format\n") - ("n_threads", value(), "\nSets the thread pool size on each MPI rank. When the default" - " value of -1 is used TECA will coordinate the thread pools across ranks such each" - " thread is bound to a unique physical core.\n") + ("n_threads", value()->default_value(-1), "\nSets the thread pool size on each MPI" + " rank. When the default value of -1 is used TECA will coordinate the thread pools" + " across ranks such each thread is bound to a unique physical core.\n") ("verbose", "\nenable extra terminal output\n") @@ -140,6 +160,38 @@ int main(int argc, char **argv) p_teca_multi_cf_reader mcf_reader = teca_multi_cf_reader::New(); mcf_reader->get_properties_description("mcf_reader", advanced_opt_defs); + p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); + norm_coords->get_properties_description("norm_coords", advanced_opt_defs); + + p_teca_valid_value_mask vv_mask = teca_valid_value_mask::New(); + vv_mask->get_properties_description("vv_mask", advanced_opt_defs); + + p_teca_unpack_data unpack = teca_unpack_data::New(); + unpack->get_properties_description("unpack", advanced_opt_defs); + + p_teca_cf_reader elev_reader = teca_cf_reader::New(); + elev_reader->get_properties_description("elev_reader", advanced_opt_defs); + elev_reader->set_t_axis_variable(""); + + p_teca_normalize_coordinates elev_coords = teca_normalize_coordinates::New(); + elev_coords->get_properties_description("elev_coords", advanced_opt_defs); + elev_coords->set_enable_periodic_shift_x(1); + + p_teca_indexed_dataset_cache elev_cache = teca_indexed_dataset_cache::New(); + elev_cache->get_properties_description("elev_cache", advanced_opt_defs); + elev_cache->set_max_cache_size(1); + + p_teca_cartesian_mesh_source elev_mesh = teca_cartesian_mesh_source::New(); + elev_mesh->get_properties_description("elev_mesh", advanced_opt_defs); + + p_teca_cartesian_mesh_regrid elev_regrid = teca_cartesian_mesh_regrid::New(); + elev_regrid->get_properties_description("elev_regrid", advanced_opt_defs); + + p_teca_elevation_mask elev_mask = teca_elevation_mask::New(); + elev_mask->get_properties_description("elev_mask", advanced_opt_defs); + elev_mask->set_surface_elevation_variable("Z"); + elev_mask->set_mesh_height_variable("ZG"); + p_teca_integrated_vapor_transport ivt_int = teca_integrated_vapor_transport::New(); ivt_int->get_properties_description("ivt_integral", advanced_opt_defs); ivt_int->set_specific_humidity_variable("Q"); @@ -154,9 +206,6 @@ int main(int argc, char **argv) l2_norm->set_component_1_variable("IVT_V"); l2_norm->set_l2_norm_variable("IVT"); - p_teca_valid_value_mask vv_mask = teca_valid_value_mask::New(); - vv_mask->get_properties_description("vv_mask", advanced_opt_defs); - // Add an executive for the writer p_teca_index_executive exec = teca_index_executive::New(); @@ -165,16 +214,21 @@ int main(int argc, char **argv) cf_writer->get_properties_description("cf_writer", advanced_opt_defs); cf_writer->set_verbose(0); cf_writer->set_steps_per_file(128); + cf_writer->set_layout(teca_cf_writer::monthly); // package basic and advanced options for display - options_description all_opt_defs(-1, -1); + options_description all_opt_defs(help_width, help_width - 4); all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -183,7 +237,15 @@ int main(int argc, char **argv) // options will override them cf_reader->set_properties("cf_reader", opt_vals); mcf_reader->set_properties("mcf_reader", opt_vals); + norm_coords->set_properties("norm_coords", opt_vals); vv_mask->set_properties("vv_mask", opt_vals); + unpack->set_properties("unpack", opt_vals); + elev_reader->set_properties("elev_reader", opt_vals); + elev_coords->set_properties("elev_coords", opt_vals); + elev_mesh->set_properties("elev_mesh", opt_vals); + elev_cache->set_properties("elev_cache", opt_vals); + elev_regrid->set_properties("elev_regrid", opt_vals); + elev_mask->set_properties("elev_mask", opt_vals); ivt_int->set_properties("ivt_integral", opt_vals); l2_norm->set_properties("ivt_magnitude", opt_vals); cf_writer->set_properties("cf_writer", opt_vals); @@ -191,23 +253,33 @@ int main(int argc, char **argv) // now pass in the basic options, these are processed // last so that they will take precedence // configure the pipeline from the command line options. - p_teca_algorithm head; + p_teca_algorithm reader; // configure the reader bool have_file = opt_vals.count("input_file"); bool have_regex = opt_vals.count("input_regex"); + if ((have_file && have_regex) || !(have_file || have_regex)) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " + "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " + "and --input_regex to activate the cf_reader (CAM like datasets)") + } + return -1; + } if (opt_vals.count("input_file")) { mcf_reader->set_input_file(opt_vals["input_file"].as()); - head = mcf_reader; + reader = mcf_reader; } else if (opt_vals.count("input_regex")) { cf_reader->set_files_regex(opt_vals["input_regex"].as()); - head = cf_reader; + reader = cf_reader; } - p_teca_algorithm reader = head; + p_teca_algorithm head = reader; if (!opt_vals["x_axis_variable"].defaulted()) { @@ -221,6 +293,13 @@ int main(int argc, char **argv) mcf_reader->set_y_axis_variable(opt_vals["y_axis_variable"].as()); } + std::string z_var = "plev"; + if (!opt_vals["z_axis_variable"].defaulted()) + z_var = opt_vals["z_axis_variable"].as(); + + cf_reader->set_z_axis_variable(z_var); + mcf_reader->set_z_axis_variable(z_var); + // set the inputs to the integrator if (!opt_vals["wind_u"].defaulted()) { @@ -257,24 +336,88 @@ int main(int argc, char **argv) } // add the valid value mask stage - vv_mask->set_input_connection(head->get_output_port()); - head = vv_mask; + norm_coords->set_input_connection(head->get_output_port()); + vv_mask->set_input_connection(norm_coords->get_output_port()); + unpack->set_input_connection(vv_mask->get_output_port()); + head = unpack; // add the ivt caluation stages if needed bool do_ivt = opt_vals["write_ivt"].as(); bool do_ivt_magnitude = opt_vals["write_ivt_magnitude"].as(); + if (!(do_ivt || do_ivt_magnitude)) - std::string z_var = "plev"; - if (!opt_vals["z_axis_variable"].defaulted()) - z_var = opt_vals["z_axis_variable"].as(); + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("At least one of --write_ivt or --write_ivt_magnitude " + " must be set.") + } + return -1; + } - cf_reader->set_z_axis_variable(z_var); - mcf_reader->set_z_axis_variable(z_var); + // add the elevation mask stages + teca_metadata md; + if (opt_vals.count("dem")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Generating elevation mask") + + elev_reader->set_files_regex(opt_vals["dem"].as()); + + elev_coords->set_input_connection(elev_reader->get_output_port()); + + md = head->update_metadata(); + + elev_mesh->set_spatial_bounds(md, false); + elev_mesh->set_spatial_extents(md, false); + elev_mesh->set_x_axis_variable(md); + elev_mesh->set_y_axis_variable(md); + elev_mesh->set_z_axis_variable(md); + elev_mesh->set_t_axis_variable(md); + elev_mesh->set_t_axis(md); + + elev_regrid->set_input_connection(0, elev_mesh->get_output_port()); + elev_regrid->set_input_connection(1, elev_coords->get_output_port()); + + elev_cache->set_input_connection(elev_regrid->get_output_port()); + + /*p_teca_cartesian_mesh_writer rdw = teca_cartesian_mesh_writer::New(); + rdw->set_input_connection(elev_cache->get_output_port()); + rdw->set_file_name("regrid_dem_%t%.vtk");*/ + + elev_mask->set_input_connection(0, head->get_output_port()); + elev_mask->set_input_connection(1, elev_cache->get_output_port()); + //elev_mask->set_input_connection(1, rdw->get_output_port()); + + if (!opt_vals["dem_variable"].defaulted()) + elev_mask->set_surface_elevation_variable( + opt_vals["dem_variable"].as()); + + if (!opt_vals["mesh_height"].defaulted()) + elev_mask->set_mesh_height_variable( + opt_vals["mesh_height"].as()); + + elev_mask->set_mask_variables({ + ivt_int->get_specific_humidity_variable() + "_valid", + ivt_int->get_wind_u_variable() + "_valid", + ivt_int->get_wind_v_variable() + "_valid"}); + + /*p_teca_cartesian_mesh_writer emw = teca_cartesian_mesh_writer::New(); + emw->set_input_connection(elev_mask->get_output_port()); + emw->set_file_name("elev_mask_%t%.vtk"); + emw->set_binary(1); + head = emw;*/ + + head = elev_mask; + } ivt_int->set_input_connection(head->get_output_port()); if (do_ivt_magnitude) { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Computing IVT magnitude") + l2_norm->set_input_connection(ivt_int->get_output_port()); head = l2_norm; } @@ -303,39 +446,23 @@ int main(int argc, char **argv) if (!opt_vals["last_step"].defaulted()) cf_writer->set_last_step(opt_vals["last_step"].as()); + if (!opt_vals["file_layout"].defaulted() && + cf_writer->set_layout(opt_vals["file_layout"].as())) + { + TECA_ERROR("An invalid file layout was provided \"" + << opt_vals["file_layout"].as() << "\"") + return -1; + } + if (opt_vals.count("verbose")) { cf_writer->set_verbose(1); exec->set_verbose(1); } - if (!opt_vals["n_threads"].defaulted()) - cf_writer->set_thread_pool_size(opt_vals["n_threads"].as()); - else - cf_writer->set_thread_pool_size(-1); + cf_writer->set_thread_pool_size(opt_vals["n_threads"].as()); // some minimal check for missing options - if ((have_file && have_regex) || !(have_file || have_regex)) - { - if (mpi_man.get_comm_rank() == 0) - { - TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " - "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " - "and --input_regex to activate the cf_reader (CAM like datasets)") - } - return -1; - } - - if (!(do_ivt || do_ivt_magnitude)) - { - if (mpi_man.get_comm_rank() == 0) - { - TECA_ERROR("AT least one of --write_ivt or --write_ivt_magnitude " - " must be set.") - } - return -1; - } - if (cf_writer->get_file_name().empty()) { if (mpi_man.get_comm_rank() == 0) @@ -355,7 +482,8 @@ int main(int argc, char **argv) if (parse_start_date || parse_end_date) { // run the reporting phase of the pipeline - teca_metadata md = reader->update_metadata(); + if (md.empty()) + md = reader->update_metadata(); teca_metadata atrs; if (md.get("attributes", atrs)) @@ -376,10 +504,8 @@ int main(int argc, char **argv) } teca_metadata coords; - p_teca_double_array time; - if (md.get("coordinates", coords) || - !(time = std::dynamic_pointer_cast( - coords.get("t")))) + p_teca_variant_array time; + if (md.get("coordinates", coords) || !(time = coords.get("t"))) { TECA_ERROR("failed to determine time coordinate") return -1; diff --git a/apps/teca_integrated_water_vapor.cpp b/apps/teca_integrated_water_vapor.cpp new file mode 100644 index 000000000..405a83aa2 --- /dev/null +++ b/apps/teca_integrated_water_vapor.cpp @@ -0,0 +1,454 @@ +#include "teca_config.h" +#include "teca_cf_reader.h" +#include "teca_cf_writer.h" +#include "teca_index_executive.h" +#include "teca_normalize_coordinates.h" +#include "teca_metadata.h" +#include "teca_integrated_water_vapor.h" +#include "teca_binary_segmentation.h" +#include "teca_l2_norm.h" +#include "teca_multi_cf_reader.h" +#include "teca_integrated_water_vapor.h" +#include "teca_valid_value_mask.h" +#include "teca_unpack_data.h" +#include "teca_cartesian_mesh_source.h" +#include "teca_cartesian_mesh_regrid.h" +#include "teca_elevation_mask.h" +#include "teca_indexed_dataset_cache.h" +#include "teca_mpi_manager.h" +#include "teca_coordinate_util.h" +#include "teca_table.h" +#include "teca_dataset_source.h" +#include "teca_app_util.h" +#include "teca_calcalcs.h" + +#include +#include +#include +#include + +#include "teca_cartesian_mesh_writer.h" + +using namespace std; + +using boost::program_options::value; + +// -------------------------------------------------------------------------- +int main(int argc, char **argv) +{ + // initialize mpi + teca_mpi_manager mpi_man(argc, argv); + + // initialize command line options description + // set up some common options to simplify use for most + // common scenarios + int help_width = 100; + options_description basic_opt_defs( + "Basic usage:\n\n" + "The following options are the most commonly used. Information\n" + "on advanced options can be displayed using --advanced_help\n\n" + "Basic command line options", help_width, help_width - 4 + ); + basic_opt_defs.add_options() + ("input_file", value(), "\na teca_multi_cf_reader configuration file" + " identifying the set of NetCDF CF2 files to process. When present data is" + " read using the teca_multi_cf_reader. Use one of either --input_file or" + " --input_regex.\n") + + ("input_regex", value(), "\na teca_cf_reader regex identifying the" + " set of NetCDF CF2 files to process. When present data is read using the" + " teca_cf_reader. Use one of either --input_file or --input_regex.\n") + + ("specific_humidity", value()->default_value("Q"), + "\nname of variable with the 3D specific humidity field.\n") + + ("iwv", value()->default_value("IWV"), + "\nname to use for the longitudinal component of the integrated vapor transport vector.\n") + + ("output_file", value()->default_value("IWV_%t%.nc"), + "\nA path and file name pattern for the output NetCDF files. %t% is replaced with a" + " human readable date and time corresponding to the time of the first time step in" + " the file. Use --cf_writer::date_format to change the formatting\n") + ("file_layout", value()->default_value("monthly"), + "\nSelects the size and layout of the set of output files. May be one of" + " number_of_steps, daily, monthly, seasonal, or yearly. Files are structured" + " such that each file contains one of the selected interval. For the number_of_steps" + " option use --steps_per_file.\n") + ("steps_per_file", value()->default_value(128), + "\nThe number of time steps per output file when --file_layout number_of_steps is" + " specified.\n") + + ("x_axis_variable", value()->default_value("lon"), + "\nname of x coordinate variable\n") + ("y_axis_variable", value()->default_value("lat"), + "\nname of y coordinate variable\n") + ("z_axis_variable", value()->default_value("plev"), + "\nname of z coordinate variable\n") + + ("dem", value(), "\nA teca_cf_reader regex identifying the" + " file containing surface elevation field or DEM.\n") + + ("dem_variable", value()->default_value("Z"), + "\nSets the name of the variable containing the surface elevation field\n") + + ("mesh_height", value()->default_value("Zg"), + "\nSets the name of the variable containing the point wise vertical height" + " in meters above mean sea level\n") + + ("first_step", value()->default_value(0), "\nfirst time step to process\n") + ("last_step", value()->default_value(-1), "\nlast time step to process\n") + + ("start_date", value(), "\nThe first time to process in 'Y-M-D h:m:s'" + " format. Note: There must be a space between the date and time specification\n") + ("end_date", value(), "\nThe last time to process in 'Y-M-D h:m:s' format\n") + + ("n_threads", value()->default_value(-1), "\nSets the thread pool size on each MPI" + " rank. When the default value of -1 is used TECA will coordinate the thread pools" + " across ranks such each thread is bound to a unique physical core.\n") + + ("verbose", "\nenable extra terminal output\n") + + ("help", "\ndisplays documentation for application specific command line options\n") + ("advanced_help", "\ndisplays documentation for algorithm specific command line options\n") + ("full_help", "\ndisplays both basic and advanced documentation together\n") + ; + + // add all options from each pipeline stage for more advanced use + options_description advanced_opt_defs( + "Advanced usage:\n\n" + "The following list contains the full set options giving one full\n" + "control over all runtime modifiable parameters. The basic options\n" + "(see" "--help) map to these, and will override them if both are\n" + "specified.\n\n" + "integrated vapor transport pipeline:\n\n" + " (cf / mcf_reader)\n" + " \\\n" + " (iwv_integral)--(iwv_magnitude)\n" + " \\\n" + " (cf_writer)\n\n" + "Advanced command line options", help_width, help_width - 4 + ); + + // create the pipeline stages here, they contain the + // documentation and parse command line. + // objects report all of their properties directly + // set default options here so that command line options override + // them. while we are at it connect the pipeline + p_teca_cf_reader cf_reader = teca_cf_reader::New(); + cf_reader->get_properties_description("cf_reader", advanced_opt_defs); + + p_teca_multi_cf_reader mcf_reader = teca_multi_cf_reader::New(); + mcf_reader->get_properties_description("mcf_reader", advanced_opt_defs); + + p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); + norm_coords->get_properties_description("norm_coords", advanced_opt_defs); + + p_teca_valid_value_mask vv_mask = teca_valid_value_mask::New(); + vv_mask->get_properties_description("vv_mask", advanced_opt_defs); + + p_teca_unpack_data unpack = teca_unpack_data::New(); + unpack->get_properties_description("unpack", advanced_opt_defs); + + p_teca_cf_reader elev_reader = teca_cf_reader::New(); + elev_reader->get_properties_description("elev_reader", advanced_opt_defs); + elev_reader->set_t_axis_variable(""); + + p_teca_normalize_coordinates elev_coords = teca_normalize_coordinates::New(); + elev_coords->get_properties_description("elev_coords", advanced_opt_defs); + elev_coords->set_enable_periodic_shift_x(1); + + p_teca_indexed_dataset_cache elev_cache = teca_indexed_dataset_cache::New(); + elev_cache->get_properties_description("elev_cache", advanced_opt_defs); + elev_cache->set_max_cache_size(1); + + p_teca_cartesian_mesh_source elev_mesh = teca_cartesian_mesh_source::New(); + elev_mesh->get_properties_description("elev_mesh", advanced_opt_defs); + + p_teca_cartesian_mesh_regrid elev_regrid = teca_cartesian_mesh_regrid::New(); + elev_regrid->get_properties_description("elev_regrid", advanced_opt_defs); + + p_teca_elevation_mask elev_mask = teca_elevation_mask::New(); + elev_mask->get_properties_description("elev_mask", advanced_opt_defs); + elev_mask->set_surface_elevation_variable("Z"); + elev_mask->set_mesh_height_variable("ZG"); + + p_teca_integrated_water_vapor iwv_int = teca_integrated_water_vapor::New(); + iwv_int->get_properties_description("iwv_integral", advanced_opt_defs); + iwv_int->set_specific_humidity_variable("Q"); + iwv_int->set_iwv_variable("IWV"); + + // Add an executive for the writer + p_teca_index_executive exec = teca_index_executive::New(); + + // Add the writer + p_teca_cf_writer cf_writer = teca_cf_writer::New(); + cf_writer->get_properties_description("cf_writer", advanced_opt_defs); + cf_writer->set_verbose(0); + cf_writer->set_steps_per_file(128); + cf_writer->set_layout(teca_cf_writer::monthly); + + // package basic and advanced options for display + options_description all_opt_defs(help_width, help_width - 4); + all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); + + // parse the command line + int ierr = 0; + variables_map opt_vals; + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) + { + if (ierr == 1) + return 0; + return -1; + } + + // pass command line arguments into the pipeline objects + // advanced options are processed first, so that the basic + // options will override them + cf_reader->set_properties("cf_reader", opt_vals); + mcf_reader->set_properties("mcf_reader", opt_vals); + norm_coords->set_properties("norm_coords", opt_vals); + vv_mask->set_properties("vv_mask", opt_vals); + unpack->set_properties("unpack", opt_vals); + elev_reader->set_properties("elev_reader", opt_vals); + elev_coords->set_properties("elev_coords", opt_vals); + elev_mesh->set_properties("elev_mesh", opt_vals); + elev_cache->set_properties("elev_cache", opt_vals); + elev_regrid->set_properties("elev_regrid", opt_vals); + elev_mask->set_properties("elev_mask", opt_vals); + iwv_int->set_properties("iwv_integral", opt_vals); + cf_writer->set_properties("cf_writer", opt_vals); + + // now pass in the basic options, these are processed + // last so that they will take precedence + // configure the pipeline from the command line options. + p_teca_algorithm reader; + + // configure the reader + bool have_file = opt_vals.count("input_file"); + bool have_regex = opt_vals.count("input_regex"); + if ((have_file && have_regex) || !(have_file || have_regex)) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " + "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " + "and --input_regex to activate the cf_reader (CAM like datasets)") + } + return -1; + } + + if (opt_vals.count("input_file")) + { + mcf_reader->set_input_file(opt_vals["input_file"].as()); + reader = mcf_reader; + } + else if (opt_vals.count("input_regex")) + { + cf_reader->set_files_regex(opt_vals["input_regex"].as()); + reader = cf_reader; + } + p_teca_algorithm head = reader; + + if (!opt_vals["x_axis_variable"].defaulted()) + { + cf_reader->set_x_axis_variable(opt_vals["x_axis_variable"].as()); + mcf_reader->set_x_axis_variable(opt_vals["x_axis_variable"].as()); + } + + if (!opt_vals["y_axis_variable"].defaulted()) + { + cf_reader->set_y_axis_variable(opt_vals["y_axis_variable"].as()); + mcf_reader->set_y_axis_variable(opt_vals["y_axis_variable"].as()); + } + + std::string z_var = "plev"; + if (!opt_vals["z_axis_variable"].defaulted()) + z_var = opt_vals["z_axis_variable"].as(); + + cf_reader->set_z_axis_variable(z_var); + mcf_reader->set_z_axis_variable(z_var); + + // set the inputs to the integrator + if (!opt_vals["specific_humidity"].defaulted()) + { + iwv_int->set_specific_humidity_variable( + opt_vals["specific_humidity"].as()); + } + + // set all that use or produce iwv + if (!opt_vals["iwv"].defaulted()) + iwv_int->set_iwv_variable(opt_vals["iwv"].as()); + + // add the valid value mask stage + norm_coords->set_input_connection(head->get_output_port()); + vv_mask->set_input_connection(norm_coords->get_output_port()); + unpack->set_input_connection(vv_mask->get_output_port()); + head = unpack; + + // add the elevation mask stages + teca_metadata md; + if (opt_vals.count("dem")) + { + if (mpi_man.get_comm_rank() == 0) + TECA_STATUS("Generating elevation mask") + + elev_reader->set_files_regex(opt_vals["dem"].as()); + + elev_coords->set_input_connection(elev_reader->get_output_port()); + + md = head->update_metadata(); + + elev_mesh->set_spatial_bounds(md, false); + elev_mesh->set_spatial_extents(md, false); + elev_mesh->set_x_axis_variable(md); + elev_mesh->set_y_axis_variable(md); + elev_mesh->set_z_axis_variable(md); + elev_mesh->set_t_axis_variable(md); + elev_mesh->set_t_axis(md); + + elev_regrid->set_input_connection(0, elev_mesh->get_output_port()); + elev_regrid->set_input_connection(1, elev_coords->get_output_port()); + + elev_cache->set_input_connection(elev_regrid->get_output_port()); + + elev_mask->set_input_connection(0, head->get_output_port()); + elev_mask->set_input_connection(1, elev_cache->get_output_port()); + + if (!opt_vals["dem_variable"].defaulted()) + elev_mask->set_surface_elevation_variable( + opt_vals["dem_variable"].as()); + + if (!opt_vals["mesh_height"].defaulted()) + elev_mask->set_mesh_height_variable( + opt_vals["mesh_height"].as()); + + elev_mask->set_mask_variables({ + iwv_int->get_specific_humidity_variable() + "_valid"}); + + head = elev_mask; + } + + iwv_int->set_input_connection(head->get_output_port()); + head = iwv_int; + + // tell the writer to write iwv if needed + std::vector point_arrays; + point_arrays.push_back(iwv_int->get_iwv_variable()); + + cf_writer->set_point_arrays(point_arrays); + + cf_writer->set_file_name(opt_vals["output_file"].as()); + + if (!opt_vals["steps_per_file"].defaulted()) + cf_writer->set_steps_per_file(opt_vals["steps_per_file"].as()); + + if (!opt_vals["first_step"].defaulted()) + cf_writer->set_first_step(opt_vals["first_step"].as()); + + if (!opt_vals["last_step"].defaulted()) + cf_writer->set_last_step(opt_vals["last_step"].as()); + + if (!opt_vals["file_layout"].defaulted() && + cf_writer->set_layout(opt_vals["file_layout"].as())) + { + TECA_ERROR("An invalid file layout was provided \"" + << opt_vals["file_layout"].as() << "\"") + return -1; + } + + if (opt_vals.count("verbose")) + { + cf_writer->set_verbose(1); + exec->set_verbose(1); + } + + cf_writer->set_thread_pool_size(opt_vals["n_threads"].as()); + + // some minimal check for missing options + if (cf_writer->get_file_name().empty()) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("missing file name pattern for netcdf writer. " + "See --help for a list of command line options.") + } + return -1; + } + + // connect the fixed stages of the pipeline + cf_writer->set_input_connection(head->get_output_port()); + + // look for requested time step range, start + bool parse_start_date = opt_vals.count("start_date"); + bool parse_end_date = opt_vals.count("end_date"); + if (parse_start_date || parse_end_date) + { + // run the reporting phase of the pipeline + if (md.empty()) + md = reader->update_metadata(); + + teca_metadata atrs; + if (md.get("attributes", atrs)) + { + TECA_ERROR("metadata missing attributes") + return -1; + } + + teca_metadata time_atts; + std::string calendar; + std::string units; + if (atrs.get("time", time_atts) + || time_atts.get("calendar", calendar) + || time_atts.get("units", units)) + { + TECA_ERROR("failed to determine the calendaring parameters") + return -1; + } + + teca_metadata coords; + p_teca_variant_array time; + if (md.get("coordinates", coords) || !(time = coords.get("t"))) + { + TECA_ERROR("failed to determine time coordinate") + return -1; + } + + // convert date string to step, start date + if (parse_start_date) + { + unsigned long first_step = 0; + std::string start_date = opt_vals["start_date"].as(); + if (teca_coordinate_util::time_step_of(time, true, true, calendar, + units, start_date, first_step)) + { + TECA_ERROR("Failed to locate time step for start date \"" + << start_date << "\"") + return -1; + } + cf_writer->set_first_step(first_step); + } + + // and end date + if (parse_end_date) + { + unsigned long last_step = 0; + std::string end_date = opt_vals["end_date"].as(); + if (teca_coordinate_util::time_step_of(time, false, true, calendar, + units, end_date, last_step)) + { + TECA_ERROR("Failed to locate time step for end date \"" + << end_date << "\"") + return -1; + } + cf_writer->set_last_step(last_step); + } + } + + // run the pipeline + cf_writer->set_executive(exec); + cf_writer->update(); + + return 0; +} diff --git a/apps/teca_metadata_probe.cpp b/apps/teca_metadata_probe.cpp index 6d9f3d192..992d8d4da 100644 --- a/apps/teca_metadata_probe.cpp +++ b/apps/teca_metadata_probe.cpp @@ -3,6 +3,7 @@ #include "teca_netcdf_util.h" #include "teca_cf_reader.h" #include "teca_multi_cf_reader.h" +#include "teca_normalize_coordinates.h" #include "teca_array_collection.h" #include "teca_variant_array.h" #include "teca_coordinate_util.h" @@ -11,7 +12,7 @@ #include "teca_app_util.h" #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #include @@ -93,15 +94,22 @@ int main(int argc, char **argv) p_teca_multi_cf_reader mcf_reader = teca_multi_cf_reader::New(); mcf_reader->get_properties_description("mcf_reader", advanced_opt_defs); + p_teca_normalize_coordinates norm_coords = teca_normalize_coordinates::New(); + norm_coords->get_properties_description("norm_coords", advanced_opt_defs); + // package basic and advanced options for display options_description all_opt_defs(help_width, help_width - 4); all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -110,6 +118,7 @@ int main(int argc, char **argv) // options will override them cf_reader->set_properties("cf_reader", opt_vals); mcf_reader->set_properties("mcf_reader", opt_vals); + norm_coords->set_properties("norm_coords", opt_vals); // now pas in the basic options, these are procesed // last so that they will take precedence @@ -138,6 +147,18 @@ int main(int argc, char **argv) bool have_file = opt_vals.count("input_file"); bool have_regex = opt_vals.count("input_regex"); + // validate the input method + if ((have_file && have_regex) || !(have_file || have_regex)) + { + if (rank == 0) + { + TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " + "Use --input_file to activate the multi_cf_reader (CMIP6 datasets) " + "and --input_regex to activate the cf_reader (CAM like datasets)") + } + return -1; + } + p_teca_algorithm reader; if (opt_vals.count("input_file")) { @@ -155,6 +176,8 @@ int main(int argc, char **argv) z_var = cf_reader->get_z_axis_variable(); reader = cf_reader; } + norm_coords->set_input_connection(reader->get_output_port()); + norm_coords->set_verbose(2); std::string time_i; if (opt_vals.count("start_date")) @@ -164,20 +187,8 @@ int main(int argc, char **argv) if (opt_vals.count("end_date")) time_j = opt_vals["end_date"].as(); - // some minimal check for mising options - if ((have_file && have_regex) || !(have_file || have_regex)) - { - if (rank == 0) - { - TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " - "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " - "and --input_regex to activate the cf_reader (CAM like datasets)") - } - return -1; - } - // run the reporting phase of the pipeline - teca_metadata md = reader->update_metadata(); + teca_metadata md = norm_coords->update_metadata(); // from here on out just rank 0 if (rank == 0) @@ -248,7 +259,7 @@ int main(int argc, char **argv) int Y = 0, M = 0, D = 0, h = 0, m = 0; double s = 0; #if defined(TECA_HAS_UDUNITS) - if (calcalcs::date(time->get(i0), &Y, &M, &D, &h, &m, &s, + if (teca_calcalcs::date(time->get(i0), &Y, &M, &D, &h, &m, &s, units.c_str(), calendar.c_str())) { TECA_ERROR("failed to detmine the first available time in the file") @@ -264,7 +275,7 @@ int main(int argc, char **argv) // human readbale last time available Y = 0, M = 0, D = 0, h = 0, m = 0, s = 0; #if defined(TECA_HAS_UDUNITS) - if (calcalcs::date(time->get(i1), &Y, &M, &D, &h, &m, &s, + if (teca_calcalcs::date(time->get(i1), &Y, &M, &D, &h, &m, &s, units.c_str(), calendar.c_str())) { TECA_ERROR("failed to detmine the last available time in the file") @@ -344,26 +355,56 @@ int main(int argc, char **argv) } std::cerr << std::endl; + // report extents + long extent[6] = {0l}; + if (!md.get("whole_extent", extent, 6)) + { + std::cerr << "Mesh extents: " << extent[0] << ", " << extent[1] + << ", " << extent[2] << ", " << extent[3]; + if (!z_var.empty()) + { + std::cerr << ", " << extent[4] << ", " << extent[5]; + } + std::cerr << std::endl; + } + + // report bounds + double bounds[6] = {0.0}; + if (!md.get("bounds", bounds, 6)) + { + std::cerr << "Mesh bounds: " << bounds[0] << ", " << bounds[1] + << ", " << bounds[2] << ", " << bounds[3]; + if (!z_var.empty()) + { + std::cerr << ", " << bounds[4] << ", " << bounds[5]; + } + std::cerr << std::endl; + } + + // report the arrays size_t n_arrays = atrs.size(); // column widths - int aiw = 0; - int anw = 0; - int atw = 0; - int adw = 0; - int asw = 0; + int aiw = 4; + int anw = 8; + int atw = 8; + int acw = 14; + int adw = 15; + int asw = 9; // column data std::vector ai; std::vector an; std::vector at; + std::vector ac; std::vector ad; std::vector as; ai.reserve(n_arrays); an.reserve(n_arrays); at.reserve(n_arrays); + ac.reserve(n_arrays); ad.reserve(n_arrays); as.reserve(n_arrays); @@ -378,10 +419,14 @@ int main(int argc, char **argv) int id = 0; p_teca_size_t_array dims; p_teca_string_array dim_names; + int centering = 0; + int n_active_dims = 0; if (atrs.get(array, atts) || atts.get("cf_type_code", 0, type) || atts.get("cf_id", 0, id) + || atts.get("centering", centering) + || atts.get("n_active_dims", n_active_dims) || !(dims = std::dynamic_pointer_cast(atts.get("cf_dims"))) || !(dim_names = std::dynamic_pointer_cast(atts.get("cf_dim_names")))) { @@ -404,6 +449,11 @@ int main(int argc, char **argv) ) atw = std::max(atw, at.back().size() + 4); + // centering + ac.push_back(teca_array_attributes::centering_to_string(centering) + + std::string(" ") + std::to_string(n_active_dims) + "D"); + acw = std::max(acw, ac.back().size() + 4); + // dims int n_dims = dim_names->size(); @@ -444,10 +494,11 @@ int main(int argc, char **argv) << std::setw(aiw) << std::left << "Id" << std::setw(anw) << std::left << "Name" << std::setw(atw) << std::left << "Type" + << std::setw(acw) << std::left << "Centering" << std::setw(adw) << std::left << "Dimensions" << std::setw(asw) << std::left << "Shape" << std::endl; - int tw = anw + atw + adw + asw; + int tw = aiw + anw + atw + adw + acw + asw; for (int i = 0; i < tw; ++i) std::cerr << '-'; std::cerr << std::endl; @@ -459,6 +510,7 @@ int main(int argc, char **argv) << std::setw(aiw) << std::left << ai[i] << std::setw(anw) << std::left << an[i] << std::setw(atw) << std::left << at[i] + << std::setw(acw) << std::left << ac[i] << std::setw(adw) << std::left << ad[i] << std::setw(asw) << std::left << as[i] << std::endl; diff --git a/apps/teca_tc_detect.cpp b/apps/teca_tc_detect.cpp index f8483f171..8171d27eb 100644 --- a/apps/teca_tc_detect.cpp +++ b/apps/teca_tc_detect.cpp @@ -19,7 +19,7 @@ #include "teca_mpi_manager.h" #include "teca_coordinate_util.h" #include "teca_app_util.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" #include #include @@ -204,14 +204,18 @@ int main(int argc, char **argv) track_writer->get_properties_description("track_writer", advanced_opt_defs); // package basic and advanced options for display - options_description all_opt_defs(-1, -1); + options_description all_opt_defs(help_width, help_width - 4); all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -267,22 +271,21 @@ int main(int argc, char **argv) surf_wind->set_component_1_variable( opt_vals["surface_wind_v"].as()); - std::vector dep_var; - core_temp->get_dependent_variables(dep_var); if (!opt_vals["500mb_temp"].defaulted()) - dep_var[0] = opt_vals["500mb_temp"].as(); + core_temp->set_dependent_variable(0, + opt_vals["500mb_temp"].as()); + if (!opt_vals["200mb_temp"].defaulted()) - dep_var[1] = opt_vals["200mb_temp"].as(); - core_temp->set_dependent_variables(dep_var); - dep_var.clear(); + core_temp->set_dependent_variable(1, + opt_vals["200mb_temp"].as()); - thickness->get_dependent_variables(dep_var); if (!opt_vals["1000mb_height"].defaulted()) - dep_var[0] = opt_vals["1000mb_height"].as(); + thickness->set_dependent_variable(0, + opt_vals["1000mb_height"].as()); + if (!opt_vals["200mb_height"].defaulted()) - dep_var[1] = opt_vals["200mb_height"].as(); - thickness->set_dependent_variables(dep_var); - dep_var.clear(); + thickness->set_dependent_variable(1, + opt_vals["200mb_height"].as()); if (!opt_vals["sea_level_pressure"].defaulted()) candidates->set_sea_level_pressure_variable( @@ -375,29 +378,31 @@ int main(int argc, char **argv) // now that command line opts have been parsed we can create // the programmable algorithms' functors - core_temp->get_dependent_variables(dep_var); - if (dep_var.size() != 2) + size_t n_var = core_temp->get_number_of_dependent_variables(); + if (n_var != 2) { TECA_ERROR("core temperature calculation requires 2 " - "variables. given " << dep_var.size()) + "variables. given " << n_var) return -1; } core_temp->set_execute_callback( - point_wise_average(dep_var[0], dep_var[1], - core_temp->get_derived_variable())); - dep_var.clear(); + point_wise_average( + core_temp->get_dependent_variable(0), + core_temp->get_dependent_variable(1), + core_temp->get_derived_variable())); - thickness->get_dependent_variables(dep_var); - if (dep_var.size() != 2) + n_var = thickness->get_number_of_dependent_variables(); + if (n_var != 2) { TECA_ERROR("thickness calculation requires 2 " - "variables. given " << dep_var.size()) + "variables. given " << n_var) return -1; } thickness->set_execute_callback( - point_wise_difference(dep_var[0], dep_var[1], - thickness->get_derived_variable())); - dep_var.clear(); + point_wise_difference( + thickness->get_dependent_variable(0), + thickness->get_dependent_variable(1), + thickness->get_derived_variable())); // and tell the candidate stage what variables the functors produce candidates->set_surface_wind_speed_variable(surf_wind->get_l2_norm_variable()); @@ -432,10 +437,8 @@ int main(int argc, char **argv) } teca_metadata coords; - p_teca_double_array time; - if (md.get("coordinates", coords) || - !(time = std::dynamic_pointer_cast( - coords.get("t")))) + p_teca_variant_array time; + if (md.get("coordinates", coords) || !(time = coords.get("t"))) { TECA_ERROR("failed to determine time coordinate") return -1; diff --git a/apps/teca_tc_trajectory.cpp b/apps/teca_tc_trajectory.cpp index 6e9c190dd..a7a945b98 100644 --- a/apps/teca_tc_trajectory.cpp +++ b/apps/teca_tc_trajectory.cpp @@ -16,7 +16,7 @@ #include "teca_table_writer.h" #include "teca_mpi_manager.h" #include "teca_coordinate_util.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" #include "teca_app_util.h" #include @@ -105,10 +105,14 @@ int main(int argc, char **argv) all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } diff --git a/apps/teca_tc_wind_radii.cpp b/apps/teca_tc_wind_radii.cpp index 4ded8748e..8654d748a 100644 --- a/apps/teca_tc_wind_radii.cpp +++ b/apps/teca_tc_wind_radii.cpp @@ -42,7 +42,7 @@ int main(int argc, char **argv) "Basic command line options", help_width, help_width - 4 ); basic_opt_defs.add_options() - ("track_file", value(), "\na file containing cyclone tracks (tracks.bin)\n") + ("track_file", value(), "\na file containing cyclone tracks\n") ("input_file", value(), "\na teca_multi_cf_reader configuration file" " identifying the set of NetCDF CF2 files to process. When present data is" @@ -135,10 +135,14 @@ int main(int argc, char **argv) all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); // parse the command line + int ierr = 0; variables_map opt_vals; - if (teca_app_util::process_command_line_help(mpi_man.get_comm_rank(), - argc, argv, basic_opt_defs, advanced_opt_defs, all_opt_defs, opt_vals)) + if ((ierr = teca_app_util::process_command_line_help( + mpi_man.get_comm_rank(), argc, argv, basic_opt_defs, + advanced_opt_defs, all_opt_defs, opt_vals))) { + if (ierr == 1) + return 0; return -1; } @@ -156,12 +160,33 @@ int main(int argc, char **argv) // now pass in the basic options, these are processed // last so that they will take precedence - if (!opt_vals["track_file"].defaulted()) - track_reader->set_file_name(opt_vals["track_file"].as()); + if (!opt_vals.count("track_file")) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("A file with previously calculated storm tracks must be " + "specified with --track_file") + } + return -1; + } + + track_reader->set_file_name(opt_vals["track_file"].as()); bool have_file = opt_vals.count("input_file"); bool have_wind_files = opt_vals.count("wind_files"); bool have_regex = opt_vals.count("input_regex"); + + if ((have_file && have_regex) || !(have_file || have_regex)) + { + if (mpi_man.get_comm_rank() == 0) + { + TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " + "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " + "and --input_regex to activate the cf_reader (CAM like datasets)") + } + return -1; + } + p_teca_algorithm wind_reader; if (have_file) { @@ -236,18 +261,6 @@ int main(int argc, char **argv) else map_reduce->set_thread_pool_size(-1); - // some minimal check for missing options - if ((have_file && have_regex) || !(have_file || have_regex)) - { - if (mpi_man.get_comm_rank() == 0) - { - TECA_ERROR("Extacly one of --input_file or --input_regex can be specified. " - "Use --input_file to activate the multi_cf_reader (HighResMIP datasets) " - "and --input_regex to activate the cf_reader (CAM like datasets)") - } - return -1; - } - // connect the pipeline wind_coords->set_input_connection(wind_reader->get_output_port()); wind_radii->set_input_connection(0, track_input->get_output_port()); diff --git a/apps/teca_temporal_reduction.in b/apps/teca_temporal_reduction.in index f14d653df..22178f0f3 100755 --- a/apps/teca_temporal_reduction.in +++ b/apps/teca_temporal_reduction.in @@ -66,6 +66,14 @@ parser.add_argument('--output_file', type=str, required=True, ' step in the file. Use --date_format to change' ' the formatting') +parser.add_argument('--file_layout', type=str, default='yearly', + help='Selects the size and layout of the set of output' + ' files. May be one of number_of_steps, daily,' + ' monthly, seasonal, or yearly. Files are structured' + ' such that each file contains one of the selected' + ' interval. For the number_of_steps option use' + ' --steps_per_file.') + parser.add_argument('--steps_per_file', type=int, default=128, help='number of time steps to write to each output ' 'file') @@ -85,7 +93,7 @@ parser.add_argument('--t_axis_variable', type=str, default='time', help='name of the variable to use for t-coordinates') parser.add_argument('--n_threads', type=int, default=2, - help='Number of threads to use when stremaing the ' + help='Number of threads to use when streaming the ' 'reduction') parser.add_argument('--verbose', type=int, default=0, @@ -99,6 +107,7 @@ except Exception: in_files = args.input_regex out_files = args.output_file +layout = args.file_layout steps_per_file = args.steps_per_file n_threads = args.n_threads interval = args.interval @@ -115,6 +124,7 @@ verbose = args.verbose if verbose and rank == 0: sys.stderr.write('running on %d ranks' % (n_ranks)) sys.stderr.write('n_threads=%d\n' % (n_threads)) + sys.stderr.write('file_layout=%s\n'%(layout)) sys.stderr.write('steps_per_file=%d\n' % (steps_per_file)) sys.stderr.write('interval=%s\n' % (interval)) sys.stderr.write('operator=%s\n' % (operator)) @@ -152,6 +162,7 @@ cfw = teca_cf_writer.New() cfw.set_input_connection(mav.get_output_port()) cfw.set_verbose(verbose) cfw.set_thread_pool_size(1) +cfw.set_layout(layout) cfw.set_steps_per_file(steps_per_file) cfw.set_file_name(out_files) cfw.set_point_arrays(point_arrays) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 525e3bcef..c1251e2db 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -7,7 +7,6 @@ set(teca_core_srcs teca_algorithm_executive.cxx teca_bad_cast.cxx teca_binary_stream.cxx - teca_calendar.cxx teca_common.cxx teca_dataset.cxx teca_dataset_source.cxx @@ -45,7 +44,7 @@ endif() if (TECA_HAS_UDUNITS) include_directories(SYSTEM ${UDUNITS_INCLUDE_DIR}) list(APPEND teca_core_link ${UDUNITS_LIBRARY}) - list(APPEND teca_core_srcs calcalcs.cxx) + list(APPEND teca_core_srcs teca_calcalcs.cxx) endif() list(APPEND teca_core_link pthread) diff --git a/core/teca_algorithm.cxx b/core/teca_algorithm.cxx index 8fe7136e9..d1801f035 100644 --- a/core/teca_algorithm.cxx +++ b/core/teca_algorithm.cxx @@ -12,6 +12,10 @@ #include #include +#if defined(TECA_HAS_BOOST) +#include +#endif + using std::vector; using std::map; using std::string; @@ -382,7 +386,8 @@ void teca_algorithm_internals::from_stream(istream &is) // -------------------------------------------------------------------------- -teca_algorithm::teca_algorithm() : internals(new teca_algorithm_internals) +teca_algorithm::teca_algorithm() : verbose(0), + internals(new teca_algorithm_internals) {} // -------------------------------------------------------------------------- @@ -403,6 +408,25 @@ MPI_Comm teca_algorithm::get_communicator() return this->internals->comm; } +#if defined(TECA_HAS_BOOST) +// -------------------------------------------------------------------------- +void teca_algorithm::get_properties_description( + const string &prefix, options_description &opts) +{ + opts.add_options() + TECA_POPTS_GET(int, prefix, verbose, + "Set to non-zero to send diagnostic messages to the terminal") + ; +} + +// -------------------------------------------------------------------------- +void teca_algorithm::set_properties( + const string &prefix, variables_map &opts) +{ + TECA_POPTS_SET(opts, int, prefix, verbose) +} +#endif + // -------------------------------------------------------------------------- teca_algorithm_output_port teca_algorithm::get_output_port( unsigned int port) diff --git a/core/teca_algorithm.h b/core/teca_algorithm.h index 9a8dbb9ac..4fd2dc709 100644 --- a/core/teca_algorithm.h +++ b/core/teca_algorithm.h @@ -2,25 +2,238 @@ #define teca_algorithm_h #include "teca_config.h" - -// forward delcaration of ref counted types -#include "teca_dataset_fwd.h" -#include "teca_algorithm_fwd.h" -#include "teca_algorithm_executive_fwd.h" -class teca_algorithm_internals; - -// for types used in the api +#include "teca_shared_object.h" +#include "teca_dataset.h" +#include "teca_algorithm_executive.h" #include "teca_metadata.h" -#include "teca_algorithm_output_port.h" #include "teca_program_options.h" #include "teca_mpi.h" #include #include #include +#include +#include -// interface to teca pipeline architecture. all sources/readers -// filters, sinks/writers will implement this interface +class teca_algorithm_internals; + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_algorithm) + +/// An output port packages an algorithm and a port number +using teca_algorithm_output_port + = std::pair; + +/// get the algorithm from the output port +inline +p_teca_algorithm &get_algorithm(teca_algorithm_output_port &op) +{ return op.first; } + +/// get port number from the output port +inline +unsigned int &get_port(teca_algorithm_output_port &op) +{ return op.second; } + +/* this is a convenience macro to be used to declare a static New method that + * will be used to construct new objects in shared_ptr's. This manages the + * details of interoperability with std C++11 shared pointer + */ +#define TECA_ALGORITHM_STATIC_NEW(T) \ + \ +/** Returns an instance of T */ \ +static p_##T New() \ +{ \ + return p_##T(new T); \ +} \ + \ +/** Enables the static constructor */ \ +std::shared_ptr shared_from_this() \ +{ \ + return std::static_pointer_cast( \ + teca_algorithm::shared_from_this()); \ +} \ + \ +/** Enables the static constructor */ \ +std::shared_ptr shared_from_this() const \ +{ \ + return std::static_pointer_cast( \ + teca_algorithm::shared_from_this()); \ +} + +#define TECA_ALGORITHM_CLASS_NAME(T) \ +/** returns the name of the class */ \ +const char *get_class_name() const override \ +{ \ + return #T; \ +} + +/** this convenience macro removes copy and assignment operators + * which generally should not be defined for reference counted types + */ +#define TECA_ALGORITHM_DELETE_COPY_ASSIGN(T) \ + \ + T(const T &src) = delete; \ + T(T &&src) = delete; \ + \ + T &operator=(const T &src) = delete; \ + T &operator=(T &&src) = delete; + +/** convenience macro to declare standard set_NAME/get_NAME methods + * where NAME is the name of a class member. will manage the + * algorithm's modified state for the user. + */ +#define TECA_ALGORITHM_PROPERTY(T, NAME) \ + \ +/** Set the value of the NAME algorithm property */ \ +void set_##NAME(const T &v) \ +{ \ + if (this->NAME != v) \ + { \ + this->NAME = v; \ + this->set_modified(); \ + } \ +} \ + \ +/** Get the value of the NAME algorithm property */ \ +const T &get_##NAME() const \ +{ \ + return this->NAME; \ +} + +/** similar to TECA_ALGORITHM_PROPERTY but prior to setting NAME + * will call the member function int valididate_NAME(T v). If + * the value v is valid the fucntion should return 0. If the value + * is not zero the function should invoke TECA_ERROR with a + * descriptive message and return non-zero. + */ +#define TECA_ALGORITHM_PROPERTY_V(T, NAME) \ + \ +/** Set the value of the NAME algorithm property */ \ +void set_##NAME(const T &v) \ +{ \ + if (this->validate_ ## NAME (v)) \ + return; \ + \ + if (this->NAME != v) \ + { \ + this->NAME = v; \ + this->set_modified(); \ + } \ +} \ + \ +/** Get the value of the NAME algorithm property */ \ +const T &get_##NAME() const \ +{ \ + return this->NAME; \ +} + +/** convenience macro to declare standard set_NAME/get_NAME methods + * where NAME is the name of a class member. will manage the + * algorithm's modified state for the user. + */ +#define TECA_ALGORITHM_VECTOR_PROPERTY(T, NAME) \ + \ +/** get the size of the NAME algorithm vector property */ \ +size_t get_number_of_##NAME##s () \ +{ \ + return this->NAME##s.size(); \ +} \ + \ +/** append to the NAME algorithm vector property */ \ +void append_##NAME(const T &v) \ +{ \ + this->NAME##s.push_back(v); \ + this->set_modified(); \ +} \ + \ +/** set the i-th element of the NAME algorithm vector property */ \ +void set_##NAME(size_t i, const T &v) \ +{ \ + if (this->NAME##s[i] != v) \ + { \ + this->NAME##s[i] = v; \ + this->set_modified(); \ + } \ +} \ + \ +/** set the NAME algorithm vector property */ \ +void set_##NAME##s(const std::vector &v) \ +{ \ + if (this->NAME##s != v) \ + { \ + this->NAME##s = v; \ + this->set_modified(); \ + } \ +} \ + \ +/** set the NAME algorithm vector property */ \ +void set_##NAME##s(const std::initializer_list &&l) \ +{ \ + std::vector v(l); \ + if (this->NAME##s != v) \ + { \ + this->NAME##s = v; \ + this->set_modified(); \ + } \ +} \ + \ +/** get the i-th element of the NAME algorithm vector property */ \ +const T &get_##NAME(size_t i) const \ +{ \ + return this->NAME##s[i]; \ +} \ + \ +/** get the NAME algorithm vector property */ \ +const std::vector &get_##NAME##s() const \ +{ \ + return this->NAME##s; \ +} \ + \ +/** clear the NAME algorithm vector property */ \ +void clear_##NAME##s() \ +{ \ + this->NAME##s.clear(); \ +} + +/// helper that allows us to use std::function as a TECA_ALGORITHM_PROPERTY +template +bool operator!=(const std::function &lhs, const std::function &rhs) +{ + return &rhs != &lhs; +} + +/** This is a work around for older versions of Apple clang + * Apple LLVM version 4.2 (clang-425.0.28) (based on LLVM 3.2svn) + * Target: x86_64-apple-darwin12.6.0 + */ +#define TECA_ALGORITHM_CALLBACK_PROPERTY(T, NAME) \ + \ +/** Set the NAME algorithm property */ \ +void set_##NAME(const T &v) \ +{ \ + /*if (this->NAME != v)*/ \ + /*{*/ \ + this->NAME = v; \ + this->set_modified(); \ + /*}*/ \ +} \ + \ +/** Get the NAME algorithm property */ \ +const T &get_##NAME() const \ +{ \ + return this->NAME; \ +} \ + \ +/** Get the NAME algorithm property */ \ +T &get_##NAME() \ +{ \ + return this->NAME; \ +} + + +/// The interface to TECA pipeline architecture. +/** + * All sources/readers filters, sinks/writers will implement this interface. + */ class teca_algorithm : public std::enable_shared_from_this { public: @@ -29,98 +242,131 @@ class teca_algorithm : public std::enable_shared_from_this TECA_ALGORITHM_DELETE_COPY_ASSIGN(teca_algorithm) - // return the name of the class. + /// return the name of the class. virtual const char *get_class_name() const = 0; - // set/get the communicator to use at this stage of the pipeline this has - // no influence on other stages. We duplicate the passed in communicator - // providing an isolated communication space for subsequent operations. By - // default the communicator is initialized to MPI_COMM_WORLD, here it is not - // duplicated. Thus to put an algorithm into a unique communication space - // one should explicitly set a communicator. When an algorithm should not - // use MPI, for instance when it is in a nested pipeline, one may set the - // communicator to MPI_COMM_SELF. + /** set the communicator to use at this stage of the pipeline this has + * no influence on other stages. We duplicate the passed in communicator + * providing an isolated communication space for subsequent operations. By + * default the communicator is initialized to MPI_COMM_WORLD, here it is not + * duplicated. Thus to put an algorithm into a unique communication space + * one should explicitly set a communicator. When an algorithm should not + * use MPI, for instance when it is in a nested pipeline, one may set the + * communicator to MPI_COMM_SELF. + */ void set_communicator(MPI_Comm comm); + + /// get the active communicator MPI_Comm get_communicator(); #if defined(TECA_HAS_BOOST) - // initialize the given options description - // with algorithm's properties - virtual void get_properties_description(const std::string &, options_description &) - {} - - // initialize the algorithm from the given options - // variable map. - virtual void set_properties(const std::string &, variables_map &) - {} + /** initialize the given options description with algorithm's properties + * implementors should call the base implementation when overriding. + * this should be called after the override adds its options. + */ + virtual void get_properties_description(const std::string &prefix, + options_description &opts); + + /** initialize the algorithm from the given options variable map. + * implementors should call the base implementation when overriding. + * this should be called before the override sets its properties. + */ + virtual void set_properties(const std::string &prefix, + variables_map &opts); #endif - // get an output port from the algorithm. to be used - // during pipeline building + /** @name verbose + * if set to a non-zero value, rank 0 will send status information to the + * terminal. The default setting of zero results in no output. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, verbose) + ///@} + + /** get an output port from the algorithm. to be used during pipeline + * building + */ virtual teca_algorithm_output_port get_output_port(unsigned int port = 0); - // set an input to this algorithm + /// set an input to this algorithm void set_input_connection(const teca_algorithm_output_port &port) { this->set_input_connection(0, port); } + /// set an input to this algorithm virtual void set_input_connection(unsigned int id, const teca_algorithm_output_port &port); - // remove input connections + /// remove input connections virtual void remove_input_connection(unsigned int id); - // remove all input connections + /// remove all input connections void clear_input_connections(); - // access the cached data produced by this algorithm. when no - // request is specified the dataset on the top(most recent) of - // the cache is returned. When a request is specified it may - // optionally be filtered by the implementations cache key filter. - // see also get_cache_key (threadsafe) + /** access the cached data produced by this algorithm. when no request is + * specified the dataset on the top(most recent) of the cache is returned. + * When a request is specified it may optionally be filtered by the + * implementations cache key filter. see also get_cache_key (threadsafe) + */ const_p_teca_dataset get_output_data(unsigned int port = 0); - // remove a dataset from the top/bottom of the cache. the - // top of the cache has the most recently created dataset. - // top or bottom is selected via the boolean argument. - // (threadsafe) + /** remove a dataset from the top/bottom of the cache. the top of the cache + * has the most recently created dataset. top or bottom is selected via + * the boolean argument. (threadsafe) + */ void pop_cache(unsigned int port = 0, int top = 0); - // set the cache size. the default is 1. (threadsafe) + /// set the cache size. the default is 1. (threadsafe) void set_cache_size(unsigned int n); - // execute the pipeline from this instance up. + /// execute the pipeline from this instance up. virtual int update(); + + /// execute the pipeline from this instance up. virtual int update(unsigned int port); - // get meta data considering this instance up. + /// get meta data considering this instance up. virtual teca_metadata update_metadata(unsigned int port = 0); - // set the executive + /// set the executive void set_executive(p_teca_algorithm_executive exe); + + /// get the executive p_teca_algorithm_executive get_executive(); - // serialize the configuration to a stream. this should - // store the public user modifiable properties so that - // runtime configuration may be saved and restored.. + /** serialize the configuration to a stream. this should store the public + * user modifiable properties so that runtime configuration may be saved + * and restored. + */ virtual void to_stream(std::ostream &s) const; + + /// deserialize from the stream. virtual void from_stream(std::istream &s); protected: teca_algorithm(); - // implementations should call this from their constructors - // to setup the internal caches and data structures required - // for execution. + /** Set the number of input connections. implementations should call this + * from their constructors to setup the internal caches and data structures + * required for execution. + */ void set_number_of_input_connections(unsigned int n); + + /** Set the number of output ports. implementations should call this from + * their constructors to setup the internal caches and data structures + * required for execution. + */ void set_number_of_output_ports(unsigned int n); - // set the modified flag on the given output port's cache. - // should be called when user modifies properties on the - // object that require the output to be regenerated. + /** set the modified flag on the given output port's cache. should be + * called when user modifies properties on the object that require the + * output to be regenerated. + */ virtual void set_modified(); + + /// an overload to set_modified by port void set_modified(unsigned int port); protected: @@ -129,46 +375,43 @@ class teca_algorithm : public std::enable_shared_from_this // teca_algorithm's such as reader, filters, and // writers. - // implementations must override this method to provide - // information to downstream consumers about what data - // will be produced on each output port. The port to - // provide information about is named in the first argument - // the second argument contains a list of the metadata - // describing data on all of the inputs. + /** implementations must override this method to provide information to + * downstream consumers about what data will be produced on each output + * port. The port to provide information about is named in the first + * argument the second argument contains a list of the metadata describing + * data on all of the inputs. + */ virtual teca_metadata get_output_metadata(unsigned int port, const std::vector &input_md); - // implementations must override this method and - // generate a set of requests describing the data - // required on the inputs to produce data for the - // named output port, given the upstream meta data - // and request. If no data is needed on an input - // then the list should contain a null request. + /** implementations must override this method and generate a set of + * requests describing the data required on the inputs to produce data for + * the named output port, given the upstream meta data and request. If no + * data is needed on an input then the list should contain a null request. + */ virtual std::vector get_upstream_request( unsigned int port, const std::vector &input_md, const teca_metadata &request); - // implementations must override this method and - // produce the output dataset for the port named - // in the first argument. The second argument is - // a list of all of the input datasets. See also - // get_request. The third argument contains a request - // from the consumer which can spcify information - // such as arrays, subset region, timestep etc. - // The implementation is free to handle the request - // as it sees fit. + /** implementations must override this method and produce the output dataset + * for the port named in the first argument. The second argument is a list + * of all of the input datasets. See also get_request. The third argument + * contains a request from the consumer which can specify information such + * as arrays, subset region, timestep etc. The implementation is free to + * handle the request as it sees fit. + */ virtual const_p_teca_dataset execute(unsigned int port, const std::vector &input_data, const teca_metadata &request); - // implementations may choose to override this method - // to gain control of keys used in the cache. By default - // the passed in request is used as the key. This overide - // gives implementor the chance to filter the passed in - // request. + /** implementations may choose to override this method to gain control of + * keys used in the cache. By default the passed in request is used as the + * key. This override gives implementor the chance to filter the passed in + * request. + */ virtual teca_metadata get_cache_key(unsigned int port, const teca_metadata &request) const; @@ -176,62 +419,71 @@ class teca_algorithm : public std::enable_shared_from_this protected: // this section contains methods that control the // pipeline's behavior. these would typically only -// need to be overriden when designing a new class +// need to be overridden when designing a new class // of algorithms. - // driver function that manage meta data reporting phase - // of pipeline execution. + /** driver function that manage meta data reporting phase of pipeline + * execution. + */ virtual teca_metadata get_output_metadata( teca_algorithm_output_port ¤t); - // driver function that manages execution of the given - // requst on the named port + /* driver function that manages execution of the given request on the named + * port + */ virtual const_p_teca_dataset request_data( teca_algorithm_output_port &port, const teca_metadata &request); - // driver function that clears the output data cache - // where modified flag has been set from the current - // port upstream. + /** driver function that clears the output data cache where modified flag + * has been set from the current port upstream. + */ virtual int validate_cache(teca_algorithm_output_port ¤t); - // driver function that clears the modified flag on the - // named port and all of it's upstream connections. + /** driver function that clears the modified flag on the named port and all + * of it's upstream connections. + */ virtual void clear_modified(teca_algorithm_output_port current); protected: // api exposing internals for use in driver methods - // search the given port's cache for the dataset associated - // with the given request. see also get_cache_key. (threadsafe) + /** search the given port's cache for the dataset associated + * with the given request. see also get_cache_key. (threadsafe) + */ const_p_teca_dataset get_output_data(unsigned int port, const teca_metadata &request); - // add or update the given request , dataset pair in the cache. - // see also get_cache_key. (threadsafe) + /** add or update the given request , dataset pair in the cache. see also + * get_cache_key. (threadsafe) + */ int cache_output_data(unsigned int port, const teca_metadata &request, const_p_teca_dataset &data); - // clear the cache on the given output port + /// clear the cache on the given output port void clear_cache(unsigned int port); - // get the number of input connections + /// get the number of input connections unsigned int get_number_of_input_connections(); - // get the output port associated with this algorithm's - // i'th input connection. + /** get the output port associated with this algorithm's i'th input + * connection. + */ teca_algorithm_output_port &get_input_connection(unsigned int i); - // clear the modified flag on the i'th output + /// clear the modified flag on the i'th output void clear_modified(unsigned int port); - // return the output port's modified flag value + /// return the output port's modified flag value int get_modified(unsigned int port) const; +protected: + int verbose; + private: teca_algorithm_internals *internals; diff --git a/core/teca_algorithm_executive.cxx b/core/teca_algorithm_executive.cxx index 95e5fc865..e5361354f 100644 --- a/core/teca_algorithm_executive.cxx +++ b/core/teca_algorithm_executive.cxx @@ -13,6 +13,7 @@ int teca_algorithm_executive::initialize(MPI_Comm comm, const teca_metadata &md) return -1; } + m_request.set("index_request_key", request_key); m_request.set(request_key, 0); return 0; diff --git a/core/teca_algorithm_executive.h b/core/teca_algorithm_executive.h index 949ae4d3d..239480438 100644 --- a/core/teca_algorithm_executive.h +++ b/core/teca_algorithm_executive.h @@ -1,25 +1,57 @@ #ifndef teca_algorithm_executive_h #define teca_algorithm_executive_h -#include "teca_algorithm_executive_fwd.h" #include "teca_metadata.h" #include "teca_mpi.h" +#include "teca_shared_object.h" -// base class and default implementation for executives. algorithm -// executives can control pipeline execution by providing a series -// of requests. this allows for the executive to act as a load -// balancer. the executive can for example partition requests across -// spatial data, time steps, or file names. in an MPI parallel -// setting the executive could coordinate this partitioning amongst -// the ranks. However, the only requirement of an algorithm executive -// is that it provide at least one non-empty request. -// -// the default implementation creates a single trivially non-empty -// request containing the key "__request_empty = 0". This will cause -// the pipeline to be executed once but will result in no data being -// requested. Therefore when the default implementation is used -// upstream algorithms must fill in the requests further to pull -// data as needed. +TECA_SHARED_OBJECT_FORWARD_DECL(teca_algorithm_executive) + +/* this is a convenience macro to be used to declare a static + * New method that will be used to construct new objects in + * shared_ptr's. This manages the details of interoperability + * with std C++11 shared pointer + */ +#define TECA_ALGORITHM_EXECUTIVE_STATIC_NEW(T) \ + \ + \ +/** Allocate a new T */ \ +static p_##T New() \ +{ \ + return p_##T(new T); \ +} \ + \ +std::shared_ptr shared_from_this() \ +{ \ + return std::static_pointer_cast( \ + teca_algorithm_executive::shared_from_this()); \ +} \ + \ +std::shared_ptr shared_from_this() const \ +{ \ + return std::static_pointer_cast( \ + teca_algorithm_executive::shared_from_this()); \ +} + + + +/// Base class and default implementation for executives. +/** + * Algorithm executives can control pipeline execution by providing + * a series of requests. this allows for the executive to act as a load + * balancer. the executive can for example partition requests across + * spatial data, time steps, or file names. in an MPI parallel + * setting the executive could coordinate this partitioning amongst + * the ranks. However, the only requirement of an algorithm executive + * is that it provide at least one non-empty request. + * + * The default implementation creates a single trivially non-empty + * request containing the key "__request_empty = 0". This will cause + * the pipeline to be executed once but will result in no data being + * requested. Therefore when the default implementation is used + * upstream algorithms must fill in the requests further to pull + * data as needed. + */ class teca_algorithm_executive : public std::enable_shared_from_this { diff --git a/core/teca_algorithm_executive_fwd.h b/core/teca_algorithm_executive_fwd.h deleted file mode 100644 index 599cab2b5..000000000 --- a/core/teca_algorithm_executive_fwd.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef teca_algorithm_executive_fwd_h -#define teca_algorithm_executive_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_algorithm_executive) - -// this is a convenience macro to be used to declare a static -// New method that will be used to construct new objects in -// shared_ptr's. This manages the details of interoperability -// with std C++11 shared pointer -#define TECA_ALGORITHM_EXECUTIVE_STATIC_NEW(T) \ - \ -static p_##T New() \ -{ \ - return p_##T(new T); \ -} \ - \ -std::shared_ptr shared_from_this() \ -{ \ - return std::static_pointer_cast( \ - teca_algorithm_executive::shared_from_this()); \ -} \ - \ -std::shared_ptr shared_from_this() const \ -{ \ - return std::static_pointer_cast( \ - teca_algorithm_executive::shared_from_this()); \ -} - -#endif diff --git a/core/teca_algorithm_fwd.h b/core/teca_algorithm_fwd.h deleted file mode 100644 index ee0a58b21..000000000 --- a/core/teca_algorithm_fwd.h +++ /dev/null @@ -1,173 +0,0 @@ -#ifndef teca_algorithm_fwd_h -#define teca_algorithm_fwd_h - -#include "teca_shared_object.h" - -#include -#include - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_algorithm) - -// this is a convenience macro to be used to declare a static -// New method that will be used to construct new objects in -// shared_ptr's. This manages the details of interoperability -// with std C++11 shared pointer -#define TECA_ALGORITHM_STATIC_NEW(T) \ - \ -static p_##T New() \ -{ \ - return p_##T(new T); \ -} \ - \ -std::shared_ptr shared_from_this() \ -{ \ - return std::static_pointer_cast( \ - teca_algorithm::shared_from_this()); \ -} \ - \ -std::shared_ptr shared_from_this() const \ -{ \ - return std::static_pointer_cast( \ - teca_algorithm::shared_from_this()); \ -} - -#define TECA_ALGORITHM_CLASS_NAME(T) \ -const char *get_class_name() const override \ -{ \ - return #T; \ -} - -// this convenience macro removes copy and aassignment operators -// which generally should not be defined for reference counted types -#define TECA_ALGORITHM_DELETE_COPY_ASSIGN(T) \ - \ - T(const T &src) = delete; \ - T(T &&src) = delete; \ - \ - T &operator=(const T &src) = delete; \ - T &operator=(T &&src) = delete; - -// convenience macro to declare standard set_X/get_X methods -// where X is the name of a class member. will manage the -// algorithm's modified state for the user. -#define TECA_ALGORITHM_PROPERTY(T, NAME) \ - \ -void set_##NAME(const T &v) \ -{ \ - if (this->NAME != v) \ - { \ - this->NAME = v; \ - this->set_modified(); \ - } \ -} \ - \ -const T &get_##NAME() const \ -{ \ - return this->NAME; \ -} - -// convenience macro to declare standard set_X/get_X methods -// where X is the name of a class member. will manage the -// algorithm's modified state for the user. -#define TECA_ALGORITHM_VECTOR_PROPERTY(T, NAME) \ - \ -size_t get_number_of_##NAME##s () \ -{ \ - return this->NAME##s.size(); \ -} \ - \ -void append_##NAME(const T &v) \ -{ \ - this->NAME##s.push_back(v); \ - this->set_modified(); \ -} \ - \ -void set_##NAME(size_t i, const T &v) \ -{ \ - if (this->NAME##s[i] != v) \ - { \ - this->NAME##s[i] = v; \ - this->set_modified(); \ - } \ -} \ - \ -void set_##NAME##s(const std::vector &v) \ -{ \ - if (this->NAME##s != v) \ - { \ - this->NAME##s = v; \ - this->set_modified(); \ - } \ -} \ - \ -void set_##NAME##s(const std::initializer_list &&l) \ -{ \ - std::vector v(l); \ - if (this->NAME##s != v) \ - { \ - this->NAME##s = v; \ - this->set_modified(); \ - } \ -} \ - \ -void set_##NAME##s(const const_p_teca_variant_array &v) \ -{ \ - v->get(this->NAME##s); \ - this->set_modified(); \ -} \ - \ -const T &get_##NAME(size_t i) const \ -{ \ - return this->NAME##s[i]; \ -} \ - \ -void get_##NAME##s(std::vector &v) const \ -{ \ - v = this->NAME##s; \ -} \ - \ -void get_##NAME##s(const p_teca_variant_array &v) const \ -{ \ - v->set(this->NAME##s); \ -} \ - \ -void clear_##NAME##s() \ -{ \ - this->NAME##s.clear(); \ -} - - -// helper that allows us to use std::function -// as a TECA_ALGORITHM_PROPERTY -template -bool operator!=(const std::function &lhs, const std::function &rhs) -{ - return &rhs != &lhs; -} - -// TODO -- this is a work around for older versions -// of Apple clang -// Apple LLVM version 4.2 (clang-425.0.28) (based on LLVM 3.2svn) -// Target: x86_64-apple-darwin12.6.0 -#define TECA_ALGORITHM_CALLBACK_PROPERTY(T, NAME) \ - \ -void set_##NAME(const T &v) \ -{ \ - /*if (this->NAME != v)*/ \ - /*{*/ \ - this->NAME = v; \ - this->set_modified(); \ - /*}*/ \ -} \ - \ -const T &get_##NAME() const \ -{ \ - return this->NAME; \ -} \ - \ -T &get_##NAME() \ -{ \ - return this->NAME; \ -} - -#endif diff --git a/core/teca_algorithm_output_port.h b/core/teca_algorithm_output_port.h index 3e7921148..70cce15f1 100644 --- a/core/teca_algorithm_output_port.h +++ b/core/teca_algorithm_output_port.h @@ -1,15 +1,18 @@ #ifndef teca_algorithm_output_port_h #define teca_algorithm_output_port_h +/// @file + +/// An output port packages an algorithm and a port number using teca_algorithm_output_port = std::pair; -// convenience functions for accessing port and algorithm -// from an output port +/// get the algorithm from the output port inline p_teca_algorithm &get_algorithm(teca_algorithm_output_port &op) { return op.first; } +/// get port number from the output port inline unsigned int &get_port(teca_algorithm_output_port &op) { return op.second; } diff --git a/core/teca_bad_cast.h b/core/teca_bad_cast.h index bf4138c6e..9c8fcf889 100644 --- a/core/teca_bad_cast.h +++ b/core/teca_bad_cast.h @@ -4,6 +4,10 @@ #include #include +/** @brief + * An exception that maybe thrown when a conversion between two data types + * fails. + */ class teca_bad_cast : public std::exception { public: @@ -18,6 +22,9 @@ class teca_bad_cast : public std::exception std::string m_what; }; +/** returns the class name of the teca_algorithm or the string "nullptr" + * if the algorithm is a nullptr. + */ template const std::string safe_class_name(const class_t &o) { diff --git a/core/teca_binary_stream.h b/core/teca_binary_stream.h index dac237602..22225707b 100644 --- a/core/teca_binary_stream.h +++ b/core/teca_binary_stream.h @@ -11,7 +11,7 @@ #include -// Serialize objects into a binary stream. +/// Serialize objects into a binary stream. class teca_binary_stream { public: @@ -35,7 +35,7 @@ class teca_binary_stream // state. void clear() noexcept; - // Alolocate n_bytes for the stream. + // Allocate n_bytes for the stream. void resize(unsigned long n_bytes); // ensures space for n_bytes more to the stream. @@ -53,7 +53,7 @@ class teca_binary_stream unsigned long size() const noexcept { return m_write_p - m_data; } - // Get the sise of the internal buffer allocated + // Get the size of the internal buffer allocated // for the stream. unsigned long capacity() const noexcept { return m_size; } diff --git a/core/calcalcs.cxx b/core/teca_calcalcs.cxx similarity index 99% rename from core/calcalcs.cxx rename to core/teca_calcalcs.cxx index d76612055..1e93e8cad 100644 --- a/core/calcalcs.cxx +++ b/core/teca_calcalcs.cxx @@ -1,4 +1,7 @@ /* +A threadsafe port of the calcalcs library +Burlen Loring Thu Apr 22 06:22:16 PM PDT 2021 + The CalCalcs routines, a set of C-language routines to perform calendar calculations. @@ -35,9 +38,9 @@ along with this program. If not, see . #endif #include "udunits2.h" -#include "calcalcs.h" +#include "teca_calcalcs.h" -namespace calcalcs +namespace teca_calcalcs { static int c_isleap_gregorian ( int year, int *leap ); diff --git a/core/calcalcs.h b/core/teca_calcalcs.h similarity index 92% rename from core/calcalcs.h rename to core/teca_calcalcs.h index 48fa55cad..4c837fb7c 100644 --- a/core/calcalcs.h +++ b/core/teca_calcalcs.h @@ -1,4 +1,7 @@ /* +A threadsafe port of the calcalcs library +Burlen Loring Thu Apr 22 06:22:16 PM PDT 2021 + The CalCalcs routines, a set of C-language routines to perform calendar calculations. @@ -22,11 +25,17 @@ along with this program. If not, see . #ifndef calcalcs_h #define calcalcs_h +/// @file + #define CALCALCS_VERSION_NUMBER 1.0 -namespace calcalcs + +/// A threadsafe port of the calcalcs library. +namespace teca_calcalcs { +/// @cond + struct cccalendar { int sig; char *name; @@ -212,24 +221,22 @@ char *ccs_err_str(int ccs_errno); #define UT_ENOINIT -10 #define UT_EINVALID -11 -/*-------------------------------------------------------------------------- - * high level thread safe initialize the library and select a calendar +/// @endcond + +/** high level thread safe initialize the library and select a calendar * to use in subsequent calls. * return 0 upon success */ int set_current_calendar( const char *calendar, const char *units ); -/*-------------------------------------------------------------------------- - * is_leap_year: determine if the specified year is a leap year in - * the specified calendar. this wraps ccs_isleap such that initialization - * is automatically handled and optimizes for repeat calls. return 0 - * if successful. +/** Determine if the specified year is a leap year in the specified calendar. + * this wraps ccs_isleap such that initialization is automatically handled and + * optimizes for repeat calls. @returns 0 if successful. */ int is_leap_year( const char *calendar, const char *units, int year, int &leap ); -/*-------------------------------------------------------------------------- - * days_in_month: returns the days per month for the given year/month. +/** Returns the days per month for the given year/month. * Note that during the month that transitions from a Julian to a * Gregorian calendar, this might be a strange number of days. this * wraps ccs_dpm such that initialization is automatically handled and @@ -238,8 +245,7 @@ int is_leap_year( const char *calendar, const char *units, int days_in_month( const char *calendar, const char *units, int year, int month, int &dpm ); -/*-------------------------------------------------------------------------- - * date : given a floating point offset in the given calendar return +/** Given a floating point offset in the given calendar return * year, month, day, hour, minute, seconds. returns 0 upon success. */ int date( double val, int *year, int *month, int *day, int *hour, @@ -247,8 +253,7 @@ int date( double val, int *year, int *month, int *day, int *hour, const char *calendar_name ); -/*-------------------------------------------------------------------------- - * given a year, month, day, hour, minute, second and calendar find +/** given a year, month, day, hour, minute, second and calendar find * the floating point offset. returns 0 upon success. */ int coordinate( int year, int month, int day, int hour, int minute, diff --git a/core/teca_calendar.cxx b/core/teca_calendar.cxx deleted file mode 100644 index 5d2cdd5de..000000000 --- a/core/teca_calendar.cxx +++ /dev/null @@ -1,43 +0,0 @@ -#include "teca_calendar.h" - -// -------------------------------------------------------------------------- -long gregorian_number(long y, long m, long d) -{ - m = (m + 9) % 12; - y = y - m/10; - return 365*y + y/4 - y/100 + y/400 + (m*306 + 5)/10 + (d - 1); -} - -// -------------------------------------------------------------------------- -void date_from_gregorian_number(long g, long &y, long &m, long &d) -{ - y = (10000*g + 14780)/3652425; - long ddd = g - (365*y + y/4 - y/100 + y/400); - if (ddd < 0) - { - y = y - 1; - ddd = g - (365*y + y/4 - y/100 + y/400); - } - - long mi = (100*ddd + 52)/3060; - - m = (mi + 2)%12 + 1; - y = y + (mi + 2)/12; - d = ddd - (mi*306 + 5)/10 + 1; -} - -// -------------------------------------------------------------------------- -bool valid_gregorian_date(long y, long m, long d) -{ - long g = gregorian_number(y,m,d); - if (g < 578027) // 578027 = gergorian_number(1582,10,1); - return false; - - long yy, mm, dd; - date_from_gregorian_number(g, yy, mm, dd); - - if ((y != yy) || (m != mm) || (d != dd)) - return false; - - return true; -} diff --git a/core/teca_calendar.h b/core/teca_calendar.h deleted file mode 100644 index 350eb42a7..000000000 --- a/core/teca_calendar.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef teca_calendar_h -#define teca_calendar_h - -// functions for date computations in gregorian calendar. -// to use convert the origin to a gergorian_number do the -// calculation and convert the number back into a date useing -// date_from_gregorian_number. for details about the math and -// an explanation of the errors see -// http://alcor.concordia.ca/~gpkatch/gdate-algorithm.html -// y -- 4 digit year -// m -- 2 digit month -// d -- 2 digit day -// returns a date number for the given date that can be used -// in computations -long gregorian_number(long y, long m, long d); - -// input -// g -- date number computed from gregorian_number -// returns -// y -- 4 digit year -// m -- 2 digit month -// d -- 2 digit day -// in the values provided -void date_from_gregorian_number(long g, long &y, long &m, long &d); - -// return true if the date is valid in the gregorian -// calendar and our conversion algorithm. -bool valid_gregorian_date(long y, long m, long d); - -#endif diff --git a/core/teca_common.cxx b/core/teca_common.cxx index 52baaccef..b3c9f0dc2 100644 --- a/core/teca_common.cxx +++ b/core/teca_common.cxx @@ -1,5 +1,7 @@ #include "teca_common.h" +namespace std +{ // ************************************************************************** std::ostream &operator<<(std::ostream &os, const std::vector &vec) { @@ -12,6 +14,7 @@ std::ostream &operator<<(std::ostream &os, const std::vector &vec) } return os; } +} // ************************************************************************** int have_tty() diff --git a/core/teca_common.h b/core/teca_common.h index 61d06afcf..04e0c77cf 100644 --- a/core/teca_common.h +++ b/core/teca_common.h @@ -3,13 +3,18 @@ #include "teca_config.h" #include "teca_parallel_id.h" + #include #include #include #include #include -// send a vector to a stream +// the operator<< overloads have to be namespace std in order for +// boost to find them. they are needed for mutitoken program options +namespace std +{ +/// send a vector to a stream template std::ostream &operator<<(std::ostream &os, const std::vector &vec) { @@ -23,11 +28,26 @@ std::ostream &operator<<(std::ostream &os, const std::vector &vec) return os; } -// send a vector of strings to a stream +/// send a vector of strings to a stream std::ostream &operator<<(std::ostream &os, const std::vector &vec); +} + +#ifndef SWIG +/// send a fixed length c-array to the stream +template ::value,bool>::type> +std::ostream &operator<<(std::ostream &os, const num_t (& data)[len]) +{ + os << data[0]; + for (int i = 1; i < len; ++i) + os << ", " << data[i]; + return os; +} +#endif -// detect if we are writing to a tty, if not then -// we should not use ansi color codes +/** Return true if we are writing to a TTY. If we are not then we should not + * use ansi color codes. + */ int have_tty(); #define ANSI_RED "\033[1;31;40m" diff --git a/core/teca_dataset.h b/core/teca_dataset.h index 3f5b167a0..b8db4f646 100644 --- a/core/teca_dataset.h +++ b/core/teca_dataset.h @@ -1,21 +1,224 @@ #ifndef teca_dataset_h #define teca_dataset_h +#include "teca_common.h" +#include "teca_shared_object.h" #include "teca_variant_array.h" -#include "teca_dataset_fwd.h" + +#include #include + class teca_binary_stream; class teca_metadata; -/** -interface for teca datasets. -*/ +TECA_SHARED_OBJECT_FORWARD_DECL(teca_dataset) + +// this is a convenience macro to be used to +// declare New and enable seamless operation +// with std C++11 shared pointer +#define TECA_DATASET_STATIC_NEW(T) \ + \ +static p_##T New() \ +{ \ + return p_##T(new T); \ +} \ + \ +std::shared_ptr shared_from_this() \ +{ \ + return std::static_pointer_cast( \ + teca_dataset::shared_from_this()); \ +} \ + \ +std::shared_ptr shared_from_this() const \ +{ \ + return std::static_pointer_cast( \ + teca_dataset::shared_from_this()); \ +} + +// convenience macro implementing new_instance method +#define TECA_DATASET_NEW_INSTANCE() \ +virtual p_teca_dataset new_instance() const override\ +{ \ + return this->New(); \ +} + +// convenience macro implementing new_copy method +#define TECA_DATASET_NEW_COPY() \ +virtual p_teca_dataset new_copy() const override \ +{ \ + p_teca_dataset o = this->new_instance(); \ + o->copy(this->shared_from_this()); \ + return o; \ +} \ + \ +virtual p_teca_dataset new_shallow_copy() override \ +{ \ + p_teca_dataset o = this->new_instance(); \ + o->shallow_copy(this->shared_from_this()); \ + return o; \ +} + +// convenience macro for adding properties to dataset +// objects +#define TECA_DATASET_PROPERTY(T, name) \ + \ +void set_##name(const T &val) \ +{ \ + this->name = val; \ +} \ + \ +const T &get_##name() const \ +{ \ + return this->name; \ +} \ + \ +T &get_##name() \ +{ \ + return this->name; \ +} + +// convenience set get methods for dataset metadata +#define TECA_DATASET_METADATA(key, T, len) \ +TECA_DATASET_METADATA_V(T, key, len) \ +TECA_DATASET_METADATA_A(T, key, len) \ +TECA_DATASET_METADATA_ ## len (T, key) + + +#define TECA_DATASET_METADATA_1(T, key) \ +void set_##key(const T & val_1) \ +{ \ + this->get_metadata().set(#key, val_1); \ +} \ + \ +int get_##key(T &val_1) const \ +{ \ + return this->get_metadata().get( \ + #key, val_1); \ +} + +#define TECA_DATASET_METADATA_2(T, key) \ +void set_##key(const T & val_1, const T & val_2) \ +{ \ + this->get_metadata().set( \ + #key, {val_1, val_2}); \ +} \ + \ +int get_##key(T &val_1, T &val_2) const \ +{ \ + std::vector vals; \ + if (this->get_metadata().get(#key, vals)) \ + return -1; \ + val_1 = vals[0]; \ + val_2 = vals[1]; \ + return 0; \ +} + +#define TECA_DATASET_METADATA_3(T, key) \ +void set_##key(const T & val_1, const T & val_2, \ + const T & val_3) \ +{ \ + this->get_metadata().set(#key, \ + {val_1, val_2, val_3}); \ +} \ + \ +int get_##key(T &val_1, T &val_2, T &val_3) const \ +{ \ + std::vector vals; \ + if (this->get_metadata().get(#key, vals)) \ + return -1; \ + val_1 = vals[0]; \ + val_2 = vals[1]; \ + val_3 = vals[2]; \ + return 0; \ +} + +#define TECA_DATASET_METADATA_4(T, key) \ +void set_##key(const T & val_1, const T & val_2, \ + const T & val_3, const T & val_4) \ +{ \ + this->get_metadata().set(#key, \ + {val_1, val_2, val_3, val_4}); \ +} + +#define TECA_DATASET_METADATA_6(T, key) \ +void set_##key(const T & val_1, const T & val_2, \ + const T & val_3, const T & val_4, \ + const T & val_5, const T & val_6) \ +{ \ + this->get_metadata().set(#key, \ + {val_1, val_2, val_3, \ + val_4, val_5, val_6}); \ +} + +#define TECA_DATASET_METADATA_8(T, key) \ +void set_##key(const T & val_1, const T & val_2, \ + const T & val_3, const T & val_4, \ + const T & val_5, const T & val_6, \ + const T & val_7, const T & val_8) \ +{ \ + this->get_metadata().set(#key, \ + {val_1, val_2, val_3, val_4, val_5, \ + val_6, val_7, val_8}); \ +} + +#define TECA_DATASET_METADATA_V(T, key, len) \ +void set_##key(const std::vector &vals) \ +{ \ + if (vals.size() != len) \ + { \ + TECA_ERROR(#key " requires " #len " values") \ + } \ + this->get_metadata().set(#key, vals); \ +} \ + \ +int get_##key(std::vector &vals) const \ +{ \ + return this->get_metadata().get(#key, vals); \ +} \ + \ +void set_##key(const p_teca_variant_array &vals) \ +{ \ + if (vals->size() != len) \ + { \ + TECA_ERROR(#key " requires " #len " values") \ + } \ + this->get_metadata().set(#key, vals); \ +} \ + \ +int get_##key(p_teca_variant_array vals) const \ +{ \ + return this->get_metadata().get(#key, vals); \ +} \ + \ +void set_##key(const std::initializer_list &l) \ +{ \ + std::vector vals(l); \ + if (vals.size() != len) \ + { \ + TECA_ERROR(#key " requires " #len " values") \ + } \ + this->get_metadata().set(#key, vals); \ +} \ + +#define TECA_DATASET_METADATA_A(T, key, len) \ +void set_##key(const T *vals) \ +{ \ + this->get_metadata().set(#key, vals, len); \ +} \ + \ +int get_##key(T *vals) const \ +{ \ + return this->get_metadata().get( \ + #key, vals, len); \ +} + +/// Interface for TECA datasets. class teca_dataset : public std::enable_shared_from_this { public: virtual ~teca_dataset(); - // the name of the key that holds the index identifing this dataset + // the name of the key that holds the index identifying this dataset // this should be set by the algorithm that creates the dataset. TECA_DATASET_METADATA(index_request_key, std::string, 1) @@ -26,7 +229,7 @@ class teca_dataset : public std::enable_shared_from_this virtual int set_request_index(const std::string &key, long val); virtual int set_request_index(long val); - // covert to bool. true if the dataset is not empty. + // covert to boolean. true if the dataset is not empty. // otherwise false. explicit operator bool() const noexcept { return !this->empty(); } diff --git a/core/teca_dataset_capture.h b/core/teca_dataset_capture.h index 04015cd26..52e8007c0 100644 --- a/core/teca_dataset_capture.h +++ b/core/teca_dataset_capture.h @@ -3,17 +3,19 @@ #include "teca_algorithm.h" #include "teca_metadata.h" -#include "teca_dataset_fwd.h" +#include "teca_dataset.h" #include "teca_shared_object.h" TECA_SHARED_OBJECT_FORWARD_DECL(teca_dataset_capture) -/** -An algorithm that takes a reference to dataset produced -by the upstream algorithm it is connected to. The dataset -is passed through so that this can be inserted anywhere -giving one access to the intermediate data. -*/ +/** @brief + * An algorithm that takes a reference to dataset produced + * by the upstream algorithm it is connected to. + * + * @details + * The dataset is passed through so that this can be inserted + * anywhere giving one access to the intermediate data. + */ class teca_dataset_capture : public teca_algorithm { public: diff --git a/core/teca_dataset_fwd.h b/core/teca_dataset_fwd.h deleted file mode 100644 index 46fd125d9..000000000 --- a/core/teca_dataset_fwd.h +++ /dev/null @@ -1,210 +0,0 @@ -#ifndef teca_dataset_fwd_h -#define teca_dataset_fwd_h - -#include "teca_common.h" -#include "teca_shared_object.h" - -#include - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_dataset) - -// this is a convenience macro to be used to -// declare New and enable seamless operation -// with std C++11 shared pointer -#define TECA_DATASET_STATIC_NEW(T) \ - \ -static p_##T New() \ -{ \ - return p_##T(new T); \ -} \ - \ -std::shared_ptr shared_from_this() \ -{ \ - return std::static_pointer_cast( \ - teca_dataset::shared_from_this()); \ -} \ - \ -std::shared_ptr shared_from_this() const \ -{ \ - return std::static_pointer_cast( \ - teca_dataset::shared_from_this()); \ -} - -// convenience macro implementing new_instance method -#define TECA_DATASET_NEW_INSTANCE() \ -virtual p_teca_dataset new_instance() const override\ -{ \ - return this->New(); \ -} - -// convenience macro implementing new_copy method -#define TECA_DATASET_NEW_COPY() \ -virtual p_teca_dataset new_copy() const override \ -{ \ - p_teca_dataset o = this->new_instance(); \ - o->copy(this->shared_from_this()); \ - return o; \ -} \ - \ -virtual p_teca_dataset new_shallow_copy() override \ -{ \ - p_teca_dataset o = this->new_instance(); \ - o->shallow_copy(this->shared_from_this()); \ - return o; \ -} - -// convenience macro for adding properties to dataset -// objects -#define TECA_DATASET_PROPERTY(T, name) \ - \ -void set_##name(const T &val) \ -{ \ - this->name = val; \ -} \ - \ -const T &get_##name() const \ -{ \ - return this->name; \ -} \ - \ -T &get_##name() \ -{ \ - return this->name; \ -} - -// convenience set get methods for dataset metadata -#define TECA_DATASET_METADATA(key, T, len) \ -TECA_DATASET_METADATA_V(T, key, len) \ -TECA_DATASET_METADATA_A(T, key, len) \ -TECA_DATASET_METADATA_ ## len (T, key) - - -#define TECA_DATASET_METADATA_1(T, key) \ -void set_##key(const T & val_1) \ -{ \ - this->get_metadata().set(#key, val_1); \ -} \ - \ -int get_##key(T &val_1) const \ -{ \ - return this->get_metadata().get( \ - #key, val_1); \ -} - -#define TECA_DATASET_METADATA_2(T, key) \ -void set_##key(const T & val_1, const T & val_2) \ -{ \ - this->get_metadata().set( \ - #key, {val_1, val_2}); \ -} \ - \ -int get_##key(T &val_1, T &val_2) const \ -{ \ - std::vector vals; \ - if (this->get_metadata().get(#key, vals)) \ - return -1; \ - val_1 = vals[0]; \ - val_2 = vals[1]; \ - return 0; \ -} - -#define TECA_DATASET_METADATA_3(T, key) \ -void set_##key(const T & val_1, const T & val_2, \ - const T & val_3) \ -{ \ - this->get_metadata().set(#key, \ - {val_1, val_2, val_3}); \ -} \ - \ -int get_##key(T &val_1, T &val_2, T &val_3) const \ -{ \ - std::vector vals; \ - if (this->get_metadata().get(#key, vals)) \ - return -1; \ - val_1 = vals[0]; \ - val_2 = vals[1]; \ - val_3 = vals[2]; \ - return 0; \ -} - -#define TECA_DATASET_METADATA_4(T, key) \ -void set_##key(const T & val_1, const T & val_2, \ - const T & val_3, const T & val_4) \ -{ \ - this->get_metadata().set(#key, \ - {val_1, val_2, val_3, val_4}); \ -} - -#define TECA_DATASET_METADATA_6(T, key) \ -void set_##key(const T & val_1, const T & val_2, \ - const T & val_3, const T & val_4, \ - const T & val_5, const T & val_6) \ -{ \ - this->get_metadata().set(#key, \ - {val_1, val_2, val_3, \ - val_4, val_5, val_6}); \ -} - -#define TECA_DATASET_METADATA_8(T, key) \ -void set_##key(const T & val_1, const T & val_2, \ - const T & val_3, const T & val_4, \ - const T & val_5, const T & val_6, \ - const T & val_7, const T & val_8) \ -{ \ - this->get_metadata().set(#key, \ - {val_1, val_2, val_3, val_4, val_5, \ - val_6, val_7, val_8}); \ -} - -#define TECA_DATASET_METADATA_V(T, key, len) \ -void set_##key(const std::vector &vals) \ -{ \ - if (vals.size() != len) \ - { \ - TECA_ERROR(#key " requires " #len " values") \ - } \ - this->get_metadata().set(#key, vals); \ -} \ - \ -int get_##key(std::vector &vals) const \ -{ \ - return this->get_metadata().get(#key, vals); \ -} \ - \ -void set_##key(const p_teca_variant_array &vals) \ -{ \ - if (vals->size() != len) \ - { \ - TECA_ERROR(#key " requires " #len " values") \ - } \ - this->get_metadata().set(#key, vals); \ -} \ - \ -int get_##key(p_teca_variant_array vals) const \ -{ \ - return this->get_metadata().get(#key, vals); \ -} \ - \ -void set_##key(const std::initializer_list &l) \ -{ \ - std::vector vals(l); \ - if (vals.size() != len) \ - { \ - TECA_ERROR(#key " requires " #len " values") \ - } \ - this->get_metadata().set(#key, vals); \ -} \ - -#define TECA_DATASET_METADATA_A(T, key, len) \ -void set_##key(const T *vals) \ -{ \ - this->get_metadata().set(#key, vals, len); \ -} \ - \ -int get_##key(T *vals) const \ -{ \ - return this->get_metadata().get( \ - #key, vals, len); \ -} - -#endif diff --git a/core/teca_dataset_source.h b/core/teca_dataset_source.h index 7269633f9..b7f96cf6d 100644 --- a/core/teca_dataset_source.h +++ b/core/teca_dataset_source.h @@ -3,18 +3,18 @@ #include "teca_algorithm.h" #include "teca_metadata.h" -#include "teca_dataset_fwd.h" +#include "teca_dataset.h" #include "teca_shared_object.h" #include TECA_SHARED_OBJECT_FORWARD_DECL(teca_dataset_source) +/// An algorithm that serves up user provided data and metadata. /** -An algorithm that serves up user provided data and metadata. -This algorithm can be used to inject a dataset constructed -on outside of TECA into a TECA pipleine. -*/ + * This algorithm can be used to inject a dataset constructed + * on outside of TECA into a TECA pipeline. + */ class teca_dataset_source : public teca_algorithm { public: diff --git a/core/teca_index_executive.cxx b/core/teca_index_executive.cxx index d864cec5a..ec83dd3f1 100644 --- a/core/teca_index_executive.cxx +++ b/core/teca_index_executive.cxx @@ -166,6 +166,7 @@ int teca_index_executive::initialize(MPI_Comm comm, const teca_metadata &md) if ((index % this->stride) == 0) { this->requests.push_back(base_req); + this->requests.back().set("index_request_key", this->index_request_key); this->requests.back().set(this->index_request_key, index); } } diff --git a/core/teca_index_executive.h b/core/teca_index_executive.h index 7a8c8b290..8c70d9ece 100644 --- a/core/teca_index_executive.h +++ b/core/teca_index_executive.h @@ -10,27 +10,23 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_index_executive) -/// -/** -An executive that generates requests using a upstream -or user defined index. an extent and list of arrays -can be optionally set. - -meta data keys: - - requires: - - index_initializer_key -- holds the name of the key that tells how - many indices are available. the named key - must also be present and should conatin the - number of indices available - - index_request_key -- holds the name of the key used to request - a specific index. request are generated with this - name set to a specific index to be processed some - upstream algorithm is expected to produce the data - associated with the given index - +/// An executive that generates requests using a upstream or user defined index. +/** An extent or bounds to subset by, and list of arrays can be optionally set. + * + * metadata keys: + * + * requires: + * + * index_initializer_key -- holds the name of the key that tells how + * many indices are available. the named key + * must also be present and should contain the + * number of indices available + * + * index_request_key -- holds the name of the key used to request + * a specific index. request are generated with this + * name set to a specific index to be processed some + * upstream algorithm is expected to produce the + * data associated with the given index */ class teca_index_executive : public teca_algorithm_executive { @@ -40,33 +36,33 @@ class teca_index_executive : public teca_algorithm_executive int initialize(MPI_Comm comm, const teca_metadata &md) override; teca_metadata get_next_request() override; - // set the index to process + /// set the index to process void set_index(long s); - // set the first time step in the series to process. - // default is 0. + // Set the first time step in the series to process. The default is 0. void set_start_index(long s); - // set the last time step in the series to process. - // default is -1. negative number results in the last - // available time step being used. + /** Set the last time step in the series to process. default is -1. + * negative number results in the last available time step being used. + */ void set_end_index(long s); - // set the stride to process time steps at. default - // is 1 + /// Set the stride to process time steps at. The default is 1 void set_stride(long s); - // set the extent to process. the default is the - // whole_extent. + /// Set the extent to process. The default is taken from whole_extent key. void set_extent(unsigned long *ext); + + /// @copydoc set_extent void set_extent(const std::vector &ext); - // set the bounds to process. If nothging is set then - // extent as provided by set_extent is used. + /** Set the bounds to process. If nothing is set then extent as provided by + * set_extent is used. + */ void set_bounds(double *bounds); void set_bounds(const std::vector &bounds); - // set the list of arrays to process + /// Set the list of arrays to process void set_arrays(const std::vector &arrays); protected: diff --git a/core/teca_index_reduce.cxx b/core/teca_index_reduce.cxx index 741ec2c11..de17ad266 100644 --- a/core/teca_index_reduce.cxx +++ b/core/teca_index_reduce.cxx @@ -138,9 +138,9 @@ void teca_index_reduce::get_properties_description(const std::string &prefix, + (prefix.empty()?"teca_index_reduce":prefix)); opts.add_options() - TECA_POPTS_GET(long, prefix, start_index, "first index to process (0)") + TECA_POPTS_GET(long, prefix, start_index, "first index to process") TECA_POPTS_GET(long, prefix, end_index, "last index to process. " - "If set to -1 all indices are processed. (-1)") + "If set to -1 all indices are processed.") ; global_opts.add(opts); diff --git a/core/teca_index_reduce.h b/core/teca_index_reduce.h index 818cb282c..85dae1786 100644 --- a/core/teca_index_reduce.h +++ b/core/teca_index_reduce.h @@ -1,38 +1,40 @@ #ifndef teca_index_reduce_h #define teca_index_reduce_h -#include "teca_dataset_fwd.h" -#include "teca_index_reduce_fwd.h" - -#include "teca_threaded_algorithm.h" +#include "teca_dataset.h" #include "teca_metadata.h" +#include "teca_shared_object.h" +#include "teca_threaded_algorithm.h" #include -// base class for MPI + threads map reduce reduction over an index. the available -// indices are partitioned across MPI ranks and threads. one can restrict -// operation to a range of time steps by setting first and last indeces to -// process. -// -// meta data keys: -// -// requires: -// -// index_initializer_key -- holds the name of the key that tells how -// many indices are available. the named key -// must also be present and should conatin the -// number of indices available -// -// index_request_key -- holds the name of the key used to request -// a specific index. request are generated with this -// name set to a specific index to be processed some -// upstream algorithm is expected to produce the data -// associated with the given index -// -// consumes: -// -// the key named by index_request_key -// +TECA_SHARED_OBJECT_FORWARD_DECL(teca_index_reduce) + +/// Base class for MPI + threads map reduce reduction over an index. +/** + * The available indices are partitioned across MPI ranks and threads. One can + * restrict operation to a range of time steps by setting first and last + * indices to process. + * + * metadata keys: + * + * requires: + * + * index_initializer_key -- holds the name of the key that tells how + * many indices are available. the named key + * must also be present and should contain the + * number of indices available + * + * index_request_key -- holds the name of the key used to request + * a specific index. request are generated with this + * name set to a specific index to be processed some + * upstream algorithm is expected to produce the + * data associated with the given index + * + * consumes: + * + * The key named by index_request_key + */ class teca_index_reduce : public teca_threaded_algorithm { public: diff --git a/core/teca_index_reduce_fwd.h b/core/teca_index_reduce_fwd.h deleted file mode 100644 index 6f977040a..000000000 --- a/core/teca_index_reduce_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_index_reduce_fwd_h -#define teca_index_reduce_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_index_reduce) - -#endif diff --git a/core/teca_memory_profiler.h b/core/teca_memory_profiler.h index 911f0eea3..4fe4c3ebc 100644 --- a/core/teca_memory_profiler.h +++ b/core/teca_memory_profiler.h @@ -6,14 +6,14 @@ extern "C" void *profile(void *argp); -// MemoryProfiler - A sampling memory use profiler +/// MemoryProfiler - A sampling memory use profiler. /** -The class samples process memory usage at the specified interval -given in seconds. For each sample the time is aquired. Calling -Initialize starts profiling, and Finalize ends it. During -Finaliziation the buffers are written using MPI-I/O to the -file name provided -*/ + * The class samples process memory usage at the specified interval + * given in seconds. For each sample the time is acquired. Calling + * Initialize starts profiling, and Finalize ends it. During + * Finalization the buffers are written using MPI-I/O to the + * file name provided. + */ class teca_memory_profiler { public: @@ -27,12 +27,12 @@ class teca_memory_profiler int initialize(); int finalize(); - // Set the interval in seconds between querrying + // Set the interval in seconds between querying // the processes memory use. void set_interval(double interval); double get_interval() const; - // Set the comunicator for parallel I/O + // Set the communicator for parallel I/O void set_communicator(MPI_Comm comm); // Set the file name to write the data to diff --git a/core/teca_metadata.h b/core/teca_metadata.h index 2168aae12..4272af62a 100644 --- a/core/teca_metadata.h +++ b/core/teca_metadata.h @@ -9,10 +9,12 @@ #include #include "teca_variant_array.h" -// a generic container for meta data in the form -// of name=value pairs. value arrays are supported. -// see meta data producer-consumer documentation for -// information about what names are valid. +/// A generic container for meta data in the form of name=value pairs. +/** + * Value arrays are supported. See metadata + * producer-consumer documentation for + * information about what names are valid. + */ class teca_metadata { public: @@ -129,6 +131,10 @@ class teca_metadata int get(const std::string &name, T *val, unsigned int n) const; + template + int get(const std::string &name, T (&val)[N]) const + { return this->get(name, val, N); } + // copy prop values from the named prop into the passed in vector. // return 0 if successful template @@ -177,7 +183,7 @@ class teca_metadata int to_stream(teca_binary_stream &s) const; int from_stream(teca_binary_stream &s); - // serialize to/from ascii + // serialize to/from ASCII int to_stream(std::ostream &os) const; int from_stream(std::ostream &) { return -1; } diff --git a/core/teca_metadata_util.cxx b/core/teca_metadata_util.cxx index 8ac1498ef..9c27bda5a 100644 --- a/core/teca_metadata_util.cxx +++ b/core/teca_metadata_util.cxx @@ -1,33 +1,37 @@ #include "teca_metadata_util.h" +#include "teca_metadata.h" +#include "teca_common.h" + namespace teca_metadata_util { -// remove post-fix from the arrays in get_upstream_request if -// the post-fix is set. For example if post-fix is set to "_filtered" -// then we remove all the variables in the "arrays" set that end with -// this post-fix, and replace it with the actual requested array. -void remove_post_fix(std::set &arrays, std::string post_fix) + +// ************************************************************************** +int get_array_extent(const teca_metadata &array_attributes, + const unsigned long mesh_extent[6], unsigned long array_extent[6]) { - size_t postfix_len = post_fix.length(); + for (int i = 0; i < 6; ++i) + array_extent[i] = mesh_extent[i]; - std::set::iterator arrays_it; - for (arrays_it=arrays.begin(); arrays_it!=arrays.end(); ++arrays_it) + unsigned long dim_active[4] = {0ul}; + if (array_attributes.get("mesh_dim_active", dim_active, 4)) { - std::string array_var = *arrays_it; - size_t array_var_len = array_var.length(); - - if (array_var_len > postfix_len) - { - size_t postfix_pos = array_var.find(post_fix, - array_var_len - postfix_len); - if (postfix_pos != std::string::npos) - { - array_var.erase(array_var_len - postfix_len, postfix_len); - - arrays.erase(arrays_it); - arrays.insert(array_var); - } - } + //TECA_ERROR("metadata issue. The array attributes collection is" + // " missing the mesh_dim_active key") + return -1; } + + // make the extent 1 in any direction that this array is undefined in + if (!dim_active[0]) + array_extent[1] = array_extent[0] = 0; + + if (!dim_active[1]) + array_extent[3] = array_extent[2] = 0; + + if (!dim_active[2]) + array_extent[5] = array_extent[4] = 0; + + return 0; } -}; \ No newline at end of file + +}; diff --git a/core/teca_metadata_util.h b/core/teca_metadata_util.h index 14213c971..ab3a61159 100644 --- a/core/teca_metadata_util.h +++ b/core/teca_metadata_util.h @@ -1,15 +1,21 @@ #ifndef teca_metadata_util_h #define teca_metadata_util_h -#include -#include +/// @file +class teca_metadata; + +/// Codes for dealing with teca_metadata namespace teca_metadata_util { -// given a set of names, where names end with a common string, here called -// a post-fix, modifies the set of names by removing the post fix from each -// name. -void remove_post_fix(std::set &names, std::string post_fix); - +/** Given a collection of array attributes (following the conventions used by + * the teca_cf_reader) and a mesh extent, compute and return the valid extent + * of the array. This takes into account 1d and 2d arrays on a 3d mesh. Return + * zero if successful. The mesh_dims_active key is required, if not found 1 + * is returned and the array_extent is set to the mesh_extent. + */ +int get_array_extent(const teca_metadata &array_attributes, + const unsigned long mesh_extent[6], unsigned long array_extent[6]); }; + #endif diff --git a/core/teca_mpi_util.cxx b/core/teca_mpi_util.cxx index 8e9673dfd..df67b6ab1 100644 --- a/core/teca_mpi_util.cxx +++ b/core/teca_mpi_util.cxx @@ -22,7 +22,7 @@ int equipartition_communicator(MPI_Comm comm, if (n_ranks < new_comm_size) { - // can't increase beyond the original sizew + // can't increase beyond the original size return 0; } @@ -51,4 +51,56 @@ int equipartition_communicator(MPI_Comm comm, #endif return 0; } + +// ************************************************************************** +int split_communicator(MPI_Comm world_comm, + int group_size, MPI_Comm *group_comm) +{ +#if defined(TECA_HAS_MPI) + int is_init = 0; + MPI_Initialized(&is_init); + if (is_init) + { + int world_rank = 0; + int world_size = 1; + + MPI_Comm_rank(world_comm, &world_rank); + MPI_Comm_size(world_comm, &world_size); + + MPI_Group world_group = MPI_GROUP_EMPTY; + MPI_Comm_group(world_comm, &world_group); + + int group_id = world_rank / group_size; + int group_start = group_id * group_size; + int group_end = std::min(world_size, group_start + group_size); + int group_range[3] = {group_start, group_end, 1}; + + MPI_Group sub_group = MPI_GROUP_EMPTY; + MPI_Group_range_incl(world_group, 1, &group_range, &sub_group); + + MPI_Comm_create(world_comm, sub_group, group_comm); + + MPI_Group_free(&world_group); + MPI_Group_free(&sub_group); + } +#endif + return 0; +} + +// ************************************************************************** +int mpi_rank_0(MPI_Comm comm) +{ + int rank = 0; +#if defined(TECA_HAS_MPI) + int is_init = 0; + MPI_Initialized(&is_init); + if (is_init) + { + MPI_Comm_rank(comm, &rank); + } +#endif + if (rank == 0) + return 1; + return 0; +} } diff --git a/core/teca_mpi_util.h b/core/teca_mpi_util.h index 2b22c5b89..ccaa717c5 100644 --- a/core/teca_mpi_util.h +++ b/core/teca_mpi_util.h @@ -1,16 +1,28 @@ #ifndef teca_mpi_util_h #define teca_mpi_util_h +/// @file + #include "teca_mpi.h" +/// Codes dealing with MPI namespace teca_mpi_util { -// subset the the communicator comm into a new communicator with -// new_comm_size ranks. ranks are selected from comm with a uniform -// stride spreading them approximatelyt equally across nodes. +/** Subset the the communicator comm into a new communicator with new_comm_size + * ranks. ranks are selected from comm with a uniform stride spreading them + * approximatelyt equally across nodes. + */ int equipartition_communicator(MPI_Comm comm, int new_comm_size, MPI_Comm *new_comm); +/** Split the communicator into a number of new communicators such that each + * new communicator has group_size ranks. + */ +int split_communicator(MPI_Comm comm, + int group_size, MPI_Comm *new_comm); + +/// return non-zero if this process is MPI rank 0 +int mpi_rank_0(MPI_Comm comm); }; #endif diff --git a/core/teca_output_port.h b/core/teca_output_port.h deleted file mode 100644 index 6afca8ffb..000000000 --- a/core/teca_output_port.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef teca_output_port_h -#define teca_output_port_h - -class teca_output_port -{ -public: - teca_output_port(int port_id, - -private: -} - -#endif diff --git a/core/teca_parallel_id.h b/core/teca_parallel_id.h index 05e70549a..cbaab317d 100644 --- a/core/teca_parallel_id.h +++ b/core/teca_parallel_id.h @@ -1,13 +1,15 @@ #ifndef teca_parallel_id_h #define teca_parallel_id_h +/// @file + #include -// a helper class for debug and error messages +/// A helper class for debug and error messages. class teca_parallel_id {}; -// print the callers rank and thread id to the given stream. this is a +// Prints the callers rank and thread id to the given stream. This is a // debug/diagnostic message and hence rank will always be reported relative to // the WORLD communicator. std::ostream &operator<<( diff --git a/core/teca_profiler.h b/core/teca_profiler.h index 670b960a8..b491e87b4 100644 --- a/core/teca_profiler.h +++ b/core/teca_profiler.h @@ -9,9 +9,11 @@ #include -// A class containing methods managing memory and time profiling -// Each timed event logs rank, event name, start and end time, and -// duration. +/// A class containing methods managing memory and time profiling. +/** + * Each timed event logs rank, event name, start and end time, and + * duration. + */ class teca_profiler { public: @@ -37,7 +39,7 @@ class teca_profiler static int finalize(); // this can occur after MPI_Finalize. It should only be called by rank 0. - // Any remaining events will be appeneded to the log file. This is necessary + // Any remaining events will be appended to the log file. This is necessary // to time MPI_Initialize/Finalize and log associated I/O. static int flush(); @@ -47,26 +49,26 @@ class teca_profiler static void set_communicator(MPI_Comm comm); // Sets the path to write the timer log to - // overriden by PROFILER_LOG_FILE environment variable + // overridden by PROFILER_LOG_FILE environment variable // default value; Timer.csv static void set_timer_log_file(const std::string &file_name); // Sets the path to write the timer log to - // overriden by MEMPROF_LOG_FILE environment variable + // overridden by MEMPROF_LOG_FILE environment variable // default value: MemProfLog.csv static void set_mem_prof_log_file(const std::string &file_name); // Sets the number of seconds in between memory use recordings - // overriden by MEMPROF_INTERVAL environment variable. + // overridden by MEMPROF_INTERVAL environment variable. static void set_mem_prof_interval(int interval); - // Enable/Disable logging. Overriden by PROFILER_ENABLE environment + // Enable/Disable logging. Overridden by PROFILER_ENABLE environment // variable. In the default format a CSV file is generated capturing each // ranks timer events. default value: disabled static void enable(int arg = 0x03); static void disable(); - // return true if loggin is enabled. + // return true if logging is enabled. static bool enabled(); // @brief Log start of an event. @@ -99,10 +101,12 @@ class teca_profiler static int to_stream(std::ostream &os); }; -// teca_time_event -- A helper class that times it's life. -// A timer event is created that starts at the object's construction and ends -// at its destruction. The pointer to the event name must be valid throughout -// the objects life. +/// A helper class that times it's life. +/** + * A timer event is created that starts at the object's construction and ends + * at its destruction. The pointer to the event name must be valid throughout + * the objects life. + */ template class teca_time_event { diff --git a/core/teca_program_options.h b/core/teca_program_options.h index be4340fe2..83a9f1771 100644 --- a/core/teca_program_options.h +++ b/core/teca_program_options.h @@ -1,7 +1,11 @@ #ifndef teca_program_options_h #define teca_program_options_h +/// @file + #include "teca_config.h" +#include "teca_common.h" +#include "teca_mpi_util.h" #if defined(TECA_HAS_BOOST) && !defined(SWIG) namespace boost @@ -13,25 +17,21 @@ namespace boost } }; -using options_description - = boost::program_options::options_description; - -using variables_map - = boost::program_options::variables_map; +using options_description = boost::program_options::options_description; +using variables_map = boost::program_options::variables_map; -// initialize the given options description -// with algorithm's properties -#define TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() \ - void get_properties_description( \ - const std::string &prefix, \ - options_description &opts) override; \ +/// initialize the given options description with algorithm's properties +#define TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() \ + \ + /** Adds the class algorithm properties to the description object */ \ + void get_properties_description(const std::string &prefix, \ + boost::program_options::options_description &opts) override; \ -// initialize the algorithm from the given options -// variable map. -#define TECA_SET_ALGORITHM_PROPERTIES() \ - void set_properties( \ - const std::string &prefix, \ - variables_map &opts) override; \ +/// initialize the algorithm from the given options variable map. +#define TECA_SET_ALGORITHM_PROPERTIES() \ + /** Sets the class algorithm properties from the map object */ \ + void set_properties(const std::string &prefix, \ + boost::program_options::variables_map &opts) override; \ // helpers for implementation dealing with Boost // program options. NOTE: because the above declarations @@ -40,21 +40,30 @@ using variables_map // . These need to be // included in your cxx files. // -#define TECA_POPTS_GET(_type, _prefix, _name, _desc) \ - (((_prefix.empty()?"":_prefix+"::") + #_name).c_str(), \ - boost::program_options::value<_type>(), "\n" _desc "\n") - -#define TECA_POPTS_MULTI_GET(_type, _prefix, _name, _desc) \ +#define TECA_POPTS_GET(_type, _prefix, _name, _desc) \ (((_prefix.empty()?"":_prefix+"::") + #_name).c_str(), \ - boost::program_options::value<_type>()->multitoken(), \ + boost::program_options::value<_type>()->default_value \ + (this->get_ ## _name()), "\n" _desc "\n") + +#define TECA_POPTS_MULTI_GET(_type, _prefix, _name, _desc) \ + (((_prefix.empty()?"":_prefix+"::") + #_name).c_str(), \ + boost::program_options::value<_type>()->multitoken \ + ()->default_value(this->get_ ## _name()), \ "\n" _desc "\n") -#define TECA_POPTS_SET(_opts, _type, _prefix, _name) \ - {std::string opt_name = \ - (_prefix.empty()?"":_prefix+"::") + #_name; \ - if (_opts.count(opt_name)) \ - { \ - this->set_##_name(_opts[opt_name].as<_type>()); \ +#define TECA_POPTS_SET(_opts, _type, _prefix, _name) \ + {std::string opt_name = \ + (_prefix.empty()?"":_prefix+"::") + #_name; \ + bool defd = _opts[opt_name].defaulted(); \ + if (!defd) \ + { \ + _type val = _opts[opt_name].as<_type>(); \ + if (this->verbose && \ + teca_mpi_util::mpi_rank_0(this->get_communicator())) \ + { \ + TECA_STATUS("Setting " << opt_name << " = " << val) \ + } \ + this->set_##_name(val); \ }} #else diff --git a/core/teca_programmable_algorithm.h b/core/teca_programmable_algorithm.h index 43cb58dc9..bb4f0a449 100644 --- a/core/teca_programmable_algorithm.h +++ b/core/teca_programmable_algorithm.h @@ -2,57 +2,86 @@ #define teca_programmable_algorithm_h #include "teca_algorithm.h" +#include "teca_shared_object.h" #include "teca_metadata.h" -#include "teca_dataset_fwd.h" -#include "teca_programmable_algorithm_fwd.h" - -/// an algorithm implemented with user provided callbacks -/** -The user can provide a callback for each of the three phases -of pipeline execution. The number of input and output ports -can also be set for filters (1 or more inputs, 1 or more outputs) -sources, (no inputs, 1 or more outputs), or sinks (1 or more -inputs, no outputs). - -1) report phase. the report callback returns metadata - describing data that can be produced. The report callback - is optional. It's only needed if the algorithm will produce - new data or transform metadata. - - the report callback must be callable with signature: - teca_metadata(unsigned int) - -2) request phase. the request callback generates a vector - of requests(metadata objects) that inform the upstream of - what data to generate. The request callback is optional. - It's only needed if the algorithm needs data from the - upstream or transform metadata. - - the request callback must be callable with the signature: - std::vector( - unsigned int, - const std::vector &, - const teca_metadata &) - -3) execute phase. the execute callback is used to do useful - work on incoming or outgoing data. Examples include - generating new datasets, processing datasets, reading - and writing data to/from disk, and so on. The execute - callback is optional. - - the execute callback must be callable with the signature: - const_p_teca_dataset( +#include "teca_dataset.h" + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_programmable_algorithm) +TECA_SHARED_OBJECT_FORWARD_DECL(teca_threaded_programmable_algorithm) + +#ifdef SWIG +typedef void* report_callback_t; +typedef void* request_callback_t; +typedef void* execute_callback_t; +typedef void* threaded_execute_callback_t; +#else +/// A callable implementing the report phase of pipeline execution +using report_callback_t = std::function&)>; + +/// A callable implementing the request phase of pipeline execution +using request_callback_t = std::function( + unsigned int, const std::vector &, + const teca_metadata &)>; + +/// A callable implementing the execute phase of pipeline execution +using execute_callback_t = std::function &, - const teca_metadata &) + const teca_metadata &)>; -see also: +/// A callable implementing the streaming execute phase of pipeline execution +using threaded_execute_callback_t = std::function &, + const teca_metadata &, int)>; +#endif -set_number_of_input_connections -set_number_of_output_ports -set_report_callback -set_request_callback -set_execute_callback -*/ +/// An algorithm implemented with user provided callbacks. +/** + * The user can provide a callback for each of the three phases + * of pipeline execution. The number of input and output ports + * can also be set for filters (1 or more inputs, 1 or more outputs) + * sources, (no inputs, 1 or more outputs), or sinks (1 or more + * inputs, no outputs). + * + * 1) report phase. the report callback returns metadata + * describing data that can be produced. The report callback + * is optional. It's only needed if the algorithm will produce + * new data or transform metadata. + * + * the report callback must be callable with signature: + * teca_metadata(unsigned int) + * + * 2) request phase. the request callback generates a vector + * of requests(metadata objects) that inform the upstream of + * what data to generate. The request callback is optional. + * It's only needed if the algorithm needs data from the + * upstream or transform metadata. + * + * the request callback must be callable with the signature: + * std::vector( + * unsigned int, + * const std::vector &, + * const teca_metadata &) + * + * 3) execute phase. the execute callback is used to do useful + * work on incoming or outgoing data. Examples include + * generating new datasets, processing datasets, reading + * and writing data to/from disk, and so on. The execute + * callback is optional. + * + * the execute callback must be callable with the signature: + * const_p_teca_dataset( + * unsigned int, const std::vector &, + * const teca_metadata &) + * + * see also: + * + * set_number_of_input_connections + * set_number_of_output_ports + * set_report_callback + * set_request_callback + * set_execute_callback + */ class teca_programmable_algorithm : public teca_algorithm { public: diff --git a/core/teca_programmable_algorithm_fwd.h b/core/teca_programmable_algorithm_fwd.h deleted file mode 100644 index 626fcfd57..000000000 --- a/core/teca_programmable_algorithm_fwd.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef teca_program_algorithm_fwd_h -#define teca_program_algorithm_fwd_h - -#include "teca_shared_object.h" -#include "teca_metadata.h" -#include "teca_dataset_fwd.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_programmable_algorithm) -TECA_SHARED_OBJECT_FORWARD_DECL(teca_threaded_programmable_algorithm) - -#ifdef SWIG -typedef void* report_callback_t; -typedef void* request_callback_t; -typedef void* execute_callback_t; -typedef void* threaded_execute_callback_t; -#else -using report_callback_t = std::function&)>; - -using request_callback_t = std::function( - unsigned int, const std::vector &, - const teca_metadata &)>; - -using execute_callback_t = std::function &, - const teca_metadata &)>; - -using threaded_execute_callback_t = std::function &, - const teca_metadata &, int)>; -#endif -#endif diff --git a/core/teca_programmable_reduce.h b/core/teca_programmable_reduce.h index a0e14dc99..f4bade029 100644 --- a/core/teca_programmable_reduce.h +++ b/core/teca_programmable_reduce.h @@ -1,23 +1,39 @@ #ifndef teca_programmable_reduce_h #define teca_programmable_reduce_h -#include "teca_programmable_reduce_fwd.h" -#include "teca_programmable_algorithm_fwd.h" -#include "teca_index_reduce.h" -#include "teca_dataset_fwd.h" +#include "teca_dataset.h" #include "teca_metadata.h" +#include "teca_shared_object.h" +#include "teca_programmable_algorithm.h" +#include "teca_index_reduce.h" #include #include +#include + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_programmable_reduce) + +#ifdef SWIG +typedef void* reduce_callback_t; +typedef void* finalize_callback_t; +#else +/// A callable that can reduce two datasets into one. +using reduce_callback_t = std::function; + +/// A callable that can finalize the reduction. +using finalize_callback_t = std::function; +#endif -// callbacks implement a user defined reduction over time steps +/// Callbacks implement a user defined reduction over time steps. /** -callbacks implement a reduction on teca_datasets over time steps. -user provides reduce callable that takes 2 datasets and produces -a thrid reduced dataset. callbacks should be threadsafe as this is -a parallel operation. see teca_index_reduce for details of -parallelization. -*/ + * Callbacks implement a reduction on teca_datasets over time steps. + * User provides reduce callable that takes 2 datasets and produces + * a third reduced dataset. Callbacks should be threadsafe as this is + * a parallel operation. See teca_index_reduce for details of + * parallelization. + */ class teca_programmable_reduce : public teca_index_reduce { public: @@ -26,7 +42,7 @@ class teca_programmable_reduce : public teca_index_reduce ~teca_programmable_reduce(){} // set the implementation name, this is used in logging to - // identify the specific instance of programmable redeuce + // identify the specific instance of programmable reduce int set_name(const std::string &name); const char *get_class_name() const override diff --git a/core/teca_programmable_reduce_fwd.h b/core/teca_programmable_reduce_fwd.h deleted file mode 100644 index e17706fae..000000000 --- a/core/teca_programmable_reduce_fwd.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef teca_program_reduce_fwd_h -#define teca_program_reduce_fwd_h - -#include "teca_shared_object.h" -#include "teca_dataset_fwd.h" -#include - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_programmable_reduce) - -#ifdef SWIG -typedef void* reduce_callback_t; -typedef void* finalize_callback_t; -#else -using reduce_callback_t = std::function; - -using finalize_callback_t = std::function; -#endif -#endif diff --git a/core/teca_shared_object.h b/core/teca_shared_object.h index 015c7d5af..739561127 100644 --- a/core/teca_shared_object.h +++ b/core/teca_shared_object.h @@ -1,14 +1,19 @@ #ifndef teca_shared_object_h #define teca_shared_object_h +/// @file + #include // convenience macro. every teca_algrotihm/dataset // should have the following forward declarations + #ifdef SWIG + // SWIG doesn't handle alias templates yet. OK for the // shared object forward but the shared object template // forward has no direct mapping into c++03. + #define TECA_SHARED_OBJECT_FORWARD_DECL(_cls) \ class _cls; \ typedef std::shared_ptr<_cls> p_##_cls; \ @@ -16,19 +21,28 @@ #define TECA_SHARED_OBJECT_TEMPLATE_FORWARD_DECL(_cls) \ template class _cls; + #else + #define TECA_SHARED_OBJECT_FORWARD_DECL(_cls) \ class _cls; \ + \ + /** a shared pointer to an instance of _cls */ \ using p_##_cls = std::shared_ptr<_cls>; \ + \ + /** A shared pointer to a const instance of _cls */ \ using const_p_##_cls = std::shared_ptr; #define TECA_SHARED_OBJECT_TEMPLATE_FORWARD_DECL(_cls) \ template class _cls; \ \ + /** a shared pointer to an instance of _cls */ \ template \ using p_##_cls = std::shared_ptr<_cls>; \ \ + /** A shared pointer to a const instance of _cls */ \ template \ using const_p_##_cls = std::shared_ptr>; + #endif #endif diff --git a/core/teca_string_util.cxx b/core/teca_string_util.cxx index d1288eb29..39eb1b754 100644 --- a/core/teca_string_util.cxx +++ b/core/teca_string_util.cxx @@ -85,5 +85,30 @@ int tokenize(char *istr, char delim, int n_cols, char **ostr) return 0; } +// ************************************************************************** +void remove_post_fix(std::set &arrays, std::string post_fix) +{ + size_t postfix_len = post_fix.length(); + + std::set::iterator arrays_it; + for (arrays_it=arrays.begin(); arrays_it!=arrays.end(); ++arrays_it) + { + std::string array_var = *arrays_it; + size_t array_var_len = array_var.length(); + + if (array_var_len > postfix_len) + { + size_t postfix_pos = array_var.find(post_fix, + array_var_len - postfix_len); + if (postfix_pos != std::string::npos) + { + array_var.erase(array_var_len - postfix_len, postfix_len); + + arrays.erase(arrays_it); + arrays.insert(array_var); + } + } + } } +} diff --git a/core/teca_string_util.h b/core/teca_string_util.h index e1d7ff54f..14d47897d 100644 --- a/core/teca_string_util.h +++ b/core/teca_string_util.h @@ -1,6 +1,8 @@ #ifndef teca_string_util_h #define teca_string_util_h +/// @file + #include "teca_common.h" #include @@ -8,25 +10,30 @@ #include #include #include +#include +/// Codes for dealing with string processing namespace teca_string_util { -// convert the characters between the first and second double -// quote to a std::string. Escaped characters are skipped. Return -// 0 if successful. +/** Convert the characters between the first and second double + * quote to a std::string. Escaped characters are skipped. Return + * 0 if successful. + */ int extract_string(const char *istr, std::string &field); -// scan the input string (istr) for the given a delimiter (delim). push a pointer -// to the first non-delimiter character and the first character after each -// instance of the delimiter. return zero if successful. when successful there -// will be at least one value. +/** Scan the input string (istr) for the given a delimiter (delim). push a pointer + * to the first non-delimiter character and the first character after each + * instance of the delimiter. return zero if successful. when successful there + * will be at least one value. + */ int tokenize(char *istr, char delim, int n_cols, char **ostr); -// scan the input string (istr) for the given a delimiter (delim). push a point -// to the first non-delimiter character and the first character after each -// instance of the delimiter. return zero if successful. when successful there -// will be at least one value. +/** Scan the input string (istr) for the given a delimiter (delim). push a point + * to the first non-delimiter character and the first character after each + * instance of the delimiter. return zero if successful. when successful there + * will be at least one value. + */ template > int tokenize(char *istr, char delim, container_t &ostr) { @@ -62,8 +69,9 @@ int tokenize(char *istr, char delim, container_t &ostr) return 0; } -// skip space, tabs, and new lines. return non-zero if the end of the string -// is reached before a non-pad character is encountered +/** Skip space, tabs, and new lines. return non-zero if the end of the string + * is reached before a non-pad character is encountered + */ inline int skip_pad(char *&buf) { @@ -73,7 +81,7 @@ int skip_pad(char *&buf) return *buf == '\0' ? -1 : 0; } -// return 0 if the first non-pad character is # +/// return 0 if the first non-pad character is # inline int is_comment(char *buf) { @@ -83,15 +91,17 @@ int is_comment(char *buf) return 0; } +/// A traits class for scanf conversion codes. template struct scanf_tt {}; -#define DECLARE_SCANF_TT(_CPP_T, _FMT_STR) \ -template<> \ -struct scanf_tt<_CPP_T> \ -{ \ - static \ - const char *format() { return _FMT_STR; } \ +#define DECLARE_SCANF_TT(_CPP_T, _FMT_STR) \ +template<> \ +/** A traits class for scanf conversion codes, specialized fo _CPP_T */ \ +struct scanf_tt<_CPP_T> \ +{ \ + static \ + const char *format() { return _FMT_STR; } \ }; DECLARE_SCANF_TT(float," %g") DECLARE_SCANF_TT(double," %lg") @@ -107,63 +117,66 @@ DECLARE_SCANF_TT(unsigned long, " %lu") DECLARE_SCANF_TT(unsigned long long, "%llu") DECLARE_SCANF_TT(std::string, " \"%128s") +/// A traits class for conversion from text to numbers template struct string_tt {}; -#define DECLARE_STR_CONVERSION_I(_CPP_T, _FUNC) \ -template <> \ -struct string_tt<_CPP_T> \ -{ \ - static const char *type_name() { return # _CPP_T; } \ - \ - static int convert(char *str, _CPP_T &val) \ - { \ - errno = 0; \ - char *endp = nullptr; \ - _CPP_T tmp = _FUNC(str, &endp, 0); \ - if (errno != 0) \ - { \ - TECA_ERROR("Failed to convert string \"" \ - << str << "\" to a nunber." << strerror(errno)) \ - return -1; \ - } \ - else if (endp == str) \ - { \ - TECA_ERROR("Failed to convert string \"" \ - << str << "\" to a nunber. Invalid string.") \ - return -1; \ - } \ - val = tmp; \ - return 0; \ - } \ +#define DECLARE_STR_CONVERSION_I(_CPP_T, _FUNC) \ +/** A traits class for conversion from text to numbers, specialized for _CPP_T */ \ +template <> \ +struct string_tt<_CPP_T> \ +{ \ + static const char *type_name() { return # _CPP_T; } \ + \ + static int convert(char *str, _CPP_T &val) \ + { \ + errno = 0; \ + char *endp = nullptr; \ + _CPP_T tmp = _FUNC(str, &endp, 0); \ + if (errno != 0) \ + { \ + TECA_ERROR("Failed to convert string \"" \ + << str << "\" to a nunber." << strerror(errno)) \ + return -1; \ + } \ + else if (endp == str) \ + { \ + TECA_ERROR("Failed to convert string \"" \ + << str << "\" to a nunber. Invalid string.") \ + return -1; \ + } \ + val = tmp; \ + return 0; \ + } \ }; -#define DECLARE_STR_CONVERSION_F(_CPP_T, _FUNC) \ -template <> \ -struct string_tt<_CPP_T> \ -{ \ - static const char *type_name() { return # _CPP_T; } \ - \ - static int convert(const char *str, _CPP_T &val) \ - { \ - errno = 0; \ - char *endp = nullptr; \ - _CPP_T tmp = _FUNC(str, &endp); \ - if (errno != 0) \ - { \ - TECA_ERROR("Failed to convert string \"" \ - << str << "\" to a nunber." << strerror(errno)) \ - return -1; \ - } \ - else if (endp == str) \ - { \ - TECA_ERROR("Failed to convert string \"" \ - << str << "\" to a nunber. Invalid string.") \ - return -1; \ - } \ - val = tmp; \ - return 0; \ - } \ +#define DECLARE_STR_CONVERSION_F(_CPP_T, _FUNC) \ +/** A traits class for conversion from text to numbers, specialized for _CPP_T */ \ +template <> \ +struct string_tt<_CPP_T> \ +{ \ + static const char *type_name() { return # _CPP_T; } \ + \ + static int convert(const char *str, _CPP_T &val) \ + { \ + errno = 0; \ + char *endp = nullptr; \ + _CPP_T tmp = _FUNC(str, &endp); \ + if (errno != 0) \ + { \ + TECA_ERROR("Failed to convert string \"" \ + << str << "\" to a nunber." << strerror(errno)) \ + return -1; \ + } \ + else if (endp == str) \ + { \ + TECA_ERROR("Failed to convert string \"" \ + << str << "\" to a nunber. Invalid string.") \ + return -1; \ + } \ + val = tmp; \ + return 0; \ + } \ }; DECLARE_STR_CONVERSION_F(float, strtof) @@ -174,6 +187,7 @@ DECLARE_STR_CONVERSION_I(int, strtol) DECLARE_STR_CONVERSION_I(long, strtoll) DECLARE_STR_CONVERSION_I(long long, strtoll) +/// A traits class for conversion from text to numbers, specialized for bool template <> struct string_tt { @@ -206,6 +220,7 @@ struct string_tt } }; +/// A traits class for conversion from text to numbers, specialized for std::string template <> struct string_tt { @@ -218,8 +233,9 @@ struct string_tt } }; - -// watch out for memory leak, val needs to be free'd +/** A traits class for conversion from text to numbers, specialized for char* + * watch out for memory leak, val needs to be free'd + */ template <> struct string_tt { @@ -232,9 +248,10 @@ struct string_tt } }; -// extract the value in a "name = value" pair. -// an error occurs if splitting the input on '=' doesn't produce 2 tokens -// or if the conversion to val_t fails. returns 0 if successful. +/** Extract the value in a "name = value" pair. + * an error occurs if splitting the input on '=' doesn't produce 2 tokens + * or if the conversion to val_t fails. returns 0 if successful. + */ template int extract_value(char *l, val_t &val) { @@ -256,6 +273,18 @@ int extract_value(char *l, val_t &val) return 0; } +/** Given a collection of strings, where some of the strings end with a common + * substring, the post-fix, this function visits each string in the collection + * and removes the post-fix from each string that it is found in. + */ +void remove_post_fix(std::set &names, std::string post_fix); + +/// When passed the string "" return empty string otherwise return the passed string +inline std::string emptystr(const std::string &in) +{ + return (in == "\"\"" ? std::string() : in); +} + } #endif diff --git a/core/teca_system_util.h b/core/teca_system_util.h index 2139501ef..dd7946c25 100644 --- a/core/teca_system_util.h +++ b/core/teca_system_util.h @@ -1,21 +1,25 @@ #ifndef teca_system_util_h #define teca_system_util_h +/// @file + #include "teca_common.h" #include "teca_string_util.h" #include +/// Codes for dealing with low level system API's namespace teca_system_util { -// initialize val with the environment variable named by var converted to a -// numeric type. Only floating point and signed integers are implemented. For -// unsigned types, check that the return is greater or equal to zero. -// -// returns: -// 0 if the variable was found and val was initialized from it -// 1 if the varibale was not found -// -1 if the variable was found but conversion from string failed +/** initialize val with the environment variable named by var converted to a + * numeric type. Only floating point and signed integers are implemented. For + * unsigned types, check that the return is greater or equal to zero. + * + * returns: + * 0 if the variable was found and val was initialized from it + * 1 if the varibale was not found + * -1 if the variable was found but conversion from string failed + */ template int get_environment_variable(const char *var, T &val) { @@ -33,14 +37,16 @@ int get_environment_variable(const char *var, T &val) return 1; } -// extract the value of the named command line argument. -// return 0 if successful. If require is not zero then an error -// will be reported if the argument is not present. +/** extract the value of the named command line argument. return 0 if + * successful. If require is not zero then an error will be reported if the + * argument is not present. + */ int get_command_line_option(int argc, char **argv, const char *arg_name, int require, std::string &arg_val); -// check for the presence of the name command line option. -// return non-zero if it is found. +/** check for the presence of the name command line option. return non-zero if + * it is found. + */ int command_line_option_check(int argc, char **argv, const char *arg_name); } diff --git a/core/teca_thread_pool.h b/core/teca_thread_pool.h index 55721111b..9c31bf704 100644 --- a/core/teca_thread_pool.h +++ b/core/teca_thread_pool.h @@ -2,7 +2,7 @@ #define teca_thread_pool_h #include "teca_common.h" -#include "teca_algorithm_fwd.h" +#include "teca_algorithm.h" #include "teca_thread_util.h" #include "teca_threadsafe_queue.h" #include "teca_mpi.h" @@ -26,8 +26,7 @@ class teca_thread_pool; template using p_teca_thread_pool = std::shared_ptr>; -// a class to manage a fixed size pool of threads that dispatch -// I/O work +/// A class to manage a fixed size pool of threads that dispatch I/O work. template class teca_thread_pool { @@ -113,8 +112,16 @@ void teca_thread_pool::create_threads(MPI_Comm comm, int n_threads = n_requested; std::deque core_ids; - teca_thread_util::thread_parameters(comm, -1, - n_requested, bind, verbose, n_threads, core_ids); + + if (teca_thread_util::thread_parameters(comm, -1, + n_requested, bind, verbose, n_threads, core_ids)) + { + TECA_WARNING("Failed to detetermine thread parameters." + " Falling back to 1 thread, affinity disabled.") + + n_threads = 1; + bind = false; + } // allocate the threads for (int i = 0; i < n_threads; ++i) diff --git a/core/teca_thread_util.cxx b/core/teca_thread_util.cxx index 3a7fe5fd7..e0c2c46e7 100644 --- a/core/teca_thread_util.cxx +++ b/core/teca_thread_util.cxx @@ -296,7 +296,7 @@ int thread_parameters(MPI_Comm comm, int base_core_id, int n_requested, (void)affinity; if (n_requested < 1) { - TECA_WARNING("Cannot autmatically detect threading parameters " + TECA_WARNING("Can not automatically detect threading parameters " "on this platform. The default is 1 thread per process.") n_threads = 1; } @@ -368,14 +368,17 @@ int thread_parameters(MPI_Comm comm, int base_core_id, int n_requested, } // if the user runs more MPI ranks than cores some of the ranks - // will have no cores to use. fallback to 1 thread on core 0 - if (n_threads < 1) + // will have no cores to use. + if (n_procs > cores_per_node) { + TECA_WARNING(<< n_procs << " MPI ranks running on this node but only " + << cores_per_node << " CPU cores are available. Performance will" + " be degraded.") + n_threads = 1; - affinity.push_back(0); - TECA_WARNING("CPU cores are unavailable, performance will be degraded. " - "This can occur when running more MPI ranks than there are CPU " - "cores. Launching 1 thread on core 0.") + affinity.push_back(base_core_id); + + return -1; } // stop now if we are not binding threads to cores @@ -401,7 +404,7 @@ int thread_parameters(MPI_Comm comm, int base_core_id, int n_requested, // there are enough cores that each thread can have it's own core // mark the cores which have the root thread as used so that we skip them. - // if we always did this in the fully apcked case we'd always be assigning + // if we always did this in the fully packed case we'd always be assigning // hyperthreads off core. it is better to keep them local. if (((n_threads+1)*n_procs) < cores_per_node) { diff --git a/core/teca_thread_util.h b/core/teca_thread_util.h index 44fb40f13..a0f45ce81 100644 --- a/core/teca_thread_util.h +++ b/core/teca_thread_util.h @@ -1,62 +1,66 @@ #ifndef teca_thread_utils_h #define teca_thread_utils_h +/// @file + #include "teca_common.h" #include "teca_mpi.h" #include +/// Codes for dealing with threading namespace teca_thread_util { -// load balances threads across an MPI communication space such that on the -// individual nodes physical cores each receive the same number of threads. -// This is an MPI collective call. Building the affinity map relies on -// features available only in _GNU_SOURCE. On systems where these features are -// unavailable, when automated detection of the number of threads is requested, -// the call will fail and the n_threads will be set to 1, -// -// comm - an MPI communcation space to load balance threads across. -// the communicator is used to coordinate affinity mapping such that -// each rank can allocate a number of threads bound to unique cores. -// -// base_core_id - identifies the core in use by this MPI rank's main -// thread. if -1 is passed this will be automatically -// determined. -// -// n_requested - the number of requested threads per rank. Passing a value of -// -1 results in use of all the cores on the node such that each -// physical core is assigned exactly 1 thread. Note that for -// performance reasons hyperthreads are not used here. The -// suggested number of threads is retruned in n_threads, and the -// returned affinity map specifies which core the thread should -// be bound to to acheive this. Passing n_requested >= 1 -// specifies a run time override. This indicates that caller -// wants to use a specific number of threads, rather than one per -// physical core. In this case the affinity map is also -// constructed. -// -// bind - if true extra work is done to determine an affinity map such -// that each thread can be bound to a unique core on the node. -// -// verbose - prints a report decribing the affinity map. -// -// n_threads - if n_requested is -1, this will be set to the number of threads -// one can use such that there is one thread per phycial core -// taking into account all ranks running on the node. if -// n_requested is >= 1 n_threads will be set to n_requested. This -// allows a run time override for cases when the caller knows how -// she wants to schedule things. if an error occurs and n_requested -// is -1 this will be set to 1. -// -// affinity - an affinity map, describing for each of n_threads, -// a core id that the thread can be bound to. if n_requested is -1 -// then the map will conatin an entry for each of n_threads where -// each of the threads is assigned a unique phyical core. when -// n_requested is >= 1 the map contains an enrty for each of the -// n_requested threads such that when more threads are requested -// than cores each core is assigned approximately the same number of -// threads. -// -// return 0 on success +/** load balances threads across an MPI communication space such that on the + * individual nodes physical cores each receive the same number of threads. + * This is an MPI collective call. Building the affinity map relies on + * features available only in _GNU_SOURCE. On systems where these features are + * unavailable, when automated detection of the number of threads is requested, + * the call will fail and the n_threads will be set to 1, + * + * @param[in] comm an MPI communcation space to load balance threads across. + * the communicator is used to coordinate affinity mapping such that + * each rank can allocate a number of threads bound to unique cores. + * + * @param[in] base_core_id identifies the core in use by this MPI rank's main + * thread. if -1 is passed this will be automatically + * determined. + * + * @param[in] n_requested the number of requested threads per rank. Passing a value of + * -1 results in use of all the cores on the node such that each + * physical core is assigned exactly 1 thread. Note that for + * performance reasons hyperthreads are not used here. The + * suggested number of threads is retruned in n_threads, and the + * returned affinity map specifies which core the thread should + * be bound to to acheive this. Passing n_requested >= 1 + * specifies a run time override. This indicates that caller + * wants to use a specific number of threads, rather than one per + * physical core. In this case the affinity map is also + * constructed. + * + * @param[in] bind if true extra work is done to determine an affinity map such + * that each thread can be bound to a unique core on the node. + * + * @param[in] verbose prints a report decribing the affinity map. + * + * @param[in,out] n_threads if n_requested is -1, this will be set to the number of threads + * one can use such that there is one thread per phycial core + * taking into account all ranks running on the node. if + * n_requested is >= 1 n_threads will be set to n_requested. This + * allows a run time override for cases when the caller knows how + * she wants to schedule things. if an error occurs and n_requested + * is -1 this will be set to 1. + * + * @param[out] affinity an affinity map, describing for each of n_threads, + * a core id that the thread can be bound to. if n_requested is -1 + * then the map will conatin an entry for each of n_threads where + * each of the threads is assigned a unique phyical core. when + * n_requested is >= 1 the map contains an enrty for each of the + * n_requested threads such that when more threads are requested + * than cores each core is assigned approximately the same number of + * threads. + * + * @returns 0 on success + */ int thread_parameters(MPI_Comm comm, int base_core_id, int n_requested, bool bind, bool verbose, int &n_threads, std::deque &affinity); }; diff --git a/core/teca_threaded_algorithm.cxx b/core/teca_threaded_algorithm.cxx index 42848e1ea..0578aef3e 100644 --- a/core/teca_threaded_algorithm.cxx +++ b/core/teca_threaded_algorithm.cxx @@ -73,7 +73,7 @@ void teca_threaded_algorithm_internals::thread_pool_resize(MPI_Comm comm, // -------------------------------------------------------------------------- -teca_threaded_algorithm::teca_threaded_algorithm() : verbose(0), +teca_threaded_algorithm::teca_threaded_algorithm() : bind_threads(1), stream_size(-1), poll_interval(1000000), internals(new teca_threaded_algorithm_internals) { @@ -95,20 +95,22 @@ void teca_threaded_algorithm::get_properties_description( opts.add_options() TECA_POPTS_GET(int, prefix, bind_threads, - "bind software threads to hardware cores (1)") + "bind software threads to hardware cores") TECA_POPTS_GET(int, prefix, verbose, - "print a run time report of settings (0)") + "print a run time report of settings") TECA_POPTS_GET(int, prefix, thread_pool_size, "number of threads in pool. When n == -1, 1 thread per core is " - "created (-1)") + "created") TECA_POPTS_GET(int, prefix, stream_size, "number of datasests to pass per execute call. -1 means wait " - "for all. (-1)") + "for all.") TECA_POPTS_GET(long, prefix, poll_interval, "number of nanoseconds to wait between scans of the thread pool " - "for completed tasks (1.0e6)") + "for completed tasks") ; + this->teca_algorithm::get_properties_description(prefix, opts); + global_opts.add(opts); } @@ -116,6 +118,8 @@ void teca_threaded_algorithm::get_properties_description( void teca_threaded_algorithm::set_properties(const std::string &prefix, variables_map &opts) { + this->teca_algorithm::set_properties(prefix, opts); + TECA_POPTS_SET(opts, int, prefix, bind_threads) TECA_POPTS_SET(opts, int, prefix, verbose) diff --git a/core/teca_threaded_algorithm.h b/core/teca_threaded_algorithm.h index 7f02f670a..77451662b 100644 --- a/core/teca_threaded_algorithm.h +++ b/core/teca_threaded_algorithm.h @@ -2,9 +2,11 @@ #define teca_threaded_algorithm_h #include "teca_algorithm.h" -#include "teca_threaded_algorithm_fwd.h" -#include "teca_algorithm_output_port.h" #include "teca_dataset.h" +#include "teca_shared_object.h" + +#include +#include template class teca_thread_pool; @@ -12,24 +14,35 @@ class teca_thread_pool; class teca_metadata; class teca_threaded_algorithm_internals; -#include -#include +TECA_SHARED_OBJECT_FORWARD_DECL(teca_threaded_algorithm) -// declare the thread pool type +/// Task type for tasks returing a pointer to teca_dataset using teca_data_request_task = std::packaged_task; class teca_data_request; + +/// A thread pool for processing teca_data_request_task using teca_data_request_queue = teca_thread_pool; +/// A pointer to teca_data_request_queue using p_teca_data_request_queue = std::shared_ptr; +/** Allocate and initialize a new thread pool. + * comm [in] The communicator to allocate thread across + * n [in] The number of threads to create per MPI rank. Use -1 to + * map one thread per physical core on each node. + * bind [in] If set then thread will be bound to a specific core. + * verbose [in] If set then the mapping is sent to the stderr + */ p_teca_data_request_queue new_teca_data_request_queue(MPI_Comm comm, int n, bool bind, bool verbose); -// this is the base class defining a threaded algorithm. -// the stratgey employed is to parallelize over upstream -// data requests using a thread pool. +/// This is the base class defining a threaded algorithm. +/** + * The strategy employed is to parallelize over upstream + * data requests using a thread pool. + */ class teca_threaded_algorithm : public teca_algorithm { public: @@ -43,29 +56,46 @@ class teca_threaded_algorithm : public teca_algorithm TECA_GET_ALGORITHM_PROPERTIES_DESCRIPTION() TECA_SET_ALGORITHM_PROPERTIES() - // set/get the number of threads in the pool. setting - // to -1 results in a thread per core factoring in all MPI - // ranks running on the node. the default is -1. + /** Set the number of threads in the pool. setting to -1 results in a + * thread per core factoring in all MPI ranks running on the node. + */ void set_thread_pool_size(int n_threads); - unsigned int get_thread_pool_size() const noexcept; - - // set/get the verbosity level. - TECA_ALGORITHM_PROPERTY(int, verbose); - - // set/get thread affinity mode. When 0 threads are not bound - // CPU cores, allowing for migration among all cores. This will - // likely degrade performance. Default is 1. - TECA_ALGORITHM_PROPERTY(int, bind_threads); - // set the smallest number of datasets to gather per call to - // execute. the default (-1) results in all datasets being - // gathered. In practice more datasets will be returned if - // ready - TECA_ALGORITHM_PROPERTY(int, stream_size); + /// Get the number of threads in the pool. + unsigned int get_thread_pool_size() const noexcept; - // set the duration in nano seconds to wait between checking - // for completed tasks - TECA_ALGORITHM_PROPERTY(long long, poll_interval); + /** @name verbose + * set/get the verbosity level. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, verbose) + ///@} + + /** @name bind_threads + * set/get thread affinity mode. When 0 threads are not bound CPU cores, + * allowing for migration among all cores. This will likely degrade + * performance. Default is 1. + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, bind_threads) + ///@} + + /** @name stream_size + * set the smallest number of datasets to gather per call to execute. the + * default (-1) results in all datasets being gathered. In practice more + * datasets will be returned if ready + */ + ///@{ + TECA_ALGORITHM_PROPERTY(int, stream_size) + ///@} + + /** @name poll_interval + * set the duration in nanoseconds to wait between checking for completed + * tasks + */ + ///@{ + TECA_ALGORITHM_PROPERTY(long long, poll_interval) + ///@} // explicitly set the thread pool to submit requests to void set_data_request_queue(const p_teca_data_request_queue &queue); @@ -88,12 +118,12 @@ class teca_threaded_algorithm : public teca_algorithm const teca_metadata &request) override; // driver function that manages execution of the given - // requst on the named port. each upstream request issued + // request on the named port. each upstream request issued // will be executed by the thread pool. const_p_teca_dataset request_data(teca_algorithm_output_port &port, const teca_metadata &request) override; + private: - int verbose; int bind_threads; int stream_size; long long poll_interval; diff --git a/core/teca_threaded_algorithm_fwd.h b/core/teca_threaded_algorithm_fwd.h deleted file mode 100644 index 5d3eb6fef..000000000 --- a/core/teca_threaded_algorithm_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_threaded_algorithm_fwd_h -#define teca_threaded_algorithm_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_threaded_algorithm) - -#endif diff --git a/core/teca_threaded_programmable_algorithm.h b/core/teca_threaded_programmable_algorithm.h index a120a3a9c..444748b0a 100644 --- a/core/teca_threaded_programmable_algorithm.h +++ b/core/teca_threaded_programmable_algorithm.h @@ -2,67 +2,66 @@ #define teca_threaded_programmable_algorithm_h #include "teca_metadata.h" +#include "teca_dataset.h" #include "teca_threaded_algorithm.h" -#include "teca_dataset_fwd.h" +#include "teca_programmable_algorithm.h" -#include "teca_programmable_algorithm_fwd.h" - -/// an algorithm implemented with user provided callbacks +/// An threaded algorithm implemented with user provided callbacks. /** -This version of the teca_programmable_algorithm is threaded. A thread pool -(call set_thread_pool_size to initialize) executes the upstream pipeline -asynchronously for each request made. Hence, this version of the prpogrammable -algorithm is most useful when there are multiple requests to be processed. Data -from the set of requests can be processed incrementally when streaming (see -set_stream_size to initialize). If one doesn't need these features it is -better to use the teca_programmable_algorithm instead. See -teca_threaded_algorithm for more details about threaded execution. - -The user can provide a callback for each of the three phases -of pipeline execution. The number of input and output ports -can also be set for filters (1 or more inputs, 1 or more outputs) -sources, (no inputs, 1 or more outputs), or sinks (1 or more -inputs, no outputs). - -1) report phase. the report callback returns metadata - describing data that can be produced. The report callback - is optional. It's only needed if the algorithm will produce - new data or transform metadata. - - the report callback must be callable with signature: - teca_metadata(unsigned int) - -2) request phase. the request callback generates a vector of - requests(metadata objects) that inform the upstream of - what data to generate. The request callback is optional. - It's only needed if the algorithm needs data from the - upstream or transform metadata. - - the request callback must be callable with the signature: - std::vector( - unsigned int, - const std::vector &, - const teca_metadata &) - -3) execute phase. the execute callback is used to do useful - work on incoming or outgoing data. Examples include - generating new datasets, processing datasets, reading - and writing data to/from disk, and so on. The execute - callback is optional. - - the execute callback must be callable with the signature: - const_p_teca_dataset( - unsigned int, const std::vector &, - const teca_metadata &, int) - -see also: - -set_number_of_input_connections -set_number_of_output_ports -set_report_callback -set_request_callback -set_execute_callback -*/ + * This version of the teca_programmable_algorithm is threaded. A thread pool + * (call set_thread_pool_size to initialize) executes the upstream pipeline + * asynchronously for each request made. Hence, this version of the + * programmable algorithm is most useful when there are multiple requests to + * be processed. Data from the set of requests can be processed incrementally + * when streaming (see set_stream_size to initialize). If one doesn't need + * these features it is better to use the teca_programmable_algorithm instead. + * See teca_threaded_algorithm for more details about threaded execution. + * + * The user can provide a callback for each of the three phases + * of pipeline execution. The number of input and output ports + * can also be set for filters (1 or more inputs, 1 or more outputs) + * sources, (no inputs, 1 or more outputs), or sinks (1 or more + * inputs, no outputs). + * + * 1) report phase. the report callback returns metadata + * describing data that can be produced. The report callback + * is optional. It's only needed if the algorithm will produce + * new data or transform metadata. + * + * the report callback must be callable with signature: + * teca_metadata(unsigned int) + * + * 2) request phase. the request callback generates a vector of + * requests(metadata objects) that inform the upstream of + * what data to generate. The request callback is optional. + * It's only needed if the algorithm needs data from the + * upstream or transform metadata. + * + * the request callback must be callable with the signature: + * std::vector( + * unsigned int, + * const std::vector &, + * const teca_metadata &) + * + * 3) execute phase. the execute callback is used to do useful + * work on incoming or outgoing data. Examples include + * generating new datasets, processing datasets, reading + * and writing data to/from disk, and so on. The execute + * callback is optional. + * + * the execute callback must be callable with the signature: + * const_p_teca_dataset( + * unsigned int, const std::vector &, + * const teca_metadata &, int) + * + * see also: + * + * set_number_of_input_connections + * set_number_of_output_ports + * set_report_callback + * set_request_callback + * set_execute_callback + */ class teca_threaded_programmable_algorithm : public teca_threaded_algorithm { public: @@ -80,7 +79,7 @@ class teca_threaded_programmable_algorithm : public teca_threaded_algorithm using teca_algorithm::set_number_of_input_connections; using teca_algorithm::set_number_of_output_ports; - // set the number of threads. The default is -1. + // set the number of threads. The default is -1. using teca_threaded_algorithm::set_thread_pool_size; // set the stream size. the default -1 disables streaming. diff --git a/core/teca_threadsafe_queue.h b/core/teca_threadsafe_queue.h index 27d10cdae..a15769686 100644 --- a/core/teca_threadsafe_queue.h +++ b/core/teca_threadsafe_queue.h @@ -5,6 +5,7 @@ #include #include +/// A thread safe queue template class teca_threadsafe_queue { diff --git a/core/teca_type_select.h b/core/teca_type_select.h index c480465d6..a372f706d 100644 --- a/core/teca_type_select.h +++ b/core/teca_type_select.h @@ -1,28 +1,45 @@ #ifndef teca_type_elevate_h #define teca_type_elevate_h +/// @file + +/// Select a type based on input type(s) namespace teca_type_select { -// given two arguments, an elevate cast, selects -// the type of or casts to the higher precision -// type. note that given a signed and unsigned -// argument, signed type is selected. +/// Select the higher precision type. +/** Given two arguments, an elevate cast, selects the type of or casts to the + * higher precision type. note that given a signed and unsigned argument, + * signed type is selected. + * + * @tparam t1 first input type + * @tparam t2 second input type + */ template struct elevate {}; -// given two areuments, a decay cast, selects the -// type of or casts to the lower precision -// type. note that given a signed and unsigned -// argument, unsigned type is selected. +/// Select the lower precision type. +/** Given two arguments, a decay cast, selects the type of or casts to the lower + * precision type. note that given a signed and unsigned argument, unsigned + * type is selected. + * + * @tparam t1 first input type + * @tparam t2 second input type + */ template struct decay {}; #define teca_type_select(_class, _ret, _t1, _t2) \ +/** Given _t1 and _t2 _class to _ret */ \ template <> \ struct _class<_t1, _t2> \ { \ + /** _class result type */ \ using type = _ret; \ + \ + /** cast to _ret */ \ static _ret cast(_t1 arg){ return arg; } \ + \ + /** name of the _class result type */ \ static constexpr const char *type_name() \ { return #_ret; } \ }; diff --git a/core/teca_uuid.h b/core/teca_uuid.h index e8ba10eca..09002f755 100644 --- a/core/teca_uuid.h +++ b/core/teca_uuid.h @@ -5,7 +5,7 @@ #include -// a universally uniquer identifier +/// A universally uniquer identifier. class teca_uuid : public boost::uuids::uuid { public: diff --git a/core/teca_variant_array.h b/core/teca_variant_array.h index e68869138..4d34243b2 100644 --- a/core/teca_variant_array.h +++ b/core/teca_variant_array.h @@ -1,6 +1,8 @@ #ifndef teca_variant_array_h #define teca_variant_array_h +/// @file + #include #include #include @@ -14,27 +16,273 @@ #include "teca_common.h" #include "teca_binary_stream.h" -#include "teca_variant_array_fwd.h" #include "teca_bad_cast.h" +#include "teca_shared_object.h" + +TECA_SHARED_OBJECT_FORWARD_DECL(teca_variant_array) +TECA_SHARED_OBJECT_TEMPLATE_FORWARD_DECL(teca_variant_array_impl) + +#ifndef SWIG +using teca_string_array = teca_variant_array_impl; +using p_teca_string_array = std::shared_ptr>; +using const_p_teca_string_array = std::shared_ptr>; + +using teca_float_array = teca_variant_array_impl; +using p_teca_float_array = std::shared_ptr>; +using const_p_teca_float_array = std::shared_ptr>; + +using teca_double_array = teca_variant_array_impl; +using p_teca_double_array = std::shared_ptr>; +using const_p_teca_double_array = std::shared_ptr>; + +using teca_char_array = teca_variant_array_impl; +using p_teca_char_array = std::shared_ptr>; +using const_p_teca_char_array = std::shared_ptr>; + +using teca_unsigned_char_array = teca_variant_array_impl; +using p_teca_unsigned_char_array = std::shared_ptr>; +using const_p_teca_unsigned_char_array = std::shared_ptr>; + +using teca_short_array = teca_variant_array_impl; +using p_teca_short_array = std::shared_ptr>; +using const_p_teca_short_array = std::shared_ptr>; + +using teca_unsigned_short_array = teca_variant_array_impl; +using p_teca_unsigned_short_array = std::shared_ptr>; +using const_p_teca_unsigned_short_array = std::shared_ptr>; + +using teca_int_array = teca_variant_array_impl; +using p_teca_int_array = std::shared_ptr>; +using const_p_teca_int_array = std::shared_ptr>; + +using teca_unsigned_int_array = teca_variant_array_impl; +using p_teca_unsigned_int_array = std::shared_ptr>; +using const_p_teca_unsigned_int_array = std::shared_ptr>; + +using teca_long_array = teca_variant_array_impl; +using p_teca_long_array = std::shared_ptr>; +using const_p_teca_long_array = std::shared_ptr>; + +using teca_unsigned_long_array = teca_variant_array_impl; +using p_teca_unsigned_long_array = std::shared_ptr>; +using const_p_teca_unsigned_long_array = std::shared_ptr>; + +using teca_long_long_array = teca_variant_array_impl; +using p_teca_long_long_array = std::shared_ptr>; +using const_p_teca_long_long_array = std::shared_ptr>; + +using teca_unsigned_long_long_array = teca_variant_array_impl; +using p_teca_unsigned_long_long_array = std::shared_ptr>; +using const_p_teca_unsigned_long_long_array = std::shared_ptr>; + +using teca_size_t_array = teca_variant_array_impl; +using p_teca_size_t_array = std::shared_ptr>; +using const_p_teca_size_t_array = std::shared_ptr>; +#endif + +/** this is a convenience macro to be used to declare a static + * New method that will be used to construct new objects in + * shared_ptr's. This manages the details of interoperability + * with std C++11 shared pointer + */ +#define TECA_VARIANT_ARRAY_STATIC_NEW(T, t) \ + \ +/** Allocate a T */ \ +static std::shared_ptr> New() \ +{ \ + return std::shared_ptr>(new T); \ +} \ + \ +/** Allocate a T of size n */ \ +static std::shared_ptr> New(size_t n) \ +{ \ + return std::shared_ptr>(new T(n)); \ +} \ + \ +/** Allocate a T of size n initialized with v */ \ +static std::shared_ptr> New(size_t n, const t &v) \ +{ \ + return std::shared_ptr>(new T(n, v)); \ +} \ + \ +/** Allocate a T initialized with n values from vals */ \ +static std::shared_ptr> New(const t *vals, size_t n) \ +{ \ + return std::shared_ptr>(new T(vals, n)); \ +} \ + \ +using teca_variant_array::shared_from_this; \ + \ +std::shared_ptr shared_from_this() \ +{ \ + return std::static_pointer_cast(shared_from_this()); \ +} \ + \ +std::shared_ptr shared_from_this() const \ +{ \ + return std::static_pointer_cast(shared_from_this()); \ +} + + + -// tag for ops on POD data + + + + +/// @cond +/// A tag for dispatching operations on POD data template struct pod_dispatch : std::integral_constant::value> {}; -// tag for ops on classes +/// A tag for disp[atching operations on classes template struct object_dispatch : std::integral_constant::value> {}; +/// @endcond + +/** Executes the code in body if p is a tt + * @param tt derived container + * @param nt contained type + * @param p base class pointer + * @param body the code to execute if the type matches + * + * The following aliases are provided to know the type within the code to execute. + * + * using TT = tt; + * using NT = nt; + * + */ +#define TEMPLATE_DISPATCH_CASE(tt, nt, p, body) \ + if (dynamic_cast*>(p)) \ + { \ + using TT = tt; \ + using NT = nt; \ + body \ + } + +/** Executes the code in body if p is a tt an idnetifier disambiguates type + * aliases when nested + * + * @param tt derived container + * @param nt contained type + * @param p base class pointer + * @param i identifier + * @param body the code to execute if the type matches + * + * The following aliases are provided to know the type within the code to execute. + * + * using TT##i = tt; + * using NT##i = nt; + * + */ +#define NESTED_TEMPLATE_DISPATCH_CASE(tt, nt, p, i, body) \ + if (dynamic_cast*>(p)) \ + { \ + using TT##i = tt; \ + using NT##i = nt; \ + body \ + } + +/// Executes the code in body if p is a t where nt is a floating point type +#define TEMPLATE_DISPATCH_FP(t, p, body) \ + TEMPLATE_DISPATCH_CASE(t, float, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, double, p, body) + +/// Executes the code in body if p is a t where nt is a signed inetegral type +#define TEMPLATE_DISPATCH_SI(t, p, body) \ + TEMPLATE_DISPATCH_CASE(t, long long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, short int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, char, p, body) + +/// Executes the code in body if p is a t where nt is either a signed integral or floating point type +#define TEMPLATE_DISPATCH_FP_SI(t, p, body) \ + TEMPLATE_DISPATCH_CASE(t, float, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, double, p, body) \ + else TEMPLATE_DISPATCH_SI(t, p, body) + +/// Executes the code in body if p is a t where nt is an integral type +#define TEMPLATE_DISPATCH_I(t, p, body) \ + TEMPLATE_DISPATCH_CASE(t, long long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, unsigned long long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, unsigned int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, unsigned long, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, short int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, short unsigned int, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, char, p, body) \ + else TEMPLATE_DISPATCH_CASE(t, unsigned char, p, body) + +/** A macro for accessing the typed contents of a teca_variant_array + * @param t container type + * @param p pointer to an instance to match on + * @param body code to execute on match + * + * See #TEMPLATE_DISPATCH_CASE for details. + */ +#define TEMPLATE_DISPATCH(t, p, body) \ + TEMPLATE_DISPATCH_FP(t, p, body) \ + else TEMPLATE_DISPATCH_I(t, p, body) + +/** A macro for accessing the floating point typed contents of a teca_variant_array + * @param t container type + * @param p pointer to an instance to match on + * @param i an indentifier to use with type aliases + * @param body code to execute on match + * + * See #NESTED_TEMPLATE_DISPATCH_CASE for details. + */ +#define NESTED_TEMPLATE_DISPATCH_FP(t, p, i, body) \ + NESTED_TEMPLATE_DISPATCH_CASE(t, float, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, double, p, i, body) + +/** A macro for accessing the inetgral typed contents of a teca_variant_array + * @param t container type + * @param p pointer to an instance to match on + * @param i an indentifier to use with type aliases + * @param body code to execute on match + * + * See #NESTED_TEMPLATE_DISPATCH_CASE for details. + */ +#define NESTED_TEMPLATE_DISPATCH_I(t, p, i, body) \ + NESTED_TEMPLATE_DISPATCH_CASE(t, long long, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned long long, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, long, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, int, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned int, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned long, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, short int, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, short unsigned int, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, char, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned char, p, i, body) -/// type agnostic container for array based data -/** -type agnostic container for array based data. -*/ +/** \def NESTED_TEMPLATE_DISPATCH(t, p, i, body) + * A macro for accessing the typed contents of a teca_variant_array + * @param t container type + * @param p pointer to an instance to match on + * @param i an indentifier to use with type aliases + * @param body code to execute on match + * + * See #NESTED_TEMPLATE_DISPATCH_CASE for details. + */ +#define NESTED_TEMPLATE_DISPATCH(t, p, i, body) \ + NESTED_TEMPLATE_DISPATCH_FP(t, p, i, body) \ + else NESTED_TEMPLATE_DISPATCH_I(t, p, i, body) + + + +/// A type agnostic container for array based data. +/** See #TEMPLATE_DISPATCH and #NESTED_TEMPLATE_DISPATCH for details on how to + * apply type specific code to an instance of teca_variant_array. + */ class teca_variant_array : public std::enable_shared_from_this { public: @@ -55,12 +303,12 @@ class teca_variant_array : public std::enable_shared_from_thisswap(other); return *this; } - // virtual constructor. return a new'ly allocated + // virtual constructor. return a newly allocated // empty object of the same type. virtual p_teca_variant_array new_instance() const = 0; virtual p_teca_variant_array new_instance(size_t n) const = 0; - // virtual copy construct. return a new'ly allocated object, + // virtual copy construct. return a newly allocated object, // initialized copy from this. caller must delete. virtual p_teca_variant_array new_copy() const = 0; virtual p_teca_variant_array new_copy(size_t start, size_t end) const = 0; @@ -119,7 +367,7 @@ class teca_variant_array : public std::enable_shared_from_thisappend(*other.get()); } // swap the contents of this and the other object. - // an excpetion is thrown when no conversion + // an exception is thrown when no conversion // between the two types exists. virtual void swap(teca_variant_array &other) = 0; void swap(const p_teca_variant_array &other) @@ -232,7 +480,7 @@ class teca_variant_array : public std::enable_shared_from_this @@ -259,86 +507,98 @@ struct pack_object !std::is_pointer::value && !pack_object_ptr::value> {}; +/// @endcond - -// implementation of our type agnostic container -// for simple arrays +/** @brief + * The concrete implementation of our type agnostic container for contiguous + * arrays. + */ template class teca_variant_array_impl : public teca_variant_array { public: - // construct + /** @name Array constructors + * Constructs a new instance containing the templated type. + */ + ///@{ TECA_VARIANT_ARRAY_STATIC_NEW(teca_variant_array_impl, T) - // destruct - virtual ~teca_variant_array_impl() noexcept; - - // virtual constructor + /// Returns a new instance initialized with a copy of this one. p_teca_variant_array new_copy() const override; + + /// Returns a new instance initialized with a copy of a subset of this one. p_teca_variant_array new_copy(size_t start, size_t end) const override; + + /// Returns a new instance of the same type. p_teca_variant_array new_instance() const override; + + /// Returns a new instance of the same type sized to hold n elements. p_teca_variant_array new_instance(size_t n) const override; + ///@} - // return the name of the class in a human readable form + virtual ~teca_variant_array_impl() noexcept; + + /// Returns the name of the class in a human readable form std::string get_class_name() const override; - // intialize with T() + /// Initialize all elements with T() void initialize() override; - // copy + /// Copy from the other array const teca_variant_array_impl & operator=(const teca_variant_array_impl &other); + /// Copy from the other array template const teca_variant_array_impl & operator=(const teca_variant_array_impl &other); - // move + /// Move the contents of the other array teca_variant_array_impl(teca_variant_array_impl &&other); + /// Move the contents of the other array const teca_variant_array_impl & operator=(teca_variant_array_impl &&other); - // get the ith value + /// Get the ith value T &get(unsigned long i) { return m_data[i]; } + /// Get the ith value const T &get(unsigned long i) const { return m_data[i]; } - // get the ith value + /// Get the ith value template void get(unsigned long i, U &val) const; - // get a range of values decribed by [start end] + // get a range of values described by [start end] // inclusive template void get(size_t start, size_t end, U *vals) const; - // copy the data out into the passed in vector + /// Copy the data out into the passed in vector template void get(std::vector &val) const; - // pointer to the data + /// Get a pointer to the data T *get(){ return &m_data[0]; } const T *get() const { return &m_data[0]; } - // set the ith value + /// Set the ith value template void set(unsigned long i, const U &val); - // set a range opf values described by [start end] - // inclusive + /// Set a range of values described by [start end] inclusive template void set(size_t start, size_t end, const U *vals); - // copy data, replacing contents with the passed in - // vector + /// Copy data, replacing contents with the passed in vector template void set(const std::vector &val); - // insert from the passed in vector at the back + /// Insert from the passed in vector at the back template void append(const std::vector &val); @@ -346,54 +606,57 @@ class teca_variant_array_impl : public teca_variant_array template void append(const U &val); - // get the current size of the data + /// Get the current size of the data virtual unsigned long size() const noexcept override; - // resize the data + /// Resize the data virtual void resize(unsigned long n) override; void resize(unsigned long n, const T &val); - // reserve space + /// Reserve space virtual void reserve(unsigned long n) override; - // clear the data + /// Clear the data virtual void clear() noexcept override; - // copy. This method is not virtual so that - // string can be handled as a special case in - // the base class. + /** copy. This method is not virtual so that string can be handled as a + * special case in the base class. + */ void copy(const teca_variant_array &other); - // append. This method is not virtual so that - // string can be handled as a special case in - // the base class. + /** append. This method is not virtual so that + * string can be handled as a special case in the base class. + */ void append(const teca_variant_array &other); - // virtual swap + /// virtual swap void swap(teca_variant_array &other) override; - // virtual equavalince test + /// virtual equivalence test bool equal(const teca_variant_array &other) const override; - // serialize to/from stream + /// Serialize to the stream int to_stream(teca_binary_stream &s) const override { this->to_binary(s); return 0; } + /// Deserialize from the stream int from_stream(teca_binary_stream &s) override { this->from_binary(s); return 0; } + /// Serialize to the stream int to_stream(std::ostream &s) const override { this->to_ascii(s); return 0; } + /// Deserialize from the stream int from_stream(std::ostream &s) override { this->from_ascii(s); @@ -448,7 +711,7 @@ class teca_variant_array_impl : public teca_variant_array void from_binary(teca_binary_stream &s, typename std::enable_if::value, U>::type* = 0); - // tag dispatch array of poniter to other objects + // tag dispatch array of pointer to other objects template void to_binary(teca_binary_stream &s, typename std::enable_if::value, U>::type* = 0) @@ -488,7 +751,7 @@ class teca_variant_array_impl : public teca_variant_array void from_ascii(std::ostream &s, typename std::enable_if::value, U>::type* = 0); - // for serializaztion + // for serialization unsigned int type_code() const noexcept override; private: std::vector m_data; @@ -497,119 +760,9 @@ class teca_variant_array_impl : public teca_variant_array template friend class teca_variant_array_impl; }; - #pragma GCC diagnostic ignored "-Wunknown-pragmas" #pragma GCC diagnostic ignored "-Wunused-local-typedefs" -// tt - derived container -// nt - contained type -// tt - complete derived type -// p - base class pointer -// body - code to execute -#define TEMPLATE_DISPATCH_CASE(tt, nt, p, body) \ - if (dynamic_cast*>(p)) \ - { \ - using TT = tt; \ - using NT = nt; \ - body \ - } - -// tt - derived container -// nt - contained type -// tt - complete derived type -// p - base class pointer -// i - id suffix -// body - code to execute -#define NESTED_TEMPLATE_DISPATCH_CASE(tt, nt, p, i, body) \ - if (dynamic_cast*>(p)) \ - { \ - using TT##i = tt; \ - using NT##i = nt; \ - body \ - } - -// variant that limits dispatch to floating point types -// for use in numerical compuatation where integer types -// are not supported (ie, math operations from std library) -#define TEMPLATE_DISPATCH_FP(t, p, body) \ - TEMPLATE_DISPATCH_CASE(t, float, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, double, p, body) - -// variant that limits dispatch to signed integer types -// for use in numerical compuatation where signed integer types -// are not supported -#define TEMPLATE_DISPATCH_SI(t, p, body) \ - TEMPLATE_DISPATCH_CASE(t, long long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, short int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, char, p, body) - -// variant that limits dispatch to floating point types -// for use in numerical compuatation where integer types -// are not supported (ie, math operations from std library) -#define TEMPLATE_DISPATCH_FP_SI(t, p, body) \ - TEMPLATE_DISPATCH_CASE(t, float, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, double, p, body) \ - else TEMPLATE_DISPATCH_SI(t, p, body) - -// variant that limits dispatch to integer types -// for use in numerical compuatation where floating point types -// are not supported -#define TEMPLATE_DISPATCH_I(t, p, body) \ - TEMPLATE_DISPATCH_CASE(t, long long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, unsigned long long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, unsigned int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, unsigned long, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, short int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, short unsigned int, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, char, p, body) \ - else TEMPLATE_DISPATCH_CASE(t, unsigned char, p, body) - -// macro for helping downcast to POD types -// don't add classes to this. -// t - derived container type -// p - pointer to base class -// body - code to execute on match -#define TEMPLATE_DISPATCH(t, p, body) \ - TEMPLATE_DISPATCH_FP(t, p, body) \ - else TEMPLATE_DISPATCH_I(t, p, body) - -// variant that limits dispatch to floating point types -// for use in numerical compuatation where integer types -// are not supported (ie, math operations from std library) -#define NESTED_TEMPLATE_DISPATCH_FP(t, p, i, body) \ - NESTED_TEMPLATE_DISPATCH_CASE(t, float, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, double, p, i, body) - -// variant that limits dispatch to integer types -// for use in numerical compuatation where integer types -// are not supported (ie, math operations from std library) -#define NESTED_TEMPLATE_DISPATCH_I(t, p, i, body) \ - NESTED_TEMPLATE_DISPATCH_CASE(t, long long, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned long long, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, long, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, int, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned int, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned long, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, short int, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, short unsigned int, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, char, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_CASE(t, unsigned char, p, i, body) - -// macro for helping downcast to POD types -// don't add classes to this. -// t - templated derived type -// p - base class pointer -// i - id for nesting -// body - code to execute on match -#define NESTED_TEMPLATE_DISPATCH(t, p, i, body) \ - NESTED_TEMPLATE_DISPATCH_FP(t, p, i, body) \ - else NESTED_TEMPLATE_DISPATCH_I(t, p, i, body) - - // -------------------------------------------------------------------------- template void teca_variant_array::get_dispatch(std::vector &vals, @@ -1243,6 +1396,7 @@ void teca_variant_array_impl::from_ascii( // TODO } +/// @cond template struct teca_variant_array_code {}; @@ -1256,7 +1410,7 @@ struct teca_variant_array_type {}; template <> \ struct teca_variant_array_code \ { \ - static unsigned int get() noexcept \ + static constexpr unsigned int get() \ { return v; } \ }; \ template <> \ @@ -1269,6 +1423,9 @@ template <> \ struct teca_variant_array_type \ { \ using type = T; \ + \ + static constexpr const char *name() \ + { return #T; } \ }; #define TECA_VARIANT_ARRAY_FACTORY_NEW(_v) \ @@ -1293,7 +1450,9 @@ TECA_VARIANT_ARRAY_TT_SPEC(double, 12) TECA_VARIANT_ARRAY_TT_SPEC(std::string, 13) TECA_VARIANT_ARRAY_TT_SPEC(teca_metadata, 14) TECA_VARIANT_ARRAY_TT_SPEC(p_teca_variant_array, 15) +/// @endcond +/// Creates an instance of teca_variant_array_impl where T is determined from the type code. struct teca_variant_array_factory { static p_teca_variant_array New(unsigned int type_code) @@ -1324,6 +1483,7 @@ struct teca_variant_array_factory } }; +/// @cond #define CODE_DISPATCH_CASE(_v, _c, _code) \ if (_v == _c) \ { \ @@ -1357,6 +1517,7 @@ struct teca_variant_array_factory CODE_DISPATCH_I(_v, _code) \ else CODE_DISPATCH_FP(_v, _code) +/// @endcond // -------------------------------------------------------------------------- template diff --git a/core/teca_variant_array_fwd.h b/core/teca_variant_array_fwd.h deleted file mode 100644 index 734f640c5..000000000 --- a/core/teca_variant_array_fwd.h +++ /dev/null @@ -1,107 +0,0 @@ -#ifndef teca_variant_array_fwd_h -#define teca_variant_array_fwd_h - -#include -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_variant_array) -TECA_SHARED_OBJECT_TEMPLATE_FORWARD_DECL(teca_variant_array_impl) - -#ifndef SWIG -// convenience defs for POD types -// these should not be used in API, use teca_variant_array instead -using teca_string_array = teca_variant_array_impl; -using p_teca_string_array = std::shared_ptr>; -using const_p_teca_string_array = std::shared_ptr>; - -using teca_float_array = teca_variant_array_impl; -using p_teca_float_array = std::shared_ptr>; -using const_p_teca_float_array = std::shared_ptr>; - -using teca_double_array = teca_variant_array_impl; -using p_teca_double_array = std::shared_ptr>; -using const_p_teca_double_array = std::shared_ptr>; - -using teca_char_array = teca_variant_array_impl; -using p_teca_char_array = std::shared_ptr>; -using const_p_teca_char_array = std::shared_ptr>; - -using teca_unsigned_char_array = teca_variant_array_impl; -using p_teca_unsigned_char_array = std::shared_ptr>; -using const_p_teca_unsigned_char_array = std::shared_ptr>; - -using teca_short_array = teca_variant_array_impl; -using p_teca_short_array = std::shared_ptr>; -using const_p_teca_short_array = std::shared_ptr>; - -using teca_unsigned_short_array = teca_variant_array_impl; -using p_teca_unsigned_short_array = std::shared_ptr>; -using const_p_teca_unsigned_short_array = std::shared_ptr>; - -using teca_int_array = teca_variant_array_impl; -using p_teca_int_array = std::shared_ptr>; -using const_p_teca_int_array = std::shared_ptr>; - -using teca_unsigned_int_array = teca_variant_array_impl; -using p_teca_unsigned_int_array = std::shared_ptr>; -using const_p_teca_unsigned_int_array = std::shared_ptr>; - -using teca_long_array = teca_variant_array_impl; -using p_teca_long_array = std::shared_ptr>; -using const_p_teca_long_array = std::shared_ptr>; - -using teca_unsigned_long_array = teca_variant_array_impl; -using p_teca_unsigned_long_array = std::shared_ptr>; -using const_p_teca_unsigned_long_array = std::shared_ptr>; - -using teca_long_long_array = teca_variant_array_impl; -using p_teca_long_long_array = std::shared_ptr>; -using const_p_teca_long_long_array = std::shared_ptr>; - -using teca_unsigned_long_long_array = teca_variant_array_impl; -using p_teca_unsigned_long_long_array = std::shared_ptr>; -using const_p_teca_unsigned_long_long_array = std::shared_ptr>; - -using teca_size_t_array = teca_variant_array_impl; -using p_teca_size_t_array = std::shared_ptr>; -using const_p_teca_size_t_array = std::shared_ptr>; -#endif - -// this is a convenience macro to be used to declare a static -// New method that will be used to construct new objects in -// shared_ptr's. This manages the details of interoperability -// with std C++11 shared pointer -#define TECA_VARIANT_ARRAY_STATIC_NEW(T, t) \ - \ -static std::shared_ptr> New() \ -{ \ - return std::shared_ptr>(new T); \ -} \ - \ -static std::shared_ptr> New(size_t n) \ -{ \ - return std::shared_ptr>(new T(n)); \ -} \ - \ -static std::shared_ptr> New(size_t n, const t &v) \ -{ \ - return std::shared_ptr>(new T(n, v)); \ -} \ - \ -static std::shared_ptr> New(const t *vals, size_t n) \ -{ \ - return std::shared_ptr>(new T(vals, n)); \ -} \ - \ -using teca_variant_array::shared_from_this; \ - \ -std::shared_ptr shared_from_this() \ -{ \ - return std::static_pointer_cast(shared_from_this()); \ -} \ - \ -std::shared_ptr shared_from_this() const \ -{ \ - return std::static_pointer_cast(shared_from_this()); \ -} -#endif diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 22c186805..72532f094 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -9,6 +9,7 @@ set(teca_data_srcs teca_arakawa_c_grid.cxx teca_array_attributes.cxx teca_array_collection.cxx + teca_calendar_util.cxx teca_cartesian_mesh.cxx teca_coordinate_util.cxx teca_curvilinear_mesh.cxx diff --git a/data/teca_arakawa_c_grid.h b/data/teca_arakawa_c_grid.h index f7e695a62..8d08eb446 100644 --- a/data/teca_arakawa_c_grid.h +++ b/data/teca_arakawa_c_grid.h @@ -11,56 +11,60 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_arakawa_c_grid) -/// data on an Arkawa C Grid -// -// The C grid is defined by combinations of horizontal and vertical centerings. -// -// The horzontal centerings occur at so called mass or M points, U points, and -// V points. These centerings are depicted in the following diagram: -// -// *-------V-------* -// | | -// | | -// | | -// U M U -// | | -// | | -// | | -// *-------V-------* -// -// The horizontal coordinates are stored in 2d arrays. Assuming the mass -// corrinate arrays have dimension [nx, ny], then the U coordinate arrays have -// dimension [nx + 1, ny], and the V coordinate arrays have dimension -// [nx, ny + 1]. -// -// The vertical centerings occur at so called mass and w points -// -// *-------W-------* -// | | -// | | -// | | -// | M | -// | | -// | | -// | | -// *-------W-------* -// -// The vertical coordinates are stored in 1d arrays. Assuming the M vertical -// coordinate has the dimension [nz], then the W coordinate has dimension -// [nz + 1]. -// -// The 3d mesh dimensions can be obtained from mesh metadata, as well as coordinate -// array names, and array attributes describing the data type, units, etc. -// -// Variables may exist on one of a number of permutations of horizontal and -// vertical centerings, array attributes contains the centering metadata. -// -// See also: -// "A Description of the Advanced Research WRF Model Version 4", -// NCAR/TN-556+STR -// -// "Grids in Numerical Weather and Climate Models" -// http://dx.doi.org/10.5772/55922 +/// A representation of mesh based data on an Arkawa C Grid. +/** + * The Arakawa C grid is defined by various combinations of horizontal and + * vertical centerings. + * + * The horizontal centerings occur at so called mass or M points, U points, + * and V points. These centerings are depicted in the following diagram: + * + * > *-------V-------* + * > | | + * > | | + * > | | + * > U M U + * > | | + * > | | + * > | | + * > *-------V-------* + * + * The horizontal coordinates are stored in 2d arrays. Assuming the mass + * coordinate arrays have dimension [nx, ny], then the U coordinate arrays + * have dimension [nx + 1, ny], and the V coordinate arrays have dimension + * [nx, ny + 1]. + * + * The vertical centerings occur at so called M points and W points. These + * centerings are depicted in the following diagram. + * + * > *-------W-------* + * > | | + * > | | + * > | | + * > | M | + * > | | + * > | | + * > | | + * > *-------W-------* + * + * The vertical coordinates are stored in 1d arrays. Assuming the M vertical + * coordinate has the dimension [nz], then the W coordinate has dimension + * [nz + 1]. + * + * The 3d mesh dimensions can be obtained from mesh metadata, as well as + * coordinate array names, and array attributes describing the data type, + * units, etc. + * + * Variables may exist on one of a number of permutations of horizontal and + * vertical centerings, array attributes contains the centering metadata. + * + * See also: + * "A Description of the Advanced Research WRF Model Version 4", + * NCAR/TN-556+STR + * + * "Grids in Numerical Weather and Climate Models" + * http://dx.doi.org/10.5772/55922 + */ class teca_arakawa_c_grid : public teca_mesh { public: @@ -75,7 +79,7 @@ class teca_arakawa_c_grid : public teca_mesh TECA_DATASET_METADATA(extent, unsigned long, 6) TECA_DATASET_METADATA(bounds, double, 6) - // flag set if the boundary in the given direction is perdiodic + // flag set if the boundary in the given direction is periodic TECA_DATASET_METADATA(periodic_in_x, int, 1) TECA_DATASET_METADATA(periodic_in_y, int, 1) TECA_DATASET_METADATA(periodic_in_z, int, 1) diff --git a/data/teca_array_attributes.cxx b/data/teca_array_attributes.cxx index b1b8fcd72..a1191bcbf 100644 --- a/data/teca_array_attributes.cxx +++ b/data/teca_array_attributes.cxx @@ -149,7 +149,7 @@ int teca_array_attributes::from(const teca_metadata &md) } // -------------------------------------------------------------------------- -void teca_array_attributes::to_stream(std::ostream &os) +void teca_array_attributes::to_stream(std::ostream &os) const { os << "type_code=" << type_code << ", centering=" << centering << ", size=" << size << ", units=\"" << units @@ -171,3 +171,49 @@ void teca_array_attributes::to_stream(std::ostream &os) os << "None"; } } + +// -------------------------------------------------------------------------- +const char *teca_array_attributes::centering_to_string(int cen) +{ + const char *cen_str = "invalid"; + switch (cen) + { + case cell_centering: + cen_str = "cell"; + break; + + case x_face_centering: + cen_str = "x-face"; + break; + + case y_face_centering: + cen_str = "y-face"; + break; + + case z_face_centering: + cen_str = "z-face"; + break; + + case x_edge_centering: + cen_str = "x-edge"; + break; + + case y_edge_centering: + cen_str = "y-edge"; + break; + + case z_edge_centering: + cen_str = "z-edge"; + break; + + case point_centering: + cen_str = "point"; + break; + + case no_centering: + cen_str = "none"; + break; + } + + return cen_str; +} diff --git a/data/teca_array_attributes.h b/data/teca_array_attributes.h index c7b976003..5052da669 100644 --- a/data/teca_array_attributes.h +++ b/data/teca_array_attributes.h @@ -6,26 +6,25 @@ #include #include -// a convenience container for conventional array attributes. -// the attributes listed here are used for CF I/O. -// -// type_code - storage type as defined by teca_variant_array::type_code() -// -// centering - one of: no_centering, point_centering, cell_centering, -// edge_centering, or face_centering -// -// size - number of elements in the array -// -// units - string describing the uints that the variable is in. -// -// long name - a more descriptive name -// -// description - text describing the data -// -// have_fill_value - set non-zero to indicate that a fill_value -// has been provided. -// -// fill_value - value used to identify missing or invalid data +/** @brief + * A convenience container for conventional array attributes necessary and/or + * useful when producing NetCDF CF format files using the teca_cf_writer. + * + * @details + * + * | Member | Description | + * | ------ | ----------- | + * | type_code | storage type as defined by teca_variant_array::type_code() | + * | centering | one of: no_centering, point_centering, cell_centering, | + * | | edge_centering, or face_centering | + * | size | number of elements in the array | + * | units | string describing the units that the variable is in. | + * | long name | a more descriptive name | + * | description | text describing the data | + * | have_fill_value | set non-zero to indicate that a fill_value has been | + * | | provided. | + * | fill_value | value used to identify missing or invalid data | + */ struct teca_array_attributes { teca_array_attributes() : type_code(0), @@ -44,7 +43,7 @@ struct teca_array_attributes teca_array_attributes(const teca_array_attributes &) = default; teca_array_attributes &operator=(const teca_array_attributes &) = default; - // converts from metadata object. + /// Convert a from metadata object. teca_array_attributes(const teca_metadata &md) : type_code(0), centering(0), size(0), units(), long_name(), description(), have_fill_value(0), fill_value(1.e20f) @@ -52,47 +51,47 @@ struct teca_array_attributes from(md); } - // convert from metadata object + /// Convert from metadata object. teca_array_attributes &operator=(const teca_metadata &md); - // converts to a metadata object + /// Converts to a metadata object. operator teca_metadata() const; - // adds current values to the metadata object + /// Adds current values to the metadata object int to(teca_metadata &md) const; - // adds current values to the metadata object, - // only if they don't exist + /// Adds the current values to the metadata object, only if they don't exist. int merge_to(teca_metadata &md) const; - // intializes values from the metadata object + /// Intializes values from the metadata object. int from(const teca_metadata &md); - // send to the stream in human readable form - void to_stream(std::ostream &os); - - // possible centrings - // - // for coordinate system with orthogonal axes x,y,z - // relative to cell centering: - // - // If A is one of x,y or z then A_face_centering data is located on the - // low A face i.e. shifted in the -A direction and arrays will be longer - // by 1 value in the A direction. - // - // If A is one of x,y or z then A_edge_centering data is located on the - // low side edge parallel to A corrdinate axis. i.e. shifted in the -B - // and -C directions and arrays will be longer by 1 value in the B and C - // directions. - // - // point_centering data is located on the low corner. i.e. shifted - // in the -A,-B, and -C directions and arrays will be longer - // by 1 value in the A, B and C directions. - // - // arrays that are not associated with geometric locations should - // be identified as no_centering. - // - // the default centering is cell centering. + /// Send to the stream in human readable form. + void to_stream(std::ostream &os) const; + + /** The possible mesh centrings. + * + * For coordinate system with orthogonal axes x,y,z relative to cell + * centering: + * + * > If A is one of x,y or z then A_face_centering data is located on the + * > low A face i.e. shifted in the -A direction and arrays will be longer + * > by 1 value in the A direction. + * > + * > If A is one of x,y or z then A_edge_centering data is located on the + * > low side edge parallel to A corrdinate axis. i.e. shifted in the -B + * > and -C directions and arrays will be longer by 1 value in the B and C + * > directions. + * > + * > point_centering data is located on the low corner. i.e. shifted + * > in the -A,-B, and -C directions and arrays will be longer + * > by 1 value in the A, B and C directions. + * + * Arrays that are not associated with geometric locations should + * be identified as no_centering. + * + * The default centering is cell centering. + */ enum { invalid_value = 0, @@ -107,6 +106,9 @@ struct teca_array_attributes no_centering = 0x1000, }; + /// convert the centering code to a string + static const char *centering_to_string(int cen); + using fill_value_t = std::variant #include #include -// a collection of named arrays -/** -A collection of named arrays -*/ +TECA_SHARED_OBJECT_FORWARD_DECL(teca_array_collection) + +/// A collection of named arrays. class teca_array_collection { public: - // construct on heap + + /// construct on heap static p_teca_array_collection New() { return p_teca_array_collection(new teca_array_collection()); } - // reset to empty state + /// reset to empty state void clear(); - // declare a set of arrays. requires name,type pairs - // for ex. define("c1",int(),"c2",float()) creates - // 2 arrays in the collection the first storing int's - // the second storing float's. + /** declare a set of arrays. requires name,type pairs for ex. + * define("c1",int(),"c2",float()) creates 2 arrays in the collection the + * first storing int the second storing float. + */ template void declare_set(nT &&a_name, aT a_type, oT &&...args); - // declare a single array + /// declare a single array template void declare(nT &&a_name, aT a_type); - // add, return the index of the new entry. - // or -1 if the array name already exists. + /** add, return the index of the new entry, or -1 if the array name already + * exists. + */ int append(p_teca_variant_array array); + + /** add, return the index of the new entry, or -1 if the array name already + * exists. + */ int append(const std::string &name, p_teca_variant_array array); - // set, return 0 on success. + /** replace the ith array, return 0 on success. the name of the array is + * not changed. + */ int set(unsigned int i, p_teca_variant_array array); + + /// add or replace the named array, returns 0 on success. int set(const std::string &name, p_teca_variant_array array); - // remove + /// remove the ith array int remove(unsigned int i); + + /// remove the named array int remove(const std::string &name); - // number of + /// Return the number of arrays unsigned int size() const noexcept { return m_arrays.size(); } - // access by id + /// access an array by its by id p_teca_variant_array get(unsigned int i) { return m_arrays[i]; } + /// access an array by its by id const_p_teca_variant_array get(unsigned int i) const { return m_arrays[i]; } - // test for array + /// test for array bool has(const std::string &name) const; - // access by name + /// access an array by name p_teca_variant_array get(const std::string &name); + + /// access an array by name const_p_teca_variant_array get(const std::string &name) const; + /// access an array by name p_teca_variant_array operator[](const std::string &name) { return this->get(name); } + /// access an array by name const_p_teca_variant_array operator[](const std::string &name) const { return this->get(name); } - // access names + // Get the name of the ith array std::string &get_name(unsigned int i) { return m_names[i]; } + // Get the name of the ith array const std::string &get_name(unsigned int i) const { return m_names[i]; } - // return a unique string identifier + // Get the list of names + std::vector &get_names() + { return m_names; } + + // Get the list of names + const std::vector &get_names() const + { return m_names; } + + /// Return the name of the class std::string get_class_name() const { return "teca_array_collection"; } - // return an integer identifier uniquely naming the dataset type + /// return an integer identifier uniquely naming the dataset type int get_type_code() const { return -1; } - // copy + /// copy void copy(const const_p_teca_array_collection &other); + + /// shallow copy void shallow_copy(const p_teca_array_collection &other); - // append + /// append int append(const const_p_teca_array_collection &other); + + /// shallow append int shallow_append(const p_teca_array_collection &other); - // swap + /// swap void swap(p_teca_array_collection &other); - // serialize the data to/from the given stream - // for I/O or communication + /// serialize the data to the given stream for I/O or communication int to_stream(teca_binary_stream &s) const; + + /// serialize the data from the given stream for I/O or communication int from_stream(teca_binary_stream &s); - // stream to/from human readable representation + /// stream to a human readable representation int to_stream(std::ostream &) const; protected: diff --git a/data/teca_array_collection_fwd.h b/data/teca_array_collection_fwd.h deleted file mode 100644 index d014188c4..000000000 --- a/data/teca_array_collection_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_array_collection_fwd_h -#define teca_array_collection_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_array_collection) - -#endif diff --git a/data/teca_calendar_util.cxx b/data/teca_calendar_util.cxx new file mode 100644 index 000000000..5d8954faa --- /dev/null +++ b/data/teca_calendar_util.cxx @@ -0,0 +1,857 @@ +#include "teca_calendar_util.h" + +#include "teca_common.h" +#include "teca_variant_array.h" +#include "teca_coordinate_util.h" +#include "teca_calcalcs.h" + +#include + +// TODO - With 23:59:59, sometimes we select the next day. What is the right +// end-of-day time so that all use cases are satisfied without selecting the +// next day? +#define END_OF_DAY_HH 23 +#define END_OF_DAY_MM 30 +#define END_OF_DAY_SS 0 +#define END_OF_DAY "23:30:00" + +// -------------------------------------------------------------------------- +std::ostream &operator<<(std::ostream &os, + const teca_calendar_util::time_point &tpt) +{ + os << "time[" << tpt.index << "] = " << tpt.time << ", \"" + << tpt.year << "-" << tpt.month << "-" + << tpt.day << " " << tpt.hour << ":" << tpt.minute + << ":" << tpt.second << "\""; + return os; +} + + + +namespace teca_calendar_util +{ + +// ************************************************************************** +long gregorian_number(long y, long m, long d) +{ + m = (m + 9) % 12; + y = y - m/10; + return 365*y + y/4 - y/100 + y/400 + (m*306 + 5)/10 + (d - 1); +} + +// ************************************************************************** +void date_from_gregorian_number(long g, long &y, long &m, long &d) +{ + y = (10000*g + 14780)/3652425; + long ddd = g - (365*y + y/4 - y/100 + y/400); + if (ddd < 0) + { + y = y - 1; + ddd = g - (365*y + y/4 - y/100 + y/400); + } + + long mi = (100*ddd + 52)/3060; + + m = (mi + 2)%12 + 1; + y = y + (mi + 2)/12; + d = ddd - (mi*306 + 5)/10 + 1; +} + +// ************************************************************************** +bool valid_gregorian_date(long y, long m, long d) +{ + long g = gregorian_number(y,m,d); + if (g < 578027) // 578027 = gergorian_number(1582,10,1); + return false; + + long yy, mm, dd; + date_from_gregorian_number(g, yy, mm, dd); + + if ((y != yy) || (m != mm) || (d != dd)) + return false; + + return true; +} + + +/// returns one of DJF,MAM,JJA,SON based on the month passed in +const char *get_season_name(int month) +{ + if ((month == 12) || ((month >= 1) && (month <= 2))) + { + return "DJF"; + } + else if ((month >= 3) && (month <= 5)) + { + return "MAM"; + } + else if ((month >= 6) && (month <= 8)) + { + return "JJA"; + } + else if ((month >= 9) && (month <= 11)) + { + return "SON"; + } + + TECA_ERROR("Failed to get the season name for month " << month) + return "invalid"; +} + +// -------------------------------------------------------------------------- +time_point::time_point(long i, double t, const std::string &units, + const std::string &calendar) : index(i), time(t), year(0), month(0), + day(0), hour(0), minute(0), second(0) + +{ + if (teca_calcalcs::date(t, &this->year, &this->month, &this->day, + &this->hour, &this->minute, &this->second, units.c_str(), + calendar.c_str())) + { + TECA_ERROR("Failed to convert the time value " << t + << " \"" << units << "\" in the \"" << calendar << "\"") + } +} + +// -------------------------------------------------------------------------- +bool season_iterator::is_valid() const +{ + if (!this->valid) + return false; + + // get the end of the current season + int ey = -1; + int em = -1; + int ed = -1; + if (this->get_season_end(this->year, this->month, ey, em, ed)) + { + TECA_ERROR("Failed to get season end") + return false; + } + + // verify that we have data for the current season + if ((ey > this->end.year) || + ((ey == this->end.year) && (em > this->end.month)) || + ((ey == this->end.year) && (em == this->end.month) && + (ed > this->end.day))) + { + return false; + } + + return true; +} + +// -------------------------------------------------------------------------- +int season_iterator::initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) +{ + this->time = t; + this->units = units; + this->calendar = calendar; + + if (t->size() == 0) + { + TECA_ERROR("The array of time values can't be empty") + return -1; + } + + if (first_step >= (long)t->size()) + { + TECA_ERROR("first_step " << first_step + << " output of bounds with " << t->size() << " time values") + return -1; + } + + if (last_step < 0) + last_step = t->size() - 1; + + if ((last_step < first_step) || (last_step >= (long)t->size())) + { + TECA_ERROR("invalid last_step " << last_step << " with first_step " + << first_step << " and " << t->size() << " time values") + return -1; + } + + // initialize the time range to iterate over + TEMPLATE_DISPATCH(const teca_variant_array_impl, + t.get(), + + const NT *p_t = dynamic_cast(t.get())->get(); + + this->begin = time_point(first_step, p_t[first_step], this->units, this->calendar); + this->end = time_point(last_step, p_t[last_step], this->units, this->calendar); + ) + + // skip ahead to the first season. + if (this->get_first_season(this->begin.year, + this->begin.month, this->year, this->month)) + { + TECA_ERROR("Failed to determine the first season") + return -1; + } + + this->valid = true; + + return 0; +} + + +// -------------------------------------------------------------------------- +int season_iterator::get_first_season(int y_in, int m_in, int &y_out, + int &m_out) const +{ + if ((m_in == 12) or (m_in == 3) or (m_in == 6) or (m_in == 9)) + { + y_out = y_in; + m_out = m_in; + return 0; + } + + return this->get_next_season(y_in, m_in, y_out, m_out); +} + +// -------------------------------------------------------------------------- +int season_iterator::get_season_end(int y_in, int m_in, int &y_out, + int &m_out, int &d_out) const +{ + if (m_in == 12) + { + y_out = y_in + 1; + m_out = 2; + } + else if ((m_in >= 1) and (m_in <= 2)) + { + y_out = y_in; + m_out = 2; + } + else if ((m_in >= 3) and (m_in <= 5)) + { + y_out = y_in; + m_out = 5; + } + else if ((m_in >= 6) and (m_in <= 8)) + { + y_out = y_in; + m_out = 8; + } + else if ((m_in >= 9) and (m_in <= 11)) + { + y_out = y_in; + m_out = 11; + } + else + { + TECA_ERROR("Failed to get the end of the season from month " + << m_in) + return -1; + } + + if (teca_calcalcs::days_in_month(this->calendar.c_str(), + this->units.c_str(), y_out, m_out, d_out)) + { + TECA_ERROR("Failed to get the last day of the month " + << y_out << " " << m_out) + return -1; + } + + return 0; +} + +// -------------------------------------------------------------------------- +int season_iterator::get_next_season(int y_in, int m_in, int &y_out, + int &m_out) const +{ + if (m_in == 12) + { + y_out = y_in + 1; + m_out = 3; + } + else if ((m_in >= 1) and (m_in <= 2)) + { + y_out = y_in; + m_out = 3; + } + else if ((m_in >= 3) and (m_in <= 5)) + { + y_out = y_in; + m_out = 6; + } + else if ((m_in >= 6) and (m_in <= 8)) + { + y_out = y_in; + m_out = 9; + } + else if ((m_in >= 9) and (m_in <= 11)) + { + y_out = y_in; + m_out = 12; + } + else + { + TECA_ERROR("Failed to get the next season from m_in " + << m_in) + return -1; + } + + return 0; +} + +// -------------------------------------------------------------------------- +int season_iterator::get_next_interval(time_point &first_step, + time_point &last_step) +{ + // get the end of the current season + int end_year = -1; + int end_month = -1; + int end_day = -1; + if (this->get_season_end(this->year, this->month, end_year, end_month, end_day)) + { + TECA_ERROR("Failed to get season end") + return -1; + } + + // verify that we have data for the current season + if ((end_year > this->end.year) || + ((end_year == this->end.year) && (end_month > this->end.month)) || + ((end_year == this->end.year) && (end_month == this->end.month) && + (end_day > this->end.day))) + { + return -1; + } + + // find the time step of the first day + int sy = this->year; + int sm = this->month; + + char t0[21] = {'\0'}; + snprintf(t0, 21, "%04d-%02d-01 00:00:00", this->year, this->month); + + unsigned long i0 = 0; + if (teca_coordinate_util::time_step_of(this->time, + false, true, this->calendar, this->units, t0, i0)) + { + TECA_ERROR("Failed to get the tme step of " << t0) + return -1; + } + + double ti = 0.0; + this->time->get(i0, ti); + first_step = time_point(i0, ti, this->year, this->month); + + // find the time step of the last day + char t1[21] = {'\0'}; + snprintf(t1, 21, "%04d-%02d-%02d " END_OF_DAY, end_year, end_month, end_day); + + unsigned long i1 = 0; + if (teca_coordinate_util::time_step_of(this->time, + true, true, this->calendar, this->units, t1, i1)) + { + TECA_ERROR("Failed to get the time step of " << t1) + return -1; + } + + this->time->get(i1, ti); + last_step = time_point(i1, ti, end_year, end_month, + end_day, END_OF_DAY_HH, END_OF_DAY_MM, END_OF_DAY_SS); + + // move to next season + if (this->get_next_season(sy, sm, this->year, this->month)) + { + TECA_ERROR("Failed to get the next season from " + << sy << "-" << sm) + return -1; + } + + if (!this->is_valid()) + this->valid = false; + + return 0; +} + +// -------------------------------------------------------------------------- +bool year_iterator::is_valid() const +{ + if (!this->valid) + return false; + + // check for more months to process + if (this->year > this->end.year) + { + return false; + } + + return true; +} + +// -------------------------------------------------------------------------- +int year_iterator::initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) +{ + this->time = t; + this->units = units; + this->calendar = calendar; + + if (t->size() == 0) + { + TECA_ERROR("The array of time values can't be empty") + return -1; + } + + if (first_step >= (long)t->size()) + { + TECA_ERROR("first_step " << first_step + << " output of bounds with " << t->size() << " time values") + return -1; + } + + if (last_step < 0) + last_step = t->size() - 1; + + if ((last_step < first_step) || (last_step >= (long)t->size())) + { + TECA_ERROR("invalid last_step " << last_step << " with first_step " + << first_step << " and " << t->size() << " time values") + return -1; + } + + // current time state + TEMPLATE_DISPATCH(const teca_variant_array_impl, + t.get(), + + const NT *p_t = dynamic_cast(t.get())->get(); + + this->begin = time_point(first_step, p_t[first_step], this->units, this->calendar); + this->end = time_point(last_step, p_t[last_step], this->units, this->calendar); + ) + + this->valid = true; + + this->year = this->begin.year; + + return 0; +} + +// -------------------------------------------------------------------------- +int year_iterator::get_next_interval(time_point &first_step, + time_point &last_step) +{ + // check for more months to process + if (!this->is_valid()) + return -1; + + // find the time step of the first day + char t0[21] = {'\0'}; + snprintf(t0, 21, "%04d-01-01 00:00:00", this->year); + + unsigned long i0 = 0; + if (teca_coordinate_util::time_step_of(this->time, + false, true, this->calendar, this->units, t0, i0)) + { + TECA_ERROR("Failed to locate a time step for " << t0) + return -1; + } + + double ti = 0.0; + this->time->get(i0, ti); + first_step = time_point(i0, ti, this->year); + + // find the time step of the last day + int n_days = 0; + if (teca_calcalcs::days_in_month(this->calendar.c_str(), + this->units.c_str(), this->year, 12, n_days)) + { + TECA_ERROR("Failed to get the last day of the month " + << this->year << " 12") + return -1; + } + + char t1[21] = {'\0'}; + snprintf(t1, 21, "%04d-12-%02d " END_OF_DAY, this->year, n_days); + + unsigned long i1 = 0; + if (teca_coordinate_util::time_step_of(this->time, + true, true, this->calendar, this->units, t1, i1)) + { + TECA_ERROR("Failed to locate a time step for " << t1) + return -1; + } + + this->time->get(i1, ti); + last_step = time_point(i1, ti, this->year, 12, + n_days, END_OF_DAY_HH, END_OF_DAY_MM, END_OF_DAY_SS); + + // move to next year + this->year += 1; + + // if we're at the end of the sequence mark the iterator invalid + if (!this->is_valid()) + this->valid = false; + + return 0; +} + +// -------------------------------------------------------------------------- +bool month_iterator::is_valid() const +{ + if (!this->valid) + return false; + + // check for more months to process + if ((this->year > this->end.year) || + ((this->year == this->end.year) && + (this->month > this->end.month))) + { + return false; + } + + return true; +} + +// -------------------------------------------------------------------------- +int month_iterator::initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) +{ + this->time = t; + this->units = units; + this->calendar = calendar; + + if (t->size() == 0) + { + TECA_ERROR("The array of time values can't be empty") + return -1; + } + + if (first_step >= (long)t->size()) + { + TECA_ERROR("first_step " << first_step + << " output of bounds with " << t->size() << " time values") + return -1; + } + + if (last_step < 0) + last_step = t->size() - 1; + + if ((last_step < first_step) || (last_step >= (long)t->size())) + { + TECA_ERROR("invalid last_step " << last_step << " with first_step " + << first_step << " and " << t->size() << " time values") + return -1; + } + + // time point's to iterate between + TEMPLATE_DISPATCH(const teca_variant_array_impl, + t.get(), + + const NT *p_t = dynamic_cast(t.get())->get(); + + this->begin = time_point(first_step, p_t[first_step], this->units, this->calendar); + this->end = time_point(last_step, p_t[last_step], this->units, this->calendar); + ) + + this->valid = true; + + this->year = this->begin.year; + this->month = this->begin.month; + + return 0; +} + +// -------------------------------------------------------------------------- +int month_iterator::get_next_interval(time_point &first_step, + time_point &last_step) +{ + // check for more months to process + if (!this->is_valid()) + return -1; + + // find the time step of the first day + char t0[21] = {'\0'}; + snprintf(t0, 21, "%04d-%02d-01 00:00:00", this->year, this->month); + + unsigned long i0 = 0; + if (teca_coordinate_util::time_step_of(this->time, + false, true, this->calendar, this->units, t0, i0)) + { + TECA_ERROR("Failed to locate a time step for " << t0) + return -1; + } + + double ti = 0.0; + this->time->get(i0, ti); + first_step = time_point(i0, ti, this->year, this->month); + + // find the time step of the last day + int n_days = 0; + if (teca_calcalcs::days_in_month(this->calendar.c_str(), + this->units.c_str(), this->year, this->month, n_days)) + { + TECA_ERROR("Failed to get the last day of the month " + << this->year << " " << this->month) + return -1; + } + + char t1[21] = {'\0'}; + snprintf(t1, 21, "%04d-%02d-%02d " END_OF_DAY, this->year, this->month, n_days); + + unsigned long i1 = 0; + if (teca_coordinate_util::time_step_of(this->time, + true, true, this->calendar, this->units, t1, i1)) + { + TECA_ERROR("Failed to locate a time step for " << t1) + return -1; + } + + this->time->get(i1, ti); + last_step = time_point(i1, ti, this->year, this->month, + n_days, END_OF_DAY_HH, END_OF_DAY_MM, END_OF_DAY_SS); + + // move to next month + this->month += 1; + + // move to next year + if (this->month == 13) + { + this->month = 1; + this->year += 1; + } + + // if we're at the end of the sequence mark the iterator invalid + if (!this->is_valid()) + this->valid = false; + + return 0; +} + +// -------------------------------------------------------------------------- +bool day_iterator::is_valid() const +{ + if (!this->valid) + return false; + + // check for more days to process + if ((this->year > this->end.year) || + ((this->year == this->end.year) && (this->month > this->end.month)) || + ((this->year == this->end.year) && (this->month == this->end.month) && + (this->day > this->end.day))) + { + return false; + } + + return true; +} + +// -------------------------------------------------------------------------- +int day_iterator::initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) +{ + this->time = t; + this->units = units; + this->calendar = calendar; + + if (t->size() == 0) + { + TECA_ERROR("The array of time values can't be empty") + return -1; + } + + if (first_step >= (long)t->size()) + { + TECA_ERROR("first_step " << first_step + << " output of bounds with " << t->size() << " time values") + return -1; + } + + if (last_step < 0) + last_step = t->size() - 1; + + if ((last_step < first_step) || (last_step >= (long)t->size())) + { + TECA_ERROR("invalid last_step " << last_step << " with first_step " + << first_step << " and " << t->size() << " time values") + return -1; + } + + // current time state + TEMPLATE_DISPATCH(const teca_variant_array_impl, + t.get(), + + const NT *p_t = dynamic_cast(t.get())->get(); + + this->begin = time_point(first_step, p_t[first_step], this->units, this->calendar); + this->end = time_point(last_step, p_t[last_step], this->units, this->calendar); + ) + + this->valid = true; + + // current time state + this->year = this->begin.year; + this->month = this->begin.month; + this->day = this->begin.day; + + return 0; +} + +// -------------------------------------------------------------------------- +int day_iterator::get_next_interval(time_point &first_step, + time_point &last_step) +{ + // check for more days to process + if (!this->is_valid()) + return -1; + + // find the time step of the first hour of the day + char tstr0[21] = {'\0'}; + snprintf(tstr0, 21, "%04d-%02d-%02d 00:00:00", + this->year, this->month, this->day); + + unsigned long i0 = 0; + if (teca_coordinate_util::time_step_of(this->time, + false, true, this->calendar, this->units, tstr0, i0)) + { + TECA_ERROR("Failed to locate a time step for " << tstr0) + return -1; + } + + double ti = 0.0; + this->time->get(i0, ti); + first_step = time_point(i0, ti, this->year, this->month, this->day); + + // find the time step of the last hour of the day + char t1[21] = {'\0'}; + snprintf(t1, 21, "%04d-%02d-%02d " END_OF_DAY, + this->year, this->month, this->day); + + unsigned long i1 = 0; + if (teca_coordinate_util::time_step_of(this->time, + true, true, this->calendar, this->units, t1, i1)) + { + TECA_ERROR("Failed to locate a time step for " << t1) + return -1; + } + + this->time->get(i1, ti); + last_step = time_point(i1, ti, this->year, this->month, + this->day, END_OF_DAY_HH, END_OF_DAY_MM, END_OF_DAY_SS); + + // move to next day + int n_days = 0; + if (teca_calcalcs::days_in_month(this->calendar.c_str(), + this->units.c_str(), this->year, this->month, n_days)) + { + TECA_ERROR("Failed to get the last day of the month " + << this->year << " " << this->month) + return -1; + } + + this->day += 1; + + // move to next month + if (this->day > n_days) + { + this->month += 1; + this->day = 1; + } + + // move to next year + if (this->month == 13) + { + this->month = 1; + this->year += 1; + } + + // if we're at the end of the sequence mark the iterator invalid + if (!this->is_valid()) + this->valid = false; + + return 0; +} + +// -------------------------------------------------------------------------- +int interval_iterator::initialize(const teca_metadata &md) +{ + return this->initialize(md, 0, -1); +} + +// -------------------------------------------------------------------------- +int interval_iterator::initialize(const teca_metadata &md, + long first_step, long last_step) +{ + // get the time axis and calendar + teca_metadata attributes; + teca_metadata t_atts; + std::string calendar; + std::string units; + teca_metadata coords; + p_teca_variant_array t; + if (md.get("attributes", attributes) || attributes.get("time", t_atts) || + t_atts.get("calendar", calendar) || t_atts.get("units", units) || + md.get("coordinates", coords) || !(t = coords.get("t"))) + { + TECA_ERROR("Failed to get the time axis from the available metadata." + << (attributes.empty() ? "missing" : "has") << " attributes. " + << (t_atts.empty() ? "missing" : "has") << " time attributes. " + << (calendar.empty() ? "missing" : "has") << " calendar ." + << (units.empty() ? "missing" : "has") << " time units. " + << (coords.empty() ? "missing" : "has") << " coordinates. " + << (t ? "has" : "missing") << " time values.") + return -1; + } + + return this->initialize(t, units, calendar, first_step, last_step); +} + +// -------------------------------------------------------------------------- +p_interval_iterator interval_iterator_factory::New(const std::string &interval) +{ + if (interval == "daily") + { + return std::make_shared(); + } + else if (interval == "monthly") + { + return std::make_shared(); + } + else if (interval == "seasonal") + { + return std::make_shared(); + } + else if (interval == "yearly") + { + return std::make_shared(); + } + + TECA_ERROR("Failed to construct a \"" + << interval << "\" interval iterator") + return nullptr; +} + +// -------------------------------------------------------------------------- +p_interval_iterator interval_iterator_factory::New(int interval) +{ + if (interval == daily) + { + return std::make_shared(); + } + else if (interval == monthly) + { + return std::make_shared(); + } + else if (interval == seasonal) + { + return std::make_shared(); + } + else if (interval == yearly) + { + return std::make_shared(); + } + + TECA_ERROR("Failed to construct a \"" + << interval << "\" interval iterator") + return nullptr; +} + +} diff --git a/data/teca_calendar_util.h b/data/teca_calendar_util.h new file mode 100644 index 000000000..5d8255ad9 --- /dev/null +++ b/data/teca_calendar_util.h @@ -0,0 +1,387 @@ +#ifndef teca_calendar_h +#define teca_calendar_h + +/// @file + +#include "teca_variant_array.h" +#include "teca_metadata.h" + +#include +#include +#include +#include + +/// Codes dealing with calendaring +namespace teca_calendar_util +{ + +/** @name Gregorian calendar + * functions for date computations in gregorian calendar. to use convert the + * origin to a gergorian_number do the calculation and convert the number back + * into a date useing date_from_gregorian_number. for details about the math + * and an explanation of the errors see + * http://alcor.concordia.ca/~gpkatch/gdate-algorithm.html + */ +///@{ +/** return a date number for the given date that can be used in computations. + * input: + * + * > y : 4 digit year + * > m : 2 digit month + * > d : 2 digit day + * + */ +long gregorian_number(long y, long m, long d); + +/** input: + * + * > g : date number computed from gregorian_number + * + * returns: + * + * > y : 4 digit year + * > m : 2 digit month + * > d : 2 digit day + * + */ +void date_from_gregorian_number(long g, long &y, long &m, long &d); + +/** + * input: + * + * > y : 4 digit year + * > m : 2 digit month + * > d : 2 digit day + * + * returns: + * + * true if the date is valid in the gregorian calendar and our conversion + * algorithm. +*/ +bool valid_gregorian_date(long y, long m, long d); +///@} + + +/// returns one of DJF,MAM,JJA,SON based on the month passed in +const char *get_season_name(int month); + +/** brief + * A floating point time value and its corresponding year, month day, hour + * minute and second + */ +struct time_point +{ + time_point() : index(-1), time(0.0), year(0), month(1), day(1), + hour(0), minute(0), second(0.0) + {} + + /** Initialize explicitly. + * @param[in] i the index of the time value + * @param[in] t the time value + * @param[in] YYYY the year + * @param[in] MM the month + * @param[in] DD the day + * @param[in] hh the hour + * @param[in] mm the minute + * @param[in] ss the second + */ + time_point(long i, double t, int YYYY=0, int MM=1, int DD=1, + int hh=0, int mm=0, double ss=0.0) : index(i), time(t), + year(YYYY), month(MM), day(DD), hour(hh), minute(mm), + second(ss) + {} + + + /** Initialize from a floating point time value. The calendar and units + * must be provided. + * @param[in] i the index of the time value + * @param[in] t the time value + * @param[in] units the units t is in + * @param[in] calendar the calendar system the units are in + */ + time_point(long i, double t, + const std::string &units, const std::string &calendar); + + long index; + double time; + int year; + int month; + int day; + int hour; + int minute; + double second; +}; + +/// An iterator over a series of time intervals +class interval_iterator +{ +public: + + interval_iterator() : time(), units(), calendar(), + begin(), end(), valid(false) + {} + + virtual ~interval_iterator() {} + + /** Initialize the iterator from a metadata object following the + * conventions defined by the teca_cf_reader. + * @returns 0 if successfully initialized + */ + virtual int initialize(const teca_metadata &md); + + /** Initialize the iterator from a metadata object following the + * conventions defined by the teca_cf_reader. + * @param[in] md a metadata object + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + virtual int initialize(const teca_metadata &md, + long first_step, long last_step); + + /** Initialize the iterator. + * @param[in] t An array of time values + * @param[in] units A string units of the time values + * @param[in] calendar A string name of the calendar system + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + virtual int initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) = 0; + + /// return true if there are more time steps in the sequence + virtual bool is_valid() const = 0; + + /** Get the next interval in the series. + * @param[out] first_step The first step in the next element of the series + * @param[out] last_step The last step in the next element of the series + * @returns 0 if successfully initialized + */ + virtual int get_next_interval(time_point &first_step, + time_point &last_step) = 0; + + /// @returns true if there are more intervals in the series + operator bool() const + { + return this->is_valid(); + } + + /// return the first time point in the series + const time_point &get_begin() const { return this->begin; } + + /// return the last time point in the series + const time_point &get_end() const { return this->end; } + +protected: + const_p_teca_variant_array time; + std::string units; + std::string calendar; + time_point begin; + time_point end; + bool valid; +}; + +/// Enumerate ranges of time steps bracketing seasons +/** + * An iterator over seasons (DJF, MAM, JJA, SON) between 2 time_point's. A + * pair of time steps bracketing the current season are returned at each + * iteration. Only full seasonal intervals are processed. If the input data + * doesn't start or end on a seasonal boundary, the data from the start to the + * first full season, and the data from the end of the last full season to the + * end is skipped. + */ +class season_iterator : public interval_iterator +{ +public: + season_iterator() : year(-1), month(-1) {} + + /// return true if there are more time steps in the sequence + bool is_valid() const override; + + /** Initialize the iterator. + * + * @param[in] t An array of time values + * @param[in] units A string units of the time values + * @param[in] calendar A string name of the calendar system + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + int initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) override; + + /** return a pair of time steps bracketing the current season. + * both returned time steps belong to the current season. + */ + int get_next_interval(time_point &first_step, + time_point &last_step) override; + +private: + /** given a year and month, checks that the values fall on a seasonal + * boundary. if not, returns the year and month of the start of the next + * season. + */ + int get_first_season(int y_in, int m_in, int &y_out, int &m_out) const; + + /** Given a year and month returns the year month and day of the end of the + * season. the input month need not be on a seasonal boundary. + */ + int get_season_end(int y_in, int m_in, + int &y_out, int &m_out, int &d_out) const; + + /** Given a year and month returns the year and month of the next season. + * the input momnth doesn't need to be on a seasonal boundary. + */ + int get_next_season(int y_in, int m_in, int &y_out, int &m_out) const; + +protected: + int year; + int month; +}; + +/// Enumerate ranges of time steps bracketing months +/** An iterator over all months between 2 time_point's. A pair + * of time steps bracketing the current month are returned at + * each iteration. + */ +class year_iterator : public interval_iterator +{ +public: + year_iterator() : year(-1) {} + + /// return true if there are more time steps in the sequence + bool is_valid() const override; + + /** Initialize the iterator. + * + * @param[in] t An array of time values + * @param[in] units A string units of the time values + * @param[in] calendar A string name of the calendar system + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + int initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) override; + + /** return a pair of time steps bracketing the current season. + * both returned time steps belong to the current season. + */ + int get_next_interval(time_point &first_step, + time_point &last_step) override; + +protected: + int year; +}; + +/// Enumerate ranges of time steps bracketing months +/** An iterator over all months between 2 time_point's. A pair + * of time steps bracketing the current month are returned at + * each iteration. + */ +class month_iterator : public interval_iterator +{ +public: + month_iterator() : year(-1), month(-1) {} + + /// return true if there are more time steps in the sequence + bool is_valid() const override; + + /** Initialize the iterator. + * + * @param[in] t An array of time values + * @param[in] units A string units of the time values + * @param[in] calendar A string name of the calendar system + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + int initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) override; + + /** return a pair of time steps bracketing the current season. + * both returned time steps belong to the current season. + */ + int get_next_interval(time_point &first_step, + time_point &last_step) override; + +protected: + int year; + int month; +}; + +/// Enumerate ranges of time steps bracketing days +/** An iterator over all days between 2 time_point's. A pair + * of time steps bracketing the current day are returned at + * each iteration. + */ +class day_iterator : public interval_iterator +{ +public: + day_iterator() : year(-1), month(-1), day(-1) {} + + /// return true if there are more time steps in the sequence + bool is_valid() const override; + + /** Initialize the iterator. + * + * @param[in] t An array of time values + * @param[in] units A string units of the time values + * @param[in] calendar A string name of the calendar system + * @param[in] first_step the first step to include in the series or 0 to use all + * @param[in] last_step the last step to include in the series or -1 to use all + * @returns 0 if successfully initialized + */ + int initialize(const const_p_teca_variant_array &t, + const std::string &units, const std::string &calendar, + long first_step, long last_step) override; + + /** return a pair of time steps bracketing the current season. + * both returned time steps belong to the current season. + */ + int get_next_interval(time_point &first_step, + time_point &last_step) override; + +protected: + int year; + int month; + int day; +}; + + +using p_interval_iterator = std::shared_ptr; + +/// A factory for interval_iterator +class interval_iterator_factory +{ +public: + /** Allocate and return an instance of the named iterator + * @param[in] interval Name of the desired interval iterator. One of daily, + * monthly, seasonal, or yearly + * @returns an instance of interval_iterator + */ + static p_interval_iterator New(const std::string &interval); + + /// The available intervals + enum {invalid = 0, daily = 2, monthly = 3, seasonal = 4, yearly = 5}; + + /** Allocate and return an instance of the named iterator + * @param[in] interval Id of the desired interval iterator. One of daily, + * monthly, seasonal, or yearly + * @returns an instance of interval_iterator + */ + static p_interval_iterator New(int interval); +}; + +} + +/// send the time_point to a stream in humnan readable form +std::ostream &operator<<(std::ostream &os, + const teca_calendar_util::time_point &tpt); + +#endif diff --git a/data/teca_cartesian_mesh.cxx b/data/teca_cartesian_mesh.cxx index 441af030a..036185b0c 100644 --- a/data/teca_cartesian_mesh.cxx +++ b/data/teca_cartesian_mesh.cxx @@ -1,6 +1,8 @@ #include "teca_cartesian_mesh.h" #include "teca_dataset_util.h" #include "teca_bad_cast.h" +#include "teca_metadata.h" +#include "teca_metadata_util.h" #include @@ -98,6 +100,32 @@ void teca_cartesian_mesh::set_z_coordinates(const std::string &var, m_coordinate_arrays->set("z", array); } +// -------------------------------------------------------------------------- +int teca_cartesian_mesh::get_array_extent(const std::string &array_name, + unsigned long array_extent[6]) +{ + teca_metadata atts; + teca_metadata array_atts; + unsigned long mesh_extent[6] = {0}; + if (this->get_extent(mesh_extent) || this->get_attributes(atts) || + atts.get(array_name, array_atts)) + { + TECA_ERROR("Cartesian mesh dataset metadata issue. extent," + "attributes, and array attributes for \"" + << array_name << "\" are required") + return -1; + } + + if (teca_metadata_util::get_array_extent(array_atts, + mesh_extent, array_extent)) + { + // not necessarily an error + return 1; + } + + return 0; +} + // -------------------------------------------------------------------------- int teca_cartesian_mesh::to_stream(teca_binary_stream &s) const { diff --git a/data/teca_cartesian_mesh.h b/data/teca_cartesian_mesh.h index 3dee6ee72..587b60d77 100644 --- a/data/teca_cartesian_mesh.h +++ b/data/teca_cartesian_mesh.h @@ -3,9 +3,10 @@ #include "teca_mesh.h" #include "teca_shared_object.h" + TECA_SHARED_OBJECT_FORWARD_DECL(teca_cartesian_mesh) -/// data on a physically uniform cartesian mesh +/// An object representing data on a stretched Cartesian mesh. class teca_cartesian_mesh : public teca_mesh { public: @@ -15,7 +16,7 @@ class teca_cartesian_mesh : public teca_mesh virtual ~teca_cartesian_mesh() = default; - // set/get metadata + // Set/get metadata TECA_DATASET_METADATA(whole_extent, unsigned long, 6) TECA_DATASET_METADATA(extent, unsigned long, 6) TECA_DATASET_METADATA(bounds, double, 6) @@ -27,58 +28,70 @@ class teca_cartesian_mesh : public teca_mesh TECA_DATASET_METADATA(z_coordinate_variable, std::string, 1) TECA_DATASET_METADATA(t_coordinate_variable, std::string, 1) - // get x coordinate array + /** Get the extent of the named array, taking into account the variable's + * dimensions as opposed to the mesh's dimensions. For instance the mesh + * extent may represent a volume while a variables extent may represent a + * slice. returns 0 if successful, -1 if an error occurred, 1 if the + * have_mesh_dims flag is missing. The latter is not necessarily an error. + */ + int get_array_extent(const std::string &array_name, + unsigned long array_extent[6]); + + /// Get the x coordinate array p_teca_variant_array get_x_coordinates() { return m_coordinate_arrays->get("x"); } const_p_teca_variant_array get_x_coordinates() const { return m_coordinate_arrays->get("x"); } - // get y coordinate array + /// Get the y coordinate array p_teca_variant_array get_y_coordinates() { return m_coordinate_arrays->get("y"); } const_p_teca_variant_array get_y_coordinates() const { return m_coordinate_arrays->get("y"); } - // get z coordinate array + /// Get the z coordinate array p_teca_variant_array get_z_coordinates() { return m_coordinate_arrays->get("z"); } const_p_teca_variant_array get_z_coordinates() const { return m_coordinate_arrays->get("z"); } - // set coordinate arrays + /// Set the x coordinate array void set_x_coordinates(const std::string &name, const p_teca_variant_array &array); + /// Set the y coordinate array void set_y_coordinates(const std::string &name, const p_teca_variant_array &array); + /// Set the z coordinate array void set_z_coordinates(const std::string &name, const p_teca_variant_array &array); - - // return a unique string identifier + /// Return the name of the class std::string get_class_name() const override { return "teca_cartesian_mesh"; } - // return an integer identifier uniquely naming the dataset type + /// return an integer identifier uniquely naming the dataset type int get_type_code() const override; - // copy data and metadata. shallow copy uses reference - // counting, while copy duplicates the data. + /** Copy data and metadata. Shallow copy uses reference + * counting, while copy duplicates the data. + */ void copy(const const_p_teca_dataset &) override; void shallow_copy(const p_teca_dataset &) override; - // copy metadata. always a deep copy. + /// Copy metadata. This is always a deep copy. void copy_metadata(const const_p_teca_dataset &other) override; - // swap internals of the two objects + /// Swap the internals of the two objects void swap(p_teca_dataset &) override; - // serialize the dataset to/from the given stream - // for I/O or communication + /** Serialize the dataset to/from the given stream + * for I/O or communication + */ int to_stream(teca_binary_stream &) const override; int from_stream(teca_binary_stream &) override; diff --git a/data/teca_coordinate_util.cxx b/data/teca_coordinate_util.cxx index 3a04d4385..e00b177d1 100644 --- a/data/teca_coordinate_util.cxx +++ b/data/teca_coordinate_util.cxx @@ -2,7 +2,7 @@ #include "teca_common.h" #if defined(TECA_HAS_UDUNITS) -#include "calcalcs.h" +#include "teca_calcalcs.h" #endif #include @@ -15,9 +15,10 @@ namespace teca_coordinate_util { // ************************************************************************** -int time_step_of(p_teca_variant_array time, bool lower, bool clamp, - const std::string &calendar, const std::string &units, - const std::string &date, unsigned long &step) +int time_step_of(const const_p_teca_variant_array &time, + bool lower, bool clamp, const std::string &calendar, + const std::string &units, const std::string &date, + unsigned long &step) { #if defined(TECA_HAS_UDUNITS) step = 0; @@ -36,7 +37,7 @@ int time_step_of(p_teca_variant_array time, bool lower, bool clamp, // apply calendaring to get a time offset double t = 0.0; - if (calcalcs::coordinate(Y, M, D, h, m, s, + if (teca_calcalcs::coordinate(Y, M, D, h, m, s, units.c_str(), calendar.c_str(), &t)) { TECA_ERROR("failed to convert date \"" << date @@ -48,9 +49,9 @@ int time_step_of(p_teca_variant_array time, bool lower, bool clamp, // locate the nearest time value in the time axis unsigned long last = time->size() - 1; - TEMPLATE_DISPATCH_FP_SI(teca_variant_array_impl, + TEMPLATE_DISPATCH_FP_SI(const teca_variant_array_impl, time.get(), - NT *p_time = std::dynamic_pointer_cast(time)->get(); + const NT *p_time = std::dynamic_pointer_cast(time)->get(); if (clamp && (t <= p_time[0])) { step = 0; @@ -91,10 +92,10 @@ int time_to_string(double val, const std::string &calendar, const std::string &units, const std::string &format, std::string &date) { #if defined(TECA_HAS_UDUNITS) - // use calcalcs to convert val to a set of year/month/day/etc. + // use teca_calcalcs to convert val to a set of year/month/day/etc. struct tm timedata = {}; double seconds = 0.0; - if (calcalcs::date(val, &timedata.tm_year, &timedata.tm_mon, + if (teca_calcalcs::date(val, &timedata.tm_year, &timedata.tm_mon, &timedata.tm_mday, &timedata.tm_hour, &timedata.tm_min, &seconds, units.c_str(), calendar.c_str())) { @@ -139,7 +140,7 @@ int bounds_to_extent(const double *bounds, const teca_metadata &md, teca_metadata coords; if (md.get("coordinates", coords)) { - TECA_ERROR("missing cooridnates") + TECA_ERROR("Metadata issue, missing cooridnates") return -1; } @@ -149,18 +150,26 @@ int bounds_to_extent(const double *bounds, const teca_metadata &md, if (!x || !y || !z) { - TECA_ERROR("empty coordinate axes") + TECA_ERROR("Metadata issue, empty coordinate axes") return -1; } - return bounds_to_extent(bounds, x, y, z, extent); -} + if (bounds_to_extent(bounds, x, y, z, extent) || + validate_extent(x->size(), y->size(), z->size(), extent, true)) + { + TECA_ERROR("Invalid bounds raequested [" << bounds[0] << ", " + << bounds[1] << ", " << bounds[2] << ", " << bounds[3] << ", " + << bounds[4] << ", " << bounds[5] << "]") + return -1; + } + return 0; +} // ************************************************************************** int bounds_to_extent(const double *bounds, - const_p_teca_variant_array x, const_p_teca_variant_array y, - const_p_teca_variant_array z, unsigned long *extent) + const const_p_teca_variant_array &x, const const_p_teca_variant_array &y, + const const_p_teca_variant_array &z, unsigned long *extent) { TEMPLATE_DISPATCH_FP( const teca_variant_array_impl, @@ -238,7 +247,56 @@ int bounds_to_extent(const double *bounds, return 0; ) - TECA_ERROR("invalid coordinate array type") + TECA_ERROR("invalid coordinate array type \"" << x->get_class_name() << "\"") + return -1; +} + +// ************************************************************************** +int bounds_to_extent(const double *bounds, + const const_p_teca_variant_array &x, unsigned long *extent) +{ + TEMPLATE_DISPATCH_FP( + const teca_variant_array_impl, + x.get(), + + // in the following, for each side (low, high) of the bounds in + // each cooridnate direction we are searching for the index that + // is either just below, just above, or exactly at the given value. + // special cases include: + // * x,y,z in descending order. we check for that and + // invert the compare functions that define the bracket + // * bounds describing a plane. we test for this and + // so that both high and low extent return the same value. + // * x,y,z are length 1. we can skip the search in that + // case. + + const NT eps8 = NT(8)*std::numeric_limits::epsilon(); + + unsigned long nx = x->size(); + unsigned long high_i = nx - 1; + extent[0] = 0; + extent[1] = high_i; + const NT *px = std::dynamic_pointer_cast(x)->get(); + NT low_x = static_cast(bounds[0]); + NT high_x = static_cast(bounds[1]); + bool slice_x = equal(low_x, high_x, eps8); + + if (((nx > 1) && (((px[high_i] > px[0]) && + (teca_coordinate_util::index_of(px, 0, high_i, low_x, true, extent[0]) + || teca_coordinate_util::index_of(px, 0, high_i, high_x, slice_x, extent[1]))) || + ((px[high_i] < px[0]) && + (teca_coordinate_util::index_of>(px, 0, high_i, low_x, false, extent[0]) + || teca_coordinate_util::index_of>(px, 0, high_i, high_x, !slice_x, extent[1])))))) + { + TECA_ERROR(<< "requested subset [" << bounds[0] << ", " << bounds[1] << ", " + << "] is not contained in the current dataset bounds [" << px[0] << ", " + << px[high_i] << "]") + return -1; + } + return 0; + ) + + TECA_ERROR("invalid coordinate array type \"" << x->get_class_name() << "\"") return -1; } @@ -284,6 +342,25 @@ int validate_centering(int centering) return ret; } +// ************************************************************************** +int get_cartesian_mesh_bounds(const const_p_teca_variant_array x, + const const_p_teca_variant_array y, const const_p_teca_variant_array z, + double *bounds) +{ + unsigned long x1 = x->size() - 1; + unsigned long y1 = y->size() - 1; + unsigned long z1 = z->size() - 1; + + x->get(0, bounds[0]); + x->get(x1, bounds[1]); + y->get(0, bounds[2]); + y->get(y1, bounds[3]); + z->get(0, bounds[4]); + z->get(z1, bounds[5]); + + return 0; +} + // ************************************************************************** int get_cartesian_mesh_extent(const teca_metadata &md, unsigned long *whole_extent, double *bounds) @@ -327,4 +404,94 @@ int get_cartesian_mesh_extent(const teca_metadata &md, return 0; } +// ************************************************************************** +int validate_extent(unsigned long nx_max, unsigned long ny_max, + unsigned long nz_max, unsigned long *extent, bool verbose) +{ + // validate x + if ((extent[1] >= nx_max) || (extent[1] < extent[0])) + { + if (verbose) + { + TECA_ERROR("The x-axis extent [" << extent[0] << ", " + << extent[1] << "] is invalid, the x-axis coordinate" + " array has " << nx_max << " values") + } + return -1; + } + + // validate y + if ((extent[3] >= ny_max) || (extent[3] < extent[2])) + { + if (verbose) + { + TECA_ERROR("The y-axis extent [" << extent[2] << ", " + << extent[3] << "] is invalid, the y-axis coordinate" + " array has " << ny_max << " values") + } + return -1; + } + + // validate z + if ((extent[5] >= nz_max) || (extent[5] < extent[4])) + { + if (verbose) + { + TECA_ERROR("The z-axis extent [" << extent[4] << ", " + << extent[5] << "] is invalid, the z-axis coordinate" + " array has " << nz_max << " values") + } + return -1; + } + + return 0; +} + +// ************************************************************************** +int clamp_dimensions_of_one(unsigned long nx_max, unsigned long ny_max, + unsigned long nz_max, unsigned long *extent, bool verbose) +{ + int clamped = 0; + + // clamp x + if ((nx_max == 1) && (extent[1] != 0)) + { + if (verbose) + { + TECA_WARNING("The requested x-axis extent [" << extent[0] << ", " + << extent[1] << "] is invalid and was clamped to [0, 0]") + } + extent[0] = 0; + extent[1] = 0; + clamped = 1; + } + + // clamp y + if ((ny_max == 1) && (extent[3] != 0)) + { + if (verbose) + { + TECA_WARNING("The requested y-axis extent [" << extent[2] << ", " + << extent[3] << "] is invalid and was clamped to [0, 0]") + } + extent[2] = 0; + extent[3] = 0; + clamped = 1; + } + + // clamp z + if ((nz_max == 1) && (extent[5] != 0)) + { + if (verbose) + { + TECA_WARNING("The requested z-axis extent [" << extent[4] << ", " + << extent[5] << "] is invalid and was clamped to [0, 0]") + } + extent[4] = 0; + extent[5] = 0; + clamped = 1; + } + + return clamped; +} }; diff --git a/data/teca_coordinate_util.h b/data/teca_coordinate_util.h index d14b93c20..01fb8d0fb 100644 --- a/data/teca_coordinate_util.h +++ b/data/teca_coordinate_util.h @@ -1,6 +1,8 @@ #ifndef teca_cartesian_mesh_util_h #define teca_cartesian_mesh_util_h +/// @file + #include "teca_cartesian_mesh.h" #include "teca_variant_array.h" #include "teca_metadata.h" @@ -12,26 +14,35 @@ #include #include +/// For printing data as ASCII with the maximum supported numerical precision #define max_prec(T) \ std::setprecision(std::numeric_limits::digits10 + 1) +/// Codes dealing with operations on coordinate systems namespace teca_coordinate_util { -// traits classes used to get default tolerances for comparing numbers -// of a given precision. A relative tolerance is used for comparing large -// numbers and an absolute tolerance is used for comparing small numbers. -// these defaults are not universal and will not work well in all situations. -// see also: -// https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ +/** @brief + * traits classes used to get default tolerances for comparing numbers + * of a given precision. + * + * @details + * A relative tolerance is used for comparing large + * numbers and an absolute tolerance is used for comparing small numbers. + * these defaults are not universal and will not work well in all situations. + * + * see also: + * https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ + */ template struct equal_tt {}; -#define declare_equal_tt(cpp_t, atol, rtol) \ -template <> \ -struct equal_tt \ -{ \ - static cpp_t absTol() { return atol; } \ - static cpp_t relTol() { return rtol; } \ +#define declare_equal_tt(cpp_t, atol, rtol) \ +/** Specialization for cpp_t with default absTol and relTol */ \ +template <> \ +struct equal_tt \ +{ \ + static cpp_t absTol() { return atol; } \ + static cpp_t relTol() { return rtol; } \ }; declare_equal_tt(float, 10.0f*std::numeric_limits::epsilon(), @@ -43,9 +54,9 @@ declare_equal_tt(double, 10.0*std::numeric_limits::epsilon(), declare_equal_tt(long double, std::numeric_limits::epsilon(), std::numeric_limits::epsilon()) -// compare two floating point numbers. -// absTol handles comparing numbers very close to zero. -// relTol handles comparing larger values. +/** Compare two floating point numbers. absTol handles comparing numbers very + * close to zero. relTol handles comparing larger values. + */ template bool equal(T a, T b, T relTol = equal_tt::relTol(), T absTol = equal_tt::absTol(), @@ -65,7 +76,7 @@ bool equal(T a, T b, return false; } -// a specialization for integer types +/// Compare two integral numbers. template bool equal(T a, T b, T relTol = 0, T absTol = 0, typename std::enable_if::value>::type* = 0) @@ -75,8 +86,10 @@ bool equal(T a, T b, T relTol = 0, T absTol = 0, return a == b; } -// an overload for use in regression tests. If the numbers are not equal then a diagnostic -// message is returned. +/** Compare two floating point numbers. This overload may be used in regression + * tests or other contexts where a diagnostic error message should be reported + * if the numbers are not equal. + */ template bool equal(T a, T b, std::string &diagnostic, T relTol = equal_tt::relTol(), T absTol = equal_tt::absTol(), @@ -109,7 +122,10 @@ bool equal(T a, T b, std::string &diagnostic, return false; } -// a specialization for integer types +/** Compare two integral numbers. This overload may be used in regression + * tests or other contexts where a diagnostic error message should be reported + * if the numbers are not equal. + */ template bool equal(T a, T b, std::string &diagnostic, T relTol = 0, T absTol = 0, typename std::enable_if::value>::type* = 0) @@ -126,44 +142,76 @@ bool equal(T a, T b, std::string &diagnostic, T relTol = 0, T absTol = 0, } -// comparators implementing bracket for ascending and -// descending input arrays +/// Less than or equal to predicate template struct leq { static bool eval(const data_t &l, const data_t &r) { return l <= r; } }; +/// Greater than or equal to predicate template struct geq { static bool eval(const data_t &l, const data_t &r) { return l >= r; } }; +/// Less than predicate template struct lt { static bool eval(const data_t &l, const data_t &r) { return l < r; } }; +/// Greater than predicate template struct gt { static bool eval(const data_t &l, const data_t &r) { return l > r; } }; +/// comparator implementing bracket for ascending input arrays template struct ascend_bracket { + // m_0 is an index into the data, m_1 = m_0 + 1 + // comparitors defining the bracket orientation. for data in + // ascending order: val >= data[m_0] && val <= data[m_1] using comp0_t = geq; using comp1_t = leq; + + // m_0 is an index into the data, m_1 = m_0 + 1 + // get the id of the smaller value (lower == true) + // or the larger value (lower == false) + static unsigned long get_id(bool lower, + unsigned long m_0, unsigned long m_1) + { + if (lower) + return m_0; + return m_1; + } }; +/// comparator implementing bracket for descending input arrays template struct descend_bracket { + // m_0 is an index into the data, m_1 = m_0 + 1 + // comparitors defining the bracket orientation. for data in + // descending order: val <= data[m_0] && val >= data[m_1] using comp0_t = leq; using comp1_t = geq; + + // m_0 is an index into the data, m_1 = m_0 + 1 + // get the id of the smaller value (lower == true) + // or the larger value (lower == false) + static unsigned long get_id(bool lower, + unsigned long m_0, unsigned long m_1) + { + if (lower) + return m_1; + return m_0; + } }; -// binary search that will locate index bounding the value -// above or below such that data[i] <= val or val <= data[i+1] -// depending on the value of lower. return 0 if the value is -// found. the comp0 and comp1 template parameters let us -// operate on both ascending and descending input. defaults -// are set for ascending inputs. +/** binary search that will locate index bounding the value above or below + * such that data[i] <= val or val <= data[i+1] depending on the value of + * lower. return 0 if the value is found. the comp0 and comp1 template + * parameters let us operate on both ascending and descending input. defaults + * are set for ascending inputs. + */ template > int index_of(const data_t *data, unsigned long l, unsigned long r, data_t val, bool lower, unsigned long &id) @@ -192,10 +240,7 @@ int index_of(const data_t *data, unsigned long l, unsigned long r, if (equal(val, data[m_1])) id = m_1; else - if (lower) - id = m_0; - else - id = m_1; + id = bracket_t::get_id(lower, m_0, m_1); return 0; } else @@ -216,8 +261,9 @@ int index_of(const data_t *data, unsigned long l, unsigned long r, return -1; } -// binary search that will locate index of the given value. -// return 0 if the value is found. +/** binary search that will locate index of the given value. return 0 if the + * value is found. + */ template int index_of(const T *data, size_t l, size_t r, T val, unsigned long &id) { @@ -263,19 +309,22 @@ int index_of(const T *data, size_t l, size_t r, T val, unsigned long &id) return -1; } -// convert bounds to extents -// return non-zero if the requested bounds are not in -// the given coordinate arrays. coordinate arrays must -// not be empty. +/** Convert bounds to extents. return non-zero if the requested bounds are + * not in the given coordinate arrays. coordinate arrays must not be empty. + */ +int bounds_to_extent(const double *bounds, + const const_p_teca_variant_array &x, const const_p_teca_variant_array &y, + const const_p_teca_variant_array &z, unsigned long *extent); + int bounds_to_extent(const double *bounds, - const_p_teca_variant_array x, const_p_teca_variant_array y, - const_p_teca_variant_array z, unsigned long *extent); + const const_p_teca_variant_array &x, unsigned long *extent); int bounds_to_extent(const double *bounds, const teca_metadata &md, unsigned long *extent); -// get the i,j,k cell index of point x,y,z in the given mesh. -// return 0 if successful. +/** Get the i,j,k cell index of point x,y,z in the given mesh. return 0 if + * successful. + */ template int index_of(const const_p_teca_cartesian_mesh &mesh, T x, T y, T z, unsigned long &i, unsigned long &j, unsigned long &k) @@ -312,29 +361,31 @@ int index_of(const const_p_teca_cartesian_mesh &mesh, T x, T y, T z, return -1; } -// given a human readable date string in YYYY-MM-DD hh:mm:ss format -// amd a list of floating point offset times inthe specified calendar -// and units find the closest time step. return 0 if successful -// see index_of for a description of lower, if clamp is true then -// when the date falls outside of the time values either the first -// or last time step is returned. -int time_step_of(p_teca_variant_array time, bool lower, bool clamp, - const std::string &calendar, const std::string &units, - const std::string &date, unsigned long &step); - -// given a time value (val), associated time units (units), and calendar -// (calendar), return a human-readable rendering of the date (date) in a -// strftime-format (format). return 0 if successful. +/** given a human readable date string in YYYY-MM-DD hh:mm:ss format and a + * list of floating point offset times in the specified calendar and units find + * the closest time step. return 0 if successful see index_of for a description + * of lower, if clamp is true then when the date falls outside of the time + * values either the first or last time step is returned. + */ +int time_step_of(const const_p_teca_variant_array &time, + bool lower, bool clamp, const std::string &calendar, + const std::string &units, const std::string &date, + unsigned long &step); + +/** given a time value (val), associated time units (units), and calendar + * (calendar), return a human-readable rendering of the date (date) in a + * strftime-format (format). return 0 if successful. + */ int time_to_string(double val, const std::string &calendar, const std::string &units, const std::string &format, std::string &date); -// build random access data structures for an indexed table. -// the index column gives each entity a unique id. the index is -// used to identify rows that belong in the entity. it is assumed -// that an entity ocupies consecutive rows. the returns are: -// n_entities, the number of entities found; counts, the number of -// rows used by each entity; offsets, the starting row of each -// entity; ids, a new set of ids for the entities starting from 0 +/** build random access data structures for an indexed table. the index column + * gives each entity a unique id. the index is used to identify rows that + * belong in the entity. it is assumed that an entity occupies consecutive rows. + * the returns are: n_entities, the number of entities found; counts, the + * number of rows used by each entity; offsets, the starting row of each + * entity; ids, a new set of ids for the entities starting from 0 + */ template void get_table_offsets(const int_t *index, unsigned long n_rows, unsigned long &n_entities, std::vector &counts, @@ -374,8 +425,16 @@ void get_table_offsets(const int_t *index, unsigned long n_rows, offsets[i] = offsets[i-1] + counts[i-1]; } -// 0 order (nearest neighbor) interpolation -// for nodal data on stretched cartesian mesh. +/** 0th order (nearest neighbor) interpolation for nodal data on a stretched + * Cartesian mesh. This overload implements the general 3D case. + * cx, cy, cz is the location to interpolate to + * p_x, p_y, p_z array arrays containing the source coordinates with extents + * [0, ihi, 0, jhi, 0, khi] + * p_data is the field to interpolate from + * val is the result + * returns 0 if successful, an error occurs if cx, cy, cz is outside of the + * source coordinate system + */ template int interpolate_nearest(CT cx, CT cy, CT cz, const CT *p_x, const CT *p_y, const CT *p_z, @@ -411,8 +470,58 @@ int interpolate_nearest(CT cx, CT cy, CT cz, return 0; } -// 1 order (linear) interpolation -// for nodal data on stretched cartesian mesh. +/** 0th order (nearest neighbor) interpolation for nodal data on a stretched + * Cartesian mesh. This overload implements the special case where both source + * and target mesh data are in a 2D x-y plane using fewer operations than the + * general 3D implementation. + * cx, cy, cz is the location to interpolate to + * p_x, p_y, p_z array arrays containing the source coordinates with extents + * [0, ihi, 0, jhi, 0, khi] + * p_data is the field to interpolate from + * val is the result + * returns 0 if successful, an error occurs if cx, cy, cz is outside of the + * source coordinate system + */ +template +int interpolate_nearest(coord_t cx, coord_t cy, const coord_t *p_x, + const coord_t *p_y, const data_t *p_data, unsigned long ihi, + unsigned long jhi, unsigned long nx, data_t &val) +{ + // get i,j of node less than cx,cy + unsigned long i = 0; + unsigned long j = 0; + + if ((ihi && teca_coordinate_util::index_of(p_x, 0, ihi, cx, true, i)) + || (jhi && teca_coordinate_util::index_of(p_y, 0, jhi, cy, true, j))) + { + // cx,cy is outside the coordinate axes + return -1; + } + + // get i,j of node greater than cx,cy + unsigned long ii = std::min(i + 1, ihi); + unsigned long jj = std::min(j + 1, jhi); + + // get index of nearest node + unsigned long p = (cx - p_x[i]) <= (p_x[ii] - cx) ? i : ii; + unsigned long q = (cy - p_y[j]) <= (p_y[jj] - cy) ? j : jj; + + // assign value from nearest node + val = p_data[p + nx*q]; + + return 0; +} + +/** 1st order (linear) interpolation for nodal data on stretched Cartesian + * mesh. This overload implements the general 3D case. + * cx, cy, cz is the location to interpolate to + * p_x, p_y, p_z array arrays containing the source coordinates with extents + * [0, ihi, 0, jhi, 0, khi] + * p_data is the field to interpolate from + * val is the result + * returns 0 if successful, an error occurs if cx, cy, cz is outside of the + * source coordinate system + */ template int interpolate_linear(CT cx, CT cy, CT cz, const CT *p_x, const CT *p_y, const CT *p_z, @@ -433,15 +542,15 @@ int interpolate_linear(CT cx, CT cy, CT cz, return -1; } - // get i,j of node greater than cx,cy + // get i,j of node greater than cx,cy,cz unsigned long ii = std::min(i + 1, ihi); unsigned long jj = std::min(j + 1, jhi); unsigned long kk = std::min(k + 1, khi); // compute weights - CT wx = (cx - p_x[i])/(p_x[ii] - p_x[i]); - CT wy = (cy - p_y[i])/(p_y[ii] - p_y[i]); - CT wz = (cz - p_z[i])/(p_z[ii] - p_z[i]); + CT wx = ii == i ? 0 : (cx - p_x[i])/(p_x[ii] - p_x[i]); + CT wy = jj == j ? 0 : (cy - p_y[j])/(p_y[jj] - p_y[j]); + CT wz = kk == k ? 0 : (cz - p_z[k])/(p_z[kk] - p_z[k]); CT vx = CT(1) - wx; CT vy = CT(1) - wy; @@ -460,11 +569,61 @@ int interpolate_linear(CT cx, CT cy, CT cz, return 0; } -// functor templated on order of accuracy for above Cartesian mesh interpolants +/** 1st order (linear) interpolation for nodal data on stretched Cartesian mesh. + * This overload implements the special case where both source and target data + * are in a 2D x-y plane using fewer operations than the general 3D + * implementation. + * cx, cy, cz is the location to interpolate to + * p_x, p_y, p_z array arrays containing the source coordinates with extents + * [0, ihi, 0, jhi, 0, khi] + * p_data is the field to interpolate from + * val is the result + * returns 0 if successful, an error occurs if cx, cy, cz is outside of the + * source coordinate system + */ +template +int interpolate_linear(CT cx, CT cy, const CT *p_x, const CT *p_y, + const DT *p_data, unsigned long ihi, unsigned long jhi, + unsigned long nx, DT &val) +{ + // get i,j of node less than cx,cy + unsigned long i = 0; + unsigned long j = 0; + + if ((ihi && teca_coordinate_util::index_of(p_x, 0, ihi, cx, true, i)) + || (jhi && teca_coordinate_util::index_of(p_y, 0, jhi, cy, true, j))) + { + // cx,cy is outside the coordinate axes + return -1; + } + + // get i,j of node greater than cx,cy + unsigned long ii = std::min(i + 1, ihi); + unsigned long jj = std::min(j + 1, jhi); + + // compute weights + CT wx = ii == i ? 0 : (cx - p_x[i])/(p_x[ii] - p_x[i]); + CT wy = jj == j ? 0 : (cy - p_y[j])/(p_y[jj] - p_y[j]); + + CT vx = CT(1) - wx; + CT vy = CT(1) - wy; + + // interpolate + val = vx*vy*p_data[ i + j*nx] + + wx*vy*p_data[ii + j*nx] + + wx*wy*p_data[ii + jj*nx] + + vx*wy*p_data[ i + jj*nx]; + + return 0; +} + +/// A functor templated on order of accuracy for above Cartesian mesh interpolants template struct interpolate_t; +/// Zero'th order interpolant specialization template<> struct interpolate_t<0> { + // 3D template int operator()(CT tx, CT ty, CT tz, const CT *sx, const CT *sy, const CT *sz, const DT *sa, unsigned long ihi, unsigned long jhi, @@ -473,10 +632,22 @@ template<> struct interpolate_t<0> return teca_coordinate_util::interpolate_nearest(tx,ty,tz, sx,sy,sz,sa, ihi,jhi,khi, nx,nxy, ta); } + + // 2D x-y plane + template + int operator()(CT tx, CT ty, const CT *sx, const CT *sy, + const DT *sa, unsigned long ihi, unsigned long jhi, + unsigned long nx, DT &ta) + { + return teca_coordinate_util::interpolate_nearest(tx,ty, + sx,sy,sa, ihi,jhi, nx, ta); + } }; +/// First order interpolant specialization template<> struct interpolate_t<1> { + // 3D template int operator()(CT tx, CT ty, CT tz, const CT *sx, const CT *sy, const CT *sz, const DT *sa, unsigned long ihi, unsigned long jhi, @@ -485,14 +656,22 @@ template<> struct interpolate_t<1> return teca_coordinate_util::interpolate_linear(tx,ty,tz, sx,sy,sz,sa, ihi,jhi,khi, nx,nxy, ta); } + + // 2D x-y plane + template + int operator()(CT tx, CT ty, const CT *sx, const CT *sy, + const DT *sa, unsigned long ihi, unsigned long jhi, + unsigned long nx, DT &ta) + { + return teca_coordinate_util::interpolate_linear(tx,ty, + sx,sy,sa, ihi,jhi, nx, ta); + } }; -// return 0 if the centering is one of the values defined -// in teca_array_attributes +/// return 0 if the centering is one of the values defined in teca_array_attributes int validate_centering(int centering); -// convert from a cell extent to a face, edge or point centered -// extent +/// convert from a cell extent to a face, edge or point centered extent template int convert_cell_extent(num_t *extent, int centering) { @@ -539,12 +718,98 @@ int convert_cell_extent(num_t *extent, int centering) return 0; } -// given carteisan mesh metadata extract whole_extent and bounds -// if bounds metadata is not already present then it is initialized -// from coordinate arrays. It's an error if whole_extent or coordinate -// arrays are not present. return zero if successful. +/** Given Cartesian mesh metadata extract whole_extent and bounds + * if bounds metadata is not already present then it is initialized + * from coordinate arrays. It's an error if whole_extent or coordinate + * arrays are not present. return zero if successful. + */ int get_cartesian_mesh_extent(const teca_metadata &md, unsigned long *whole_extent, double *bounds); +/// get the mesh's bounds from the coordinate axis arrays +int get_cartesian_mesh_bounds(const const_p_teca_variant_array x, + const const_p_teca_variant_array y, const const_p_teca_variant_array z, + double *bounds); + +/** Check that one Cartesian region covers the other coordinates must be in + * ascending order. assumes that both regions are specified in ascending order. + */ +template +int covers_ascending(const num_t *whole, const num_t *part) +{ + if ((part[0] >= whole[0]) && (part[0] <= whole[1]) && + (part[1] >= whole[0]) && (part[1] <= whole[1]) && + (part[2] >= whole[2]) && (part[2] <= whole[3]) && + (part[3] >= whole[2]) && (part[3] <= whole[3]) && + (part[4] >= whole[4]) && (part[4] <= whole[5]) && + (part[5] >= whole[4]) && (part[5] <= whole[5])) + return 1; + return 0; +} + +/** Check that one Cartesian region covers the other, taking into account the + * order of the coordinates. assumes that the regions are specified in the same + * orientation. + */ +template +int covers(const num_t *whole, const num_t *part) +{ + bool x_ascend = whole[0] <= whole[1]; + bool y_ascend = whole[2] <= whole[3]; + bool z_ascend = whole[4] <= whole[5]; + if (((x_ascend && + (part[0] >= whole[0]) && (part[0] <= whole[1]) && + (part[1] >= whole[0]) && (part[1] <= whole[1])) || + (!x_ascend && + (part[0] <= whole[0]) && (part[0] >= whole[1]) && + (part[1] <= whole[0]) && (part[1] >= whole[1]))) && + ((y_ascend && + (part[2] >= whole[2]) && (part[2] <= whole[3]) && + (part[3] >= whole[2]) && (part[3] <= whole[3])) || + (!y_ascend && + (part[2] <= whole[2]) && (part[2] >= whole[3]) && + (part[3] <= whole[2]) && (part[3] >= whole[3]))) && + ((z_ascend && + (part[4] >= whole[4]) && (part[4] <= whole[5]) && + (part[5] >= whole[4]) && (part[5] <= whole[5])) || + (!z_ascend && + (part[4] <= whole[4]) && (part[4] >= whole[5]) && + (part[5] <= whole[4]) && (part[5] >= whole[5])))) + return 1; + return 0; +} + +/** check that two Cartesian regions have the same orientation ie they are + * either both specified in ascending or descending order. + */ +template +int same_orientation(const num_t *whole, const num_t *part) +{ + if ((((whole[0] <= whole[1]) && (part[0] <= part[1])) || + ((whole[0] >= whole[1]) && (part[0] >= part[1]))) && + (((whole[2] <= whole[3]) && (part[2] <= part[3])) || + ((whole[2] >= whole[3]) && (part[2] >= part[3]))) && + (((whole[4] <= whole[5]) && (part[4] <= part[5])) || + ((whole[4] >= whole[5]) && (part[4] >= part[5])))) + return 1; + return 0; +} + +/** where array dimensions specified by nx_max, ny_max, and nz_max are 1, and + * the extent would be out of bounds, set the extent to [0, 0]. If verbose is + * set, a warning is reported when the extent was clamped in one or more + * directions. The return is non zero if any direction was clamped and 0 + * otherwise. + */ +int clamp_dimensions_of_one(unsigned long nx_max, unsigned long ny_max, + unsigned long nz_max, unsigned long *extent, bool verbose); + +/** Return 0 if the passed extent does not exceed array dimensions specified in + * nx_max, ny_max, and nz_max. If verbose is set, an error is reported via + * TECA_ERROR when the extent would be out of bounds. + */ +int validate_extent(unsigned long nx_max, unsigned long ny_max, + unsigned long nz_max, unsigned long *extent, bool verbose); + }; #endif diff --git a/data/teca_curvilinear_mesh.h b/data/teca_curvilinear_mesh.h index dff79b9b4..5690c429b 100644 --- a/data/teca_curvilinear_mesh.h +++ b/data/teca_curvilinear_mesh.h @@ -5,7 +5,7 @@ #include "teca_shared_object.h" TECA_SHARED_OBJECT_FORWARD_DECL(teca_curvilinear_mesh) -/// data on a physically uniform curvilinear mesh +/// Data on a physically uniform curvilinear mesh. class teca_curvilinear_mesh : public teca_mesh { public: diff --git a/data/teca_database.h b/data/teca_database.h index 87b1f37fd..11df3a08c 100644 --- a/data/teca_database.h +++ b/data/teca_database.h @@ -3,17 +3,19 @@ #include "teca_dataset.h" #include "teca_table.h" -#include "teca_database_fwd.h" +#include "teca_shared_object.h" #include "teca_table_collection.h" #include class teca_binary_stream; -/// teca_database - A collection of named tables +TECA_SHARED_OBJECT_FORWARD_DECL(teca_database) + +/// A collection of named tables. /** -A dataset consisting of a collection of named tables. This -is a thin wrapper around the teca_table_collection implementing -the teca_dataset API. -*/ + * A dataset consisting of a collection of named tables. This + * is a thin wrapper around the teca_table_collection implementing + * the teca_dataset API. + */ class teca_database : public teca_dataset { public: diff --git a/data/teca_database_fwd.h b/data/teca_database_fwd.h deleted file mode 100644 index e8f784964..000000000 --- a/data/teca_database_fwd.h +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef teca_database_fwd_h -#define teca_database_fwd_h - -#include "teca_shared_object.h" -TECA_SHARED_OBJECT_FORWARD_DECL(teca_database) - -#endif diff --git a/data/teca_dataset_util.h b/data/teca_dataset_util.h index e2e90b515..07f4ccc0d 100644 --- a/data/teca_dataset_util.h +++ b/data/teca_dataset_util.h @@ -10,6 +10,7 @@ #include "teca_table.h" #include "teca_database.h" +/// @cond template struct teca_dataset_tt {}; @@ -41,7 +42,22 @@ DECLARE_DATASET_TT(teca_curvilinear_mesh, 6) case _code: \ return teca_dataset_new<_code>::New(); \ break; +/// @endcond + +/// Constructs a new instance of teca_dataset from the provided type code. +/** The type codes are: + * + * | code | teca_dataset | + * | ---- | ------------ | + * | 1 | teca_table | + * | 2 | teca_database | + * | 3 | teca_cartesian_mesh | + * | 4 | teca_uniform_cartesian_mesh | + * | 5 | teca_arakawa_c_grid | + * | 6 | teca_curvilinear_mesh | + * + */ struct teca_dataset_factory { static p_teca_dataset New(int code) diff --git a/data/teca_mesh.h b/data/teca_mesh.h index 51e01ec7f..8196fcf29 100644 --- a/data/teca_mesh.h +++ b/data/teca_mesh.h @@ -1,18 +1,20 @@ #ifndef teca_mesh_h #define teca_mesh_h -#include "teca_mesh_fwd.h" #include "teca_dataset.h" #include "teca_metadata.h" #include "teca_array_collection.h" +#include "teca_shared_object.h" -/// a base class for geometric data +TECA_SHARED_OBJECT_FORWARD_DECL(teca_mesh) + +/// A base class for geometric data. /** -The mesh declares containers for typical geometricly associated data -such as point, cell, face and edge centered data arrays and defines -the API's for accessing them. API's for accessing common metadata such -as time related metadata are declared here. -*/ + * The mesh declares containers for typical geometrically associated data + * such as point, cell, face and edge centered data arrays and defines + * the APIs for accessing them. APIs for accessing common metadata such + * as time related metadata are declared here. + */ class teca_mesh : public teca_dataset { public: @@ -24,8 +26,8 @@ class teca_mesh : public teca_dataset TECA_DATASET_METADATA(time_units, std::string, 1) TECA_DATASET_METADATA(time_step, unsigned long, 1) - // set/get array attribute metadata - TECA_DATASET_METADATA(array_attributes, teca_metadata, 1) + // set/get attribute metadata + TECA_DATASET_METADATA(attributes, teca_metadata, 1) // get the array collection for the given centering // the centering enumeration is defined in teca_array_attributes diff --git a/data/teca_mesh_fwd.h b/data/teca_mesh_fwd.h deleted file mode 100644 index 92b2b3aa0..000000000 --- a/data/teca_mesh_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_mesh_fwd_h -#define teca_mesh_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_mesh) - -#endif diff --git a/data/teca_priority_queue.h b/data/teca_priority_queue.h new file mode 100644 index 000000000..4a5debb9f --- /dev/null +++ b/data/teca_priority_queue.h @@ -0,0 +1,460 @@ +#ifndef teca_priority_queue_h +#define teca_priority_queue_h + +#include +#include +#include +#include +#include +#include + + +/// @cond + +// use one of the following aliases for key_map_t. key_map_t +// is the type of container used to hold the locations of user provided +// keys in the heap. +// for keys that are not ordinals 0 to N, use the mapped_key_t alias +// for contiguous keys 0 to N (faster), use the contiguous_key_t alias +template +using mapped_key_t = std::map; + +using contiguous_key_t = std::vector; + +// use one of the following objects to provide the priority for the +// given key. these objects internally point to the container index by +// key value holding the associated priority. +// for keys that are not ordinals 0 to N, use the mapped_key_priority_t alias +// for contiguous keys 0 to N (faster), use the contiguous_key_priority_t alias +template +struct mapped_key_priority +{ + using key_map_t = mapped_key_t; + + mapped_key_priority(std::map &mp) : m_map(&mp) + {} + + priority_t operator()(key_t i) + { return (*m_map)[i]; } + + std::map *m_map; +}; + +template +struct contiguous_key_priority +{ + using key_map_t = contiguous_key_t; + + contiguous_key_priority(const std::vector &vec) : + m_vec(vec.data()) + {} + + priority_t operator()(key_t i) + { return m_vec[i]; } + + const priority_t *m_vec; +}; + +// forward declare the queue +template , + typename key_map_t=contiguous_key_t> +class teca_priority_queue; + +// pointer type +template +using p_teca_priority_queue = std::shared_ptr< + teca_priority_queue>; + +/// @endcond + +/** @brief + * An indirect priority queue that supports random access modification of + * priority. + * + * @details + * an indirect priority queue that supports random access modification of + * priority the queue works with user provided keys and lookup functor that + * converts keys to priorities. + * + * ### template parameters: + * + * | name | description | + * | ---- | ----------- | + * | key_t | type of the user provided keys | + * | lookup_t | callable that implements: priority_t operator()(key_t key) | + * | comp_t | callable that implements the predicate: bool(key_t, key_t), | + * | | used to enforce heap order. (std::less) | + * | key_map_t | type of container used to track the position in the heap | + * | | of the keys. The default, a vector, is only valid for | + * | | interger ordinals from 0 to N. Use mapped_key_t | + * | | for all other cases. (contiguous_key_t) | + * + * ### typical usage: + * + * construct a container of objects to prioritize, and initialize a lookup + * object that given a key returns the priority of the coresponding object. + * create an instance of the priority_queue and push the key values. as keys + * are pushed heap ording is imposed, this is why objects need to be in place + * before pushing keys. when an object's priority has been changed one must + * call modified passing the key of the object. the location of each object is + * tracked and the queue will reprioritize itself after modification. + * + * ### recomendation: + * + * to obtain high performance, it's best to avoid using std::function for + * lookup operations. Instead, write a small functor so that the compiler + * can inline lookup calls. + * + * don't forget to change key_map_t to mapped_key_t if + * keys are not integer ordinals 0 to N. + */ +template +class teca_priority_queue +{ +public: + + ~teca_priority_queue() = default; + + // return a new instance, must pass the lookup operator that + // translates keys into priority values + static p_teca_priority_queue + New(lookup_t &lookup, unsigned long init_size=256, + unsigned long block_size=256) + { + p_teca_priority_queue ptr( + new teca_priority_queue< + key_t, lookup_t, comp_t, key_map_t>( + lookup, init_size, block_size)); + return ptr; + } + + // return true if the queue has no keys + bool empty() { return m_end == 0; } + + + /// add a value into the queue + void push(const key_t &key); + + /// free all resources and reset the queue to an empty state + void clear(); + + // restore heap condition after an id is modified + void modified(const key_t &key); + + // return the id at the top of the queue, and remove it. + // internal memory is not deallocated. + key_t pop(); + + // return the id in the top of queue + key_t peak(); + + // print the state of the queue + void to_stream(std::ostream &os, bool priorities = true); + +protected: + teca_priority_queue() = default; + + teca_priority_queue(const teca_priority_queue &) = delete; + void operator=(const teca_priority_queue &) = delete; + + // initialize the queue with an comperator, the initial size, and declare + // the amount to grow the queue by during dynamic resizing. + template + teca_priority_queue(lookup_t lookup, unsigned long init_size, + unsigned long block_size, + typename std::enable_if, u>::value>::type * = 0); + + template + teca_priority_queue(lookup_t lookup, unsigned long init_size, + unsigned long block_size, + typename std::enable_if, u>::value>::type * = 0); + + // grow the queue to the new size + template + void grow(unsigned long n, + typename std::enable_if, u>::value>::type * = 0); + + // grow the queue to the new size + template + void grow(unsigned long n, + typename std::enable_if, u>::value>::type * = 0); + + // restore the heap condition starting from here + // and working up + void up_heapify(unsigned long id); + + // restore the heap condition starting from here + // and working down + void down_heapify(unsigned long id); + + // exchange two items + void swap(unsigned long i, unsigned long j); + + // helpers for walking tree + unsigned long left_child(unsigned long a_id) + { return a_id*2; } + + unsigned long right_child(unsigned long a_id) + { return a_id*2 + 1; } + + unsigned long parent(unsigned long a_id) + { return a_id/2; } + + +private: + lookup_t m_lookup; // callable to turn keys into priority values + std::vector m_ids; // array of keys + key_map_t m_locs; // map indexed by key to find the current position in the queue + unsigned long m_size; // size of the key buffer + unsigned long m_end; // index of the last key in the queue + unsigned long m_block_size; // amount to grow the dynamically alloacted buffers by +}; + + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::push(const key_t &key) +{ + // extend the queue + ++m_end; + + // verify that there is space, if not allocate it + if (m_end >= m_size) + this->grow(m_size + m_block_size); + + // add key and it's location + m_ids[m_end] = key; + m_locs[key] = m_end; + + // restore heap condition + this->up_heapify(m_end); +} + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::clear() +{ + m_ids.clear(); + m_locs.clear(); + m_size = 0; + m_end = 0; +} + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::modified(const key_t &key) +{ + // find the loc of the modified key + unsigned long id = m_locs[key]; + // fix up then down + this->up_heapify(id); + this->down_heapify(id); +} + +// -------------------------------------------------------------------------- +template +key_t teca_priority_queue::pop() +{ + key_t id_1 = m_ids[1]; + if (m_end > 0) + { + this->swap(1, m_end); + --m_end; + this->down_heapify(1); + } + return id_1; +} + +// -------------------------------------------------------------------------- +template +key_t teca_priority_queue::peak() +{ + return m_ids[1]; +} + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::to_stream(std::ostream &os, bool priorities) +{ + long log_end = std::log2(m_end); + long n_rows = log_end + 1; + unsigned long q = 0; + for (long i = 0; i < n_rows; ++i) + { + if (q > m_end) + break; + + long n_elem = 1 << i; + long isp = (1 << (n_rows - 1 - i)) - 1; + long bsp = 2*isp + 1; + + for (long j = 0; j < isp; ++j) + os << " "; + + for (long j = 0; (j < n_elem) && (q < m_end); ++j) + { + if (priorities) + os << m_lookup(m_ids[++q]); + else + os << m_ids[++q]; + for (long k = 0; k < bsp; ++k) + os << " "; + } + + os << std::endl; + } +} + +// -------------------------------------------------------------------------- +template +template +teca_priority_queue::teca_priority_queue(lookup_t lookup, + unsigned long init_size, unsigned long block_size, + typename std::enable_if, u>::value>::type *) : + m_lookup(lookup), m_size(init_size), m_end(0), + m_block_size(block_size) +{ + m_ids.resize(init_size); + m_locs.resize(init_size); +} + +// -------------------------------------------------------------------------- +template +template +teca_priority_queue::teca_priority_queue(lookup_t lookup, + unsigned long init_size, unsigned long block_size, + typename std::enable_if, u>::value>::type *) : + m_lookup(lookup), m_size(init_size), m_end(0), + m_block_size(block_size) +{ + m_ids.resize(init_size); +} + +// -------------------------------------------------------------------------- +template +template +void teca_priority_queue::grow(unsigned long n, + typename std::enable_if, u>::value>::type *) +{ + m_ids.resize(n); + m_locs.resize(n); + m_size = n; +} + +// -------------------------------------------------------------------------- +template +template +void teca_priority_queue::grow(unsigned long n, + typename std::enable_if, u>::value>::type *) +{ + m_ids.resize(n); + m_size = n; +} + + + // -------------------------------------------------------------------------- +template +void teca_priority_queue::up_heapify(unsigned long id) +{ + // if at tree root then stop + if (id < 2) + return; + + // else find parent and enforce heap order + comp_t comp; + unsigned long id_p = parent(id); + if (comp(m_lookup(m_ids[id]), m_lookup(m_ids[id_p]))) + this->swap(id, id_p); + + // continue up toward the root + this->up_heapify(id_p); +} + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::down_heapify(unsigned long id) +{ + // if no current node then stop + if (id > m_end) + return; + + // if no left child then stop + unsigned long lc = left_child(id); + if (lc > m_end) + return; + + // find the smaller child + comp_t comp; + unsigned long smallc = lc; + unsigned long rc = right_child(id); + if (rc <= m_end) + smallc = comp(m_lookup(m_ids[lc]), + m_lookup(m_ids[rc])) ? lc : rc; + + // if in heap order then stop + if (comp(m_lookup(m_ids[id]), m_lookup(m_ids[smallc]))) + return; + + // else swap and continue + this->swap(id, smallc); + this->down_heapify(smallc); +} + +// -------------------------------------------------------------------------- +template +void teca_priority_queue::swap(unsigned long i, unsigned long j) +{ + key_t key_i = m_ids[i]; + key_t key_j = m_ids[j]; + // exchange keys + m_ids[i] = key_j; + m_ids[j] = key_i; + // update locs + m_locs[key_j] = i; + m_locs[key_i] = j; +} + +template +std::ostream & operator<<(std::ostream &os, p_teca_priority_queue &q) +{ + q->to_stream(os); + return os; +} + +#endif diff --git a/data/teca_table.h b/data/teca_table.h index a3b372eff..241f42d37 100644 --- a/data/teca_table.h +++ b/data/teca_table.h @@ -12,10 +12,10 @@ TECA_SHARED_OBJECT_FORWARD_DECL(teca_table) -/** -A collection of collumnar data with row based -accessors and communication and I/O support. -*/ +/** @brief + * A collection of columnar data with row based + * accessors and communication and I/O support. + */ class teca_table : public teca_dataset { public: @@ -34,8 +34,8 @@ class teca_table : public teca_dataset // define the table columns. requires name,type pairs // for ex. define("c1",int(),"c2",float()) creates a - // table with 2 columns the first storing int's the - // second storing float's. + // table with 2 columns the first storing int the + // second storing float. template void declare_columns(nT &&col_name, cT col_type, oT &&...args); @@ -55,7 +55,7 @@ class teca_table : public teca_dataset const_p_teca_variant_array get_column(unsigned int i) const; const_p_teca_variant_array get_column(const std::string &col_name) const; - // test for the existance of a specific column + // test for the existence of a specific column bool has_column(const std::string &col_name) const { return m_impl->columns->has(col_name); } @@ -103,7 +103,7 @@ class teca_table : public teca_dataset // return an integer identifier uniquely naming the dataset type int get_type_code() const override; - // covert to bool. true if the dataset is not empty. + // covert to boolean. true if the dataset is not empty. // otherwise false. explicit operator bool() const noexcept { return !this->empty(); } diff --git a/data/teca_table_collection.h b/data/teca_table_collection.h index c2eb9bc95..1bc2a4a68 100644 --- a/data/teca_table_collection.h +++ b/data/teca_table_collection.h @@ -1,16 +1,16 @@ #ifndef teca_table_collection_h #define teca_table_collection_h -#include "teca_table_collection_fwd.h" #include "teca_table.h" +#include "teca_shared_object.h" + #include #include #include -// a collection of named tables -/** -A collection of named tables -*/ +TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_collection) + +/// A collection of named tables. class teca_table_collection { public: diff --git a/data/teca_table_collection_fwd.h b/data/teca_table_collection_fwd.h deleted file mode 100644 index 732902648..000000000 --- a/data/teca_table_collection_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_table_collection_fwd_h -#define teca_table_collection_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_table_collection) - -#endif diff --git a/data/teca_uniform_cartesian_mesh.h b/data/teca_uniform_cartesian_mesh.h index bd7ef44d1..f6e31c66c 100644 --- a/data/teca_uniform_cartesian_mesh.h +++ b/data/teca_uniform_cartesian_mesh.h @@ -1,10 +1,12 @@ #ifndef teca_uniform_cartesian_mesh_h #define teca_uniform_cartesian_mesh_h -#include "teca_uniform_cartesian_mesh_fwd.h" #include "teca_mesh.h" +#include "teca_shared_object.h" -/// data on a uniform cartesian mesh +TECA_SHARED_OBJECT_FORWARD_DECL(teca_uniform_cartesian_mesh) + +/// Data on a uniform cartesian mesh. class teca_uniform_cartesian_mesh : public teca_mesh { public: @@ -22,19 +24,19 @@ class teca_uniform_cartesian_mesh : public teca_mesh TECA_DATASET_METADATA(extent, unsigned long, 6) TECA_DATASET_METADATA(local_extent, unsigned long, 6) - // return a unique string identifier + /// return a unique string identifier std::string get_class_name() const override { return "teca_uniform_cartesian_mesh"; } - // return a unique integer identifier + /// return a unique integer identifier int get_type_code() const override; - // copy data and metadata. shallow copy uses reference - // counting, while copy duplicates the data. + /// copy data and metadata. shallow copy uses reference + /// counting, while copy duplicates the data. void copy(const const_p_teca_dataset &) override; void shallow_copy(const p_teca_dataset &) override; - // swap internals of the two objects + /// swap internals of the two objects void swap(p_teca_dataset &) override; protected: diff --git a/data/teca_uniform_cartesian_mesh_fwd.h b/data/teca_uniform_cartesian_mesh_fwd.h deleted file mode 100644 index 79a66e960..000000000 --- a/data/teca_uniform_cartesian_mesh_fwd.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef teca_uniform_cartesian_mesh_fwd_h -#define teca_uniform_cartesian_mesh_fwd_h - -#include "teca_shared_object.h" - -TECA_SHARED_OBJECT_FORWARD_DECL(teca_uniform_cartesian_mesh) - -#endif diff --git a/doc/release/4.1.0.md b/doc/release/4.1.0.md new file mode 100644 index 000000000..ce91a2e50 --- /dev/null +++ b/doc/release/4.1.0.md @@ -0,0 +1,26 @@ +## Release Notes for TECA 4.1.0 +Thu May 6 12:10:26 PM PDT 2021 + +* new mask below surface algorithm, inetgrated into all the apps +* new unpack NetCDF packed data stage +* add coordinate normalization stage transform for longitude from -180 to 180 + to 0 to 360 +* new IWV algorithm +* new time based file layouts (daily, monthly, yearly, seasonal) +* BARD app can now generate output fields weighted by AR probabilities +* new rename variables stage +* improvements to cartesian_mesh_source for remeshing +* cf_reader correctly detects centering and per field dimensionality +* multi_cf_reader MCF file format improvements. Add support for reader + properties, globablly and per reader. +* cf_reader option to produce 2D field when the 3'rd dimension is length 1 +* Cartesian meshes can now contain both 2D and 3D arrays, metadata annotations + are used to differentiate at run time +* metadata probe improvements to report per-field centering +* new remeshing capability deployed in cf_restripe and apps that utilize + elevation mask +* improvements to the user guide +* refactored source code documentation to be compatible with Doxygen, +* published Doxygen on the rtd site : https://teca.readthedocs.io/en/integrating_breathe/doxygen/index.html +* new capabilities in the cf_restripe command line application for remeshing +* 25+ bug fixes diff --git a/doc/rtd/Doxyfile b/doc/rtd/Doxyfile new file mode 100644 index 000000000..aa8d9ea3f --- /dev/null +++ b/doc/rtd/Doxyfile @@ -0,0 +1,2547 @@ +# Doxyfile 1.8.20 + + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "TECA" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "The Toolkit for Extreme Climate Analysis" + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = _build + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. + +OUTPUT_TEXT_DIRECTION = None + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = YES + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = ../../.. + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) + +ALIASES = + +#ALIASES = "rst=\verbatim embed:rst" +#ALIASES += "endrst=\endverbatim" +#ALIASES += "rststar=\verbatim embed:rst:leading-asterisk" +#ALIASES += "endrststar=\endverbatim" + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See https://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = YES + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = NO + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which efficively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = NO + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# declarations. If set to NO, these declarations will be included in the +# documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# (including Cygwin) and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = YES + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 10000 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = DoxygenLayout.xml + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = ../../README.md \ + ../../alg \ + ../../core \ + ../../data \ + ../../io \ + ../../python + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: https://www.gnu.org/software/libiconv/) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), +# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen +# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.vhdl, *.ucf, *.qsf and *.ice. + +FILE_PATTERNS = *.h \ + *.py + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = ../../alg/teca_deeplab_ar_detect_internals.py \ + parse_xml.py + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = ../../doc/rtd/images + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = README.md + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = NO + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# entity all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see https://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html/doxygen + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = YES + +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: https://developer.apple.com/xcode/), introduced with OSX +# 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the main .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 200 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. + +FORMULA_MACROFILE = + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# https://www.mathjax.org) which uses client side JavaScript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = YES + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /