From 19726175c3c415adb5eb9e3be1d4657283792ae1 Mon Sep 17 00:00:00 2001 From: Abdelrahman Elbashandy Date: Thu, 6 Feb 2020 17:21:54 -0800 Subject: [PATCH 001/385] Closes #321 Fixing travis-ci osx to use python 3 --- .travis.yml | 4 ++-- test/travis_ci/install_osx.sh | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 36f4ffc3a..2fdd4f5c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,12 +15,12 @@ env: - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - TECA_DATA_REVISION=49 - matrix: + jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - DOCKER_IMAGE=fedora IMAGE_VERSION=28 IMAGE_NAME=fedora_28 - NO_DOCKER=TRUE -matrix: +jobs: exclude: - os: osx env: DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index cfc09bf95..b9feebed2 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -6,11 +6,10 @@ export PATH=/usr/local/bin:$PATH # install deps. note than many are included as a part of brew-core # these days. hence this list isn't comprehensive brew update -brew upgrade python brew install openmpi swig svn udunits # matplotlib currently doesn't have a formula # teca fails to locate mpi4py installed from brew -pip3 install mpi4py matplotlib +pip3 install numpy mpi4py matplotlib # install data files. # On Apple svn is very very slow. On my mac book pro From 96c62517a32bfd765535d0ba1c46f507efb5c637 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 18 Feb 2020 13:35:19 -0800 Subject: [PATCH 002/385] make ctest fail right away when any part fails --- test/travis_ci/ctest_linux.cmake | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/travis_ci/ctest_linux.cmake b/test/travis_ci/ctest_linux.cmake index 98e046f54..af9ebf377 100644 --- a/test/travis_ci/ctest_linux.cmake +++ b/test/travis_ci/ctest_linux.cmake @@ -23,13 +23,19 @@ TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data TECA_TEST_CORES=2") file(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" ${INITIAL_CACHE}) ctest_start("${DASHBOARD_TRACK}") -ctest_configure() ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}") -ctest_build() -ctest_submit(PARTS Update Configure Build Notes RETRY_DELAY 15 RETRY_COUNT 10) +ctest_configure(RETURN_VALUE terr) +ctest_submit(PARTS Update Configure RETRY_DELAY 15 RETRY_COUNT 10) +if (NOT terr EQUAL 0) + message(FATAL_ERROR "ERROR: ctest configure failed!") +endif() +ctest_build(RETURN_VALUE terr) +ctest_submit(PARTS Build RETRY_DELAY 15 RETRY_COUNT 10) +if (NOT terr EQUAL 0) + message(FATAL_ERROR "ERROR: ctest build failed!") +endif() ctest_test(RETURN_VALUE terr) ctest_submit(PARTS Test RETRY_DELAY 15 RETRY_COUNT 10) if (NOT terr EQUAL 0) - # if any tests failed abort with a fatal error - message(FATAL_ERROR "ERROR: At least one test failed!") + message(FATAL_ERROR "ERROR: ctest test failed!") endif() From 0da2e1b0558fd87de0c79fd6b66a950ffdc716b1 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 10 Feb 2020 11:50:54 -0800 Subject: [PATCH 003/385] fix copy paste doc issue in component_area_filter --- alg/teca_component_area_filter.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alg/teca_component_area_filter.h b/alg/teca_component_area_filter.h index b1e8efce3..30282f499 100644 --- a/alg/teca_component_area_filter.h +++ b/alg/teca_component_area_filter.h @@ -75,7 +75,7 @@ class teca_component_area_filter : public teca_algorithm TECA_ALGORITHM_PROPERTY(double, high_area_threshold) // a string to be appended to the name of the output variable. - // setting this to an empty string will result in the damped array + // setting this to an empty string will result in the masked array // replacing the input array in the output. default is an empty // string "" TECA_ALGORITHM_PROPERTY(std::string, variable_post_fix) From a17ac4b642d5b1bb6529b109961c319a75911a6b Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 11 Feb 2020 12:14:37 -0800 Subject: [PATCH 004/385] make the python_algorithm polymorphic the author of a new algorithm would now have to override report, request, and execute methods instead of get'ers that return functions implementing these. the get_X_callback methods remain in the python_algorithm and use the closeruse trick but now return a function that calls the above class methods (report, request, execute). from a user's pov this should be a simplification as they no longer need to know about the closure trick. this change makes it possible to use polymorphism to provide default implementations that could be called after some preprocesing of the inputs. This could be useful in teca_model_segmentation and its child classes. --- alg/teca_python_algorithm.py | 111 +-- alg/teca_tc_activity.py | 164 ++-- alg/teca_tc_stats.py | 919 +++++++++++------------ alg/teca_tc_trajectory_scalars.py | 689 +++++++++-------- alg/teca_tc_wind_radii_stats.py | 318 ++++---- doc/rtd/python.rst | 96 ++- doc/rtd/source/stats_callbacks.py | 60 +- test/python/CMakeLists.txt | 7 + test/python/test_information_array_io.py | 85 +-- test/python/test_python_algorithm.py | 116 +++ 10 files changed, 1337 insertions(+), 1228 deletions(-) create mode 100644 test/python/test_python_algorithm.py diff --git a/alg/teca_python_algorithm.py b/alg/teca_python_algorithm.py index c1d15cf51..1cdfec94b 100644 --- a/alg/teca_python_algorithm.py +++ b/alg/teca_python_algorithm.py @@ -1,17 +1,17 @@ class teca_python_algorithm(object): """ - The base class used for writing new algorithms in Python. - Contains plumbing that connects user provided callbacks - to an instance of teca_programmable_algorithm. Users are - expected to override one or more of get_report_callback, - get_request_callback, and/or get_execute_callback. These - methods return a callable with the correct signature, and - use a closure to access class state. + The base class used for writing new algorithms in Python. Contains + plumbing that connects user provided overrides to an instance of + teca_programmable_algorithm. Users are expected to override one or more of + report, request, and/or execute. """ + @classmethod def New(derived_class): - """ factory method returns an instance of the derived type """ + """ + factory method returns an instance of the derived type + """ dc = derived_class() dc.initialize_implementation() return dc @@ -36,7 +36,9 @@ def initialize_implementation(self): self.impl.set_execute_callback(self.get_execute_callback()) def __getattr__(self, name): - """ forward stuff to the programmable algorithm """ + """ + forward calls to the programmable algorithm + """ # guard against confusing infinite recursion that # occurs if impl is not present. one common way @@ -51,58 +53,73 @@ def __getattr__(self, name): # forward to the teca_programmable_algorithm return self.impl.__getattribute__(name) - def get_number_of_input_connections(self): - """ Override to change number of inputs """ - return 1 - - def get_number_of_output_ports(self): - """ Override to change number of outputs """ - return 1 - def get_report_callback(self): """ - Returns a function with the signature - - report_callback(port, md_in) -> teca_metadata - - The default implementation passes the report down stream. - Override this to customize the behavior of the report - phase of execution. + returns a callback to be used by the programmable algorithm that + forwards calls to the class method. """ def report_callback(port, md_in): - import teca_py - return teca_py.teca_metadata(md_in[0]) + return self.report(port, md_in) return report_callback def get_request_callback(self): """ - Returns a function with the signature - - request_callback(port, md_in, req_in) -> [teca_metadata] - - The default implementation passes the request up stream. - Override this to customize the behavior of the request - phase of execution. + returns a callback to be used by the programmable algorithm that + forwards calls to the class method. """ def request_callback(port, md_in, req_in): - import teca_py - return [teca_py.teca_metadata(req_in)] + return self.request(port, md_in, req_in) return request_callback def get_execute_callback(self): """ - Returns a function with the signature - - execute_callback(port, data_in, req_in) -> teca_dataset - - The default implementation shallow copies the input dataset. - Override this to customize the behavior of the execute - phase of execution. + returns a callback to be used by the programmable algorithm that + forwards calls to the class method. """ def execute_callback(port, data_in, req_in): - import teca_py - if len(data_in): - data_out = data_in[0].new_instance() - data_out.shallow_copy(teca_py.as_non_const_teca_dataset(data_out)) - return data_out + return self.execute(port, data_in, req_in) return execute_callback + + def get_number_of_input_connections(self): + """ + return the number of input connections this algorithm needs. + The default is 1, override to modify. + """ + return 1 + + def get_number_of_output_ports(self): + """ + return the number of output ports this algorithm provides. + The default is 1, override to modify. + """ + return 1 + + def report(self, port, md_in): + """ + return the metadata decribing the data available for consumption. + Override this to customize the behavior of the report phase of + execution. The default passes metadata on the first input through. + """ + import teca_py + return teca_py.teca_metadata(md_in[0]) + + def request(self, port, md_in, req_in): + """ + return the request for needed data for execution. Override this to + customize the behavior of the request phase of execution. The default + passes the request on the first input port through. + """ + import teca_py + return [teca_py.teca_metadata(req_in)] + + def execute(self, port, data_in, req_in): + """ + return the processed data. Override this to customize the behavior of + the execute phase of execution. The default passes the dataset on the + first input port through. + """ + import teca_py + if len(data_in): + data_out = data_in[0].new_instance() + data_out.shallow_copy(teca_py.as_non_const_teca_dataset(data_out)) + return data_out diff --git a/alg/teca_tc_activity.py b/alg/teca_tc_activity.py index 384297c3f..b3c7afcbb 100644 --- a/alg/teca_tc_activity.py +++ b/alg/teca_tc_activity.py @@ -51,93 +51,87 @@ def set_color_map(self, color_map): """ self.color_map = color_map - def get_execute_callback(self): + def execute(self, port, data_in, req): """ - return a teca_algorithm::execute function. a closure - is used to gain self. + expects the output of the teca_tc_classify algorithm + generates a handful of histograms, summary statistics, + and plots. returns summary table with counts of annual + storms and their categories. """ - def execute(port, data_in, req): - """ - expects the output of the teca_tc_classify algorithm - generates a handful of histograms, summary statistics, - and plots. returns summary table with counts of annual - storms and their categories. - """ - global plt - global plt_mp - global plt_tick - - import matplotlib.pyplot as plt - import matplotlib.patches as plt_mp - import matplotlib.ticker as plt_tick - - # store matplotlib state we modify - legend_frame_on_orig = plt.rcParams['legend.frameon'] - - # tweak matplotlib slightly - plt.rcParams['figure.max_open_warning'] = 0 - plt.rcParams['legend.frameon'] = 1 - - # get the input table - in_table = teca_py.as_teca_table(data_in[0]) - if in_table is None: - # TODO if this is part of a parallel pipeline then - # only rank 0 should report an error. - sys.stderr.write('ERROR: empty input, or not a table\n') - return teca_table.New() - - time_units = in_table.get_time_units() - - # get the columns of raw data - year = in_table.get_column('year').as_array() - region_id = in_table.get_column('region_id').as_array() - region_name = in_table.get_column('region_name') - region_long_name = in_table.get_column('region_long_name') - start_y = in_table.get_column('start_y').as_array() - - ACE = in_table.get_column('ACE').as_array() - PDI = in_table.get_column('PDI').as_array() - - # organize the data by year month etc... - regional_ACE = [] - regional_PDI = [] - - # get unique for use as indices etc - uyear = sorted(set(year)) - n_year = len(uyear) - ureg = sorted(set(zip(region_id, region_name, region_long_name))) + global plt + global plt_mp + global plt_tick + + import matplotlib.pyplot as plt + import matplotlib.patches as plt_mp + import matplotlib.ticker as plt_tick + + # store matplotlib state we modify + legend_frame_on_orig = plt.rcParams['legend.frameon'] + + # tweak matplotlib slightly + plt.rcParams['figure.max_open_warning'] = 0 + plt.rcParams['legend.frameon'] = 1 + + # get the input table + in_table = teca_py.as_teca_table(data_in[0]) + if in_table is None: + # TODO if this is part of a parallel pipeline then + # only rank 0 should report an error. + sys.stderr.write('ERROR: empty input, or not a table\n') + return teca_table.New() + + time_units = in_table.get_time_units() + + # get the columns of raw data + year = in_table.get_column('year').as_array() + region_id = in_table.get_column('region_id').as_array() + region_name = in_table.get_column('region_name') + region_long_name = in_table.get_column('region_long_name') + start_y = in_table.get_column('start_y').as_array() + + ACE = in_table.get_column('ACE').as_array() + PDI = in_table.get_column('PDI').as_array() + + # organize the data by year month etc... + regional_ACE = [] + regional_PDI = [] + + # get unique for use as indices etc + uyear = sorted(set(year)) + n_year = len(uyear) + ureg = sorted(set(zip(region_id, region_name, region_long_name))) - self.accum_by_year_and_region(uyear, \ - ureg, year, region_id, start_y, ACE, regional_ACE) + self.accum_by_year_and_region(uyear, \ + ureg, year, region_id, start_y, ACE, regional_ACE) - self.accum_by_year_and_region(uyear, \ - ureg, year, region_id, start_y, PDI, regional_PDI) + self.accum_by_year_and_region(uyear, \ + ureg, year, region_id, start_y, PDI, regional_PDI) - # now plot the organized data in various ways - if self.color_map is None: - self.color_map = plt.cm.jet + # now plot the organized data in various ways + if self.color_map is None: + self.color_map = plt.cm.jet - self.plot_individual(uyear, \ - ureg, regional_ACE,'ACE', '$10^4 kn^2$') + self.plot_individual(uyear, \ + ureg, regional_ACE,'ACE', '$10^4 kn^2$') - self.plot_individual(uyear, \ - ureg, regional_PDI, 'PDI', '$m^3 s^{-2}$') + self.plot_individual(uyear, \ + ureg, regional_PDI, 'PDI', '$m^3 s^{-2}$') - self.plot_cumulative(uyear, \ - ureg, regional_ACE, 'ACE', '$10^4 kn^2$') + self.plot_cumulative(uyear, \ + ureg, regional_ACE, 'ACE', '$10^4 kn^2$') - self.plot_cumulative(uyear, \ - ureg, regional_PDI, 'PDI', '$m^3 s^{-2}$') + self.plot_cumulative(uyear, \ + ureg, regional_PDI, 'PDI', '$m^3 s^{-2}$') - if (self.interactive): - plt.show() + if (self.interactive): + plt.show() - # restore matplot lib global state - plt.rcParams['legend.frameon'] = legend_frame_on_orig + # restore matplot lib global state + plt.rcParams['legend.frameon'] = legend_frame_on_orig - # send data downstream - return in_table - return execute + # send data downstream + return in_table @staticmethod def two_digit_year_fmt(x, pos): @@ -315,23 +309,3 @@ def plot_cumulative(self, uyear, ureg, var, var_name, units): return -# def write_table(state, uyear, ureg, var, var_name, units, table_out): -# -# n_reg = len(ureg) + 3 # add 2 for n & s hemi, 1 for global -# n_year = len(uyear) -# -# rnms = zip(*ureg)[2] -# rnms += ('Southern','Northern','Global') -# -# -# tab = teca_table.New() -# tab.declare_columns(['Year'] + rnms, -# ['l'] + ['d']*n_reg) -# -# for val in var: -# if -# q = 0 -# while q < n_reg: -# tmp = var[q::n_reg] -# -# plt.plot(uyear, ,'-',color=fill_col[q],linewidth=2) diff --git a/alg/teca_tc_stats.py b/alg/teca_tc_stats.py index 5f474cac3..917715c5d 100644 --- a/alg/teca_tc_stats.py +++ b/alg/teca_tc_stats.py @@ -44,500 +44,495 @@ def set_rel_axes(self, rel_axes): """ self.rel_axes = rel_axes - def get_execute_callback(self): + def execute(self, port, data_in, req): """ - return a teca_algorithm::execute function. a closure - is used to gain self. + expects the output of the teca_tc_classify algorithm + generates a handful of histograms, summary statistics, + and plots. returns summary table with counts of annual + storms and their categories. """ - def execute(port, data_in, req): - """ - expects the output of the teca_tc_classify algorithm - generates a handful of histograms, summary statistics, - and plots. returns summary table with counts of annual - storms and their categories. - """ - import matplotlib.pyplot as plt - import matplotlib.patches as plt_mp - - # store matplotlib state we modify - legend_frame_on_orig = plt.rcParams['legend.frameon'] - - # tweak matplotlib slightly - plt.rcParams['figure.max_open_warning'] = 0 - plt.rcParams['legend.frameon'] = 1 - - # get the input table - in_table = teca_py.as_teca_table(data_in[0]) - if in_table is None: - # TODO if this is part of a parallel pipeline then - # only rank 0 should report an error. - sys.stderr.write('ERROR: empty input, or not a table\n') - return teca_table.New() - - time_units = in_table.get_time_units() - - # get the columns of raw data - year = in_table.get_column('year').as_array() - month = in_table.get_column('month').as_array() - duration = in_table.get_column('duration').as_array() - length = in_table.get_column('length').as_array()/1000.0 - category = in_table.get_column('category').as_array() - region_id = in_table.get_column('region_id').as_array() - region_name = in_table.get_column('region_name') - region_long_name = in_table.get_column('region_long_name') - wind = in_table.get_column('max_surface_wind').as_array() - press = in_table.get_column('min_sea_level_pressure').as_array() - start_y = in_table.get_column('start_y').as_array() - ACE = in_table.get_column('ACE').as_array() - - # organize the data by year month etc... - annual_cat = [] - annual_count = [] - annual_wind = [] - annual_press = [] - annual_dur = [] - annual_len = [] - annual_ACE = [] - by_month = [] - by_region = [] - totals = [] - - # get unique for use as indices etc - uyear = sorted(set(year)) - n_year = len(uyear) - - ureg = sorted(set(zip(region_id, region_name, region_long_name))) - n_reg = len(ureg) + 3 # add 2 for n & s hemi, 1 for global - - for yy in uyear: - yids = np.where(year==yy) - # break these down by year - annual_count.append(len(yids[0])) - annual_cat.append(category[yids]) - annual_wind.append(wind[yids]) - annual_press.append(press[yids]) - annual_dur.append(duration[yids]) - annual_len.append(length[yids]) - annual_ACE.append(ACE[yids]) - - # global totals - tmp = [annual_count[-1]] + import matplotlib.pyplot as plt + import matplotlib.patches as plt_mp + + # store matplotlib state we modify + legend_frame_on_orig = plt.rcParams['legend.frameon'] + + # tweak matplotlib slightly + plt.rcParams['figure.max_open_warning'] = 0 + plt.rcParams['legend.frameon'] = 1 + + # get the input table + in_table = teca_py.as_teca_table(data_in[0]) + if in_table is None: + # TODO if this is part of a parallel pipeline then + # only rank 0 should report an error. + sys.stderr.write('ERROR: empty input, or not a table\n') + return teca_table.New() + + time_units = in_table.get_time_units() + + # get the columns of raw data + year = in_table.get_column('year').as_array() + month = in_table.get_column('month').as_array() + duration = in_table.get_column('duration').as_array() + length = in_table.get_column('length').as_array()/1000.0 + category = in_table.get_column('category').as_array() + region_id = in_table.get_column('region_id').as_array() + region_name = in_table.get_column('region_name') + region_long_name = in_table.get_column('region_long_name') + wind = in_table.get_column('max_surface_wind').as_array() + press = in_table.get_column('min_sea_level_pressure').as_array() + start_y = in_table.get_column('start_y').as_array() + ACE = in_table.get_column('ACE').as_array() + + # organize the data by year month etc... + annual_cat = [] + annual_count = [] + annual_wind = [] + annual_press = [] + annual_dur = [] + annual_len = [] + annual_ACE = [] + by_month = [] + by_region = [] + totals = [] + + # get unique for use as indices etc + uyear = sorted(set(year)) + n_year = len(uyear) + + ureg = sorted(set(zip(region_id, region_name, region_long_name))) + n_reg = len(ureg) + 3 # add 2 for n & s hemi, 1 for global + + for yy in uyear: + yids = np.where(year==yy) + # break these down by year + annual_count.append(len(yids[0])) + annual_cat.append(category[yids]) + annual_wind.append(wind[yids]) + annual_press.append(press[yids]) + annual_dur.append(duration[yids]) + annual_len.append(length[yids]) + annual_ACE.append(ACE[yids]) + + # global totals + tmp = [annual_count[-1]] + for c in np.arange(0,6,1): + cids = np.where(category[yids]==c) + tmp.append(len(cids[0])) + totals.append(tmp) + + # break down by year, month, and category + mm = month[yids] + mnum = np.arange(1,13,1) + monthly = [] + for m in mnum: + mids = np.where(mm==m) + mcats = category[yids][mids] + cats = [] for c in np.arange(0,6,1): - cids = np.where(category[yids]==c) - tmp.append(len(cids[0])) - totals.append(tmp) - - # break down by year, month, and category - mm = month[yids] - mnum = np.arange(1,13,1) - monthly = [] - for m in mnum: - mids = np.where(mm==m) - mcats = category[yids][mids] - cats = [] - for c in np.arange(0,6,1): - cids = np.where(mcats==c) - cats.append(len(cids[0])) - monthly.append(cats) - by_month.append(monthly) - # break down by year and region - rr = region_id[yids] - max_reg = np.max(rr) - regional = [] - for r,n,l in ureg: - rids = np.where(rr==r) - rcats = category[yids][rids] - cats = [] - for c in np.arange(0,6,1): - cids = np.where(rcats==c) - cats.append(len(cids[0])) - regional.append(cats) - by_region.append(regional) - # add north and south hemisphere regions - hemi = [] - nhids = np.where(start_y[yids] >= 0.0) - cats = category[yids][nhids] - nhcats = [] + cids = np.where(mcats==c) + cats.append(len(cids[0])) + monthly.append(cats) + by_month.append(monthly) + # break down by year and region + rr = region_id[yids] + max_reg = np.max(rr) + regional = [] + for r,n,l in ureg: + rids = np.where(rr==r) + rcats = category[yids][rids] + cats = [] for c in np.arange(0,6,1): - cids = np.where(cats==c) - nhcats.append(len(cids[0])) - by_region[-1].append(nhcats) - shids = np.where(start_y[yids] < 0.0) - cats = category[yids][shids] - shcats = [] - for c in np.arange(0,6,1): - cids = np.where(cats==c) - shcats.append(len(cids[0])) - by_region[-1].append(shcats) - # global break down - gcats = [] - cats = category[yids] - for c in np.arange(0,6,1): - cids = np.where(cats==c) - gcats.append(len(cids[0])) - by_region[-1].append(gcats) - - # dump annual totals - summary = teca_py.teca_table.New() - summary.declare_columns(['year', 'total', 'cat 0', \ - 'cat 1', 'cat 2', 'cat 3', 'cat 4', 'cat 5'], \ - ['i', 'ul', 'i', 'i', 'i', 'i', 'i', 'i']) - q = 0 - while q < n_year: - summary << int(uyear[q]) << int(totals[q][0]) \ - << int(totals[q][1]) << int(totals[q][2]) \ - << int(totals[q][3]) << int(totals[q][4]) \ - << int(totals[q][5]) << int(totals[q][6]) - q += 1 - f = open('%s_summary.csv'%(self.basename),'w') - f.write(str(summary)) - f.close() - - # now plot the organized data in various ways - n_cols = 3 - n_plots = n_year + 1 - n_left = n_plots%n_cols - n_rows = n_plots/n_cols + (1 if n_left else 0) - wid = 2.5*n_cols - ht = 2.0*n_rows - - # use this color map for Saphir-Simpson scale - red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ - '#ea4f00','#c92500','#a80300'] - - red_cmap_pats = [] - q = 0 - while q < 6: - red_cmap_pats.append( \ - plt_mp.Patch(color=red_cmap[q], label='cat %d'%(q))) - q += 1 - - # plot annual saphir-simpson distribution - page_no = 1 - cat_fig = plt.figure() - cat_fig.set_size_inches(wid, ht) - - max_y = 0 - q = 0 - while q < n_year: - max_y = max(max_y, len(np.where(annual_cat[q]==0)[0])) - q += 1 - - q = 0 - for yy in uyear: - plt.subplot(n_rows, n_cols, q+1) - ax = plt.gca() - ax.grid(zorder=0) - n,bins,pats = plt.hist(annual_cat[q], bins=np.arange(-0.5, 6.0, 1.0), \ - facecolor='steelblue', alpha=0.95, edgecolor='black', \ - linewidth=2, zorder=3) - j = 0 - while j < 6: - pats[j].set_facecolor(red_cmap[j]) - j += 1 - plt.xticks(np.arange(0,6,1)) - if self.rel_axes: - ax.set_ylim([0, max_y*1.05]) - if (q%n_cols == 0): - plt.ylabel('Count', fontweight='normal', fontsize=10) - if (q >= (n_year - n_cols)): - plt.xlabel('Category', fontweight='normal', fontsize=10) - plt.title('%d'%(yy), fontweight='bold', fontsize=11) - plt.grid(True) - - q += 1 - + cids = np.where(rcats==c) + cats.append(len(cids[0])) + regional.append(cats) + by_region.append(regional) + # add north and south hemisphere regions + hemi = [] + nhids = np.where(start_y[yids] >= 0.0) + cats = category[yids][nhids] + nhcats = [] + for c in np.arange(0,6,1): + cids = np.where(cats==c) + nhcats.append(len(cids[0])) + by_region[-1].append(nhcats) + shids = np.where(start_y[yids] < 0.0) + cats = category[yids][shids] + shcats = [] + for c in np.arange(0,6,1): + cids = np.where(cats==c) + shcats.append(len(cids[0])) + by_region[-1].append(shcats) + # global break down + gcats = [] + cats = category[yids] + for c in np.arange(0,6,1): + cids = np.where(cats==c) + gcats.append(len(cids[0])) + by_region[-1].append(gcats) + + # dump annual totals + summary = teca_py.teca_table.New() + summary.declare_columns(['year', 'total', 'cat 0', \ + 'cat 1', 'cat 2', 'cat 3', 'cat 4', 'cat 5'], \ + ['i', 'ul', 'i', 'i', 'i', 'i', 'i', 'i']) + q = 0 + while q < n_year: + summary << int(uyear[q]) << int(totals[q][0]) \ + << int(totals[q][1]) << int(totals[q][2]) \ + << int(totals[q][3]) << int(totals[q][4]) \ + << int(totals[q][5]) << int(totals[q][6]) + q += 1 + f = open('%s_summary.csv'%(self.basename),'w') + f.write(str(summary)) + f.close() + + # now plot the organized data in various ways + n_cols = 3 + n_plots = n_year + 1 + n_left = n_plots%n_cols + n_rows = n_plots/n_cols + (1 if n_left else 0) + wid = 2.5*n_cols + ht = 2.0*n_rows + + # use this color map for Saphir-Simpson scale + red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ + '#ea4f00','#c92500','#a80300'] + + red_cmap_pats = [] + q = 0 + while q < 6: + red_cmap_pats.append( \ + plt_mp.Patch(color=red_cmap[q], label='cat %d'%(q))) + q += 1 + + # plot annual saphir-simpson distribution + page_no = 1 + cat_fig = plt.figure() + cat_fig.set_size_inches(wid, ht) + + max_y = 0 + q = 0 + while q < n_year: + max_y = max(max_y, len(np.where(annual_cat[q]==0)[0])) + q += 1 + + q = 0 + for yy in uyear: plt.subplot(n_rows, n_cols, q+1) ax = plt.gca() ax.grid(zorder=0) - l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) - plt.axis('off') - - plt.suptitle('Annual Saphir-Simpson Distribution', fontweight='bold') - plt.subplots_adjust(hspace=0.4, top=0.92) - - plt.savefig('%s_annual_saphire_simpson_distribution_%d.png'%( \ - self.basename, page_no), dpi=self.dpi) - - # break annual distributions down by month - mos_fig = plt.figure() - mos_fig.set_size_inches(wid, ht) + n,bins,pats = plt.hist(annual_cat[q], bins=np.arange(-0.5, 6.0, 1.0), \ + facecolor='steelblue', alpha=0.95, edgecolor='black', \ + linewidth=2, zorder=3) + j = 0 + while j < 6: + pats[j].set_facecolor(red_cmap[j]) + j += 1 + plt.xticks(np.arange(0,6,1)) + if self.rel_axes: + ax.set_ylim([0, max_y*1.05]) + if (q%n_cols == 0): + plt.ylabel('Count', fontweight='normal', fontsize=10) + if (q >= (n_year - n_cols)): + plt.xlabel('Category', fontweight='normal', fontsize=10) + plt.title('%d'%(yy), fontweight='bold', fontsize=11) + plt.grid(True) + + q += 1 + + plt.subplot(n_rows, n_cols, q+1) + ax = plt.gca() + ax.grid(zorder=0) + l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) + plt.axis('off') + + plt.suptitle('Annual Saphir-Simpson Distribution', fontweight='bold') + plt.subplots_adjust(hspace=0.4, top=0.92) + + plt.savefig('%s_annual_saphire_simpson_distribution_%d.png'%( \ + self.basename, page_no), dpi=self.dpi) + + # break annual distributions down by month + mos_fig = plt.figure() + mos_fig.set_size_inches(wid, ht) + + max_y = 0 + q = 0 + while q < n_year: + p = 0 + while p < 12: + max_y = max(max_y, sum(by_month[q][p])) + p += 1 + q += 1 - max_y = 0 - q = 0 - while q < n_year: + q = 0 + for yy in uyear: + plt.subplot(n_rows, n_cols, q+1) + ax = plt.gca() + ax.grid(zorder=0) + # build up a stacked bar chart, each category is a layer + # copy that cat for all months into a temp array then add + # it to the plot at the right hight and color. + mcts = by_month[q] + bot = np.zeros((12)) + c = 0 + while c < 6: + tmp = [] p = 0 while p < 12: - max_y = max(max_y, sum(by_month[q][p])) + tmp.append(mcts[p][c]) p += 1 - q += 1 - - q = 0 - for yy in uyear: - plt.subplot(n_rows, n_cols, q+1) - ax = plt.gca() - ax.grid(zorder=0) - # build up a stacked bar chart, each category is a layer - # copy that cat for all months into a temp array then add - # it to the plot at the right hight and color. - mcts = by_month[q] - bot = np.zeros((12)) - c = 0 - while c < 6: - tmp = [] - p = 0 - while p < 12: - tmp.append(mcts[p][c]) - p += 1 - plt.bar(np.arange(1,13,1)-0.375, tmp, width=0.75, bottom=bot, \ - facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ - tick_label=['J','F','M','A','M','J','J','A','S','O','N','D'], \ - zorder=3) - bot += tmp - c += 1 - - plt.xticks(np.arange(1,13,1)) - if self.rel_axes: - ax.set_ylim([0, 1.05*max_y]) - if (q%n_cols == 0): - plt.ylabel('Count', fontweight='normal', fontsize=10) - if (q >= (n_year - n_cols)): - plt.xlabel('Month', fontweight='normal', fontsize=10) - plt.title('%d'%(yy), fontweight='bold', fontsize=11) - plt.grid(True) - - q += 1 - + plt.bar(np.arange(1,13,1)-0.375, tmp, width=0.75, bottom=bot, \ + facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ + tick_label=['J','F','M','A','M','J','J','A','S','O','N','D'], \ + zorder=3) + bot += tmp + c += 1 + + plt.xticks(np.arange(1,13,1)) + if self.rel_axes: + ax.set_ylim([0, 1.05*max_y]) + if (q%n_cols == 0): + plt.ylabel('Count', fontweight='normal', fontsize=10) + if (q >= (n_year - n_cols)): + plt.xlabel('Month', fontweight='normal', fontsize=10) + plt.title('%d'%(yy), fontweight='bold', fontsize=11) + plt.grid(True) + + q += 1 + + plt.subplot(n_rows, n_cols, q+1) + ax = plt.gca() + ax.grid(zorder=0) + l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) + plt.axis('off') + + plt.suptitle('Monthly Breakdown', fontweight='bold') + plt.subplots_adjust(hspace=0.4, top=0.92) + + plt.savefig('%s_monthly_breakdown_%d.png'%( \ + self.basename, page_no), dpi=self.dpi) + + # plot annual counts by region + reg_fig = plt.figure() + reg_fig.set_size_inches(wid, ht) + + rcds = list(zip(*ureg))[1] + rcds += ('NH', 'SH', 'G') + + max_y = 0 + q = 0 + while q < n_year: + j = 0 + while j < n_reg: + max_y = max(max_y, sum(by_region[q][j])) + j += 1 + q += 1 + + q = 0 + for yy in uyear: plt.subplot(n_rows, n_cols, q+1) ax = plt.gca() ax.grid(zorder=0) - l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) - plt.axis('off') - - plt.suptitle('Monthly Breakdown', fontweight='bold') - plt.subplots_adjust(hspace=0.4, top=0.92) - - plt.savefig('%s_monthly_breakdown_%d.png'%( \ - self.basename, page_no), dpi=self.dpi) - - # plot annual counts by region - reg_fig = plt.figure() - reg_fig.set_size_inches(wid, ht) - - rcds = list(zip(*ureg))[1] - rcds += ('NH', 'SH', 'G') + # build up a stacked bar chart, each category is a layer + # copy that cat for all months into a temp array then add + # it to the plot at the right height and color. + rcnts = by_region[q] + bot = np.zeros((n_reg)) + c = 0 + while c < 6: + tmp = [] + p = 0 + while p < n_reg: + tmp.append(rcnts[p][c]) + p += 1 - max_y = 0 + plt.bar(np.arange(0,n_reg,1)-0.375, tmp, width=0.75, bottom=bot, \ + facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ + tick_label=rcds, \ + zorder=3) + + bot += tmp + c += 1 + + plt.xticks(np.arange(0,n_reg,1), rotation='vertical') + if self.rel_axes: + ax.set_ylim([0, 1.05*max_y]) + if (q%n_cols == 0): + plt.ylabel('Count', fontweight='normal', fontsize=10) + if (q >= (n_year - n_cols)): + plt.xlabel('Region', fontweight='normal', fontsize=10) + plt.title('%d'%(yy), fontweight='bold', fontsize=11) + plt.grid(True) + + q += 1 + + # add the color map legend + plt.subplot(n_rows, n_cols, q+1) + ax = plt.gca() + ax.grid(zorder=0) + l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) + plt.axis('off') + + plt.suptitle('Regional Breakdown', fontweight='bold') + plt.subplots_adjust(wspace=0.3, hspace=0.6, top=0.92) + + plt.savefig('%s_regional_break_down_%d.png'%( \ + self.basename, page_no), dpi=self.dpi) + + # plot annual distributions + dist_fig = plt.figure() + + wid = n_year*0.65 + dist_fig.set_size_inches(wid, 9.0) + + ax = plt.subplot(5,1,1) + plt.boxplot(annual_wind, labels=uyear) + plt.xlabel('Year') + plt.ylabel('ms^-1') + plt.title('Peak Instantaneous Wind', fontweight='bold') + ax.get_yaxis().set_label_coords(-0.1,0.5) + + ax = plt.subplot(5,1,2) + plt.boxplot(annual_press, labels=uyear) + plt.xlabel('Year') + plt.ylabel('Pa') + plt.title('Min Instantaneous Pressure', fontweight='bold') + ax.get_yaxis().set_label_coords(-0.1,0.5) + + ax = plt.subplot(5,1,3) + plt.boxplot(annual_dur, labels=uyear) + plt.xlabel('Year') + plt.ylabel('%s'%(time_units.split()[0])) + plt.title('Track Duration', fontweight='bold') + ax.get_yaxis().set_label_coords(-0.1,0.5) + + ax = plt.subplot(5,1,4) + plt.boxplot(annual_len, labels=uyear) + plt.xlabel('Year') + plt.ylabel('km') + plt.title('Track Length', fontweight='bold') + ax.get_yaxis().set_label_coords(-0.1,0.5) + + ax = plt.subplot(5,1,5) + #plt.axhline(82,color='k',linestyle='--',alpha=0.25) + plt.boxplot(annual_ACE, labels=uyear) + plt.xlabel('Year') + plt.ylabel('10^4 kn^2') + plt.title('ACE', fontweight='bold') + ax.get_yaxis().set_label_coords(-0.1,0.5) + + plt.suptitle('Distributions', fontweight='bold') + plt.subplots_adjust(hspace=0.72, top=0.93) + + plt.savefig('%s_distribution_%d.png'%( \ + self.basename, page_no), dpi=self.dpi) + + # plot region over time + reg_t_fig = plt.figure() + + rnms = list(zip(*ureg))[2] + rnms += ('Northern', 'Southern', 'Global') + + tmp = np.array(uyear) + tmp = tmp - tmp/100*100 + ynms = [] + for t in tmp: + ynms.append('%02d'%t) + + n_plots = n_reg + 1 + n_left = n_plots%n_cols + n_rows = n_plots/n_cols + (1 if n_left else 0) + wid = 2.5*n_cols + ht = 2.0*n_rows + reg_t_fig.set_size_inches(wid, ht) + + reg_by_t = [] + p = 0 + while p < n_reg: + reg = [] q = 0 while q < n_year: - j = 0 - while j < n_reg: - max_y = max(max_y, sum(by_region[q][j])) - j += 1 + reg.append(by_region[q][p]) q += 1 + reg_by_t.append(reg) + p += 1 + + max_y_reg = -1 + max_y_hem = -1 + q = 0 + while q < n_reg: + dat = reg_by_t[q] + p = 0 + while p < n_year: + if q < n_reg-3: + max_y_reg = max(max_y_reg, sum(dat[p])) + elif q < n_reg-1: + max_y_hem = max(max_y_hem, sum(dat[p])) + p += 1 + q += 1 - q = 0 - for yy in uyear: - plt.subplot(n_rows, n_cols, q+1) - ax = plt.gca() - ax.grid(zorder=0) - # build up a stacked bar chart, each category is a layer - # copy that cat for all months into a temp array then add - # it to the plot at the right height and color. - rcnts = by_region[q] - bot = np.zeros((n_reg)) - c = 0 - while c < 6: - tmp = [] - p = 0 - while p < n_reg: - tmp.append(rcnts[p][c]) - p += 1 - - plt.bar(np.arange(0,n_reg,1)-0.375, tmp, width=0.75, bottom=bot, \ - facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ - tick_label=rcds, \ - zorder=3) - - bot += tmp - c += 1 - - plt.xticks(np.arange(0,n_reg,1), rotation='vertical') - if self.rel_axes: - ax.set_ylim([0, 1.05*max_y]) - if (q%n_cols == 0): - plt.ylabel('Count', fontweight='normal', fontsize=10) - if (q >= (n_year - n_cols)): - plt.xlabel('Region', fontweight='normal', fontsize=10) - plt.title('%d'%(yy), fontweight='bold', fontsize=11) - plt.grid(True) - - q += 1 + q = 0 + while q < n_reg: + dat = reg_by_t[q] - # add the color map legend plt.subplot(n_rows, n_cols, q+1) ax = plt.gca() ax.grid(zorder=0) - l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) - plt.axis('off') - - plt.suptitle('Regional Breakdown', fontweight='bold') - plt.subplots_adjust(wspace=0.3, hspace=0.6, top=0.92) - - plt.savefig('%s_regional_break_down_%d.png'%( \ - self.basename, page_no), dpi=self.dpi) - - # plot annual distributions - dist_fig = plt.figure() - - wid = n_year*0.65 - dist_fig.set_size_inches(wid, 9.0) - - ax = plt.subplot(5,1,1) - plt.boxplot(annual_wind, labels=uyear) - plt.xlabel('Year') - plt.ylabel('ms^-1') - plt.title('Peak Instantaneous Wind', fontweight='bold') - ax.get_yaxis().set_label_coords(-0.1,0.5) - - ax = plt.subplot(5,1,2) - plt.boxplot(annual_press, labels=uyear) - plt.xlabel('Year') - plt.ylabel('Pa') - plt.title('Min Instantaneous Pressure', fontweight='bold') - ax.get_yaxis().set_label_coords(-0.1,0.5) - - ax = plt.subplot(5,1,3) - plt.boxplot(annual_dur, labels=uyear) - plt.xlabel('Year') - plt.ylabel('%s'%(time_units.split()[0])) - plt.title('Track Duration', fontweight='bold') - ax.get_yaxis().set_label_coords(-0.1,0.5) - - ax = plt.subplot(5,1,4) - plt.boxplot(annual_len, labels=uyear) - plt.xlabel('Year') - plt.ylabel('km') - plt.title('Track Length', fontweight='bold') - ax.get_yaxis().set_label_coords(-0.1,0.5) - - ax = plt.subplot(5,1,5) - #plt.axhline(82,color='k',linestyle='--',alpha=0.25) - plt.boxplot(annual_ACE, labels=uyear) - plt.xlabel('Year') - plt.ylabel('10^4 kn^2') - plt.title('ACE', fontweight='bold') - ax.get_yaxis().set_label_coords(-0.1,0.5) - - plt.suptitle('Distributions', fontweight='bold') - plt.subplots_adjust(hspace=0.72, top=0.93) - - plt.savefig('%s_distribution_%d.png'%( \ - self.basename, page_no), dpi=self.dpi) - - # plot region over time - reg_t_fig = plt.figure() - - rnms = list(zip(*ureg))[2] - rnms += ('Northern', 'Southern', 'Global') - - tmp = np.array(uyear) - tmp = tmp - tmp/100*100 - ynms = [] - for t in tmp: - ynms.append('%02d'%t) - - n_plots = n_reg + 1 - n_left = n_plots%n_cols - n_rows = n_plots/n_cols + (1 if n_left else 0) - wid = 2.5*n_cols - ht = 2.0*n_rows - reg_t_fig.set_size_inches(wid, ht) - - reg_by_t = [] - p = 0 - while p < n_reg: - reg = [] - q = 0 - while q < n_year: - reg.append(by_region[q][p]) - q += 1 - reg_by_t.append(reg) - p += 1 - max_y_reg = -1 - max_y_hem = -1 - q = 0 - while q < n_reg: - dat = reg_by_t[q] + # build up a stacked bar chart, each category is a layer + # copy that cat for all months into a temp array then add + # it to the plot at the right height and color. + bot = np.zeros((n_year)) + c = 0 + while c < 6: + tmp = [] p = 0 while p < n_year: - if q < n_reg-3: - max_y_reg = max(max_y_reg, sum(dat[p])) - elif q < n_reg-1: - max_y_hem = max(max_y_hem, sum(dat[p])) + tmp.append(dat[p][c]) p += 1 - q += 1 - q = 0 - while q < n_reg: - dat = reg_by_t[q] - - plt.subplot(n_rows, n_cols, q+1) - ax = plt.gca() - ax.grid(zorder=0) - - # build up a stacked bar chart, each category is a layer - # copy that cat for all months into a temp array then add - # it to the plot at the right height and color. - bot = np.zeros((n_year)) - c = 0 - while c < 6: - tmp = [] - p = 0 - while p < n_year: - tmp.append(dat[p][c]) - p += 1 - - plt.bar(np.arange(0,n_year,1)-0.375, tmp, width=0.75, bottom=bot, \ - facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ - tick_label=ynms, \ - zorder=3) - - bot += tmp - c += 1 - - plt.xticks(np.arange(0,n_year,1), rotation='vertical') - if self.rel_axes and q < n_reg - 1: - ax.set_ylim([0, 1.05*(max_y_reg if q < n_reg - 3 else max_y_hem)]) - if (q%n_cols == 0): - plt.ylabel('Count', fontweight='normal', fontsize=10) - if (q >= (n_reg - n_cols)): - plt.xlabel('Year', fontweight='normal', fontsize=10) - plt.title('%s'%(rnms[q]), fontweight='bold', fontsize=11) - plt.grid(True) + plt.bar(np.arange(0,n_year,1)-0.375, tmp, width=0.75, bottom=bot, \ + facecolor=red_cmap[c], edgecolor='k', linewidth=1, \ + tick_label=ynms, \ + zorder=3) - q += 1 + bot += tmp + c += 1 - plt.suptitle('Regional Trend', fontweight='bold') - plt.subplots_adjust(wspace=0.3, hspace=0.6, top=0.92) + plt.xticks(np.arange(0,n_year,1), rotation='vertical') + if self.rel_axes and q < n_reg - 1: + ax.set_ylim([0, 1.05*(max_y_reg if q < n_reg - 3 else max_y_hem)]) + if (q%n_cols == 0): + plt.ylabel('Count', fontweight='normal', fontsize=10) + if (q >= (n_reg - n_cols)): + plt.xlabel('Year', fontweight='normal', fontsize=10) + plt.title('%s'%(rnms[q]), fontweight='bold', fontsize=11) + plt.grid(True) - # add the color map legend - plt.subplot(n_rows, n_cols, q+1) - ax = plt.gca() - ax.grid(zorder=0) - l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) - plt.axis('off') + q += 1 + + plt.suptitle('Regional Trend', fontweight='bold') + plt.subplots_adjust(wspace=0.3, hspace=0.6, top=0.92) + + # add the color map legend + plt.subplot(n_rows, n_cols, q+1) + ax = plt.gca() + ax.grid(zorder=0) + l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(0.0, 1.0)) + plt.axis('off') + + plt.savefig('%s_regional_trend_%d.png'%( \ + self.basename, page_no), dpi=self.dpi) - plt.savefig('%s_regional_trend_%d.png'%( \ - self.basename, page_no), dpi=self.dpi) + if (self.interactive): + plt.show() - if (self.interactive): - plt.show() + # restore matplotlib global state + plt.rcParams['legend.frameon'] = legend_frame_on_orig - # restore matplotlib global state - plt.rcParams['legend.frameon'] = legend_frame_on_orig + # send data downstream + return summary - # send data downstream - return summary - return execute diff --git a/alg/teca_tc_trajectory_scalars.py b/alg/teca_tc_trajectory_scalars.py index 7499f136e..0fe0f634c 100644 --- a/alg/teca_tc_trajectory_scalars.py +++ b/alg/teca_tc_trajectory_scalars.py @@ -58,362 +58,356 @@ def set_plot_peak_radius(self, plot_peak_radius): """ self.plot_peak_radius = plot_peak_radius - def get_execute_callback(self): + def execute(self, port, data_in, req): """ - return a teca_algorithm::execute function. a closure - is used to gain self. + expects the output of the teca_tc_classify algorithm + generates a handful of histograms, summary statistics, + and plots. returns summary table with counts of annual + storms and their categories. """ - def execute(port, data_in, req): - """ - expects the output of the teca_tc_classify algorithm - generates a handful of histograms, summary statistics, - and plots. returns summary table with counts of annual - storms and their categories. - """ - #sys.stderr.write('teca_tc_trajectory_scalars::execute\n') - - import matplotlib.pyplot as plt - import matplotlib.patches as plt_mp - import matplotlib.image as plt_img - import matplotlib.gridspec as plt_gridspec - - # store matplotlib state we modify - legend_frame_on_orig = plt.rcParams['legend.frameon'] - - # tweak matplotlib slightly - plt.rcParams['figure.max_open_warning'] = 0 - plt.rcParams['legend.frameon'] = 1 - - # get the input table - in_table = teca_py.as_teca_table(data_in[0]) - if in_table is None: - # TODO if this is part of a parallel pipeline then - # only rank 0 should report an error. - sys.stderr.write('ERROR: empty input, or not a table\n') - return teca_py.teca_table.New() - - # use this color map for Saphir-Simpson scale - red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ - '#ea4f00','#c92500','#a80300'] - - km_per_deg_lat = 111 - - time_units = in_table.get_time_units() - - time = in_table.get_column('time').as_array() - step = in_table.get_column('step').as_array() - track = in_table.get_column('track_id').as_array() - - lon = in_table.get_column('lon').as_array() - lat = in_table.get_column('lat').as_array() - - year = in_table.get_column('year').as_array() - month = in_table.get_column('month').as_array() - day = in_table.get_column('day').as_array() - hour = in_table.get_column('hour').as_array() - minute = in_table.get_column('minute').as_array() - - wind = in_table.get_column('surface_wind').as_array() - vort = in_table.get_column('850mb_vorticity').as_array() - psl = in_table.get_column('sea_level_pressure').as_array() - temp = in_table.get_column('core_temp').as_array() - have_temp = in_table.get_column('have_core_temp').as_array() - thick = in_table.get_column('thickness').as_array() - have_thick = in_table.get_column('have_thickness').as_array() - speed = in_table.get_column('storm_speed').as_array() - - wind_rad = [] - i = 0 - while i < 5: - col_name = 'wind_radius_%d'%(i) - if in_table.has_column(col_name): - wind_rad.append(in_table.get_column(col_name).as_array()) - i += 1 - peak_rad = in_table.get_column('peak_radius').as_array() \ - if in_table.has_column('peak_radius') else None - - # get the list of unique track ids, this is our loop index - utrack = sorted(set(track)) - nutracks = len(utrack) - - # load background image - if (self.tex is None) and self.tex_file: - self.tex = plt_img.imread(self.tex_file) - - for i in utrack: - #sys.stderr.write('processing track %d\n'%(i)) - sys.stderr.write('.') - - fig = plt.figure() - fig.set_size_inches(10,9.75) - - ii = np.where(track == i)[0] - - # get the scalar values for this storm - lon_i = lon[ii] - lat_i = lat[ii] - wind_i = wind[ii] - psl_i = psl[ii] - vort_i = vort[ii] - thick_i = thick[ii] - temp_i = temp[ii] - speed_i = speed[ii]/24.0 - - wind_rad_i = [] - for col in wind_rad: - wind_rad_i.append(col[ii]) - peak_rad_i = peak_rad[ii] if peak_rad is not None else None - - # construct the title - q = ii[0] - r = ii[-1] - - t0 = time[q] - t1 = time[r] - - s0 = step[q] - s1 = step[r] - - Y0 = year[q] - Y1 = year[r] - - M0 = month[q] - M1 = month[r] - - D0 = day[q] - D1 = day[r] - - h0 = hour[q] - h1 = hour[r] - - m0 = minute[q] - m1 = minute[r] - - tt = time[ii] - t0 - - cat = teca_py.teca_tc_saffir_simpson.classify_mps(float(np.max(wind_i))) - - plt.suptitle( \ - 'Track %d, cat %d, steps %d - %d\n%d/%d/%d %d:%d:00 - %d/%d/%d %d:%d:00'%(\ - i, cat, s0, s1, Y0, M0, D0, h0, m0, Y1, M1, D1, h1, m1), \ - fontweight='bold') - - # plot the scalars - gs = plt_gridspec.GridSpec(5, 4) - - plt.subplot2grid((5,4),(0,0),colspan=2,rowspan=2) - # prepare the texture - if self.tex is not None: - ext = [np.min(lon_i), np.max(lon_i), np.min(lat_i), np.max(lat_i)] - if self.axes_equal: - w = ext[1]-ext[0] - h = ext[3]-ext[2] - if w > h: - c = (ext[2] + ext[3])/2.0 - w2 = w/2.0 - ext[2] = c - w2 - ext[3] = c + w2 - else: - c = (ext[0] + ext[1])/2.0 - h2 = h/2.0 - ext[0] = c - h2 - ext[1] = c + h2 - border_size = 0.15 - wrimax = 0 if peak_rad_i is None else \ - max(0 if not self.plot_peak_radius else \ - np.max(peak_rad_i), np.max(wind_rad_i[0])) - dlon = max(wrimax, (ext[1] - ext[0])*border_size) - dlat = max(wrimax, (ext[3] - ext[2])*border_size) - ext[0] = max(ext[0] - dlon, 0.0) - ext[1] = min(ext[1] + dlon, 360.0) - ext[2] = max(ext[2] - dlat, -90.0) - ext[3] = min(ext[3] + dlat, 90.0) - i0 = int(self.tex.shape[1]/360.0*ext[0]) - i1 = int(self.tex.shape[1]/360.0*ext[1]) - j0 = int(-((ext[3] + 90.0)/180.0 - 1.0)*self.tex.shape[0]) - j1 = int(-((ext[2] + 90.0)/180.0 - 1.0)*self.tex.shape[0]) - plt.imshow(self.tex[j0:j1, i0:i1], extent=ext, aspect='auto') - - edge_color = '#ffff00' if self.tex is not None else 'b' - - # plot the storm size - if peak_rad_i is None: - plt.plot(lon_i, lat_i, '.', linewidth=2, color=edge_color) - else: - # compute track unit normals - npts = len(ii) - norm_x = np.zeros(npts) - norm_y = np.zeros(npts) - npts -= 1 - q = 1 - while q < npts: - norm_x[q] = lat_i[q+1] - lat_i[q-1] - norm_y[q] = -(lon_i[q+1] - lon_i[q-1]) - nmag = np.sqrt(norm_x[q]**2 + norm_y[q]**2) - norm_x[q] = norm_x[q]/nmag - norm_y[q] = norm_y[q]/nmag - q += 1 - # normal at first and last point on the track - norm_x[0] = lat_i[1] - lat_i[0] - norm_y[0] = -(lon_i[1] - lon_i[0]) - norm_x[0] = norm_x[0]/nmag - norm_y[0] = norm_y[0]/nmag - norm_x[npts] = lat_i[npts] - lat_i[npts-1] - norm_y[npts] = -(lon_i[npts] - lon_i[npts-1]) - norm_x[npts] = norm_x[npts]/nmag - norm_y[npts] = norm_y[npts]/nmag - # for each wind radius, render a polygon of width 2*wind - # centered on the track. have to break it into continuous - # segments - nwri = len(wind_rad_i) - q = nwri - 1 - while q >= 0: - self.plot_wind_rad(lon_i, lat_i, norm_x, norm_y, \ - wind_rad_i[q], '-', edge_color if q==0 else red_cmap[q], \ - 2 if q==0 else 1, red_cmap[q], 0.98, q+4) - q -= 1 - # plot the peak radius - if (self.plot_peak_radius): - # peak radius is only valid if one of the other wind radii - # exist, zero out other values - kk = wind_rad_i[0] > 1.0e-6 - q = 1 - while q < nwri: - kk = np.logical_or(kk, wind_rad_i[q] > 1.0e-6) - q += 1 - peak_rad_i[np.logical_not(kk)] = 0.0 - self.plot_wind_rad(lon_i, lat_i, norm_x, norm_y, \ - peak_rad_i, '--', (0,0,0,0.25), 1, 'none', 1.00, nwri+4) - # mark track - marks = wind_rad_i[0] <= 1.0e-6 - q = 1 - while q < nwri: - marks = np.logical_and(marks, np.logical_not(wind_rad_i[q] > 1.0e-6)) - q += 1 - kk = np.where(marks)[0] - - plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=2, \ - color=edge_color,zorder=10) + #sys.stderr.write('teca_tc_trajectory_scalars::execute\n') - plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=1, \ - color='k', zorder=10, markersize=1) - - marks = wind_rad_i[0] > 1.0e-6 + import matplotlib.pyplot as plt + import matplotlib.patches as plt_mp + import matplotlib.image as plt_img + import matplotlib.gridspec as plt_gridspec + + # store matplotlib state we modify + legend_frame_on_orig = plt.rcParams['legend.frameon'] + + # tweak matplotlib slightly + plt.rcParams['figure.max_open_warning'] = 0 + plt.rcParams['legend.frameon'] = 1 + + # get the input table + in_table = teca_py.as_teca_table(data_in[0]) + if in_table is None: + # TODO if this is part of a parallel pipeline then + # only rank 0 should report an error. + sys.stderr.write('ERROR: empty input, or not a table\n') + return teca_py.teca_table.New() + + # use this color map for Saphir-Simpson scale + red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ + '#ea4f00','#c92500','#a80300'] + + km_per_deg_lat = 111 + + time_units = in_table.get_time_units() + + time = in_table.get_column('time').as_array() + step = in_table.get_column('step').as_array() + track = in_table.get_column('track_id').as_array() + + lon = in_table.get_column('lon').as_array() + lat = in_table.get_column('lat').as_array() + + year = in_table.get_column('year').as_array() + month = in_table.get_column('month').as_array() + day = in_table.get_column('day').as_array() + hour = in_table.get_column('hour').as_array() + minute = in_table.get_column('minute').as_array() + + wind = in_table.get_column('surface_wind').as_array() + vort = in_table.get_column('850mb_vorticity').as_array() + psl = in_table.get_column('sea_level_pressure').as_array() + temp = in_table.get_column('core_temp').as_array() + have_temp = in_table.get_column('have_core_temp').as_array() + thick = in_table.get_column('thickness').as_array() + have_thick = in_table.get_column('have_thickness').as_array() + speed = in_table.get_column('storm_speed').as_array() + + wind_rad = [] + i = 0 + while i < 5: + col_name = 'wind_radius_%d'%(i) + if in_table.has_column(col_name): + wind_rad.append(in_table.get_column(col_name).as_array()) + i += 1 + peak_rad = in_table.get_column('peak_radius').as_array() \ + if in_table.has_column('peak_radius') else None + + # get the list of unique track ids, this is our loop index + utrack = sorted(set(track)) + nutracks = len(utrack) + + # load background image + if (self.tex is None) and self.tex_file: + self.tex = plt_img.imread(self.tex_file) + + for i in utrack: + #sys.stderr.write('processing track %d\n'%(i)) + sys.stderr.write('.') + + fig = plt.figure() + fig.set_size_inches(10,9.75) + + ii = np.where(track == i)[0] + + # get the scalar values for this storm + lon_i = lon[ii] + lat_i = lat[ii] + wind_i = wind[ii] + psl_i = psl[ii] + vort_i = vort[ii] + thick_i = thick[ii] + temp_i = temp[ii] + speed_i = speed[ii]/24.0 + + wind_rad_i = [] + for col in wind_rad: + wind_rad_i.append(col[ii]) + peak_rad_i = peak_rad[ii] if peak_rad is not None else None + + # construct the title + q = ii[0] + r = ii[-1] + + t0 = time[q] + t1 = time[r] + + s0 = step[q] + s1 = step[r] + + Y0 = year[q] + Y1 = year[r] + + M0 = month[q] + M1 = month[r] + + D0 = day[q] + D1 = day[r] + + h0 = hour[q] + h1 = hour[r] + + m0 = minute[q] + m1 = minute[r] + + tt = time[ii] - t0 + + cat = teca_py.teca_tc_saffir_simpson.classify_mps(float(np.max(wind_i))) + + plt.suptitle( \ + 'Track %d, cat %d, steps %d - %d\n%d/%d/%d %d:%d:00 - %d/%d/%d %d:%d:00'%(\ + i, cat, s0, s1, Y0, M0, D0, h0, m0, Y1, M1, D1, h1, m1), \ + fontweight='bold') + + # plot the scalars + gs = plt_gridspec.GridSpec(5, 4) + + plt.subplot2grid((5,4),(0,0),colspan=2,rowspan=2) + # prepare the texture + if self.tex is not None: + ext = [np.min(lon_i), np.max(lon_i), np.min(lat_i), np.max(lat_i)] + if self.axes_equal: + w = ext[1]-ext[0] + h = ext[3]-ext[2] + if w > h: + c = (ext[2] + ext[3])/2.0 + w2 = w/2.0 + ext[2] = c - w2 + ext[3] = c + w2 + else: + c = (ext[0] + ext[1])/2.0 + h2 = h/2.0 + ext[0] = c - h2 + ext[1] = c + h2 + border_size = 0.15 + wrimax = 0 if peak_rad_i is None else \ + max(0 if not self.plot_peak_radius else \ + np.max(peak_rad_i), np.max(wind_rad_i[0])) + dlon = max(wrimax, (ext[1] - ext[0])*border_size) + dlat = max(wrimax, (ext[3] - ext[2])*border_size) + ext[0] = max(ext[0] - dlon, 0.0) + ext[1] = min(ext[1] + dlon, 360.0) + ext[2] = max(ext[2] - dlat, -90.0) + ext[3] = min(ext[3] + dlat, 90.0) + i0 = int(self.tex.shape[1]/360.0*ext[0]) + i1 = int(self.tex.shape[1]/360.0*ext[1]) + j0 = int(-((ext[3] + 90.0)/180.0 - 1.0)*self.tex.shape[0]) + j1 = int(-((ext[2] + 90.0)/180.0 - 1.0)*self.tex.shape[0]) + plt.imshow(self.tex[j0:j1, i0:i1], extent=ext, aspect='auto') + + edge_color = '#ffff00' if self.tex is not None else 'b' + + # plot the storm size + if peak_rad_i is None: + plt.plot(lon_i, lat_i, '.', linewidth=2, color=edge_color) + else: + # compute track unit normals + npts = len(ii) + norm_x = np.zeros(npts) + norm_y = np.zeros(npts) + npts -= 1 + q = 1 + while q < npts: + norm_x[q] = lat_i[q+1] - lat_i[q-1] + norm_y[q] = -(lon_i[q+1] - lon_i[q-1]) + nmag = np.sqrt(norm_x[q]**2 + norm_y[q]**2) + norm_x[q] = norm_x[q]/nmag + norm_y[q] = norm_y[q]/nmag + q += 1 + # normal at first and last point on the track + norm_x[0] = lat_i[1] - lat_i[0] + norm_y[0] = -(lon_i[1] - lon_i[0]) + norm_x[0] = norm_x[0]/nmag + norm_y[0] = norm_y[0]/nmag + norm_x[npts] = lat_i[npts] - lat_i[npts-1] + norm_y[npts] = -(lon_i[npts] - lon_i[npts-1]) + norm_x[npts] = norm_x[npts]/nmag + norm_y[npts] = norm_y[npts]/nmag + # for each wind radius, render a polygon of width 2*wind + # centered on the track. have to break it into continuous + # segments + nwri = len(wind_rad_i) + q = nwri - 1 + while q >= 0: + self.plot_wind_rad(lon_i, lat_i, norm_x, norm_y, \ + wind_rad_i[q], '-', edge_color if q==0 else red_cmap[q], \ + 2 if q==0 else 1, red_cmap[q], 0.98, q+4) + q -= 1 + # plot the peak radius + if (self.plot_peak_radius): + # peak radius is only valid if one of the other wind radii + # exist, zero out other values + kk = wind_rad_i[0] > 1.0e-6 q = 1 while q < nwri: - marks = np.logical_or(marks, wind_rad_i[q] > 1.0e-6) + kk = np.logical_or(kk, wind_rad_i[q] > 1.0e-6) q += 1 - kk = np.where(marks)[0] - - plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=1, \ - color='k', zorder=10, markersize=2, alpha=0.1) - - # mark track start and end - plt.plot(lon_i[0], lat_i[0], 'o', markersize=6, markeredgewidth=2, \ - color=edge_color, markerfacecolor='g',zorder=10) - - plt.plot(lon_i[-1], lat_i[-1], '^', markersize=6, markeredgewidth=2, \ - color=edge_color, markerfacecolor='r',zorder=10) - - plt.grid(True) - plt.xlabel('deg lon') - plt.ylabel('deg lat') - plt.title('Track', fontweight='bold') - - plt.subplot2grid((5,4),(0,2),colspan=2) - plt.plot(tt, psl_i, 'b-', linewidth=2) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('millibars') - plt.title('Sea Level Pressure', fontweight='bold') - plt.xlim([0, tt[-1]]) - - plt.subplot2grid((5,4),(1,2),colspan=2) - plt.plot(tt, wind_i, 'b-', linewidth=2) + peak_rad_i[np.logical_not(kk)] = 0.0 + self.plot_wind_rad(lon_i, lat_i, norm_x, norm_y, \ + peak_rad_i, '--', (0,0,0,0.25), 1, 'none', 1.00, nwri+4) + # mark track + marks = wind_rad_i[0] <= 1.0e-6 + q = 1 + while q < nwri: + marks = np.logical_and(marks, np.logical_not(wind_rad_i[q] > 1.0e-6)) + q += 1 + kk = np.where(marks)[0] + + plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=2, \ + color=edge_color,zorder=10) + + plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=1, \ + color='k', zorder=10, markersize=1) + + marks = wind_rad_i[0] > 1.0e-6 + q = 1 + while q < nwri: + marks = np.logical_or(marks, wind_rad_i[q] > 1.0e-6) + q += 1 + kk = np.where(marks)[0] + + plt.plot(lon_i[kk], lat_i[kk], '.', linewidth=1, \ + color='k', zorder=10, markersize=2, alpha=0.1) + + # mark track start and end + plt.plot(lon_i[0], lat_i[0], 'o', markersize=6, markeredgewidth=2, \ + color=edge_color, markerfacecolor='g',zorder=10) + + plt.plot(lon_i[-1], lat_i[-1], '^', markersize=6, markeredgewidth=2, \ + color=edge_color, markerfacecolor='r',zorder=10) + + plt.grid(True) + plt.xlabel('deg lon') + plt.ylabel('deg lat') + plt.title('Track', fontweight='bold') + + plt.subplot2grid((5,4),(0,2),colspan=2) + plt.plot(tt, psl_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('millibars') + plt.title('Sea Level Pressure', fontweight='bold') + plt.xlim([0, tt[-1]]) + + plt.subplot2grid((5,4),(1,2),colspan=2) + plt.plot(tt, wind_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('ms^-1') + plt.title('Surface Wind', fontweight='bold') + plt.xlim([0, tt[-1]]) + + plt.subplot2grid((5,4),(2,0),colspan=2) + plt.plot(tt, speed_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('km d^-1') + plt.title('Propagation Speed', fontweight='bold') + plt.xlim([0, tt[-1]]) + + plt.subplot2grid((5,4),(2,2),colspan=2) + plt.plot(tt, vort_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('s^-1') + plt.title('Vorticity', fontweight='bold') + plt.xlim([0, tt[-1]]) + + plt.subplot2grid((5,4),(3,0),colspan=2) + plt.plot(tt, thick_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('meters') + plt.title('Thickness', fontweight='bold') + plt.xlim([0, tt[-1]]) + + plt.subplot2grid((5,4),(3,2),colspan=2) + plt.plot(tt, temp_i, 'b-', linewidth=2) + plt.grid(True) + plt.xlabel('time (days)') + plt.ylabel('deg K') + plt.title('Core Temperature', fontweight='bold') + plt.xlim([0, tt[-1]]) + + if peak_rad_i is not None: + plt.subplot2grid((5,4),(4,0),colspan=2) + q = len(wind_rad_i) - 1 + while q >= 0: + wr_i_q = km_per_deg_lat*wind_rad_i[q] + plt.fill_between(tt, 0, wr_i_q, color=red_cmap[q], alpha=0.9, zorder=q+3) + plt.plot(tt, wr_i_q, '-', linewidth=2, color=red_cmap[q], zorder=q+3) + q -= 1 + if (self.plot_peak_radius): + plt.plot(tt, km_per_deg_lat*peak_rad_i, 'k--', linewidth=1, zorder=10) + plt.plot(tt, np.zeros(len(tt)), 'w-', linewidth=2, zorder=10) plt.grid(True) plt.xlabel('time (days)') - plt.ylabel('ms^-1') - plt.title('Surface Wind', fontweight='bold') + plt.ylabel('radius (km)') + plt.title('Storm Size', fontweight='bold') plt.xlim([0, tt[-1]]) - - plt.subplot2grid((5,4),(2,0),colspan=2) - plt.plot(tt, speed_i, 'b-', linewidth=2) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('km d^-1') - plt.title('Propagation Speed', fontweight='bold') - plt.xlim([0, tt[-1]]) - - plt.subplot2grid((5,4),(2,2),colspan=2) - plt.plot(tt, vort_i, 'b-', linewidth=2) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('s^-1') - plt.title('Vorticity', fontweight='bold') - plt.xlim([0, tt[-1]]) - - plt.subplot2grid((5,4),(3,0),colspan=2) - plt.plot(tt, thick_i, 'b-', linewidth=2) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('meters') - plt.title('Thickness', fontweight='bold') - plt.xlim([0, tt[-1]]) - - plt.subplot2grid((5,4),(3,2),colspan=2) - plt.plot(tt, temp_i, 'b-', linewidth=2) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('deg K') - plt.title('Core Temperature', fontweight='bold') - plt.xlim([0, tt[-1]]) - - if peak_rad_i is not None: - plt.subplot2grid((5,4),(4,0),colspan=2) - q = len(wind_rad_i) - 1 - while q >= 0: - wr_i_q = km_per_deg_lat*wind_rad_i[q] - plt.fill_between(tt, 0, wr_i_q, color=red_cmap[q], alpha=0.9, zorder=q+3) - plt.plot(tt, wr_i_q, '-', linewidth=2, color=red_cmap[q], zorder=q+3) - q -= 1 - if (self.plot_peak_radius): - plt.plot(tt, km_per_deg_lat*peak_rad_i, 'k--', linewidth=1, zorder=10) - plt.plot(tt, np.zeros(len(tt)), 'w-', linewidth=2, zorder=10) - plt.grid(True) - plt.xlabel('time (days)') - plt.ylabel('radius (km)') - plt.title('Storm Size', fontweight='bold') - plt.xlim([0, tt[-1]]) - plt.ylim(ymin=0) - - plt.subplot2grid((5,4),(4,2)) - red_cmap_pats = [] - q = 0 - while q < 6: - red_cmap_pats.append( \ - plt_mp.Patch(color=red_cmap[q], label='R%d'%(q))) - q += 1 - if (self.plot_peak_radius): - red_cmap_pats.append(plt_mp.Patch(color='k', label='RP')) - l = plt.legend(handles=red_cmap_pats, loc=2, \ - bbox_to_anchor=(-0.1, 1.0), borderaxespad=0.0, \ - frameon=True, ncol=2) - plt.axis('off') - - plt.subplots_adjust(left=0.065, right=0.98, \ - bottom=0.05, top=0.9, wspace=0.6, hspace=0.7) - - plt.savefig('%s_%06d.png'%(self.basename, i), dpi=self.dpi) - if (not self.interactive): - plt.close(fig) - - if (self.interactive): - plt.show() - - out_table = teca_py.teca_table.New() - out_table.shallow_copy(in_table) - return out_table - return execute + plt.ylim(ymin=0) + + plt.subplot2grid((5,4),(4,2)) + red_cmap_pats = [] + q = 0 + while q < 6: + red_cmap_pats.append( \ + plt_mp.Patch(color=red_cmap[q], label='R%d'%(q))) + q += 1 + if (self.plot_peak_radius): + red_cmap_pats.append(plt_mp.Patch(color='k', label='RP')) + l = plt.legend(handles=red_cmap_pats, loc=2, \ + bbox_to_anchor=(-0.1, 1.0), borderaxespad=0.0, \ + frameon=True, ncol=2) + plt.axis('off') + + plt.subplots_adjust(left=0.065, right=0.98, \ + bottom=0.05, top=0.9, wspace=0.6, hspace=0.7) + + plt.savefig('%s_%06d.png'%(self.basename, i), dpi=self.dpi) + if (not self.interactive): + plt.close(fig) + + if (self.interactive): + plt.show() + + out_table = teca_py.teca_table.New() + out_table.shallow_copy(in_table) + return out_table @staticmethod def render_poly(x, y, norm_x, norm_y, rad, edge_style, \ @@ -623,3 +617,4 @@ def plot_wind_rad(self, x, y, norm_x, norm_y, wind_rad, \ p0 = -1 qq += 1 + diff --git a/alg/teca_tc_wind_radii_stats.py b/alg/teca_tc_wind_radii_stats.py index fc66e8c6d..53728ee16 100644 --- a/alg/teca_tc_wind_radii_stats.py +++ b/alg/teca_tc_wind_radii_stats.py @@ -48,170 +48,164 @@ def set_output_prefix(self, output_prefix): """ self.output_prefix = output_prefix - def get_execute_callback(self): + def execute(self, port, data_in, req): """ - return a teca_algorithm::execute function. a closure - is used to gain self. + expects a table with track data containing wind radii computed + along each point of the track. produces statistical plots showing + the global distribution of wind radii. """ - def execute(port, data_in, req): - """ - expects a table with track data containing wind radii computed - along each point of the track. produces statistical plots showing - the global distribution of wind radii. - """ - track_table = teca_py.as_teca_table(data_in[0]) - - # plot stats - import matplotlib.pyplot as plt - import matplotlib.patches as plt_mp - from matplotlib.colors import LogNorm - - red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ - '#ea4f00','#c92500','#a80300'] - - km_per_deg_lat = 111 - km_s_per_m_hr = 3.6 - - fig = plt.figure(figsize=(9.25,6.75),dpi=self.dpi) - - # scatter - plt.subplot('331') - - if not track_table.has_column(self.wind_column): - sys.stderr.write('ERROR: track table missing %s\n'%(self.wind_column)) - sys.exit(-1) - - - year = track_table.get_column('year').as_array() - month = track_table.get_column('month').as_array() - day = track_table.get_column('day').as_array() - - ws = km_s_per_m_hr*track_table.get_column(self.wind_column).as_array() - - wr = [] - nwr = 0 - while track_table.has_column('wind_radius_%d'%(nwr)): - wr.append(km_per_deg_lat*track_table.get_column('wind_radius_%d'%(nwr)).as_array()) - nwr += 1 - - i = 0 - while i < nwr: - wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) - wri = wr[i] - ii = np.where(wri > 0.0) - plt.scatter(wri[ii], ws[ii], c=red_cmap[i], alpha=0.25, marker='.', zorder=3+i) - i += 1 - - plt.ylabel('Wind speed (km/hr)', fontweight='normal', fontsize=10) - plt.title('R0 - R5 vs Wind speed', fontweight='bold', fontsize=11) - plt.grid(True) - ax = plt.gca() - ax.set_xlim([0.0, 6.0*km_per_deg_lat]) - - # all - plt.subplot('332') - i = 0 - while i < nwr: - wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) - wri = wr[i] - n,bins,pats = plt.hist(wri[np.where(wri > 0.0)], 32, range=[0,6.0*km_per_deg_lat], \ - facecolor=red_cmap[i], alpha=0.95, edgecolor='black', \ - linewidth=2, zorder=3+i) - i += 1 - plt.ylabel('Number', fontweight='normal', fontsize=10) - plt.title('All R0 - R5', fontweight='bold', fontsize=11) - plt.grid(True) - ax = plt.gca() - ax.set_xlim([0.0, 6.0*km_per_deg_lat]) - - # r0 - r5 - i = 0 - while i < nwr: - plt.subplot(333+i) - wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) - wri = wr[i] - wrii=wri[np.where(wri > 0.0)] - n,bins,pats = plt.hist(wrii, 32, \ - facecolor=red_cmap[i], alpha=1.00, edgecolor='black', \ - linewidth=2, zorder=3) - if ((i % 3) == 1): - plt.ylabel('Number', fontweight='normal', fontsize=10) - if (i >= 3): - plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) - plt.title('R%d (%0.1f km/hr)'%(i,wc), fontweight='bold', fontsize=11) - plt.grid(True) - ax = plt.gca() - try: - ax.set_xlim([np.min(wrii), np.max(wrii)]) - except: - pass - i += 1 - - # legend - plt.subplot('339') - red_cmap_pats = [] - q = 0 - while q < nwr: - red_cmap_pats.append( \ - plt_mp.Patch(color=red_cmap[q], label='R%d'%(q))) - q += 1 - l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(-0.1, 1.0), fancybox=True) - plt.axis('off') - - - plt.suptitle('Wind Radii %s/%d/%d - %s/%d/%d'%(month[0],day[0],year[0], \ - month[-1],day[-1],year[-1]), fontweight='bold', fontsize=12) - plt.subplots_adjust(hspace=0.35, wspace=0.35, top=0.90) - - plt.savefig(self.output_prefix + 'wind_radii_stats.png') - - fig = plt.figure(figsize=(7.5,4.0),dpi=100) - # peak radius - pr = km_per_deg_lat*track_table.get_column('peak_radius').as_array() - # peak radius is only valid if one of the other wind radii - # exist - kk = wr[0] > 1.0e-6 - q = 1 - while q < nwr: - kk = np.logical_or(kk, wr[q] > 1.0e-6) - q += 1 - pr = pr[kk] - - plt.subplot(121) - n,bins,pats = plt.hist(pr[np.where(pr > 0.0)], 24, \ - facecolor='steelblue', alpha=0.95, edgecolor='black', \ + track_table = teca_py.as_teca_table(data_in[0]) + + # plot stats + import matplotlib.pyplot as plt + import matplotlib.patches as plt_mp + from matplotlib.colors import LogNorm + + red_cmap = ['#ffd2a3','#ffa749','#ff7c04', \ + '#ea4f00','#c92500','#a80300'] + + km_per_deg_lat = 111 + km_s_per_m_hr = 3.6 + + fig = plt.figure(figsize=(9.25,6.75),dpi=self.dpi) + + # scatter + plt.subplot('331') + + if not track_table.has_column(self.wind_column): + sys.stderr.write('ERROR: track table missing %s\n'%(self.wind_column)) + sys.exit(-1) + + + year = track_table.get_column('year').as_array() + month = track_table.get_column('month').as_array() + day = track_table.get_column('day').as_array() + + ws = km_s_per_m_hr*track_table.get_column(self.wind_column).as_array() + + wr = [] + nwr = 0 + while track_table.has_column('wind_radius_%d'%(nwr)): + wr.append(km_per_deg_lat*track_table.get_column('wind_radius_%d'%(nwr)).as_array()) + nwr += 1 + + i = 0 + while i < nwr: + wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) + wri = wr[i] + ii = np.where(wri > 0.0) + plt.scatter(wri[ii], ws[ii], c=red_cmap[i], alpha=0.25, marker='.', zorder=3+i) + i += 1 + + plt.ylabel('Wind speed (km/hr)', fontweight='normal', fontsize=10) + plt.title('R0 - R5 vs Wind speed', fontweight='bold', fontsize=11) + plt.grid(True) + ax = plt.gca() + ax.set_xlim([0.0, 6.0*km_per_deg_lat]) + + # all + plt.subplot('332') + i = 0 + while i < nwr: + wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) + wri = wr[i] + n,bins,pats = plt.hist(wri[np.where(wri > 0.0)], 32, range=[0,6.0*km_per_deg_lat], \ + facecolor=red_cmap[i], alpha=0.95, edgecolor='black', \ + linewidth=2, zorder=3+i) + i += 1 + plt.ylabel('Number', fontweight='normal', fontsize=10) + plt.title('All R0 - R5', fontweight='bold', fontsize=11) + plt.grid(True) + ax = plt.gca() + ax.set_xlim([0.0, 6.0*km_per_deg_lat]) + + # r0 - r5 + i = 0 + while i < nwr: + plt.subplot(333+i) + wc = teca_py.teca_tc_saffir_simpson.get_upper_bound_kmph(i-1) + wri = wr[i] + wrii=wri[np.where(wri > 0.0)] + n,bins,pats = plt.hist(wrii, 32, \ + facecolor=red_cmap[i], alpha=1.00, edgecolor='black', \ linewidth=2, zorder=3) - plt.ylabel('Number', fontweight='normal', fontsize=10) - plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) - plt.title('RP (radius at peak wind)', fontweight='bold', fontsize=11) - plt.grid(True) - ax = plt.gca() - ax.set_xlim([0.0, np.max(pr)]) - - # scatter - plt.subplot('122') - ii = np.where(pr > 0.0) - cnts,xe,ye,im = plt.hist2d(pr[ii], ws[ii], bins=24, norm=LogNorm(), zorder=2) - plt.ylabel('Wind speed (km/hr)', fontweight='normal', fontsize=10) - plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) - plt.title('RP vs Wind speed', fontweight='bold', fontsize=11) + if ((i % 3) == 1): + plt.ylabel('Number', fontweight='normal', fontsize=10) + if (i >= 3): + plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) + plt.title('R%d (%0.1f km/hr)'%(i,wc), fontweight='bold', fontsize=11) plt.grid(True) ax = plt.gca() - ax.set_xlim([0.0, np.max(pr)]) - - fig.subplots_adjust(right=0.85) - cbar_ax = fig.add_axes([0.88, 0.35, 0.05, 0.5]) - fig.colorbar(im, cax=cbar_ax) - - plt.suptitle('Wind Radii %s/%d/%d - %s/%d/%d'%(month[0],day[0],year[0], \ - month[-1],day[-1],year[-1]), fontweight='bold', fontsize=12) - plt.subplots_adjust(hspace=0.3, wspace=0.3, top=0.85) - - plt.savefig(self.output_prefix + 'peak_radius_stats.png') - - if self.interactive: - plt.show() - - # send data downstream - return track_table - return execute + try: + ax.set_xlim([np.min(wrii), np.max(wrii)]) + except: + pass + i += 1 + + # legend + plt.subplot('339') + red_cmap_pats = [] + q = 0 + while q < nwr: + red_cmap_pats.append( \ + plt_mp.Patch(color=red_cmap[q], label='R%d'%(q))) + q += 1 + l = plt.legend(handles=red_cmap_pats, loc=2, bbox_to_anchor=(-0.1, 1.0), fancybox=True) + plt.axis('off') + + + plt.suptitle('Wind Radii %s/%d/%d - %s/%d/%d'%(month[0],day[0],year[0], \ + month[-1],day[-1],year[-1]), fontweight='bold', fontsize=12) + plt.subplots_adjust(hspace=0.35, wspace=0.35, top=0.90) + + plt.savefig(self.output_prefix + 'wind_radii_stats.png') + + fig = plt.figure(figsize=(7.5,4.0),dpi=100) + # peak radius + pr = km_per_deg_lat*track_table.get_column('peak_radius').as_array() + # peak radius is only valid if one of the other wind radii + # exist + kk = wr[0] > 1.0e-6 + q = 1 + while q < nwr: + kk = np.logical_or(kk, wr[q] > 1.0e-6) + q += 1 + pr = pr[kk] + + plt.subplot(121) + n,bins,pats = plt.hist(pr[np.where(pr > 0.0)], 24, \ + facecolor='steelblue', alpha=0.95, edgecolor='black', \ + linewidth=2, zorder=3) + plt.ylabel('Number', fontweight='normal', fontsize=10) + plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) + plt.title('RP (radius at peak wind)', fontweight='bold', fontsize=11) + plt.grid(True) + ax = plt.gca() + ax.set_xlim([0.0, np.max(pr)]) + + # scatter + plt.subplot('122') + ii = np.where(pr > 0.0) + cnts,xe,ye,im = plt.hist2d(pr[ii], ws[ii], bins=24, norm=LogNorm(), zorder=2) + plt.ylabel('Wind speed (km/hr)', fontweight='normal', fontsize=10) + plt.xlabel('Radius (km)', fontweight='normal', fontsize=10) + plt.title('RP vs Wind speed', fontweight='bold', fontsize=11) + plt.grid(True) + ax = plt.gca() + ax.set_xlim([0.0, np.max(pr)]) + + fig.subplots_adjust(right=0.85) + cbar_ax = fig.add_axes([0.88, 0.35, 0.05, 0.5]) + fig.colorbar(im, cax=cbar_ax) + + plt.suptitle('Wind Radii %s/%d/%d - %s/%d/%d'%(month[0],day[0],year[0], \ + month[-1],day[-1],year[-1]), fontweight='bold', fontsize=12) + plt.subplots_adjust(hspace=0.3, wspace=0.3, top=0.85) + + plt.savefig(self.output_prefix + 'peak_radius_stats.png') + + if self.interactive: + plt.show() + + # send data downstream + return track_table diff --git a/doc/rtd/python.rst b/doc/rtd/python.rst index 59b12e794..7b7919441 100644 --- a/doc/rtd/python.rst +++ b/doc/rtd/python.rst @@ -142,27 +142,59 @@ reader's metadata convention. For most use cases the reader will be TECA's NetCDF CF 2.0 reader, teca\_cf\_reader. The convention adopted by the CF reader are documented in its header file and in section \ref{sec:cf_reader}. -In C++11 polymorphism is used to provide customized behavior for each -of the three pipeline phases. In Python we use the -teca\_python\_algorithm, an adapter class that calls user provided -callback functions at the appropriate times during each phase of pipeline -execution. Hence writing a TECA algorithm purely in Python amounts to providing -three appropriate callbacks. +In C++ and Python polymorphism is used to provide customized behavior for each +of the three pipeline phases. In Python we use the teca\_python\_algorithm, an +adapter class that connects the user provided overrides such that they are +called at the appropriate times during each phase of pipeline execution. Hence +writing a TECA algorithm purely in Python amounts to providing three +appropriate overrides. .. _py_devel_code: .. literalinclude:: source/stats_callbacks.py :language: python :linenos: - :caption: Callbacks implementing the calculation of descriptive statistics over a set of variables laid out on a Cartesian lat-lon mesh. The request callback requests the variables, the execute callback makes the computations and constructs a table to store them in. + :caption: Overrides implementing the calculation of descriptive statistics over a set of variables laid out on a Cartesian lat-lon mesh. The request override requests the variables, the execute override makes the computations and constructs a table to store them in. -The Report Callback + +Python Algorithm Template ++++++++++++++++++++++++++ +To extend TECA using Python one derives from teca_python_algorithm. A template +follows: + +.. code-block:: python + + class YOUR_CLASS_NAME(teca_python_algorithm): + + # YOUR INITIALIZATION CODE + + # YOUR PROPERTY SETTING CODE + + def report(self, o_port, reports_in): + # YOUR CODE REPORTING NEW DATA PRODUCED IN + # EXECUTE. + + def request(self, o_port, reports_in, request_in): + # YOUR CODE REQUESTING SPECIFIC DATA NEEDED + # IN EXECUTE + + def execute(self, o_port, data_in, request_in): + # YOUR CODE DO A CALCULATION, TRANSFORM DATA, + # OR MAKING A SIDE AFFECT + +One overrides one por more of the three methods: report, request, and execute. +In addition one can add initialization and properties that control run time +behavior. For instance a writer may use a file_name property to specify the +locatgion on disk to put the data. Typically this would be accessed via a +set_file_name method. The overrides are described in more detail below. + +The Report Override +++++++++++++++++++ -The report callback will report the universe of what the algorithm could produce. +The report override will report the universe of what the algorithm could produce. .. code-block:: python - def report_callback(o_port, reports_in) -> report_out + def report(self, o_port, reports_in) -> report_out o\_port integer. the output port number to report for. can be ignored for single @@ -181,14 +213,14 @@ is passed through with metadata describing new data that could be produced appended as needed. This allows upstream data producers to advertise their capabilities. -The Request Callback +The Request Override ++++++++++++++++++++ -The request callback generates an up stream request requesting the minimum -amount of data actually needed to fulfill the incoming request -. +The request override generates an up stream request requesting the minimum +amount of data actually needed to fulfill the incoming request. + .. code-block:: python - def request(o_port, reports_in, request_in) -> requests_out + def request(self, o_port, reports_in, request_in) -> requests_out o\_port integer. the output port number to report for. can be ignored for single output algorithms. @@ -208,14 +240,14 @@ Typically the incoming request is passed through appending the necessary metadata as needed. This allows down stream data consumers to request data that is produced upstream. -The Execute Callback +The Execute Override ++++++++++++++++++++ -The execute callback is where the computations or I/O necessary to produce the +The execute override is where the computations or I/O necessary to produce the requested data are handled. .. code-block:: python - def execute(o_port, data_in, request_in) -> data_out + def execute(self, o_port, data_in, request_in) -> data_out o\_port integer. the output port number to report for. can be ignored for single @@ -223,7 +255,7 @@ o\_port data\_in teca\_dataset list. a dataset for each request you made in the request - callback in the same order. + override in the same order. request\_in teca\_metadata. the request being made of you. @@ -243,24 +275,22 @@ Lines 25-27 of listing :numref:`py_glue_code` illustrate the use of teca\_python\_algorithm. In this example a class derived from teca\_python\_algorithm computes descriptive statistics over a set of variables laid out on a Cartesian lat-lon mesh. The derived class, descriptive_stats, is -in a separate file, stats\_callbacks.py (listing :numref:`py_devel_code`) +in a separate file, stats\_overrides.py (listing :numref:`py_devel_code`) imported on line 4. Listing :numref:`py_devel_code` shows the class derived from teca\_python\_algorithm that is used in listing :numref:`py_glue_code`. The -class implements request and execute callbacks. Note, that we did not need to -provide a report callback as the default implementation, which passes the -report through was all that was needed. In both our request and execute -callbacks we used a closure to access class state from the callback. Our -request callback (lines 15-21 of listing :numref:`py_devel_code`) simply adds -the list of variables we need into the incoming request which it then forwards -up stream. The execute callback (lines 23-45) gets the input dataset (line 27), -creates the output table adding columns and values of time and time step (lines -29-31), then for each variable we add columns to the table for each computation -(line 35), get the array from the input dataset (line 39), compute statistics -and add them to the table (lines 41-43), and returns the table containing the -results (line 45). This data can then be processed by the next stage in the -pipeline. +class implements request and execute overrides. Note, that we did not need to +provide a report override as the default implementation, which passes the +report through was all that was needed. Our request override (lines 15-21 of +listing :numref:`py_devel_code`) simply adds the list of variables we need into +the incoming request which it then forwards up stream. The execute override +(lines 23-45) gets the input dataset (line 27), creates the output table adding +columns and values of time and time step (lines 29-31), then for each variable +we add columns to the table for each computation (line 35), get the array from +the input dataset (line 39), compute statistics and add them to the table +(lines 41-43), and returns the table containing the results (line 45). This +data can then be processed by the next stage in the pipeline. .. _data_struct: diff --git a/doc/rtd/source/stats_callbacks.py b/doc/rtd/source/stats_callbacks.py index 5c09d952b..3c81d4f5b 100644 --- a/doc/rtd/source/stats_callbacks.py +++ b/doc/rtd/source/stats_callbacks.py @@ -12,35 +12,31 @@ def __init__(self): def set_variable_names(self, vn): self.var_names = vn - def get_request_callback(self): - def request(port, md_in, req_in): - sys.stderr.write('descriptive_stats::request MPI %d\n'%(self.rank)) - req = teca_metadata(req_in) - req['arrays'] = self.var_names - return [req] - return request - - def get_execute_callback(self): - def execute(port, data_in, req): - sys.stderr.write('descriptive_stats::execute MPI %d\n'%(self.rank)) - - mesh = as_teca_cartesian_mesh(data_in[0]) - - table = teca_table.New() - table.declare_columns(['step','time'], ['ul','d']) - table << mesh.get_time_step() << mesh.get_time() - - for var_name in self.var_names: - - table.declare_columns(['min '+var_name, 'avg '+var_name, \ - 'max '+var_name, 'std '+var_name, 'low_q '+var_name, \ - 'med '+var_name, 'up_q '+var_name], ['d']*7) - - var = mesh.get_point_arrays().get(var_name).as_array() - - table << np.min(var) << np.average(var) \ - << np.max(var) << np.std(var) \ - << np.percentile(var, [25.,50.,75.]) - - return table - return execute + def request(self, port, md_in, req_in): + sys.stderr.write('descriptive_stats::request MPI %d\n'%(self.rank)) + req = teca_metadata(req_in) + req['arrays'] = self.var_names + return [req] + + def execute(self, port, data_in, req): + sys.stderr.write('descriptive_stats::execute MPI %d\n'%(self.rank)) + + mesh = as_teca_cartesian_mesh(data_in[0]) + + table = teca_table.New() + table.declare_columns(['step','time'], ['ul','d']) + table << mesh.get_time_step() << mesh.get_time() + + for var_name in self.var_names: + + table.declare_columns(['min '+var_name, 'avg '+var_name, \ + 'max '+var_name, 'std '+var_name, 'low_q '+var_name, \ + 'med '+var_name, 'up_q '+var_name], ['d']*7) + + var = mesh.get_point_arrays().get(var_name).as_array() + + table << np.min(var) << np.average(var) \ + << np.max(var) << np.std(var) \ + << np.percentile(var, [25.,50.,75.]) + + return table diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index ae42c9b99..87dff4f19 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -66,6 +66,13 @@ teca_add_test(py_test_programmable_algorithm FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) +teca_add_test(py_test_python_algorithm + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_python_algorithm.py + "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" + U850 V850 0 0 "py_test_programmable_algorithm_%t%.vtk" + FEATURES ${TECA_HAS_NETCDF} + REQ_TECA_DATA) + teca_add_test(py_test_vector_ops COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_vector_ops.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-01-10800\\.nc" diff --git a/test/python/test_information_array_io.py b/test/python/test_information_array_io.py index a7f68809c..3800e0d28 100644 --- a/test/python/test_information_array_io.py +++ b/test/python/test_information_array_io.py @@ -29,47 +29,37 @@ def print_status(self, msg): if self.verbose: sys.stderr.write('generate_info_arrays::%s\n'%(msg)) - def get_report_callback(self): - self.print_status('get_report_callback') + def report(self, port, md_in): + self.print_status('report(override)') + md_out = teca_metadata(md_in[0]) + try: + arrays = md_out['arrays'] + except: + arrays = [] + md_out['arrays'] = arrays + ['time_info', 'step_info'] + return md_out - def report_callback(port, md_in): - self.print_status('report_callback') - md_out = teca_metadata(md_in[0]) - try: - arrays = md_out['arrays'] - except: - arrays = [] - md_out['arrays'] = arrays + ['time_info', 'step_info'] - return md_out + def execute(self, port, data_in, req_in): + self.print_status('execute(override)') - return report_callback + mesh_in = as_const_teca_cartesian_mesh(data_in[0]) + t = mesh_in.get_time() + s = mesh_in.get_time_step() - def get_execute_callback(self): - self.print_status('get_execute_callback') + mesh_out = teca_cartesian_mesh.New() + mesh_out.shallow_copy(mesh_in) - def execute_callback(port, data_in, req_in): - self.print_status('execute_callback') + t_inf = teca_variant_array.New(np.array([t, t, t, t])) + mesh_out.get_information_arrays().append('time_info', t_inf) - mesh_in = as_const_teca_cartesian_mesh(data_in[0]) - t = mesh_in.get_time() - s = mesh_in.get_time_step() + s_inf = teca_variant_array.New(np.array([s,s])) + mesh_out.get_information_arrays().append('step_info', s_inf) - mesh_out = teca_cartesian_mesh.New() - mesh_out.shallow_copy(mesh_in) - - t_inf = teca_variant_array.New(np.array([t, t, t, t])) - mesh_out.get_information_arrays().append('time_info', t_inf) - - s_inf = teca_variant_array.New(np.array([s,s])) - mesh_out.get_information_arrays().append('step_info', s_inf) - - if self.verbose: - sys.stderr.write('t=%g, time_info=%s\n'%(t, str(t_inf))) - sys.stderr.write('s=%d, step_info=%s\n'%(s, str(s_inf))) - - return mesh_out + if self.verbose: + sys.stderr.write('t=%g, time_info=%s\n'%(t, str(t_inf))) + sys.stderr.write('s=%d, step_info=%s\n'%(s, str(s_inf))) - return execute_callback + return mesh_out class print_info_arrays(teca_python_algorithm): @@ -84,26 +74,21 @@ def print_status(self, msg): if self.verbose: sys.stderr.write('print_info_arrays::%s\n'%(msg)) - def get_execute_callback(self): - self.print_status('get_execute_callback') - - def execute_callback(port, data_in, req_in): - self.print_status('execute_callback') + def execute(self, port, data_in, req_in): + self.print_status('execute(override)') - mesh_in = as_const_teca_cartesian_mesh(data_in[0]) - mesh_out = teca_cartesian_mesh.New() - mesh_out.shallow_copy(mesh_in) + mesh_in = as_const_teca_cartesian_mesh(data_in[0]) + mesh_out = teca_cartesian_mesh.New() + mesh_out.shallow_copy(mesh_in) - if self.verbose: - sys.stderr.write('t=%g, time_info=%s\n'%(mesh_out.get_time(), \ - str(mesh_out.get_information_arrays().get('time_info')))) - - sys.stderr.write('s=%d, step_info=%s\n'%(mesh_out.get_time_step(), \ - str(mesh_out.get_information_arrays().get('step_info')))) + if self.verbose: + sys.stderr.write('t=%g, time_info=%s\n'%(mesh_out.get_time(), \ + str(mesh_out.get_information_arrays().get('time_info')))) - return mesh_out + sys.stderr.write('s=%d, step_info=%s\n'%(mesh_out.get_time_step(), \ + str(mesh_out.get_information_arrays().get('step_info')))) - return execute_callback + return mesh_out diff --git a/test/python/test_python_algorithm.py b/test/python/test_python_algorithm.py new file mode 100644 index 000000000..a87fcadce --- /dev/null +++ b/test/python/test_python_algorithm.py @@ -0,0 +1,116 @@ +from teca import * +import numpy as np +import sys + +set_stack_trace_on_error() + +if not len(sys.argv) == 7: + sys.stderr.write('test_python_algorithm.py [dataset regex] ' \ + '[u_var] [v_var] [first step] [last step] [out file name]\n') + sys.exit(-1) + +data_regex = sys.argv[1] +u_var = sys.argv[2] +v_var = sys.argv[3] +first_step = int(sys.argv[4]) +end_index = int(sys.argv[5]) +out_file = sys.argv[6] + +class wind_speed(teca_python_algorithm): + + def __init__(self): + self.u_var = None + self.v_var = None + + def set_u_var(self, u_var): + self.u_var = u_var + + def set_v_var(self, v_var): + self.v_var = v_var + + def report(self, port, md_in): + sys.stderr.write('wind_speed::report\n') + md = md_in[0] + md.append('variables', 'wind_speed') + return md + + def request(self, port, md_in, req_in): + sys.stderr.write('wind_speed::request\n') + req = teca_metadata(req_in) + req['arrays'] = [self.u_var, self.v_var] + return [req] + + def execute(self, port, data_in, req): + sys.stderr.write('wind_speed::execute\n') + in_mesh = as_teca_cartesian_mesh(data_in[0]) + out_mesh = teca_cartesian_mesh.New() + out_mesh.shallow_copy(in_mesh) + arrays = out_mesh.get_point_arrays() + u = arrays[self.u_var] + v = arrays[self.v_var] + w = np.sqrt(u*u + v*v) + arrays['wind_speed'] = w + return out_mesh + + + +class max_wind_speed(wind_speed): + + def execute(self, port, data_in, req): + """ + override the base class implementation. + demonstrate the use of super to call the base + class + """ + sys.stderr.write('max_wind_speed::execute\n') + + # let the base class calculate the wind speed + wdata = super().execute(port, data_in, req) + + # find the max + mesh = as_teca_cartesian_mesh(wdata) + md = mesh.get_metadata() + arrays = mesh.get_point_arrays() + max_ws = np.max(arrays['wind_speed']) + + # construct the output + table = teca_table.New() + table.copy_metadata(mesh) + + table.declare_columns(['time_step', 'time', 'max_wind'], \ + ['l', 'd', 'd']) + + table << md['time_step'] << md['time'] << max_ws + + return table + + + +cfr = teca_cf_reader.New() +cfr.set_files_regex(data_regex) +cfr.set_x_axis_variable('lon') +cfr.set_y_axis_variable('lat') +cfr.set_t_axis_variable('time') + +coords = teca_normalize_coordinates.New() +coords.set_input_connection(cfr.get_output_port()) + +mws = max_wind_speed.New() +mws.set_input_connection(coords.get_output_port()) +mws.set_u_var(u_var) +mws.set_v_var(v_var) + +mr = teca_table_reduce.New() +mr.set_input_connection(mws.get_output_port()) +mr.set_thread_pool_size(1) + +ts = teca_table_sort.New() +ts.set_input_connection(mr.get_output_port()) +ts.set_index_column('time_step') + +tw = teca_table_writer.New() +tw.set_input_connection(ts.get_output_port()) +tw.set_file_name('test_python_alg.csv') + +tw.update() + From 954e972f7bf1c33ccecb72f9f23dbfa9543f7870 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Thu, 12 Mar 2020 12:12:01 -0700 Subject: [PATCH 005/385] use mpich on Travis CI Apple Mac OS --- test/travis_ci/install_osx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index b9feebed2..b4ae0c27a 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -6,7 +6,7 @@ export PATH=/usr/local/bin:$PATH # install deps. note than many are included as a part of brew-core # these days. hence this list isn't comprehensive brew update -brew install openmpi swig svn udunits +brew install mpich swig svn udunits # matplotlib currently doesn't have a formula # teca fails to locate mpi4py installed from brew pip3 install numpy mpi4py matplotlib From 51dfa6ab3a2b8951c6c678e3a1b4897337c960ce Mon Sep 17 00:00:00 2001 From: Abdelrahman Elbashandy Date: Thu, 20 Feb 2020 15:56:16 -0800 Subject: [PATCH 006/385] Replacing missmarple to sourceforge --- doc/teca_users_guide.tex | 2 +- test/travis_ci/install_fedora_28.sh | 2 +- test/travis_ci/install_osx.sh | 4 ++-- test/travis_ci/install_ubuntu_14_04.sh | 2 +- test/travis_ci/install_ubuntu_18_04.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/teca_users_guide.tex b/doc/teca_users_guide.tex index c63719928..10c753e8b 100644 --- a/doc/teca_users_guide.tex +++ b/doc/teca_users_guide.tex @@ -215,7 +215,7 @@ \subsubsection{Installing TECA} \vspace{2mm}\hspace{0.2in}\begin{minipage}{0.8\textwidth} \begin{minted}[fontsize=\small]{bash} -svn co svn://missmarple.lbl.gov/work3/teca/TECA_data +svn co svn://svn.code.sf.net/p/teca/TECA_data \end{minted} \end{minipage}\vspace{2mm} diff --git a/test/travis_ci/install_fedora_28.sh b/test/travis_ci/install_fedora_28.sh index 98161a822..cfc9be9e0 100755 --- a/test/travis_ci/install_fedora_28.sh +++ b/test/travis_ci/install_fedora_28.sh @@ -31,4 +31,4 @@ echo ${TECA_DATA_REVISION} pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib # install data files. -svn co svn://missmarple.lbl.gov/work3/teca/TECA_data@${TECA_DATA_REVISION} TECA_data +svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index b4ae0c27a..05ad604cc 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -18,8 +18,8 @@ pip3 install numpy mpi4py matplotlib # travis will kill a build if it does not get console output # for 10 min. The following snippet sends progress marks # to the console while svn runs. -echo 'svn co svn://missmarple.lbl.gov/work3/teca/TECA_data@${TECA_DATA_REVISION} &' -svn co svn://missmarple.lbl.gov/work3/teca/TECA_data@${TECA_DATA_REVISION} & +echo 'svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data &' +svn svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data & svn_pid=$! while [ -n "$(ps -p $svn_pid -o pid=)" ] do diff --git a/test/travis_ci/install_ubuntu_14_04.sh b/test/travis_ci/install_ubuntu_14_04.sh index 754ce149e..6dd6d58b2 100755 --- a/test/travis_ci/install_ubuntu_14_04.sh +++ b/test/travis_ci/install_ubuntu_14_04.sh @@ -22,4 +22,4 @@ wget https://cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz sudo tar -C /usr -x -z -f cmake-3.5.2-Linux-x86_64.tar.gz --strip-components=1 # install data files. -svn co svn://missmarple.lbl.gov/work3/teca/TECA_data@${TECA_DATA_REVISION} TECA_data +svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data diff --git a/test/travis_ci/install_ubuntu_18_04.sh b/test/travis_ci/install_ubuntu_18_04.sh index 65bc7126a..49a7b31ea 100755 --- a/test/travis_ci/install_ubuntu_18_04.sh +++ b/test/travis_ci/install_ubuntu_18_04.sh @@ -27,4 +27,4 @@ echo ${TECA_DATA_REVISION} pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib # install data files. -svn co svn://missmarple.lbl.gov/work3/teca/TECA_data@${TECA_DATA_REVISION} TECA_data +svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data From 744d468bfcae2191765dd9aa757061548c210feb Mon Sep 17 00:00:00 2001 From: elbashandy Date: Thu, 19 Mar 2020 15:30:39 -0700 Subject: [PATCH 007/385] Implementing teca_model_segmentation * An algorithm that loads a PyTorch model and applies a field of choice like IVT or IWV to give out predictions Implementing teca_deeplabv3p_ar_detect. * It uses Google's DeepLab V3 plus NN architecture to detect Atmospheric Rivers --- .gitignore | 1 + .travis.yml | 2 +- alg/CMakeLists.txt | 2 + alg/teca_deeplabv3p_ar_detect.py | 475 ++++++++++++++++++++++ alg/teca_model_segmentation.py | 228 +++++++++++ apps/CMakeLists.txt | 1 + apps/teca_bayesian_ar_detect.cpp | 8 +- apps/teca_pytorch_deeplabv3p_ar_detect.in | 245 +++++++++++ python/teca_py_alg.i | 12 +- teca_config.h.in | 1 + test/python/CMakeLists.txt | 18 + test/python/test_bayesian_ar_detect.py | 2 +- test/python/test_deeplabv3p_ar_detect.py | 76 ++++ test/travis_ci/install_fedora_28.sh | 2 +- test/travis_ci/install_osx.sh | 2 +- test/travis_ci/install_ubuntu_14_04.sh | 2 +- test/travis_ci/install_ubuntu_18_04.sh | 2 +- 17 files changed, 1068 insertions(+), 11 deletions(-) create mode 100644 alg/teca_deeplabv3p_ar_detect.py create mode 100644 alg/teca_model_segmentation.py create mode 100644 apps/teca_pytorch_deeplabv3p_ar_detect.in create mode 100644 test/python/test_deeplabv3p_ar_detect.py diff --git a/.gitignore b/.gitignore index 3c74fb441..bf6a22fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.sw[a-z] *.patch _build +*.pt diff --git a/.travis.yml b/.travis.yml index 2fdd4f5c1..3801b6c20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - BUILD_TYPE=Debug - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - - TECA_DATA_REVISION=49 + - TECA_DATA_REVISION=53 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - DOCKER_IMAGE=fedora IMAGE_VERSION=28 IMAGE_NAME=fedora_28 diff --git a/alg/CMakeLists.txt b/alg/CMakeLists.txt index be76eb139..4b8433e38 100644 --- a/alg/CMakeLists.txt +++ b/alg/CMakeLists.txt @@ -50,6 +50,8 @@ set(teca_alg_cxx_srcs ) teca_py_install(${LIB_PREFIX} + teca_model_segmentation.py + teca_deeplabv3p_ar_detect.py teca_tc_stats.py teca_tc_activity.py teca_python_algorithm.py diff --git a/alg/teca_deeplabv3p_ar_detect.py b/alg/teca_deeplabv3p_ar_detect.py new file mode 100644 index 000000000..0998847ea --- /dev/null +++ b/alg/teca_deeplabv3p_ar_detect.py @@ -0,0 +1,475 @@ +import os +import teca_py +import math +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +# Implementation of Google's Deeplab-V3-Plus +# source: https://arxiv.org/pdf/1802.02611.pdf +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, rate=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + dilation=rate, padding=rate, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.rate = rate + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + def __init__(self, nInputChannels, block, layers, os=16, + pretrained=False, state_dict_resnet=None): + self.inplanes = 64 + super(ResNet, self).__init__() + if os == 16: + strides = [1, 2, 2, 1] + rates = [1, 1, 1, 2] + blocks = [1, 2, 4] + elif os == 8: + strides = [1, 2, 1, 1] + rates = [1, 1, 2, 2] + blocks = [1, 2, 1] + else: + raise NotImplementedError + + self.state_dict_resnet = state_dict_resnet + + # Modules + self.conv1 = nn.Conv2d(nInputChannels, 64, kernel_size=7, + stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.layer1 = self._make_layer(block, 64, layers[0], + stride=strides[0], rate=rates[0]) + self.layer2 = self._make_layer(block, 128, layers[1], + stride=strides[1], rate=rates[1]) + self.layer3 = self._make_layer(block, 256, layers[2], + stride=strides[2], rate=rates[2]) + self.layer4 = self._make_MG_unit(block, 512, blocks=blocks, + stride=strides[3], rate=rates[3]) + + self._init_weight() + + if pretrained: + self._load_pretrained_model() + + def _make_layer(self, block, planes, blocks, stride=1, rate=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, rate, downsample) + ) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def _make_MG_unit(self, block, planes, + blocks=[1, 2, 4], stride=1, rate=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block(self.inplanes, planes, stride, + rate=blocks[0]*rate, downsample=downsample) + ) + self.inplanes = planes * block.expansion + for i in range(1, len(blocks)): + layers.append( + block(self.inplanes, planes, + stride=1, rate=blocks[i]*rate) + ) + + return nn.Sequential(*layers) + + def forward(self, input): + x = self.conv1(input) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + low_level_feat = x + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + return x, low_level_feat + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _load_pretrained_model(self): + self.load_state_dict(self.state_dict_resnet) + + +def ResNet101(nInputChannels=3, os=16, pretrained=False, + state_dict_resnet=None): + model = ResNet( + nInputChannels, Bottleneck, [3, 4, 23, 3], os, + pretrained=pretrained, state_dict_resnet=state_dict_resnet + ) + return model + + +class ASPP_module(nn.Module): + def __init__(self, inplanes, planes, rate): + super(ASPP_module, self).__init__() + if rate == 1: + kernel_size = 1 + padding = 0 + else: + kernel_size = 3 + padding = rate + self.atrous_convolution = nn.Conv2d( + inplanes, planes, kernel_size=kernel_size, + stride=1, padding=padding, dilation=rate, bias=False + ) + self.bn = nn.BatchNorm2d(planes) + self.relu = nn.ReLU() + + self._init_weight() + + def forward(self, x): + x = self.atrous_convolution(x) + x = self.bn(x) + + return self.relu(x) + + def _init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + +class DeepLabv3_plus(nn.Module): + def __init__(self, nInputChannels=3, n_classes=21, os=16, + pretrained=False, state_dict_resnet=None, _print=True): + if _print: + sys.stdout.write("Constructing DeepLabv3+ model...\n") + sys.stdout.write("Number of classes: {}\n".format(n_classes)) + sys.stdout.write("Output stride: {}\n".format(os)) + sys.stdout.write( + "Number of Input Channels: {}\n".format(nInputChannels) + ) + super(DeepLabv3_plus, self).__init__() + + # Atrous Conv + self.resnet_features = ResNet101( + nInputChannels, os, pretrained=pretrained, + state_dict_resnet=state_dict_resnet + ) + + # ASPP + if os == 16: + rates = [1, 6, 12, 18] + elif os == 8: + rates = [1, 12, 24, 36] + else: + raise NotImplementedError + + self.aspp1 = ASPP_module(2048, 256, rate=rates[0]) + self.aspp2 = ASPP_module(2048, 256, rate=rates[1]) + self.aspp3 = ASPP_module(2048, 256, rate=rates[2]) + self.aspp4 = ASPP_module(2048, 256, rate=rates[3]) + + self.relu = nn.ReLU() + + self.global_avg_pool = nn.Sequential( + nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(2048, 256, 1, stride=1, bias=False), + nn.BatchNorm2d(256), + nn.ReLU() + ) + + self.conv1 = nn.Conv2d(1280, 256, 1, bias=False) + self.bn1 = nn.BatchNorm2d(256) + + # adopt [1x1, 48] for channel reduction. + self.conv2 = nn.Conv2d(256, 48, 1, bias=False) + self.bn2 = nn.BatchNorm2d(48) + + self.last_conv = nn.Sequential( + nn.Conv2d( + 304, 256, kernel_size=3, stride=1, + padding=1, bias=False + ), + nn.BatchNorm2d(256), + nn.ReLU(), + nn.Conv2d( + 256, 256, kernel_size=3, stride=1, + padding=1, bias=False + ), + nn.BatchNorm2d(256), + nn.ReLU(), + nn.Conv2d(256, n_classes, kernel_size=1, stride=1)) + + def forward(self, input): + x, low_level_features = self.resnet_features(input) + x1 = self.aspp1(x) + x2 = self.aspp2(x) + x3 = self.aspp3(x) + x4 = self.aspp4(x) + x5 = self.global_avg_pool(x) + + x5 = F.interpolate( + x5, size=x4.size()[2:], mode='bilinear', + align_corners=True + ) + + x = torch.cat((x1, x2, x3, x4, x5), dim=1) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + x = F.interpolate( + x, + size=( + int(math.ceil(input.size()[-2]/4)), + int(math.ceil(input.size()[-1]/4)) + ), + mode='bilinear', + align_corners=True + ) + + low_level_features = self.conv2(low_level_features) + low_level_features = self.bn2(low_level_features) + low_level_features = self.relu(low_level_features) + + x = torch.cat((x, low_level_features), dim=1) + x = self.last_conv(x) + + x = F.interpolate( + x, size=input.size()[2:], mode='bilinear', + align_corners=True + ) + + return x + + def freeze_bn(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + + def __init_weight(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + torch.nn.init.kaiming_normal_(m.weight) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + +# End of DeepLabv3_plus implementation + + +class teca_deeplabv3p_ar_detect(teca_py.teca_model_segmentation): + """ + This algorithm detects Atmospheric Rivers using the DeepLabv3+ + architecture. Given an input field of integrated vapor transport, + it calculates the probability of AR presence at each gridcell. + """ + def __init__(self): + super().__init__() + + # inner config data needed for pre & post processing for + # the input and its prediction + self.mesh_ny = None + self.mesh_nx = None + self.padding_amount_y = None + self.padding_amount_x = None + self.set_variable_name('IVT') + + def set_mesh_dims(self, mesh_ny, mesh_nx): + self.mesh_ny = mesh_ny + self.mesh_nx = mesh_nx + + def set_padding_atts(self): + target_shape = 128 * np.ceil(self.mesh_ny/128.0) + target_shape_diff = target_shape - self.mesh_ny + self.padding_amount_y = ( + int(np.ceil(target_shape_diff / 2.0)), + int(np.floor(target_shape_diff / 2.0)) + ) + + target_shape = 64 * np.ceil(self.mesh_nx/64.0) + target_shape_diff = target_shape - self.mesh_nx + self.padding_amount_x = ( + int(np.ceil(target_shape_diff / 2.0)), + int(np.floor(target_shape_diff / 2.0)) + ) + + def input_preprocess(self, input_data): + """ + Necessary data preprocessing before inputing it into + the model. The preprocessing is padding the data and + converting it into 3 channels + """ + self.set_padding_atts() + + input_data = np.reshape( + self.var_array, [1, self.mesh_ny, self.mesh_nx] + ) + input_data = np.pad( + input_data, ((0, 0), self.padding_amount_y, (0, 0)), + 'constant', constant_values=0 + ) + input_data = np.pad( + input_data, ((0, 0), (0, 0), self.padding_amount_x), + 'constant', constant_values=0 + ) + input_data = input_data.astype('float32') + + transformed_input_data = np.zeros( + (1, 3, input_data.shape[1], input_data.shape[2]) + ).astype('float32') + for i in range(3): + transformed_input_data[0, i, ...] = input_data + + return transformed_input_data + + def output_postprocess(self, ouput_data): + """ + post-processing the model output. This is + necessary to unpad the output to fit the netcdf + dimensions + """ + # unpadding the padded zeros + y_start = self.padding_amount_y[0] + y_end = ouput_data.shape[2] - self.padding_amount_y[1] + x_start = self.padding_amount_x[0] + x_end = ouput_data.shape[3] - self.padding_amount_x[1] + + ouput_data = ouput_data.numpy() + ouput_data = ouput_data[:, :, y_start:y_end, x_start:x_end] + + return ouput_data.ravel() + + def build_model(self, state_dict_deeplab_file=None, + state_dict_resnet_file=None): + """ + Load model from file system. If multi-threading is used rank 0 + loads the model file and broadcasts it to the other ranks + """ + if not state_dict_deeplab_file: + deeplab_sd_path = \ + "cascade_deeplab_IVT.pt" + state_dict_deeplab_file = os.path.join( + teca_py.get_teca_data_root(), + deeplab_sd_path + ) + + if not state_dict_resnet_file: + resnet_sd_path = \ + "resnet101-5d3b4d8f-state_dict.pth" + state_dict_resnet_file = os.path.join( + teca_py.get_teca_data_root(), + resnet_sd_path + ) + + comm = self.get_communicator() + + state_dict_deeplab = self.load_state_dict(state_dict_deeplab_file) + state_dict_resnet = self.load_state_dict(state_dict_resnet_file) + + model = DeepLabv3_plus(n_classes=1, pretrained=True, _print=False, + state_dict_resnet=state_dict_resnet) + model.load_state_dict(state_dict_deeplab) + + self.set_model(model) + + def get_execute_callback(self): + """ + return a teca_algorithm::execute function + """ + def execute(port, data_in, req): + """ + expects an array of an input variable to run through + the torch model and get the segmentation results as an + output. + """ + in_mesh = teca_py.as_teca_cartesian_mesh(data_in[0]) + + if in_mesh is None: + raise ValueError("ERROR: empty input, or not a mesh") + if self.model is None: + raise ValueError( + "ERROR: pretrained model has not been specified" + ) + + md = in_mesh.get_metadata() + ext = md["extent"] + + nlat = int(ext[3]-ext[2]+1) + nlon = int(ext[1]-ext[0]+1) + self.set_mesh_dims(nlat, nlon) + + arrays = in_mesh.get_point_arrays() + self.var_array = arrays[self.variable_name] + + self.torch_inference_fn = torch.sigmoid + + super_execute = super( + teca_deeplabv3p_ar_detect, self + ).get_execute_callback() + + out_mesh = super_execute(port, data_in, req) + + return out_mesh + return execute diff --git a/alg/teca_model_segmentation.py b/alg/teca_model_segmentation.py new file mode 100644 index 000000000..1ddc700eb --- /dev/null +++ b/alg/teca_model_segmentation.py @@ -0,0 +1,228 @@ +import teca_py +import numpy as np +import torch +import torch.nn.functional as F + + +class teca_model_segmentation(teca_py.teca_python_algorithm): + """ + A generic TECA algorithm that provides torch based deep + learning feature detecting algorithms with core TECA + capabilities for easy integration. + """ + def __init__(self): + self.variable_name = None + self.pred_name = None + self.var_array = None + self.pred_array = None + self.torch_inference_fn = None + self.transform_fn = None + self.transport_fn_args = None + self.model = None + self.model_path = None + self.device = 'cpu' + + def __str__(self): + ms_str = 'variable_name=%s, pred_name=%d\n\n' % ( + self.variable_name, self.pred_name) + + ms_str += 'model:\n%s\n\n' % (str(self.model)) + + ms_str += 'device:\n%s\n' % (str(self.device)) + + return ms_str + + def load_state_dict(self, state_dict_file): + """ + Load only the pytorch state_dict parameters file only + once and broadcast it to all ranks + """ + comm = self.get_communicator() + rank = comm.Get_rank() + + sd = None + if rank == 0: + sd = torch.load( + state_dict_file, + map_location=lambda storage, + loc: storage + ) + sd = comm.bcast(sd, root=0) + + return sd + + def set_variable_name(self, name): + """ + set the variable name that will be inputed to the model + """ + self.variable_name = str(name) + self.set_pred_name(self.variable_name + '_pred') + + def set_pred_name(self, name): + """ + set the variable name that will be the output to the model + """ + self.pred_name = name + + def set_torch_inference_fn(self, torch_fn): + """ + set the final torch inference function. ex. torch.sigmoid() + """ + self.torch_inference_fn = torch_fn + + def input_preprocess(self, input_data): + return input_data + + def output_postprocess(self, output_data): + return output_data + + def __set_transform_fn(self, fn, *args): + """ + if the data need to be transformed in a way then a function + could be provided to be applied on the requested data before + running it to the model. + """ + if not hasattr(fn, '__call__'): + raise TypeError( + "ERROR: The provided data transform function" + "is not a function" + ) + + if not args: + raise ValueError( + "ERROR: The provided data transform function " + "must at least have 1 argument -- the data array object to " + "apply the transformation on." + ) + + self.transform_fn = fn + self.transport_fn_args = args + + def set_num_threads(self, n): + """ + torch: Sets the number of threads used for intraop parallelism on CPU + """ + # n=-1: use default + if n != -1: + torch.set_num_threads(n) + + def set_torch_device(self, device="cpu"): + """ + Set device to either 'cuda' or 'cpu' + """ + if device[:4] == "cuda" and not torch.cuda.is_available(): + raise AttributeError( + "ERROR: Couldn\'t set device to cuda, cuda is " + "not available" + ) + + self.device = device + + def set_model(self, model): + """ + set Pytorch pretrained model + """ + self.model = model + self.model.eval() + + def get_report_callback(self): + """ + return a teca_algorithm::report function adding the output name + that will hold the output predictions of the used model. + """ + def report(port, rep_in): + rep_temp = rep_in[0] + + rep = teca_py.teca_metadata(rep_temp) + + if not rep.has('variables'): + sys.stdout.write("variables key doesn't exist\n") + rep['variables'] = teca_py.teca_variant_array.New(np.array([])) + + if self.pred_name: + rep.append("variables", self.pred_name) + + return rep + return report + + def get_request_callback(self): + """ + return a teca_algorithm::request function adding the variable name + that the pretrained model will process. + """ + def request(port, md_in, req_in): + if not self.variable_name: + raise ValueError( + "ERROR: No variable to request specifed" + ) + + req = teca_py.teca_metadata(req_in) + + arrays = [] + if req.has('arrays'): + arrays = req['arrays'] + + arrays.append(self.variable_name) + req['arrays'] = arrays + + return [req] + return request + + def get_execute_callback(self): + """ + return a teca_algorithm::execute function + """ + def execute(port, data_in, req): + """ + expects an array of an input variable to run through + the torch model and get the segmentation results as an + output. + """ + in_mesh = teca_py.as_teca_cartesian_mesh(data_in[0]) + + if in_mesh is None: + raise ValueError("ERROR: empty input, or not a mesh") + + if self.model is None: + raise ValueError( + "ERROR: pretrained model has not been specified" + ) + + if self.variable_name is None: + raise ValueError( + "ERROR: data variable name has not been specified" + ) + + if self.var_array is None: + raise ValueError( + "ERROR: data variable array has not been set" + ) + + if self.torch_inference_fn is None: + raise ValueError( + "ERROR: final torch inference layer" + "has not been set" + ) + + self.var_array = self.input_preprocess(self.var_array) + + self.var_array = torch.from_numpy(self.var_array).to(self.device) + + with torch.no_grad(): + self.pred_array = self.torch_inference_fn( + self.model(self.var_array) + ) + + if self.pred_array is None: + raise Exception("ERROR: Model failed to get predictions") + + self.pred_array = self.output_postprocess(self.pred_array) + + out_mesh = teca_py.teca_cartesian_mesh.New() + out_mesh.shallow_copy(in_mesh) + + self.pred_array = teca_py.teca_variant_array.New(self.pred_array) + out_mesh.get_point_arrays().set(self.pred_name, self.pred_array) + + return out_mesh + return execute diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index cb361726a..e099b6e56 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -33,6 +33,7 @@ teca_add_app(teca_bayesian_ar_detect LIBS ${teca_app_link} teca_py_install_apps( teca_convert_table.in teca_dataset_metadata.in + teca_pytorch_deeplabv3p_ar_detect.in teca_event_filter.in teca_profile_explorer.in teca_tc_stats.in diff --git a/apps/teca_bayesian_ar_detect.cpp b/apps/teca_bayesian_ar_detect.cpp index 9fa619c68..cd5f472d9 100644 --- a/apps/teca_bayesian_ar_detect.cpp +++ b/apps/teca_bayesian_ar_detect.cpp @@ -48,7 +48,7 @@ int main(int argc, char **argv) "probability threshold for segmenting ar_probability to produce ar_binary_tag") ("first_step", value(), "first time step to process") ("last_step", value(), "last time step to process") - ("steps_per_file", value(), "number of time steps per output filr") + ("steps_per_file", value(), "number of time steps per output file") ("start_date", value(), "first time to proces in YYYY-MM-DD hh:mm:ss format") ("end_date", value(), "first time to proces in YYYY-MM-DD hh:mm:ss format") ("n_threads", value(), "thread pool size. default is -1. -1 for all") @@ -253,7 +253,7 @@ int main(int argc, char **argv) teca_metadata atrs; if (md.get("attributes", atrs)) { - TECA_ERROR("metadata mising attributes") + TECA_ERROR("metadata missing attributes") return -1; } @@ -286,7 +286,7 @@ int main(int argc, char **argv) if (teca_coordinate_util::time_step_of(time, true, calendar, units, start_date, first_step)) { - TECA_ERROR("Failed to lcoate time step for start date \"" + TECA_ERROR("Failed to locate time step for start date \"" << start_date << "\"") return -1; } @@ -301,7 +301,7 @@ int main(int argc, char **argv) if (teca_coordinate_util::time_step_of(time, false, calendar, units, end_date, last_step)) { - TECA_ERROR("Failed to lcoate time step for end date \"" + TECA_ERROR("Failed to locate time step for end date \"" << end_date << "\"") return -1; } diff --git a/apps/teca_pytorch_deeplabv3p_ar_detect.in b/apps/teca_pytorch_deeplabv3p_ar_detect.in new file mode 100644 index 000000000..5d73db23e --- /dev/null +++ b/apps/teca_pytorch_deeplabv3p_ar_detect.in @@ -0,0 +1,245 @@ +#!/usr/bin/env python@TECA_PYTHON_VERSION@ +from teca import * +import sys +import argparse +import numpy as np +try: + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() +except ImportError: + rank = 0 + + +def main(): + # parse the command line + parser = argparse.ArgumentParser() + parser.add_argument( + '--input_file', type=str, required=False, + help='file path to the simulation to search for atmospheric rivers' + ) + + parser.add_argument( + '--input_regex', type=str, required=False, + help='regex matching simulation files to search for atmospheric rivers' + ) + + parser.add_argument( + '--output_file', type=str, required=False, + default=r'pytorch_deeplabv3p_ar_detect_%t%.nc', + help=r'file pattern for output netcdf files (%t% is the time index)' + ) + + parser.add_argument( + '--ivt', type=str, default='IVT', + help='name of variable with integrated vapor transport (IVT)' + ) + + parser.add_argument( + '--pytorch_deeplab_model', type=str, required=False, + help='the pretrained deeplabv3plus model file' + ) + + parser.add_argument( + '--pytorch_resnet_model', type=str, required=False, + help='the pretrained resnet model file' + ) + + parser.add_argument( + '--t_axis_variable', type=str, required=False, + help='time dimension name' + ) + + parser.add_argument( + '--t_calendar', type=str, required=False, + help='time calendar' + ) + + parser.add_argument( + '--t_units', type=str, required=False, + help='time unit' + ) + + parser.add_argument( + '--filename_time_template', type=str, required=False, + help='filename time template' + ) + + parser.add_argument( + '--compression_level', type=int, required=False, + help='the compression level used for each variable' + ) + + parser.add_argument( + '--date_format', type=str, required=False, + help='the format for the date to write in the filename' + ) + + parser.add_argument( + '--first_step', type=int, required=False, + help='first time step to process' + ) + + parser.add_argument( + '--last_step', type=int, required=False, + help='last time step to process' + ) + + parser.add_argument( + '--steps_per_file', type=int, required=False, + help='number of time steps per output file' + ) + + parser.add_argument( + '--start_date', type=str, required=False, + help='first time to proces in YYYY-MM-DD hh:mm:ss format' + ) + + parser.add_argument( + '--end_date', type=str, required=False, + help='end time to proces in YYYY-MM-DD hh:mm:ss format' + ) + + args = parser.parse_args() + + if not args.input_file and not args.input_regex: + if rank == 0: + raise parser.error( + "missing file name or regex for simulation reader. " + "See --help for a list of command line options." + ) + + cf_reader = teca_cf_reader.New() + + coords = teca_normalize_coordinates.New() + coords.set_input_connection(cf_reader.get_output_port()) + + deeplabv3p_ar_detect = teca_deeplabv3p_ar_detect.New() + deeplabv3p_ar_detect.set_input_connection(coords.get_output_port()) + + teca_exec = teca_index_executive.New() + + cf_writer = teca_cf_writer.New() + cf_writer.set_input_connection( + deeplabv3p_ar_detect.get_output_port() + ) + cf_writer.set_thread_pool_size(1) + + if args.input_file: + cf_reader.append_file_name( + args.input_file + ) + + if args.input_regex: + cf_reader.set_files_regex( + args.input_regex + ) + + if args.output_file: + cf_writer.set_file_name( + args.output_file + ) + + if args.ivt: + deeplabv3p_ar_detect.set_variable_name( + args.ivt + ) + + if args.pytorch_deeplab_model and args.pytorch_resnet_model: + deeplabv3p_ar_detect.build_model( + args.pytorch_deeplab_model, + args.pytorch_resnet_model + ) + else: + deeplabv3p_ar_detect.build_model() + + if args.t_axis_variable is not None: + cf_reader.set_t_axis_variable( + args.t_axis_variable + ) + + if args.t_calendar: + cf_reader.set_t_calendar( + args.t_calendar + ) + + if args.t_units: + cf_reader.set_t_units( + args.t_units + ) + + if args.filename_time_template: + cf_reader.set_filename_time_template( + args.filename_time_template + ) + + if args.compression_level: + cf_writer.set_compression_level( + args.compression_level + ) + + if args.date_format: + cf_writer.set_date_format( + args.date_format + ) + + if args.first_step: + cf_writer.set_first_step( + args.first_step + ) + + if args.last_step: + cf_writer.set_last_step( + args.last_step + ) + + if args.steps_per_file: + cf_writer.set_steps_per_file( + args.steps_per_file + ) + + # some minimal check for missing options + if (cf_reader.get_number_of_file_names() == 0 and + not cf_reader.get_files_regex()): + if rank == 0: + raise ValueError( + "missing file name or regex for simulation reader. " + "See --help for a list of command line options." + ) + + if not cf_writer.get_file_name(): + if rank == 0: + raise ValueError( + "missing file name pattern for netcdf writer. " + "See --help for a list of command line options." + ) + + if args.start_date or args.end_date: + time_atts = atrs["time"] + calendar = time_atts["calendar"] + units = time_atts["units"] + + coords = md["coordinates"] + time = coords["t"] + + # convert date string to step, start date + if args.start_date: + first_step = teca_coordinate.time_step_of( + time, True, calendar, units, args.start_date + ) + teca_exec.set_start_index(first_step) + + # and end date + if args.end_date: + last_step = teca_coordinate.time_step_of( + time, False, calendar, units, args.end_date + ) + teca_exec.set_end_index(last_step) + + # run the pipeline + cf_writer.set_executive(teca_exec) + cf_writer.update() + + +if __name__ == '__main__': + main() diff --git a/python/teca_py_alg.i b/python/teca_py_alg.i index c61bf4afd..08b735a52 100644 --- a/python/teca_py_alg.i +++ b/python/teca_py_alg.i @@ -92,7 +92,6 @@ %ignore teca_laplacian::operator=; %include "teca_laplacian.h" - /*************************************************************************** mask ***************************************************************************/ @@ -308,6 +307,17 @@ from teca_python_algorithm import * %} +/*************************************************************************** + model_segmentation & deeplabv3p_ar_detect + ***************************************************************************/ +%pythoncode %{ +try: + from teca_model_segmentation import * + from teca_deeplabv3p_ar_detect import teca_deeplabv3p_ar_detect +except ImportError: + pass +%} + /*************************************************************************** tc_activity ***************************************************************************/ diff --git a/teca_config.h.in b/teca_config.h.in index b18962d97..fba85eb79 100644 --- a/teca_config.h.in +++ b/teca_config.h.in @@ -14,6 +14,7 @@ #cmakedefine TECA_VERSION_DESCR "@TECA_VERSION_DESCR@" #cmakedefine TECA_PYTHON_VERSION @TECA_PYTHON_VERSION@ +#cmakedefine TECA_DATA_ROOT "@TECA_DATA_ROOT@" #cmakedefine TECA_ENABLE_PROFILER diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index 87dff4f19..c3d5255d0 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -230,6 +230,24 @@ teca_add_test(py_test_bayesian_ar_detect_mpi_threads FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) +teca_add_test(py_test_deeplabv3p_ar_detect + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py + "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" + "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" + "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" + "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT -1 + FEATURES ${TECA_HAS_NETCDF} + REQ_TECA_DATA) + +teca_add_test(py_test_deeplabv3p_ar_detect_mpi_threads + COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py + "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" + "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" + "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" + "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT ${HALF_CORES} + FEATURES ${TECA_HAS_NETCDF} + REQ_TECA_DATA) + teca_add_test(py_test_binary_stream COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_binary_stream.py diff --git a/test/python/test_bayesian_ar_detect.py b/test/python/test_bayesian_ar_detect.py index e3b81dcc9..0130a97cb 100644 --- a/test/python/test_bayesian_ar_detect.py +++ b/test/python/test_bayesian_ar_detect.py @@ -93,7 +93,7 @@ else: # make a baseline if rank == 0: - cerr << 'generating baseline image ' << baseline_table << endl + sys.stdout.write('generating baseline image ' + baseline_table + '\n') tts = teca_table_to_stream.New() tts.set_input_connection(sort.get_output_port()) diff --git a/test/python/test_deeplabv3p_ar_detect.py b/test/python/test_deeplabv3p_ar_detect.py new file mode 100644 index 000000000..f939d6424 --- /dev/null +++ b/test/python/test_deeplabv3p_ar_detect.py @@ -0,0 +1,76 @@ +try: + from mpi4py import * + rank = MPI.COMM_WORLD.Get_rank() + n_ranks = MPI.COMM_WORLD.Get_size() +except ImportError: + rank = 0 + n_ranks = 1 +import os +import sys +import numpy as np +from teca import * + + +def main(): + set_stack_trace_on_error() + set_stack_trace_on_mpi_error() + + if (len(sys.argv) != 7): + sys.stderr.write( + "\n\nUsage error:\n" + "test_deeplabv3p_ar_detect [deeplab model] [resnet model]" + "[mesh data regex] [baseline mesh]" + "[water vapor var] [num threads]\n\n" + ) + sys.exit(-1) + + # parse command line + deeplab_model = sys.argv[1] + resnet_model = sys.argv[2] + mesh_data_regex = sys.argv[3] + baseline_mesh = sys.argv[4] + water_vapor_var = sys.argv[5] + n_threads = int(sys.argv[6]) + + cf_reader = teca_cf_reader.New() + cf_reader.set_files_regex(mesh_data_regex) + cf_reader.set_t_axis_variable("") + cf_reader.set_periodic_in_x(1) + + deeplabv3p_ar_detect = teca_deeplabv3p_ar_detect.New() + deeplabv3p_ar_detect.set_input_connection( + cf_reader.get_output_port() + ) + deeplabv3p_ar_detect.set_variable_name(water_vapor_var) + deeplabv3p_ar_detect.set_num_threads(n_threads) + + deeplabv3p_ar_detect.build_model( + deeplab_model, + resnet_model + ) + + if os.path.exists(baseline_mesh): + # run the test + baseline_mesh_reader = teca_cartesian_mesh_reader.New() + baseline_mesh_reader.set_file_name(baseline_mesh) + + diff = teca_dataset_diff.New() + diff.set_input_connection(0, baseline_mesh_reader.get_output_port()) + diff.set_input_connection(1, deeplabv3p_ar_detect.get_output_port()) + diff.update() + else: + # make a baseline + if rank == 0: + sys.stdout.write( + 'generating baseline image ' + baseline_mesh + '\n' + ) + wri = teca_cartesian_mesh_writer.New() + wri.set_input_connection(deeplabv3p_ar_detect.get_output_port()) + wri.set_file_name(baseline_mesh) + + wri.update() + sys.exit(-1) + + +if __name__ == '__main__': + main() diff --git a/test/travis_ci/install_fedora_28.sh b/test/travis_ci/install_fedora_28.sh index cfc9be9e0..b60a7feae 100755 --- a/test/travis_ci/install_fedora_28.sh +++ b/test/travis_ci/install_fedora_28.sh @@ -28,7 +28,7 @@ echo ${IMAGE_VERSION} echo ${TECA_PYTHON_VERSION} echo ${TECA_DATA_REVISION} -pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib +pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib torch # install data files. svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index 05ad604cc..610d9cd58 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -9,7 +9,7 @@ brew update brew install mpich swig svn udunits # matplotlib currently doesn't have a formula # teca fails to locate mpi4py installed from brew -pip3 install numpy mpi4py matplotlib +pip3 install numpy mpi4py matplotlib torch # install data files. # On Apple svn is very very slow. On my mac book pro diff --git a/test/travis_ci/install_ubuntu_14_04.sh b/test/travis_ci/install_ubuntu_14_04.sh index 6dd6d58b2..aed677435 100755 --- a/test/travis_ci/install_ubuntu_14_04.sh +++ b/test/travis_ci/install_ubuntu_14_04.sh @@ -15,7 +15,7 @@ sudo apt-get install -qq -y gcc-5 g++-5 gfortran-5 swig3.0 \ libmpich-dev libhdf5-dev libnetcdf-dev libboost-program-options-dev \ python-dev subversion libudunits2-0 libudunits2-dev -pip install --user numpy mpi4py matplotlib +pip install --user numpy mpi4py matplotlib torch # install cmake manually because repo/ppa versions are too old wget https://cmake.org/files/v3.5/cmake-3.5.2-Linux-x86_64.tar.gz diff --git a/test/travis_ci/install_ubuntu_18_04.sh b/test/travis_ci/install_ubuntu_18_04.sh index 49a7b31ea..461c8eb58 100755 --- a/test/travis_ci/install_ubuntu_18_04.sh +++ b/test/travis_ci/install_ubuntu_18_04.sh @@ -24,7 +24,7 @@ echo ${IMAGE_VERSION} echo ${TECA_PYTHON_VERSION} echo ${TECA_DATA_REVISION} -pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib +pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib torch # install data files. svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data From dacfd05eb381419c450280c78ea43e95b0c6e7aa Mon Sep 17 00:00:00 2001 From: elbashandy Date: Wed, 1 Apr 2020 18:07:28 -0700 Subject: [PATCH 008/385] Making diff tolerance 1e-4 for the deeplabv3 test --- test/python/test_deeplabv3p_ar_detect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/python/test_deeplabv3p_ar_detect.py b/test/python/test_deeplabv3p_ar_detect.py index f939d6424..c99233538 100644 --- a/test/python/test_deeplabv3p_ar_detect.py +++ b/test/python/test_deeplabv3p_ar_detect.py @@ -57,6 +57,7 @@ def main(): diff = teca_dataset_diff.New() diff.set_input_connection(0, baseline_mesh_reader.get_output_port()) diff.set_input_connection(1, deeplabv3p_ar_detect.get_output_port()) + diff.set_tolerance(1e-4) diff.update() else: # make a baseline From 346e525a76dac520fc0b4a1c6a2e278793bd30c1 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Thu, 19 Mar 2020 16:02:17 -0700 Subject: [PATCH 009/385] Support time_step_of & time_step_of functions in python --- python/teca_py_data.i | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/python/teca_py_data.i b/python/teca_py_data.i index b0adf99df..3dccce04b 100644 --- a/python/teca_py_data.i +++ b/python/teca_py_data.i @@ -4,6 +4,7 @@ #include "teca_variant_array.h" #include "teca_array_collection.h" #include "teca_cartesian_mesh.h" +#include "teca_coordinate_util.h" #include "teca_mesh.h" #include "teca_table.h" #include "teca_table_collection.h" @@ -619,3 +620,45 @@ TECA_PY_CONST_CAST(teca_database) { TECA_PY_STR() } + +/*************************************************************************** + coordinate utilities + ***************************************************************************/ +%inline +%{ +struct teca_coordinate +{ +// given a human readable date string in YYYY-MM-DD hh:mm:ss format +// amd a list of floating point offset times inthe specified calendar +// and units find the closest time step. return 0 if successful +static +unsigned long time_step_of(p_teca_double_array time, bool lower, + const std::string &calendar, const std::string &units, + const std::string &date) +{ + unsigned long step = 0; + if (teca_coordinate_util::time_step_of(time, lower, calendar, units, date, step)) + { + TECA_PY_ERROR_NOW(PyExc_RuntimeError, "Failed to get time step from string") + } + return step; +} + +// given a time value (val), associated time units (units), and calendar +// (calendar), return a human-readable rendering of the date (date) in a +// strftime-format (format). return 0 if successful. +static +std::string time_to_string(double val, const std::string &calendar, + const std::string &units, const std::string &format) +{ + std::string date; + if (teca_coordinate_util::time_to_string(val, calendar, units, format, date)) + { + TECA_PY_ERROR_NOW(PyExc_RuntimeError, "Failed tp convert time to string") + } + return date; +} +}; +%} + +%include "teca_coordinate_util.h" From 0a59cafa61bd652d904b65be92ee9b4b281b6e8e Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Fri, 27 Mar 2020 10:37:44 -0700 Subject: [PATCH 010/385] fix data handling bug in ar apps --- apps/teca_bayesian_ar_detect.cpp | 6 +++--- apps/teca_pytorch_deeplabv3p_ar_detect.in | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/teca_bayesian_ar_detect.cpp b/apps/teca_bayesian_ar_detect.cpp index cd5f472d9..fef1765c7 100644 --- a/apps/teca_bayesian_ar_detect.cpp +++ b/apps/teca_bayesian_ar_detect.cpp @@ -290,7 +290,7 @@ int main(int argc, char **argv) << start_date << "\"") return -1; } - exec->set_start_index(first_step); + cf_writer->set_first_step(first_step); } // and end date @@ -305,7 +305,7 @@ int main(int argc, char **argv) << end_date << "\"") return -1; } - exec->set_end_index(last_step); + cf_writer->set_last_step(last_step); } } @@ -320,7 +320,7 @@ int main(int argc, char **argv) seg_atts.set("scheme",std::string("cascade_bard")); seg_atts.set("version",std::string("1.0")); seg_atts.set("note", - std::string("derived by thresholding ar_probability >= ") + + std::string("derived by thresholding ar_probability >= ") + std::to_string(ar_tag_threshold)); ar_tag->set_segmentation_variable_atts(seg_atts); diff --git a/apps/teca_pytorch_deeplabv3p_ar_detect.in b/apps/teca_pytorch_deeplabv3p_ar_detect.in index 5d73db23e..ff89ddf9b 100644 --- a/apps/teca_pytorch_deeplabv3p_ar_detect.in +++ b/apps/teca_pytorch_deeplabv3p_ar_detect.in @@ -199,7 +199,7 @@ def main(): ) # some minimal check for missing options - if (cf_reader.get_number_of_file_names() == 0 and + if (cf_reader.get_number_of_file_names() == 0 and not cf_reader.get_files_regex()): if rank == 0: raise ValueError( @@ -215,6 +215,7 @@ def main(): ) if args.start_date or args.end_date: + time_atts = atrs["time"] calendar = time_atts["calendar"] units = time_atts["units"] @@ -227,14 +228,14 @@ def main(): first_step = teca_coordinate.time_step_of( time, True, calendar, units, args.start_date ) - teca_exec.set_start_index(first_step) + cf_writer.set_first_step(first_step) # and end date if args.end_date: last_step = teca_coordinate.time_step_of( time, False, calendar, units, args.end_date ) - teca_exec.set_end_index(last_step) + cf_writer.set_last_step(last_step) # run the pipeline cf_writer.set_executive(teca_exec) From 5b042665b579930d1e159545adf9b481b29e0585 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 14 Apr 2020 12:58:25 -0700 Subject: [PATCH 011/385] fix output file names in cf_writer test --- test/CMakeLists.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a3cc26805..ec7a5fd12 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -89,7 +89,7 @@ teca_add_test(test_cf_writer_cam5_serial LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x lon -y lat -t time -c 2 -n 1 U850 V850 + -o "test_cf_writer_cam5_s_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n 1 U850 V850 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -103,7 +103,7 @@ teca_add_test(test_cf_writer_cam5_compression teca_add_test(test_cf_writer_cfsr_serial COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x longitude -y latitude + -o "test_cf_writer_NCEP_CFSR_s_%t%.nc" -s 0,-1 -x longitude -y latitude -b 65,110,10,55,0,0 -c 2 -n 1 elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -111,21 +111,21 @@ teca_add_test(test_cf_writer_cfsr_serial teca_add_test(test_cf_writer_era5_serial COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_%t%.%e%" -s 0,-1 -x longitude -y latitude -t time -c 2 -n 1 VIWVN + -o "test_cf_writer_era5_s_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n 1 VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cam5_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x lon -y lat -t time -c 2 -n ${TECA_TEST_CORES} U850 V850 + -o "test_cf_writer_cam5_t_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n ${TECA_TEST_CORES} U850 V850 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cfsr_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x longitude -y latitude + -o "test_cf_writer_NCEP_CFSR_t_%t%.nc" -s 0,-1 -x longitude -y latitude -b 65,110,10,55,0,0 -n ${TECA_TEST_CORES} -c 2 elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -133,21 +133,21 @@ teca_add_test(test_cf_writer_cfsr_threads teca_add_test(test_cf_writer_era5_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_%t%.%e%" -s 0,-1 -x longitude -y latitude -t time -c 2 -n ${TECA_TEST_CORES} VIWVN + -o "test_cf_writer_era5_t_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n ${TECA_TEST_CORES} VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cam5_mpi COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x lon -y lat -t time -c 2 -n 1 U850 V850 + -o "test_cf_writer_cam5_m_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n 1 U850 V850 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) teca_add_test(test_cf_writer_cfsr_mpi COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x longitude -y latitude + -o "test_cf_writer_NCEP_CFSR_m_%t%.nc" -s 0,-1 -x longitude -y latitude -b 65,110,10,55,0,0 -c 2 -n 1 elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -155,21 +155,21 @@ teca_add_test(test_cf_writer_cfsr_mpi teca_add_test(test_cf_writer_era5_mpi COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_%t%.%e%" -s 0,-1 -x longitude -y latitude -t time -c 2 -n 1 VIWVN + -o "test_cf_writer_era5_m_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n 1 VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cam5_mpi_threads COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x lon -y lat -t time -c 1 -n ${HALF_CORES} U850 V850 + -o "test_cf_writer_cam5_mt_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 1 -n ${HALF_CORES} U850 V850 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) teca_add_test(test_cf_writer_cfsr_mpi_threads COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" - -o "test_cf_writer_%t%.%e%" -s 0,-1 -x longitude -y latitude + -o "test_cf_writer_NCEP_CFSR_mt_%t%.nc" -s 0,-1 -x longitude -y latitude -b 65,110,10,55,0,0 -c 1 -n ${HALF_CORES} elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -177,7 +177,7 @@ teca_add_test(test_cf_writer_cfsr_mpi_threads teca_add_test(test_cf_writer_era5_mpi_threads COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_%t%.%e%" -s 0,-1 -x longitude -y latitude -t time -c 1 -n ${HALF_CORES} VIWVN + -o "test_cf_writer_era5_mt_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 1 -n ${HALF_CORES} VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) From 2af51d72db315aef4a6bba43e3d97dfe2e799f22 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 14 Apr 2020 12:59:39 -0700 Subject: [PATCH 012/385] cf reader fix time axis variable --- io/teca_cf_reader.cxx | 60 ++++++++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/io/teca_cf_reader.cxx b/io/teca_cf_reader.cxx index 5d8726de4..7d6e1583c 100644 --- a/io/teca_cf_reader.cxx +++ b/io/teca_cf_reader.cxx @@ -755,6 +755,12 @@ teca_metadata teca_cf_reader::get_output_metadata( read_variable_queue_t thread_pool(MPI_COMM_SELF, this->thread_pool_size, true, false); + // we rely t_axis_variable being empty to indicate either that + // there is no time axis, or that a time axis will be defined by + // other algorithm properties. This temporary is used for metadata + // consistency across those cases. + std::string t_axis_var = t_axis_variable; + std::vector step_count; if (!t_axis_variable.empty()) { @@ -762,7 +768,7 @@ teca_metadata teca_cf_reader::get_output_metadata( size_t n_files = files.size(); for (size_t i = 0; i < n_files; ++i) { - read_variable reader(path, files[i], i, this->t_axis_variable); + read_variable reader(path, files[i], i, t_axis_variable); read_variable_task_t task(reader); thread_pool.push_task(task); } @@ -821,6 +827,8 @@ teca_metadata teca_cf_reader::get_output_metadata( step_count.resize(n_t_vals, 1); t_axis = t; + + t_axis_var = "time"; } // infer the time from the filenames else if (! this->filename_time_template.empty()) @@ -946,6 +954,8 @@ teca_metadata teca_cf_reader::get_output_metadata( } t_axis = t; ) + + t_axis_var = "t"; } this->internals->metadata.set("variables", vars); @@ -953,12 +963,9 @@ teca_metadata teca_cf_reader::get_output_metadata( teca_metadata coords; coords.set("x_variable", x_axis_variable); - coords.set("y_variable", - (y_axis_variable.empty() ? "y" : y_axis_variable)); - coords.set("z_variable", - (z_axis_variable.empty() ? "z" : z_axis_variable)); - coords.set("t_variable", - (t_axis_variable.empty() ? "t" : t_axis_variable)); + coords.set("y_variable", (y_axis_variable.empty() ? "y" : y_axis_variable)); + coords.set("z_variable", (z_axis_variable.empty() ? "z" : z_axis_variable)); + coords.set("t_variable", t_axis_var); coords.set("x", x_axis); coords.set("y", y_axis); coords.set("z", z_axis); @@ -1060,6 +1067,18 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, return nullptr; } + // get names, need to be careful since some of these depend + // on run time information, ex user can specific a time axis + // via algorithm properties + std::string x_axis_var; + std::string y_axis_var; + std::string z_axis_var; + std::string t_axis_var; + coords.get("x_variable", x_axis_var); + coords.get("y_variable", y_axis_var); + coords.get("z_variable", z_axis_var); + coords.get("t_variable", t_axis_var); + // get request unsigned long time_step = 0; double t = 0.0; @@ -1175,9 +1194,9 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, // create output dataset p_teca_cartesian_mesh mesh = teca_cartesian_mesh::New(); - mesh->set_x_coordinates(x_axis_variable, out_x); - mesh->set_y_coordinates(y_axis_variable, out_y); - mesh->set_z_coordinates(z_axis_variable, out_z); + mesh->set_x_coordinates(x_axis_var, out_x); + mesh->set_y_coordinates(y_axis_var, out_y); + mesh->set_z_coordinates(z_axis_var, out_z); mesh->set_time(t); mesh->set_time_step(time_step); mesh->set_whole_extent(whole_extent); @@ -1200,7 +1219,7 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, teca_metadata time_atts; std::string calendar; std::string units; - if (!atrs.get("time", time_atts) + if (!atrs.get(t_axis_var, time_atts) && !time_atts.get("calendar", calendar) && !time_atts.get("units", units)) { @@ -1219,16 +1238,16 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, out_atrs.set(arrays[i], atrs.get(arrays[i])); // pass coordinate axes attributes - if (atrs.has(x_axis_variable)) - out_atrs.set(x_axis_variable, atrs.get(x_axis_variable)); - if (atrs.has(y_axis_variable)) - out_atrs.set(y_axis_variable, atrs.get(y_axis_variable)); - if (atrs.has(z_axis_variable)) - out_atrs.set(z_axis_variable, atrs.get(z_axis_variable)); + if (atrs.has(x_axis_var)) + out_atrs.set(x_axis_var, atrs.get(x_axis_var)); + if (atrs.has(y_axis_var)) + out_atrs.set(y_axis_var, atrs.get(y_axis_var)); + if (atrs.has(z_axis_var)) + out_atrs.set(z_axis_var, atrs.get(z_axis_var)); if (!time_atts.empty()) - out_atrs.set("time", time_atts); + out_atrs.set(t_axis_var, time_atts); - mesh->get_metadata().set("attributes", out_atrs); + md.set("attributes", out_atrs); // figure out the mapping between our extent and netcdf // representation @@ -1338,7 +1357,7 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, std::vector starts(n_dims); std::vector counts(n_dims); size_t n_vals = 1; - if (dim_names->get(0) == this->t_axis_variable) + if (!t_axis_variable.empty() && (dim_names->get(0) == t_axis_variable)) { starts[0] = offs; counts[0] = 1; @@ -1350,6 +1369,7 @@ const_p_teca_dataset teca_cf_reader::execute(unsigned int port, counts[0] = dim_len; n_vals = dim_len; } + for (unsigned int ii = 1; ii < n_dims; ++ii) { size_t dim_len = dims->get(ii); From 7a128d272fee153d3895c931c915a7c4ad3382fc Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Thu, 27 Feb 2020 10:26:32 -0800 Subject: [PATCH 013/385] document confusing code in test_descriptive_stats --- test/test_descriptive_statistics.cpp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/test/test_descriptive_statistics.cpp b/test/test_descriptive_statistics.cpp index 5db1ed2ae..52cd1b893 100644 --- a/test/test_descriptive_statistics.cpp +++ b/test/test_descriptive_statistics.cpp @@ -3,6 +3,8 @@ #include "teca_normalize_coordinates.h" #include "teca_dataset_diff.h" #include "teca_descriptive_statistics.h" +#include "teca_table.h" +#include "teca_table_to_stream.h" #include "teca_file_util.h" #include "teca_table_reader.h" #include "teca_programmable_reduce.h" @@ -12,7 +14,6 @@ #include "teca_test_util.h" #include "teca_mpi_manager.h" #include "teca_system_interface.h" -#include "teca_table.h" #include "teca_mpi.h" #include @@ -69,24 +70,32 @@ int main(int argc, char **argv) if (argc < 3) { cerr << endl << "Usage error:" << endl - << "test_map_descriptive_statistics [input regex] [test baseline] [first step = 0] " - << "[last step = -1] [num threads = 1] [array 0 =] ... [array n =]" + << "test_map_descriptive_statistics [number of files] [input regex] " + "[test baseline] [first step = 0] [last step = -1] [num threads = 1] " + "[array 0 =] ... [array n =]" << endl << endl; return -1; } + // handle a list of files on the command line. + // the first argument passed is the number of files + // or 0 if a regex is to be used. this case was added + // to test processing datasets that do not contain a + // time axis, but have one stored externally. int i = 1; int files_num = atoi(argv[i++]); + // get the list of files (if files_num != 0) vector files; vector time_values; for (; i <= (files_num + 1); ++i) files.push_back(argv[i]); + // generate some time values for the passed files for (; i <= (2*files_num + 1); ++i) time_values.push_back(atof(argv[i])); - // regex is set + // files_num == 0 so a regex should have been given if (files.empty()) files.push_back(argv[i++]); @@ -119,6 +128,7 @@ int main(int argc, char **argv) p_teca_cf_reader cf_reader = teca_cf_reader::New(); if (files_num) { + // we pass the time axis with a list of files cf_reader->set_t_axis_variable(""); cf_reader->set_t_values(time_values); cf_reader->set_t_calendar("noleap"); @@ -127,6 +137,7 @@ int main(int argc, char **argv) } else { + // time axis is obtained from CF2 compliant dataset cf_reader->set_files_regex(files[0]); } @@ -169,10 +180,13 @@ int main(int argc, char **argv) // make a baseline if (rank == 0) cerr << "generating baseline image " << baseline << endl; + + p_teca_table_to_stream tts = teca_table_to_stream::New(); + tts->set_input_connection(cal->get_output_port()); + p_teca_table_writer table_writer = teca_table_writer::New(); - table_writer->set_input_connection(cal->get_output_port()); + table_writer->set_input_connection(tts->get_output_port()); table_writer->set_file_name(baseline.c_str()); - table_writer->set_output_format_bin(); table_writer->update(); } From 83aae6bd80cfed938531f0e2f973d0fb22718f52 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Fri, 3 Apr 2020 14:07:02 -0700 Subject: [PATCH 014/385] fix build without NetCDF disable tests and applications that reuire NetCDF when NetCDF is not found --- CMake/teca_app.cmake | 17 ++++++++++++++--- CMake/teca_python.cmake | 31 +++++++++++++++++++++++++++++++ apps/CMakeLists.txt | 36 ++++++++++++++++++------------------ test/CMakeLists.txt | 10 +++++++--- 4 files changed, 70 insertions(+), 24 deletions(-) diff --git a/CMake/teca_app.cmake b/CMake/teca_app.cmake index 70ddee83b..263449b8d 100644 --- a/CMake/teca_app.cmake +++ b/CMake/teca_app.cmake @@ -5,11 +5,20 @@ # ) function (teca_add_app app_name) set(opt_args) - set(val_args FEATURES) - set(array_args SOURCES LIBS) + set(val_args) + set(array_args SOURCES LIBS FEATURES) cmake_parse_arguments(APP "${opt_args}" "${val_args}" "${array_args}" ${ARGN}) - if (APP_FEATURES) + set(APP_ENABLED ON) + if (DEFINED APP_FEATURES) + foreach(feature ${APP_FEATURES}) + if (NOT feature) + set(APP_ENABLED OFF) + endif() + endforeach() + endif() + if (APP_ENABLED) + message(STATUS "command line application ${app_name} -- enabled") if (NOT APP_SOURCES) set(APP_SOURCES "${app_name}.cpp") endif() @@ -20,5 +29,7 @@ function (teca_add_app app_name) ${APP_LIBS}) endif() install(TARGETS ${app_name} RUNTIME DESTINATION ${BIN_PREFIX}) + else() + message(STATUS "command line application ${app_name} -- disabled") endif() endfunction() diff --git a/CMake/teca_python.cmake b/CMake/teca_python.cmake index 1d676187d..6ee3f7397 100644 --- a/CMake/teca_python.cmake +++ b/CMake/teca_python.cmake @@ -35,3 +35,34 @@ function(teca_py_install_apps) # TODO compile the sources endif() endfunction() + +# teca_add_python_app(name +# SOURCES -- optional, source files to comile +# FEATURES -- optional, boolean condition decribing feature dependencies +# ) +function (teca_add_python_app app_name) + if (TECA_HAS_PYTHON) + set(opt_args) + set(val_args) + set(array_args SOURCES FEATURES) + cmake_parse_arguments(APP + "${opt_args}" "${val_args}" "${array_args}" ${ARGN}) + set(APP_ENABLED ON) + if (DEFINED APP_FEATURES) + foreach(feature ${APP_FEATURES}) + if (NOT feature) + set(APP_ENABLED OFF) + endif() + endforeach() + endif() + if (APP_ENABLED) + message(STATUS "command line application ${app_name} -- enabled") + if (NOT APP_SOURCES) + set(APP_SOURCES "${app_name}.in") + endif() + teca_py_install_apps(${APP_SOURCES}) + else() + message(STATUS "command line application ${app_name} -- disabled") + endif() + endif() +endfunction() diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index e099b6e56..52e2be65a 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -16,27 +16,27 @@ if (TECA_HAS_BOOST) endif() teca_add_app(teca_tc_detect LIBS ${teca_app_link} - FEATURES (TECA_HAS_BOOST AND TECA_HAS_NETCDF AND TECA_HAS_UDUNITS)) + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) teca_add_app(teca_tc_wind_radii LIBS ${teca_app_link} - FEATURES (TECA_HAS_BOOST AND TECA_HAS_NETCDF AND TECA_HAS_UDUNITS)) + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) teca_add_app(teca_tc_trajectory LIBS ${teca_app_link} - FEATURES (TECA_HAS_BOOST AND TECA_HAS_UDUNITS)) + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_UDUNITS}) teca_add_app(teca_metadata_probe LIBS ${teca_app_link} - FEATURES (TECA_HAS_BOOST AND TECA_HAS_NETCDF AND TECA_HAS_UDUNITS)) - -teca_add_app(teca_bayesian_ar_detect LIBS ${teca_app_link} - FEATURES (TECA_HAS_BOOST AND TECA_HAS_NETCDF AND TECA_HAS_UDUNITS)) - -teca_py_install_apps( - teca_convert_table.in - teca_dataset_metadata.in - teca_pytorch_deeplabv3p_ar_detect.in - teca_event_filter.in - teca_profile_explorer.in - teca_tc_stats.in - teca_tc_wind_radii_stats.in - teca_tc_trajectory_scalars.in - ) + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) + +teca_add_python_app(teca_convert_table) + +teca_add_python_app(teca_dataset_metadata + FEATURES ${TECA_HAS_NETCDF}) + +teca_add_python_app(teca_pytorch_deeplabv3p_ar_detect + FEATURES ${TECA_HAS_NETCDF}) + +teca_add_python_app(teca_event_filter) +teca_add_python_app(teca_profile_explorer) +teca_add_python_app(teca_tc_stats) +teca_add_python_app(teca_tc_wind_radii_stats) +teca_add_python_app(teca_tc_trajectory_scalars) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ec7a5fd12..bb7a55e8d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -419,6 +419,7 @@ teca_add_test(test_tc_wind_radii_serial "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" "32" "1" "1" "0" "-1" + FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_tc_wind_radii_mpi @@ -426,7 +427,7 @@ teca_add_test(test_tc_wind_radii_mpi "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" "32" "1" "1" "0" "-1" - FEATURES ${TECA_HAS_MPI} + FEATURES ${TECA_HAS_MPI} ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_tc_wind_radii_mpi_threads @@ -434,7 +435,7 @@ teca_add_test(test_tc_wind_radii_mpi_threads "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" "32" "1" ${HALF_CORES} "0" "-1" - FEATURES ${TECA_HAS_MPI} + FEATURES (${TECA_HAS_MPI} AND ${TECA_HAS_NETCDF}) REQ_TECA_DATA) teca_add_test(test_tc_wind_radii_threads @@ -442,6 +443,7 @@ teca_add_test(test_tc_wind_radii_threads "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" "32" "1" ${TECA_TEST_CORES} "0" "-1" + FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_type_select @@ -501,6 +503,7 @@ teca_add_test(test_binary_segmentation LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} COMMAND test_binary_segmentation "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" prw 50 75 "prw_segmentation_50-75_%t%.%e%" + FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_bayesian_ar_detect_serial @@ -511,6 +514,7 @@ teca_add_test(test_bayesian_ar_detect_serial "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" "${TECA_DATA_ROOT}/test_bayesian_ar_detect.bin" prw "bayesian_ar_detect_%t%.vtk" 1 0 4 + FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_bayesian_ar_detect_threads @@ -526,7 +530,7 @@ teca_add_test(test_bayesian_ar_detect_mpi_threads "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" "${TECA_DATA_ROOT}/test_bayesian_ar_detect.bin" prw "bayesian_ar_detect_%t%.vtk" ${HALF_CORES} 0 4 - FEATURES ${TECA_HAS_MPI} + FEATURES (${TECA_HAS_MPI} AND ${TECA_HAS_NETCDF}) REQ_TECA_DATA) teca_add_test(test_normalize_coordinates_pass_through From 38f5d4ad162b217d9b65aea10e991af3487d9710 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Fri, 3 Apr 2020 16:06:19 -0700 Subject: [PATCH 015/385] add build support for parallel NetCDF Adds cmake code to detect if NetCDF was compiled with MPI support and exposes via pre-processor define --- CMake/FindNetCDF.cmake | 157 +++++++++++----------------- CMakeLists.txt | 12 +++ io/CMakeLists.txt | 3 +- teca_config.h.in | 1 + test/travis_ci/install_fedora_28.sh | 14 +-- 5 files changed, 78 insertions(+), 109 deletions(-) diff --git a/CMake/FindNetCDF.cmake b/CMake/FindNetCDF.cmake index beccd3ff9..7f7a9957b 100644 --- a/CMake/FindNetCDF.cmake +++ b/CMake/FindNetCDF.cmake @@ -32,133 +32,96 @@ # target_link_libraries (uses_everthing ${NETCDF_LIBRARIES}) # target_link_libraries (only_uses_f90 ${NETCDF_F90_LIBRARIES}) + #search starting from user editable cache var if (NETCDF_INCLUDE_DIR AND NETCDF_LIBRARY) # Already in cache, be silent set (NETCDF_FIND_QUIETLY TRUE) -endif () - -# find the library -# first look where the user told us -if (NETCDF_DIR) - find_library (NETCDF_LIBRARY NAMES netcdf - PATHS "${NETCDF_DIR}/lib" "${NETCDF_DIR}/lib64" - NO_DEFAULT_PATH) -endif() - -# next look in LD_LIBRARY_PATH for libraries -find_library (NETCDF_LIBRARY NAMES netcdf - PATHS ENV LD_LIBRARY_PATH NO_DEFAULT_PATH) - -# finally CMake can look -find_library (NETCDF_LIBRARY NAMES netcdf) - -mark_as_advanced (NETCDF_LIBRARY) -set (NETCDF_C_LIBRARIES ${NETCDF_LIBRARY}) - -# find the header -# first look where the user told us -if (NETCDF_DIR) - find_path (NETCDF_INCLUDE_DIR netcdf.h - PATHS "${NETCDF_DIR}/include" NO_DEFAULT_PATH) endif() -# then look relative to library dir -get_filename_component(NETCDF_LIBRARY_DIR - ${NETCDF_LIBRARY} DIRECTORY) - -find_path (NETCDF_INCLUDE_DIR netcdf.h - PATHS "${NETCDF_LIBRARY_DIR}/../include" - NO_DEFAULT_PATH) - -# finally CMake can look -find_path (NETCDF_INCLUDE_DIR netcdf.h) - -mark_as_advanced (NETCDF_INCLUDE_DIR) -set (NETCDF_C_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR}) - - -#start finding requested language components -set (NetCDF_libs "") -set (NetCDF_includes "${NETCDF_INCLUDE_DIR}") - -get_filename_component (NetCDF_lib_dirs "${NETCDF_LIBRARY}" PATH) -set (NETCDF_HAS_INTERFACES "YES") # will be set to NO if we're missing any interfaces - -macro (NetCDF_check_interface lang header libs) - if (NETCDF_${lang}) - # find the library +# find the library +# use package config, this works well on systems that make +# use of modules to manage installs not in the standard +# locations that cmake knows about +find_package(PkgConfig REQUIRED) +pkg_check_modules(NC_TMP netcdf QUIET) +if (NC_TMP_FOUND AND NC_TMP_LINK_LIBRARIES AND NC_TMP_LIBRARY_DIRS AND NC_TMP_INCLUDE_DIRS) + set(NETCDF_LIBRARY_DIR ${NC_TMP_LIBRARY_DIRS}) + set(NETCDF_LIBRARY ${NC_TMP_LINK_LIBRARIES}) + set(NETCDF_INCLUDE_DIR ${NC_TMP_INCLUDE_DIRS}) +else() + # package config failed, use cmake # first look where the user told us if (NETCDF_DIR) - find_library (NETCDF_${lang}_LIBRARY NAMES netcdf + find_library(NETCDF_LIBRARY NAMES netcdf PATHS "${NETCDF_DIR}/lib" "${NETCDF_DIR}/lib64" NO_DEFAULT_PATH) endif() # next look in LD_LIBRARY_PATH for libraries - find_library (NETCDF_${lang}_LIBRARY NAMES netcdf + find_library(NETCDF_LIBRARY NAMES netcdf PATHS ENV LD_LIBRARY_PATH NO_DEFAULT_PATH) # finally CMake can look - find_library (NETCDF_${lang}_LIBRARY NAMES netcdf) + find_library(NETCDF_LIBRARY NAMES netcdf) + + message(STATUS ${NETCDF_LIBRARY}) +endif() - # find the header +# if we can find the library it is found now +# record what we have +mark_as_advanced (NETCDF_LIBRARY) +set (NETCDF_C_LIBRARIES ${NETCDF_LIBRARY}) + +# find the header +# package config failed, use cmake +if (NOT NC_TMP_FOUND OR NOT NC_TMP_LINK_LIBRARIES OR NOT NC_TMP_LIBRARY_DIRS OR NOT NC_TMP_INCLUDE_DIRS) # first look where the user told us if (NETCDF_DIR) - find_path (NETCDF_${lang}_INCLUDE_DIR netcdf.h + find_path (NETCDF_INCLUDE_DIR netcdf.h PATHS "${NETCDF_DIR}/include" NO_DEFAULT_PATH) endif() # then look relative to library dir - get_filename_component(NETCDF_${lang}_LIBRARY_DIR - ${NETCDF_${lang}_LIBRARY} DIRECTORY CACHE) + get_filename_component(NETCDF_LIBRARY_DIR + ${NETCDF_LIBRARY} DIRECTORY) - find_path (NETCDF_${lang}_INCLUDE_DIR netcdf.h - PATHS "${NETCDF_${lang}_LIBRARY_DIR}/../include" + find_path (NETCDF_INCLUDE_DIR netcdf.h + PATHS "${NETCDF_LIBRARY_DIR}/../include" NO_DEFAULT_PATH) - # finally CMake can look - find_path (NETCDF_${lang}_INCLUDE_DIR netcdf.h) - - #export to internal varS that rest of project can use directly - mark_as_advanced (NETCDF_${lang}_INCLUDE_DIR NETCDF_${lang}_LIBRARY) - - set (NETCDF_${lang}_LIBRARIES ${NETCDF_${lang}_LIBRARY}) - set (NETCDF_${lang}_INCLUDE_DIRS ${NETCDF_${lang}_INCLUDE_DIR}) - - if (NETCDF_${lang}_INCLUDE_DIR AND NETCDF_${lang}_LIBRARY) - list (APPEND NetCDF_libs ${NETCDF_${lang}_LIBRARY}) - list (APPEND NetCDF_includes ${NETCDF_${lang}_INCLUDE_DIR}) - else () - set (NETCDF_HAS_INTERFACES "NO") - message (STATUS "Failed to find NetCDF interface for ${lang}") - endif () - endif () -endmacro (NetCDF_check_interface) - -list (FIND NetCDF_FIND_COMPONENTS "CXX" _nextcomp) -if (_nextcomp GREATER -1) - set (NETCDF_CXX 1) -endif () -list (FIND NetCDF_FIND_COMPONENTS "F77" _nextcomp) -if (_nextcomp GREATER -1) - set (NETCDF_F77 1) -endif () -list (FIND NetCDF_FIND_COMPONENTS "F90" _nextcomp) -if (_nextcomp GREATER -1) - set (NETCDF_F90 1) -endif () -NetCDF_check_interface (CXX netcdfcpp.h netcdf_c++) -NetCDF_check_interface (F77 netcdf.inc netcdff) -NetCDF_check_interface (F90 netcdf.mod netcdff) + # CMake can look + find_path(NETCDF_INCLUDE_DIR netcdf.h) +endif() + +# look for header file that indicates MPI support +set(NETCDF_IS_PARALLEL FALSE) +find_file(NETCDF_PAR_INCLUDE_DIR netcdf_par.h + PATHS ${NETCDF_INCLUDE_DIR} NO_DEFAULT_PATH) +if (NETCDF_PAR_INCLUDE_DIR) + set(NETCDF_IS_PARALLEL TRUE) +endif() + +# if we can find the headers they are found now +# record what we have +mark_as_advanced(NETCDF_INCLUDE_DIR) +mark_as_advanced(NETCDF_IS_PARALLEL) +mark_as_advanced(NETCDF_PAR_INCLUDE_DIR) +set(NETCDF_C_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR}) + +#start finding requested language components +set (NetCDF_libs "") +set (NetCDF_includes "${NETCDF_INCLUDE_DIR}") + +get_filename_component (NetCDF_lib_dirs "${NETCDF_LIBRARY}" PATH) #export accumulated results to internal varS that rest of project can depend on -list (APPEND NetCDF_libs "${NETCDF_C_LIBRARIES}") -set (NETCDF_LIBRARIES ${NetCDF_libs}) -set (NETCDF_INCLUDE_DIRS ${NetCDF_includes}) +list(APPEND NetCDF_libs "${NETCDF_C_LIBRARIES}") +set(NETCDF_LIBRARIES ${NetCDF_libs}) +set(NETCDF_INCLUDE_DIRS ${NetCDF_includes}) # handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE if # all listed variables are TRUE include (FindPackageHandleStandardArgs) find_package_handle_standard_args (NetCDF - DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDE_DIRS NETCDF_HAS_INTERFACES) + DEFAULT_MSG NETCDF_LIBRARIES NETCDF_INCLUDE_DIRS) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d24e70b6..5c7967767 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,6 +114,17 @@ if (NETCDF_FOUND AND ((DEFINED TECA_HAS_NETCDF AND TECA_HAS_NETCDF) OR (NOT DEFINED TECA_HAS_NETCDF))) message(STATUS "NetCDF features -- enabled") set(tmp ON) + teca_interface_library(NetCDF SYSTEM + INCLUDES ${NETCDF_INCLUDE_DIRS} + LIBRARIES ${NETCDF_LIBRARIES}) + if (NOT NETCDF_IS_PARALLEL) + message(STATUS "Check NetcCDF for MPI support -- not found") + if (REQUIRE_NETCDF_MPI) + message(FATAL_ERROR "NetCDF MPI support -- required but not found.") + endif() + else() + message(STATUS "Check NetCDF for MPI support -- enabled") + endif() elseif (REQUIRE_NETCDF) message(FATAL_ERROR "NetCDF features -- required but not found. set NETCDF_DIR to enable.") else() @@ -121,6 +132,7 @@ else() message(WARNING "NetCDF is required for CF-2 I/O") endif() set(TECA_HAS_NETCDF ${tmp} CACHE BOOL "NetCDF features") +set(TECA_HAS_NETCDF_MPI ${NETCDF_IS_PARALLEL} CACHE BOOL "NetCDF MPI support") # configure for libxlsxwriter set(tmp OFF) diff --git a/io/CMakeLists.txt b/io/CMakeLists.txt index 68c2a3c8b..981329ee3 100644 --- a/io/CMakeLists.txt +++ b/io/CMakeLists.txt @@ -23,8 +23,7 @@ set(teca_io_link) if (TECA_HAS_NETCDF) list(APPEND teca_io_srcs teca_netcdf_util.cxx teca_cf_reader.cxx teca_cf_writer.cxx) - include_directories(SYSTEM ${NETCDF_INCLUDE_DIRS}) - list(APPEND teca_io_link ${NETCDF_LIBRARIES}) + list(APPEND teca_io_link NetCDF) if (TECA_HAS_OPENSSL) include_directories(SYSTEM ${OPENSSL_INCLUDE_DIR}) list(APPEND teca_io_link ${OPENSSL_LIBRARIES}) diff --git a/teca_config.h.in b/teca_config.h.in index fba85eb79..b8a933e0c 100644 --- a/teca_config.h.in +++ b/teca_config.h.in @@ -3,6 +3,7 @@ #cmakedefine TECA_HAS_REGEX #cmakedefine TECA_HAS_NETCDF +#cmakedefine TECA_HAS_NETCDF_MPI #cmakedefine TECA_HAS_MPI #cmakedefine TECA_HAS_BOOST #cmakedefine TECA_HAS_VTK diff --git a/test/travis_ci/install_fedora_28.sh b/test/travis_ci/install_fedora_28.sh index b60a7feae..3d8c56eae 100755 --- a/test/travis_ci/install_fedora_28.sh +++ b/test/travis_ci/install_fedora_28.sh @@ -7,19 +7,13 @@ dnf update -qq -y # install deps # use PIP for Python packages dnf install -qq -y environment-modules which git-all gcc-c++ gcc-gfortran \ - make cmake-3.11.0-1.fc28 swig mpich-devel hdf5-devel netcdf-devel boost-devel \ - python-devel python-pip python3-devel python3-pip subversion udunits2 \ - udunits2-devel zlib-devel openssl-devel wget - -git clone https://github.com/jmcnamara/libxlsxwriter.git -cd libxlsxwriter -make -make install -cd .. + make cmake-3.11.0-1.fc28 swig mpich-devel hdf5-mpich-devel \ + netcdf-mpich-devel boost-devel python3-devel python3-pip subversion \ + udunits2 udunits2-devel zlib-devel openssl-devel wget source /usr/share/Modules/init/bash -module load mpi +module load mpi/mpich-x86_64 echo ${TRAVIS_BRANCH} echo ${BUILD_TYPE} From ebc448a1b8acfd73484937437af55f5793fa5a5c Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Sun, 12 Apr 2020 11:04:16 -0700 Subject: [PATCH 016/385] use more cores in regression tests * Use CMake to detect the number of available cores. * use the number of phyisical cores for parallel tests and number of logical cores for over subscription tests by default. --- CMakeLists.txt | 28 +++++++++++++--- test/CMakeLists.txt | 66 +++++++++++++++++++------------------- test/python/CMakeLists.txt | 47 ++++++++++++++------------- 3 files changed, 80 insertions(+), 61 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c7967767..1345b46b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -346,15 +346,33 @@ add_subdirectory(paraview) set(BUILD_TESTING OFF CACHE BOOL "Enable tests") if (BUILD_TESTING) include(CTest) + include(ProcessorCount) configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake" @ONLY) - set(TECA_TEST_CORES 4 CACHE STRING - "Number of cores for use in parallel tests") - math(EXPR HALF_CORES "${TECA_TEST_CORES}/2") - if (HALF_CORES LESS 1) - message(FATAL_ERROR "Parallel test require at lest 2 cores") + + # figure out how many cores we can use for parallel tests + set(TECA_TEST_CORES 0 CACHE STRING + "Max number of cores for use in parallel tests") + if (TECA_TEST_CORES LESS 1) + ProcessorCount(LOGICAL_CORES) + if (LOGICAL_CORES EQUAL 0) + set(LOGICAL_CORES 4) + endif() + else() + math(EXPR LOGICAL_CORES "${TECA_TEST_CORES}*2") endif() + math(EXPR PHYSICAL_CORES "${LOGICAL_CORES}/2") + set(TEST_CORES ${PHYSICAL_CORES}) + math(EXPR HALF_TEST_CORES "${TEST_CORES}/2") + set(TWICE_TEST_CORES ${LOGICAL_CORES}) + if (HALF_TEST_CORES LESS 1) + message(WARNING "Parallel tests require at lest 2 phyisical cores") + endif() + message(STATUS "regression testing -- enabled (${TEST_CORES} cores).") + add_subdirectory(test) +else() + message(STATUS "regression testing -- disbaled") endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bb7a55e8d..aa773a676 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -56,14 +56,14 @@ teca_add_test(test_cf_reader_cam5 LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} COMMAND test_cf_reader -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_reader_%t%.%e%" -s 1,2 -x lon -y lat -t time U850 V850 + -o "test_cf_reader_cam5_%t%.%e%" -s 1,2 -x lon -y lat -t time U850 V850 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_reader_cfsr COMMAND test_cf_reader -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" - -o "test_cf_reader_%t%.%e%" -s 1,2 -x longitude -y latitude + -o "test_cf_reader_cfsr_%t%.%e%" -s 1,2 -x longitude -y latitude -b 65,110,10,55,0,0 elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -118,7 +118,7 @@ teca_add_test(test_cf_writer_era5_serial teca_add_test(test_cf_writer_cam5_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_cam5_t_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n ${TECA_TEST_CORES} U850 V850 + -o "test_cf_writer_cam5_t_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n ${TEST_CORES} U850 V850 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -126,26 +126,26 @@ teca_add_test(test_cf_writer_cfsr_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" -o "test_cf_writer_NCEP_CFSR_t_%t%.nc" -s 0,-1 -x longitude -y latitude - -b 65,110,10,55,0,0 -n ${TECA_TEST_CORES} -c 2 elevation + -b 65,110,10,55,0,0 -n ${TEST_CORES} -c 2 elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_era5_threads COMMAND test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_t_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n ${TECA_TEST_CORES} VIWVN + -o "test_cf_writer_era5_t_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n ${TEST_CORES} VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cam5_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" -o "test_cf_writer_cam5_m_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 2 -n 1 U850 V850 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) teca_add_test(test_cf_writer_cfsr_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" -o "test_cf_writer_NCEP_CFSR_m_%t%.nc" -s 0,-1 -x longitude -y latitude -b 65,110,10,55,0,0 -c 2 -n 1 elevation @@ -153,31 +153,31 @@ teca_add_test(test_cf_writer_cfsr_mpi REQ_TECA_DATA) teca_add_test(test_cf_writer_era5_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" -o "test_cf_writer_era5_m_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 2 -n 1 VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_cam5_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - -o "test_cf_writer_cam5_mt_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 1 -n ${HALF_CORES} U850 V850 + -o "test_cf_writer_cam5_mt_%t%.nc" -s 0,-1 -x lon -y lat -t time -c 1 -n ${HALF_TEST_CORES} U850 V850 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) teca_add_test(test_cf_writer_cfsr_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/NCEP_CFSR_0\\.5_1979\\.nc" -o "test_cf_writer_NCEP_CFSR_mt_%t%.nc" -s 0,-1 -x longitude -y latitude - -b 65,110,10,55,0,0 -c 1 -n ${HALF_CORES} elevation + -b 65,110,10,55,0,0 -c 1 -n ${HALF_TEST_CORES} elevation FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_cf_writer_era5_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_cf_writer + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_cf_writer -i "${TECA_DATA_ROOT}/e5\.oper\.an\.vinteg\.162_072_viwvn.*\.nc" - -o "test_cf_writer_era5_mt_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 1 -n ${HALF_CORES} VIWVN + -o "test_cf_writer_era5_mt_%t%.nc" -s 0,-1 -x longitude -y latitude -t time -c 1 -n ${HALF_TEST_CORES} VIWVN FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -292,14 +292,14 @@ teca_add_test(test_descriptive_statistics_threads LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} COMMAND test_descriptive_statistics 0 "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - "${TECA_DATA_ROOT}/test_descriptive_statistics.bin" 0 -1 ${TECA_TEST_CORES} + "${TECA_DATA_ROOT}/test_descriptive_statistics.bin" 0 -1 ${TEST_CORES} TMQ T200 T500 FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(test_descriptive_statistics_mpi LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_descriptive_statistics 0 + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_descriptive_statistics 0 "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" "${TECA_DATA_ROOT}/test_descriptive_statistics.bin" 0 -1 1 TMQ T200 T500 FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} @@ -307,9 +307,9 @@ teca_add_test(test_descriptive_statistics_mpi teca_add_test(test_descriptive_statistics_mpi_threads LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_descriptive_statistics 0 + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_descriptive_statistics 0 "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-[0-9][0-9]-10800\\.nc" - "${TECA_DATA_ROOT}/test_descriptive_statistics.bin" 0 -1 ${HALF_CORES} + "${TECA_DATA_ROOT}/test_descriptive_statistics.bin" 0 -1 ${HALF_TEST_CORES} TMQ T200 T500 FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) @@ -334,7 +334,7 @@ teca_add_test(test_tc_candidates_serial REQ_TECA_DATA) teca_add_test(test_tc_candidates_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_tc_candidates + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_tc_candidates "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 1 U850 V850 UBOT VBOT PSL T500 T200 Z1000 Z200 -20 20 @@ -347,15 +347,15 @@ teca_add_test(test_tc_candidates_threads LIBS teca_core teca_data teca_io teca_alg ${teca_test_link} COMMAND test_tc_candidates "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" - "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${TECA_TEST_CORES} + "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${TEST_CORES} U850 V850 UBOT VBOT PSL T500 T200 Z1000 Z200 -20 20 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} REQ_TECA_DATA) teca_add_test(test_tc_candidates_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_tc_candidates + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_tc_candidates "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" - "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${HALF_CORES} + "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${HALF_TEST_CORES} U850 V850 UBOT VBOT PSL T500 T200 Z1000 Z200 -20 20 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} REQ_TECA_DATA) @@ -386,7 +386,7 @@ teca_add_test(test_table_reader_distribute_serial REQ_TECA_DATA) teca_add_test(test_table_reader_distribute_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_table_reader_distribute + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_table_reader_distribute "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" "${TECA_DATA_ROOT}/test_table_reader_distribute_20.bin" "step" 0 -1 1 @@ -400,14 +400,14 @@ teca_add_test(test_table_reader_distribute_threads COMMAND test_table_reader_distribute "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" "${TECA_DATA_ROOT}/test_table_reader_distribute_20.bin" - "step" 0 -1 ${TECA_TEST_CORES} + "step" 0 -1 ${TEST_CORES} REQ_TECA_DATA) teca_add_test(test_table_reader_distribute_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_table_reader_distribute + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_table_reader_distribute "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" "${TECA_DATA_ROOT}/test_table_reader_distribute_20.bin" - "step" 0 -1 ${HALF_CORES} + "step" 0 -1 ${HALF_TEST_CORES} FEATURES ${TECA_HAS_MPI} REQ_TECA_DATA) @@ -423,7 +423,7 @@ teca_add_test(test_tc_wind_radii_serial REQ_TECA_DATA) teca_add_test(test_tc_wind_radii_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} test_tc_wind_radii "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800.bin" + COMMAND ${MPIEXEC} -n ${TEST_CORES} test_tc_wind_radii "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800.bin" "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" "32" "1" "1" "0" "-1" @@ -431,10 +431,10 @@ teca_add_test(test_tc_wind_radii_mpi REQ_TECA_DATA) teca_add_test(test_tc_wind_radii_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_tc_wind_radii "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800.bin" + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_tc_wind_radii "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800.bin" "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" - "32" "1" ${HALF_CORES} "0" "-1" + "32" "1" ${HALF_TEST_CORES} "0" "-1" FEATURES (${TECA_HAS_MPI} AND ${TECA_HAS_NETCDF}) REQ_TECA_DATA) @@ -442,7 +442,7 @@ teca_add_test(test_tc_wind_radii_threads COMMAND test_tc_wind_radii "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800.bin" "${TECA_DATA_ROOT}/cam5_1_amip_run2_1990s/.*\\.nc$" "${TECA_DATA_ROOT}/test_tc_wind_radii.bin" "!(((track_id==4)&&(surface_wind*3.6d>=177.0d))||((track_id==191)&&(surface_wind*3.6d>=249.0d))||((track_id==523)&&(3.6d*surface_wind>=209.0d)))" - "32" "1" ${TECA_TEST_CORES} "0" "-1" + "32" "1" ${TEST_CORES} "0" "-1" FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -521,15 +521,15 @@ teca_add_test(test_bayesian_ar_detect_threads COMMAND test_bayesian_ar_detect "${TECA_DATA_ROOT}/bayesian_ar_parameters.bin" "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" "${TECA_DATA_ROOT}/test_bayesian_ar_detect.bin" prw - "bayesian_ar_detect_%t%.vtk" ${TECA_TEST_CORES} 0 4 + "bayesian_ar_detect_%t%.vtk" ${TEST_CORES} 0 4 REQ_TECA_DATA) teca_add_test(test_bayesian_ar_detect_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} test_bayesian_ar_detect + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} test_bayesian_ar_detect "${TECA_DATA_ROOT}/bayesian_ar_parameters.bin" "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" "${TECA_DATA_ROOT}/test_bayesian_ar_detect.bin" prw - "bayesian_ar_detect_%t%.vtk" ${HALF_CORES} 0 4 + "bayesian_ar_detect_%t%.vtk" ${HALF_TEST_CORES} 0 4 FEATURES (${TECA_HAS_MPI} AND ${TECA_HAS_NETCDF}) REQ_TECA_DATA) diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index c3d5255d0..cd9258c3b 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -15,12 +15,12 @@ teca_add_test(py_test_cf_writer_serial teca_add_test(py_test_cf_writer_threads COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_cf_writer.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" - 0 -1 ${TECA_TEST_CORES} 2 "py_test_cf_writer_%t%.nc" U850 V850 + 0 -1 ${TEST_CORES} 2 "py_test_cf_writer_%t%.nc" U850 V850 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) teca_add_test(py_test_cf_writer_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_cf_writer.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" 0 -1 1 2 "py_test_cf_writer_%t%.nc" U850 V850 @@ -28,10 +28,10 @@ teca_add_test(py_test_cf_writer_mpi REQ_TECA_DATA) teca_add_test(py_test_cf_writer_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_cf_writer.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" - 0 -1 ${HALF_CORES} 1 "py_test_cf_writer_%t%.nc" U850 V850 + 0 -1 ${HALF_TEST_CORES} 1 "py_test_cf_writer_%t%.nc" U850 V850 FEATURES ${TECA_HAS_MPI} ${TECA_HAS_NETCDF} REQ_TECA_DATA) @@ -112,7 +112,7 @@ teca_add_test(py_test_programmable_map_reduce_thread REQ_TECA_DATA) teca_add_test(py_test_programmable_map_reduce_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_programmable_map_reduce.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" "${TECA_DATA_ROOT}/py_test_programmable_map_reduce.bin" 0 -1 1 TMQ T200 T500 @@ -120,10 +120,10 @@ teca_add_test(py_test_programmable_map_reduce_mpi REQ_TECA_DATA) teca_add_test(py_test_programmable_map_reduce_mpi_thread - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_programmable_map_reduce.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" - "${TECA_DATA_ROOT}/py_test_programmable_map_reduce.bin" 0 -1 ${HALF_CORES} + "${TECA_DATA_ROOT}/py_test_programmable_map_reduce.bin" 0 -1 ${HALF_TEST_CORES} TMQ T200 T500 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4PY_FOUND} REQ_TECA_DATA) @@ -137,7 +137,7 @@ teca_add_test(py_test_tc_candidates_serial REQ_TECA_DATA) teca_add_test(py_test_tc_candidates_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_candidates.py "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 1 @@ -148,16 +148,16 @@ teca_add_test(py_test_tc_candidates_mpi teca_add_test(py_test_tc_candidates_threads COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_candidates.py "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" - "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${TECA_TEST_CORES} + "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${TEST_CORES} U850 V850 UBOT VBOT PSL T500 T200 Z1000 Z200 -20 20 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} REQ_TECA_DATA) teca_add_test(py_test_tc_candidates_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_candidates.py "${TECA_DATA_ROOT}/test_tc_candidates_1990_07_0[12]\\.nc" - "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${HALF_CORES} + "${TECA_DATA_ROOT}/test_tc_candidates_20.bin" 0 3 ${HALF_TEST_CORES} U850 V850 UBOT VBOT PSL T500 T200 Z1000 Z200 -20 20 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} REQ_TECA_DATA) @@ -191,7 +191,7 @@ teca_add_test(py_test_tc_trajectory_scalars_serial REQ_TECA_DATA) teca_add_test(py_test_tc_trajectory_scalars_mpi - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tc_trajectory_scalars.py "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin" "${TECA_DATA_ROOT}/earthmap4k.png" @@ -221,12 +221,12 @@ teca_add_test(py_test_bayesian_ar_detect_threads REQ_TECA_DATA) teca_add_test(py_test_bayesian_ar_detect_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_bayesian_ar_detect.py "${TECA_DATA_ROOT}/bayesian_ar_parameters.bin" "${TECA_DATA_ROOT}/prw_hus_day_MRI.*\\.nc$" "${TECA_DATA_ROOT}/test_bayesian_ar_detect.bin" prw "bayesian_ar_detect_py_%t%.vtk" - ${HALF_CORES} 0 4 + ${HALF_TEST_CORES} 0 4 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} REQ_TECA_DATA) @@ -240,24 +240,25 @@ teca_add_test(py_test_deeplabv3p_ar_detect REQ_TECA_DATA) teca_add_test(py_test_deeplabv3p_ar_detect_mpi_threads - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py - "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" - "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" - "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" - "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT ${HALF_CORES} - FEATURES ${TECA_HAS_NETCDF} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py + "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" + "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" + "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" + "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT ${HALF_TEST_CORES} + FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) teca_add_test(py_test_binary_stream - COMMAND ${MPIEXEC} -n ${TECA_TEST_CORES} ${PYTHON_EXECUTABLE} + COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_binary_stream.py "${TECA_DATA_ROOT}/py_test_binary_stream.bin" FEATURES ${TECA_HAS_MPI} ${MPI4PY_FOUND} REQ_TECA_DATA) teca_add_test(py_test_nested_pipeline - COMMAND ${MPIEXEC} -n ${HALF_CORES} ${PYTHON_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/test_nested_pipeline.py 16 16 32 ${HALF_CORES} + COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_nested_pipeline.py 16 16 32 ${HALF_TEST_CORES} FEATURES ${TECA_HAS_MPI} ${MPI4PY_FOUND}) teca_add_test(py_test_information_array_io From 3a299fec6a7e618e39c85c2db38863de1f5c2b77 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 09:12:53 -0700 Subject: [PATCH 017/385] bump fedora version to 31 in travis ci --- .travis.yml | 4 ++-- .../{install_fedora_28.sh => install_fedora_31.sh} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename test/travis_ci/{install_fedora_28.sh => install_fedora_31.sh} (73%) diff --git a/.travis.yml b/.travis.yml index 3801b6c20..0d118d88c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ env: - TECA_DATA_REVISION=53 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - - DOCKER_IMAGE=fedora IMAGE_VERSION=28 IMAGE_NAME=fedora_28 + - DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31 - NO_DOCKER=TRUE jobs: @@ -25,7 +25,7 @@ jobs: - os: osx env: DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - os: osx - env: DOCKER_IMAGE=fedora IMAGE_VERSION=28 IMAGE_NAME=fedora_28 + env: DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31 - os: linux env: NO_DOCKER=TRUE diff --git a/test/travis_ci/install_fedora_28.sh b/test/travis_ci/install_fedora_31.sh similarity index 73% rename from test/travis_ci/install_fedora_28.sh rename to test/travis_ci/install_fedora_31.sh index 3d8c56eae..a9c521c12 100755 --- a/test/travis_ci/install_fedora_28.sh +++ b/test/travis_ci/install_fedora_31.sh @@ -7,9 +7,9 @@ dnf update -qq -y # install deps # use PIP for Python packages dnf install -qq -y environment-modules which git-all gcc-c++ gcc-gfortran \ - make cmake-3.11.0-1.fc28 swig mpich-devel hdf5-mpich-devel \ - netcdf-mpich-devel boost-devel python3-devel python3-pip subversion \ - udunits2 udunits2-devel zlib-devel openssl-devel wget + make cmake swig mpich-devel hdf5-mpich-devel netcdf-mpich-devel \ + boost-devel python3-devel python3-pip subversion udunits2 udunits2-devel \ + zlib-devel openssl-devel wget source /usr/share/Modules/init/bash From 73af3d06854b08e199b18fcd7adb7372a26d90cb Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 16:22:17 -0700 Subject: [PATCH 018/385] fix CMake 3.17 find module warnings The warning complains that variables don't match the module/file name. * rename the MPI4PY variables to MPI4Py * rename Numpy variables and file name to NumPy --- CMake/FindMPI4Py.cmake | 28 +++++++++++----------- CMake/{FindNumpy.cmake => FindNumPy.cmake} | 24 +++++++++---------- CMakeLists.txt | 8 +++---- python/CMake/teca_python.cmake | 6 ++--- test/python/CMakeLists.txt | 10 ++++---- 5 files changed, 38 insertions(+), 38 deletions(-) rename CMake/{FindNumpy.cmake => FindNumPy.cmake} (58%) diff --git a/CMake/FindMPI4Py.cmake b/CMake/FindMPI4Py.cmake index 2ac89bd9b..6dbfbb282 100644 --- a/CMake/FindMPI4Py.cmake +++ b/CMake/FindMPI4Py.cmake @@ -4,22 +4,22 @@ # Check if mpi4py is installed and configure c-api includes # # This module defines -# MPI4PY_FOUND, set TRUE if mpi4py and c-api are available -# MPI4PY_INCLUDE_DIR, where to find c-api headers -# MPI4PY_VERSION, mpi4py release version +# MPI4Py_FOUND, set TRUE if mpi4py and c-api are available +# MPI4Py_INCLUDE_DIR, where to find c-api headers +# MPI4Py_VERSION, mpi4py release version set(_TMP_PY_OUTPUT) set(_TMP_PY_RETURN) exec_program("${PYTHON_EXECUTABLE}" ARGS "-c 'import mpi4py; print(mpi4py.get_include())'" OUTPUT_VARIABLE _TMP_PY_OUTPUT RETURN_VALUE _TMP_PY_RETURN) -set(MPI4PY_INCLUDE_FOUND FALSE) +set(MPI4Py_INCLUDE_FOUND FALSE) if(NOT _TMP_PY_RETURN AND EXISTS "${_TMP_PY_OUTPUT}") - set(MPI4PY_INCLUDE_FOUND TRUE) + set(MPI4Py_INCLUDE_FOUND TRUE) else() set(_TMP_PY_OUTPUT) endif() -set(MPI4PY_INCLUDE_DIR "${_TMP_PY_OUTPUT}" CACHE PATH +set(MPI4Py_INCLUDE_DIR "${_TMP_PY_OUTPUT}" CACHE PATH "mpi4py include directories") set(_TMP_PY_OUTPUT) @@ -28,22 +28,22 @@ exec_program("${PYTHON_EXECUTABLE}" ARGS "-c 'import mpi4py; print(mpi4py.__version__)'" OUTPUT_VARIABLE _TMP_PY_OUTPUT RETURN_VALUE _TMP_PY_RETURN) -set(MPI4PY_VERSION_FOUND FALSE) +set(MPI4Py_VERSION_FOUND FALSE) if(NOT _TMP_PY_RETURN) - set(MPI4PY_VERSION_FOUND TRUE) + set(MPI4Py_VERSION_FOUND TRUE) else() set(_TMP_PY_OUTPUT) endif() -set(MPI4PY_VERSION "${_TMP_PY_OUTPUT}" CACHE STRING +set(MPI4Py_VERSION "${_TMP_PY_OUTPUT}" CACHE STRING "mpi4py version string") if (NOT ${QUIET}) - message(STATUS "MPI4PY_INCLUDE_DIR=${MPI4PY_INCLUDE_DIR}") - message(STATUS "MPI4PY_VERSION=${MPI4PY_VERSION}") + message(STATUS "MPI4Py_INCLUDE_DIR=${MPI4Py_INCLUDE_DIR}") + message(STATUS "MPI4Py_VERSION=${MPI4Py_VERSION}") endif() -mark_as_advanced(MPI4PY_INCLUDE_DIR MPI4PY_VERSION) +mark_as_advanced(MPI4Py_INCLUDE_DIR MPI4Py_VERSION) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(MPI4PY DEFAULT_MSG - MPI4PY_INCLUDE_FOUND MPI4PY_VERSION_FOUND) +find_package_handle_standard_args(MPI4Py DEFAULT_MSG + MPI4Py_INCLUDE_FOUND MPI4Py_VERSION_FOUND) diff --git a/CMake/FindNumpy.cmake b/CMake/FindNumPy.cmake similarity index 58% rename from CMake/FindNumpy.cmake rename to CMake/FindNumPy.cmake index 5e2419b90..a10ebaa78 100644 --- a/CMake/FindNumpy.cmake +++ b/CMake/FindNumPy.cmake @@ -4,22 +4,22 @@ # Check if numpy is installed and configure c-api includes # # This module defines -# NUMPY_FOUND, set TRUE if numpy and c-api are available -# NUMPY_INCLUDE_DIR, where to find c-api headers -# NUMPY_VERSION, numpy release version +# NumPy_FOUND, set TRUE if numpy and c-api are available +# NumPy_INCLUDE_DIR, where to find c-api headers +# NumPy_VERSION, numpy release version set(_TMP_PY_OUTPUT) set(_TMP_PY_RETURN) exec_program("${PYTHON_EXECUTABLE}" ARGS "-c 'import numpy; print(numpy.get_include())'" OUTPUT_VARIABLE _TMP_PY_OUTPUT RETURN_VALUE _TMP_PY_RETURN) -set(NUMPY_INCLUDE_FOUND FALSE) +set(NumPy_INCLUDE_FOUND FALSE) if(NOT _TMP_PY_RETURN AND EXISTS "${_TMP_PY_OUTPUT}") - set(NUMPY_INCLUDE_FOUND TRUE) + set(NumPy_INCLUDE_FOUND TRUE) else() set(_TMP_PY_OUTPUT) endif() -set(NUMPY_INCLUDE_DIR "${_TMP_PY_OUTPUT}") +set(NumPy_INCLUDE_DIR "${_TMP_PY_OUTPUT}") set(_TMP_PY_OUTPUT) set(_TMP_PY_RETURN) @@ -27,15 +27,15 @@ exec_program("${PYTHON_EXECUTABLE}" ARGS "-c 'import numpy; print(numpy.version.version)'" OUTPUT_VARIABLE _TMP_PY_OUTPUT RETURN_VALUE _TMP_PY_RETURN) -set(NUMPY_VERSION_FOUND FALSE) +set(NumPy_VERSION_FOUND FALSE) if(NOT _TMP_PY_RETURN) - set(NUMPY_VERSION_FOUND TRUE) + set(NumPy_VERSION_FOUND TRUE) else() set(_TMP_PY_OUTPUT) endif() -set(NUMPY_VERSION "${_TMP_PY_OUTPUT}") +set(NumPy_VERSION "${_TMP_PY_OUTPUT}") -#set(NUMPY_INCLUDE_DIR "${_TMP_PY_OUTPUT}" CACHE PATH "Numpy C API headers") -#mark_as_advanced(NUMPY_INCLUDE_DIR) +#set(NumPy_INCLUDE_DIR "${_TMP_PY_OUTPUT}" CACHE PATH "Numpy C API headers") +#mark_as_advanced(NumPy_INCLUDE_DIR) include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NUMPY DEFAULT_MSG NUMPY_INCLUDE_FOUND NUMPY_VERSION_FOUND) +find_package_handle_standard_args(NumPy DEFAULT_MSG NumPy_INCLUDE_FOUND NumPy_VERSION_FOUND) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1345b46b7..3e6766574 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -246,7 +246,7 @@ if(PYTHONINTERP_FOUND) endif() endif() find_package(PythonLibs ${TECA_PYTHON_VERSION}) -find_package(Numpy) +find_package(NumPy) if (TECA_HAS_MPI) find_package(MPI4Py) endif() @@ -256,14 +256,14 @@ if (swig_cmd) else() message(FATAL_ERROR "Found SWIG: FALSE") endif() -if (PYTHONINTERP_FOUND AND PYTHONLIBS_FOUND AND NUMPY_FOUND AND swig_cmd - AND ((TECA_HAS_MPI AND MPI4PY_FOUND) OR (NOT TECA_HAS_MPI)) +if (PYTHONINTERP_FOUND AND PYTHONLIBS_FOUND AND NumPy_FOUND AND swig_cmd + AND ((TECA_HAS_MPI AND MPI4Py_FOUND) OR (NOT TECA_HAS_MPI)) AND ((DEFINED TECA_HAS_PYTHON AND TECA_HAS_PYTHON) OR (NOT DEFINED TECA_HAS_PYTHON))) message(STATUS "Python ${TECA_PYTHON_VERSION} features -- enabled") set(tmp ON) teca_interface_library(PYTHON SYSTEM - INCLUDES ${PYTHON_INCLUDE_PATH} ${MPI4PY_INCLUDE_DIR} ${NUMPY_INCLUDE_DIR} + INCLUDES ${PYTHON_INCLUDE_PATH} ${MPI4Py_INCLUDE_DIR} ${NumPy_INCLUDE_DIR} LIBRARIES ${PYTHON_LIBRARIES}) elseif (REQUIRE_PYTHON) message(FATAL_ERROR "Python ${TECA_PYTHON_VERSION} features -- required but not found") diff --git a/python/CMake/teca_python.cmake b/python/CMake/teca_python.cmake index a05d8ae89..fe09f4900 100644 --- a/python/CMake/teca_python.cmake +++ b/python/CMake/teca_python.cmake @@ -5,7 +5,7 @@ function(depend_swig input output) add_custom_command( OUTPUT ${output_file} COMMAND ${swig_cmd} -c++ -python -MM - -I${MPI4PY_INCLUDE_DIR} + -I${MPI4Py_INCLUDE_DIR} -I${CMAKE_CURRENT_BINARY_DIR} -I${CMAKE_CURRENT_BINARY_DIR}/.. -I${CMAKE_CURRENT_SOURCE_DIR}/../core @@ -20,7 +20,7 @@ function(depend_swig input output) message(STATUS "Generating initial dependency list for ${input}") execute_process( COMMAND ${swig_cmd} -c++ -python -MM - -I${MPI4PY_INCLUDE_DIR} + -I${MPI4Py_INCLUDE_DIR} -I${CMAKE_CURRENT_BINARY_DIR} -I${CMAKE_CURRENT_BINARY_DIR}/.. -I${CMAKE_CURRENT_SOURCE_DIR}/../core @@ -41,7 +41,7 @@ function(wrap_swig input output depend) OUTPUT ${output_file} COMMAND ${swig_cmd} -c++ -python -threads -w341,325 -DSWIG_TYPE_TABLE=teca_py - -I${MPI4PY_INCLUDE_DIR} + -I${MPI4Py_INCLUDE_DIR} -I${CMAKE_CURRENT_BINARY_DIR} -I${CMAKE_CURRENT_BINARY_DIR}/.. -I${CMAKE_CURRENT_SOURCE_DIR}/../core diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index cd9258c3b..b36d15db6 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -116,7 +116,7 @@ teca_add_test(py_test_programmable_map_reduce_mpi ${CMAKE_CURRENT_SOURCE_DIR}/test_programmable_map_reduce.py "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" "${TECA_DATA_ROOT}/py_test_programmable_map_reduce.bin" 0 -1 1 TMQ T200 T500 - FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4PY_FOUND} + FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) teca_add_test(py_test_programmable_map_reduce_mpi_thread @@ -125,7 +125,7 @@ teca_add_test(py_test_programmable_map_reduce_mpi_thread "${TECA_DATA_ROOT}/cam5_1_amip_run2\\.cam2\\.h2\\.1991-10-0[12]-10800\\.nc" "${TECA_DATA_ROOT}/py_test_programmable_map_reduce.bin" 0 -1 ${HALF_TEST_CORES} TMQ T200 T500 - FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4PY_FOUND} + FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) teca_add_test(py_test_tc_candidates_serial @@ -196,7 +196,7 @@ teca_add_test(py_test_tc_trajectory_scalars_mpi "${TECA_DATA_ROOT}/tracks_1990s_3hr_mdd_4800_median_in_cat_wr.bin" "${TECA_DATA_ROOT}/earthmap4k.png" "${TECA_DATA_ROOT}/py_test_tc_trajectory_scalars.bin" 0 -1 - FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4PY_FOUND} + FEATURES ${TECA_HAS_UDUNITS} ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) teca_add_test(py_test_tc_wind_radii_stats @@ -253,13 +253,13 @@ teca_add_test(py_test_binary_stream COMMAND ${MPIEXEC} -n ${TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_binary_stream.py "${TECA_DATA_ROOT}/py_test_binary_stream.bin" - FEATURES ${TECA_HAS_MPI} ${MPI4PY_FOUND} + FEATURES ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) teca_add_test(py_test_nested_pipeline COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_nested_pipeline.py 16 16 32 ${HALF_TEST_CORES} - FEATURES ${TECA_HAS_MPI} ${MPI4PY_FOUND}) + FEATURES ${TECA_HAS_MPI} ${MPI4Py_FOUND}) teca_add_test(py_test_information_array_io COMMAND ${PYTHON_EXECUTABLE} From 3da5f4b8e7b38d9ac56b1abfdcb37d60b75ebe3f Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 16:24:04 -0700 Subject: [PATCH 019/385] fix build without udunits --- io/teca_cf_reader.cxx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/io/teca_cf_reader.cxx b/io/teca_cf_reader.cxx index 7d6e1583c..142306939 100644 --- a/io/teca_cf_reader.cxx +++ b/io/teca_cf_reader.cxx @@ -903,17 +903,17 @@ teca_metadata teca_cf_reader::get_output_metadata( second, t_units.c_str(), t_calendar.c_str(), ¤t_time)) { - TECA_ERROR( - "conversion of date inferred from filename failed"); + TECA_ERROR("conversion of date inferred from " + "filename failed"); + return teca_metadata(); } -#else - (void)date; - TECA_ERROR( - "The UDUnits package is required for this operation") - return -1; -#endif // add the current time to the list t_values.push_back(current_time); +#else + TECA_ERROR("The UDUnits package is required " + "for this operation") + return teca_metadata(); +#endif } // set the time metadata From 2657f3baeb3579b969e47739133b52f4d0264554 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 16:48:53 -0700 Subject: [PATCH 020/385] travis ci require dependencies require MPI, OpenSSL, NetCDF, Python, UDUnits, boost, and TECA_data. Configure step will fail if any of these are missing. --- CMakeLists.txt | 2 ++ test/travis_ci/ctest_linux.cmake | 9 ++++++++- test/travis_ci/ctest_osx.cmake | 10 +++++++++- test/travis_ci/install_osx.sh | 9 ++++++--- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e6766574..0c4ac9ffb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,6 +320,8 @@ set(TECA_DATA_ROOT "/path/to/TECA_data" if (EXISTS "${TECA_DATA_ROOT}") set(tmp ON) message(STATUS "TECA_data -- available") +elseif (REQUIRE_TECA_DATA) + message(FATAL_ERROR "TECA_data -- required but not found") else() message(STATUS "TECA_data -- not available") endif() diff --git a/test/travis_ci/ctest_linux.cmake b/test/travis_ci/ctest_linux.cmake index af9ebf377..7294a7dd2 100644 --- a/test/travis_ci/ctest_linux.cmake +++ b/test/travis_ci/ctest_linux.cmake @@ -20,7 +20,14 @@ BUILD_TESTING=ON TECA_ENABLE_PROFILER=ON TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION} TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data -TECA_TEST_CORES=2") +TECA_TEST_CORES=2 +REQUIRE_OPENSSL=TRUE +REQUIRE_BOOST=TRUE +REQUIRE_NETCDF=TRUE +REQUIRE_UDUNITS=TRUE +REQUIRE_MPI=TRUE +REQUIRE_PYTHON=TRUE +REQUIRE_TECA_DATA=TRUE") file(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" ${INITIAL_CACHE}) ctest_start("${DASHBOARD_TRACK}") ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}") diff --git a/test/travis_ci/ctest_osx.cmake b/test/travis_ci/ctest_osx.cmake index e7163f0cd..8a1f5623f 100644 --- a/test/travis_ci/ctest_osx.cmake +++ b/test/travis_ci/ctest_osx.cmake @@ -20,7 +20,15 @@ BUILD_TESTING=ON TECA_ENABLE_PROFILER=ON TECA_PYTHON_VERSION=$ENV{TECA_PYTHON_VERSION} TECA_DATA_ROOT=$ENV{DASHROOT}/TECA_data -TECA_TEST_CORES=2") +TECA_TEST_CORES=2 +REQUIRE_OPENSSL=TRUE +OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1 +REQUIRE_BOOST=TRUE +REQUIRE_NETCDF=TRUE +REQUIRE_UDUNITS=TRUE +REQUIRE_MPI=TRUE +REQUIRE_PYTHON=TRUE +REQUIRE_TECA_DATA=TRUE") file(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" ${INITIAL_CACHE}) ctest_start("${DASHBOARD_TRACK}") ctest_configure() diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index 610d9cd58..c93a856f6 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x # override system install export PATH=/usr/local/bin:$PATH @@ -6,7 +7,8 @@ export PATH=/usr/local/bin:$PATH # install deps. note than many are included as a part of brew-core # these days. hence this list isn't comprehensive brew update -brew install mpich swig svn udunits +brew install mpich swig svn udunits openssl@1.1 + # matplotlib currently doesn't have a formula # teca fails to locate mpi4py installed from brew pip3 install numpy mpi4py matplotlib torch @@ -18,9 +20,10 @@ pip3 install numpy mpi4py matplotlib torch # travis will kill a build if it does not get console output # for 10 min. The following snippet sends progress marks # to the console while svn runs. -echo 'svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data &' -svn svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data & +svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data & svn_pid=$! + +set +x while [ -n "$(ps -p $svn_pid -o pid=)" ] do echo -n "." From 59a52e45806b59ae5008245097aff84b9aef105f Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 16:51:27 -0700 Subject: [PATCH 021/385] make ctest fail right away when any part fails (mac os) --- test/travis_ci/ctest_osx.cmake | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/travis_ci/ctest_osx.cmake b/test/travis_ci/ctest_osx.cmake index 8a1f5623f..1c41b77b1 100644 --- a/test/travis_ci/ctest_osx.cmake +++ b/test/travis_ci/ctest_osx.cmake @@ -1,5 +1,5 @@ set(CTEST_SITE "Travis_CI") -set(CTEST_BUILD_NAME "Apple_OSX_10.95_$ENV{TRAVIS_BRANCH}_$ENV{SHA}") +set(CTEST_BUILD_NAME "Apple_Mac_OS_$ENV{TRAVIS_BRANCH}_$ENV{SHA}") set(CTEST_DASHBOARD_ROOT "$ENV{DASHROOT}") set(CTEST_BUILD_CONFIGURATION $ENV{BUILD_TYPE}) set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) @@ -31,13 +31,19 @@ REQUIRE_PYTHON=TRUE REQUIRE_TECA_DATA=TRUE") file(WRITE "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt" ${INITIAL_CACHE}) ctest_start("${DASHBOARD_TRACK}") -ctest_configure() ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}") -ctest_build() -ctest_submit(PARTS Update Configure Build Notes RETRY_DELAY 15 RETRY_COUNT 10) +ctest_configure(RETURN_VALUE terr) +ctest_submit(PARTS Update Configure RETRY_DELAY 15 RETRY_COUNT 10) +if (NOT terr EQUAL 0) + message(FATAL_ERROR "ERROR: ctest configure failed!") +endif() +ctest_build(RETURN_VALUE terr) +ctest_submit(PARTS Build RETRY_DELAY 15 RETRY_COUNT 10) +if (NOT terr EQUAL 0) + message(FATAL_ERROR "ERROR: ctest build failed!") +endif() ctest_test(RETURN_VALUE terr) ctest_submit(PARTS Test RETRY_DELAY 15 RETRY_COUNT 10) if (NOT terr EQUAL 0) - # if any tests failed abort with a fatal error - message(FATAL_ERROR "ERROR: At least one test failed!") + message(FATAL_ERROR "ERROR: ctest test failed!") endif() From a8ec648eb9c1535885a81f2ae05c78bebce3b4da Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 13 Apr 2020 21:27:16 -0700 Subject: [PATCH 022/385] use Python venv for dependencies in Travis CI --- test/travis_ci/ctest_linux.sh | 10 ++++++++-- test/travis_ci/ctest_osx.sh | 7 +++++++ test/travis_ci/install_fedora_31.sh | 13 +++++++++---- test/travis_ci/install_osx.sh | 4 ++++ test/travis_ci/install_ubuntu_18_04.sh | 8 ++++++-- 5 files changed, 34 insertions(+), 8 deletions(-) diff --git a/test/travis_ci/ctest_linux.sh b/test/travis_ci/ctest_linux.sh index 9c9540f58..33069fb36 100755 --- a/test/travis_ci/ctest_linux.sh +++ b/test/travis_ci/ctest_linux.sh @@ -1,15 +1,21 @@ #!/bin/bash -set -v +set -x + export DASHROOT=`pwd` export SHA=`git log --pretty=format:'%h' -n 1` + +set +x if [[ "$DOCKER_IMAGE" == "fedora" ]]; then source /usr/share/Modules/init/bash module load mpi fi +source `pwd`/../tci/bin/activate +set -x + export PATH=.:${PATH} export PYTHONPATH=${DASHROOT}/build/lib export LD_LIBRARY_PATH=${DASHROOT}/build/lib export MPLBACKEND=Agg mkdir build -cmake --version + ctest -S ${DASHROOT}/test/travis_ci/ctest_linux.cmake -V --timeout 180 diff --git a/test/travis_ci/ctest_osx.sh b/test/travis_ci/ctest_osx.sh index 51d1c279e..c482f76eb 100755 --- a/test/travis_ci/ctest_osx.sh +++ b/test/travis_ci/ctest_osx.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -x + export DASHROOT=`pwd` export SHA=`git log --pretty=format:'%h' -n 1` export PATH=.:/usr/local/bin:${PATH} @@ -7,5 +9,10 @@ export LD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/build/lib export DYLD_LIBRARY_PATH=${TRAVIS_BUILD_DIR}/build/lib export MPLBACKEND=Agg export TMPDIR=/tmp + +set +x +source `pwd`/../tci/bin/activate +set -x + mkdir build ctest -S ${DASHROOT}/test/travis_ci/ctest_osx.cmake -V --timeout 180 diff --git a/test/travis_ci/install_fedora_31.sh b/test/travis_ci/install_fedora_31.sh index a9c521c12..4fdd557ae 100755 --- a/test/travis_ci/install_fedora_31.sh +++ b/test/travis_ci/install_fedora_31.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -v +set -x # suck in package lists dnf update -qq -y @@ -9,11 +9,12 @@ dnf update -qq -y dnf install -qq -y environment-modules which git-all gcc-c++ gcc-gfortran \ make cmake swig mpich-devel hdf5-mpich-devel netcdf-mpich-devel \ boost-devel python3-devel python3-pip subversion udunits2 udunits2-devel \ - zlib-devel openssl-devel wget + zlib-devel openssl-devel wget redhat-rpm-config +set +x source /usr/share/Modules/init/bash - module load mpi/mpich-x86_64 +set -x echo ${TRAVIS_BRANCH} echo ${BUILD_TYPE} @@ -22,7 +23,11 @@ echo ${IMAGE_VERSION} echo ${TECA_PYTHON_VERSION} echo ${TECA_DATA_REVISION} -pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib torch +python3 -mvenv `pwd`/../tci +set +x +source `pwd`/../tci/bin/activate +set -x +pip3 install numpy mpi4py matplotlib torch # install data files. svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data diff --git a/test/travis_ci/install_osx.sh b/test/travis_ci/install_osx.sh index c93a856f6..0b57085e0 100755 --- a/test/travis_ci/install_osx.sh +++ b/test/travis_ci/install_osx.sh @@ -11,6 +11,10 @@ brew install mpich swig svn udunits openssl@1.1 # matplotlib currently doesn't have a formula # teca fails to locate mpi4py installed from brew +python3 -mvenv `pwd`/../tci +set +x +source `pwd`/../tci/bin/activate +set -x pip3 install numpy mpi4py matplotlib torch # install data files. diff --git a/test/travis_ci/install_ubuntu_18_04.sh b/test/travis_ci/install_ubuntu_18_04.sh index 461c8eb58..489d86311 100755 --- a/test/travis_ci/install_ubuntu_18_04.sh +++ b/test/travis_ci/install_ubuntu_18_04.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -v +set -x # suck in package lists apt-get update -qq @@ -24,7 +24,11 @@ echo ${IMAGE_VERSION} echo ${TECA_PYTHON_VERSION} echo ${TECA_DATA_REVISION} -pip${TECA_PYTHON_VERSION} install numpy mpi4py matplotlib torch +python3 -mvenv `pwd`/../tci +set +x +source `pwd`/../tci/bin/activate +set -x +pip3 install numpy mpi4py matplotlib torch # install data files. svn co svn://svn.code.sf.net/p/teca/TECA_data@${TECA_DATA_REVISION} TECA_data From f8dc2a50d2028fdd1903d95ba31061c446f91502 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 14 Apr 2020 12:30:37 -0700 Subject: [PATCH 023/385] fix baseline in deeplab test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0d118d88c..cc9bacec2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - BUILD_TYPE=Debug - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - - TECA_DATA_REVISION=53 + - TECA_DATA_REVISION=54 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31 From 0a52894d076fdb95d340051421b71dd1a54d4557 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 14 Apr 2020 14:46:38 -0700 Subject: [PATCH 024/385] increase fail tolerance in deeplab ar detector test Increase the tolerance from from 1e-4 to 1e-3 after observing test fail on two different platforms by a small ammount. --- test/python/test_deeplabv3p_ar_detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/python/test_deeplabv3p_ar_detect.py b/test/python/test_deeplabv3p_ar_detect.py index c99233538..88e9d4fd0 100644 --- a/test/python/test_deeplabv3p_ar_detect.py +++ b/test/python/test_deeplabv3p_ar_detect.py @@ -57,7 +57,7 @@ def main(): diff = teca_dataset_diff.New() diff.set_input_connection(0, baseline_mesh_reader.get_output_port()) diff.set_input_connection(1, deeplabv3p_ar_detect.get_output_port()) - diff.set_tolerance(1e-4) + diff.set_tolerance(1e-3) diff.update() else: # make a baseline From a549499c08d23106570afeeb30378c0663fe88fd Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Tue, 14 Apr 2020 15:00:49 -0700 Subject: [PATCH 025/385] dont over subscribe cores in deeplab ar detect test --- test/python/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index b36d15db6..c9a487f86 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -235,17 +235,17 @@ teca_add_test(py_test_deeplabv3p_ar_detect "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" - "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT -1 + "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1 FEATURES ${TECA_HAS_NETCDF} REQ_TECA_DATA) -teca_add_test(py_test_deeplabv3p_ar_detect_mpi_threads +teca_add_test(py_test_deeplabv3p_ar_detect_mpi COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" - "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT ${HALF_TEST_CORES} + "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} ${MPI4Py_FOUND} REQ_TECA_DATA) From befd5ce0687a392c9b931ce04c4cd7f06ff0b5c9 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Fri, 3 Apr 2020 15:53:30 -0700 Subject: [PATCH 026/385] Fixing minor issue mentioned in #342 with the deeplab ar detector --- alg/teca_deeplabv3p_ar_detect.py | 3 ++- alg/teca_model_segmentation.py | 4 +++- apps/CMakeLists.txt | 2 +- ...h_deeplabv3p_ar_detect.in => teca_deeplabv3p_ar_detect.in} | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) rename apps/{teca_pytorch_deeplabv3p_ar_detect.in => teca_deeplabv3p_ar_detect.in} (97%) mode change 100644 => 100755 diff --git a/alg/teca_deeplabv3p_ar_detect.py b/alg/teca_deeplabv3p_ar_detect.py index 0998847ea..d2594c8f4 100644 --- a/alg/teca_deeplabv3p_ar_detect.py +++ b/alg/teca_deeplabv3p_ar_detect.py @@ -334,7 +334,8 @@ def __init__(self): self.mesh_nx = None self.padding_amount_y = None self.padding_amount_x = None - self.set_variable_name('IVT') + self.set_variable_name("IVT") + self.set_pred_name("ar_probability") def set_mesh_dims(self, mesh_ny, mesh_nx): self.mesh_ny = mesh_ny diff --git a/alg/teca_model_segmentation.py b/alg/teca_model_segmentation.py index 1ddc700eb..033b4106d 100644 --- a/alg/teca_model_segmentation.py +++ b/alg/teca_model_segmentation.py @@ -1,3 +1,4 @@ +import sys import teca_py import numpy as np import torch @@ -56,7 +57,8 @@ def set_variable_name(self, name): set the variable name that will be inputed to the model """ self.variable_name = str(name) - self.set_pred_name(self.variable_name + '_pred') + if self.pred_name is None: + self.set_pred_name(self.variable_name + '_pred') def set_pred_name(self, name): """ diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 52e2be65a..325f3788e 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -32,7 +32,7 @@ teca_add_python_app(teca_convert_table) teca_add_python_app(teca_dataset_metadata FEATURES ${TECA_HAS_NETCDF}) -teca_add_python_app(teca_pytorch_deeplabv3p_ar_detect +teca_add_python_app(teca_deeplabv3p_ar_detect FEATURES ${TECA_HAS_NETCDF}) teca_add_python_app(teca_event_filter) diff --git a/apps/teca_pytorch_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in old mode 100644 new mode 100755 similarity index 97% rename from apps/teca_pytorch_deeplabv3p_ar_detect.in rename to apps/teca_deeplabv3p_ar_detect.in index ff89ddf9b..1f451e52c --- a/apps/teca_pytorch_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -26,8 +26,8 @@ def main(): parser.add_argument( '--output_file', type=str, required=False, - default=r'pytorch_deeplabv3p_ar_detect_%t%.nc', - help=r'file pattern for output netcdf files (%t% is the time index)' + default=r'deeplabv3p_ar_detect_%%t%%.nc', + help=r'file pattern for output netcdf files (%%t%% is the time index)' ) parser.add_argument( From af4bca2f68180cd05fa2fd91920768da788d899c Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Wed, 15 Apr 2020 07:22:38 -0700 Subject: [PATCH 027/385] Python support for nested metadata puts propper support for teca_metadata into teca_py_object. adds set/get of nested meatdata to the test. --- python/teca_py_core.i | 6 +-- python/teca_py_object.h | 85 +++++++++++++++++++++++++++++++----- test/python/test_metadata.py | 4 ++ 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/python/teca_py_core.i b/python/teca_py_core.i index 9c02d69a2..296027896 100644 --- a/python/teca_py_core.i +++ b/python/teca_py_core.i @@ -359,8 +359,7 @@ TECA_PY_DYNAMIC_VARIANT_ARRAY_CAST(unsigned long long, unsigned_long_long) else TEMPLATE_DISPATCH_CASE(const teca_variant_array_impl, teca_metadata, varr.get(), TT *varrt = static_cast(varr.get()); - return SWIG_NewPointerObj(new teca_metadata(varrt->get(0)), - SWIGTYPE_p_teca_metadata, SWIG_POINTER_OWN); + return teca_py_object::py_tt::new_object(varrt->get(0)); ) } else if (n_elem > 1) @@ -389,8 +388,7 @@ TECA_PY_DYNAMIC_VARIANT_ARRAY_CAST(unsigned long long, unsigned_long_long) for (size_t i = 0; i < n_elem; ++i) { PyList_SET_ITEM(list, i, - SWIG_NewPointerObj(new teca_metadata(varrt->get(i)), - SWIGTYPE_p_teca_metadata, SWIG_POINTER_OWN)); + teca_py_object::py_tt::new_object(varrt->get(i))); } return list; ) diff --git a/python/teca_py_object.h b/python/teca_py_object.h index 5e0554af8..43753124c 100644 --- a/python/teca_py_object.h +++ b/python/teca_py_object.h @@ -41,6 +41,30 @@ teca_py_object_cpp_tt_declare(long, long, PyLongCheck, PyLongToCLong) teca_py_object_cpp_tt_declare(float, double, PyFloat_Check, PyFloat_AsDouble) teca_py_object_cpp_tt_declare(char*, std::string, PyStringCheck, PyStringToCString) teca_py_object_cpp_tt_declare(bool, int, PyBool_Check, PyIntegerToCInt) +// teca_metadata +template <> struct cpp_tt +{ + typedef teca_metadata type; + + static bool is_type(PyObject *obj) + { + void *pv = nullptr; + int ierr = SWIG_ConvertPtr(obj, &pv, SWIGTYPE_p_teca_metadata, 0); + return SWIG_IsOK(ierr); + } + + static type value(PyObject *obj) + { + void *pv = nullptr; + int ierr = SWIG_ConvertPtr(obj, &pv, SWIGTYPE_p_teca_metadata, 0); + if (!SWIG_IsOK(ierr)) + { + TECA_PY_ERROR(PyExc_TypeError, "object is not a teca_metadata") + return teca_metadata(); + } + return *reinterpret_cast(pv); + } +}; /// py_tt, traits class for working with PyObject's /** @@ -85,14 +109,25 @@ teca_py_object_py_tt_declare(unsigned long, int, CIntUToPyInteger) teca_py_object_py_tt_declare(unsigned long long, int, CIntULLToPyInteger) teca_py_object_py_tt_declare(float, float, PyFloat_FromDouble) teca_py_object_py_tt_declare(double, float, PyFloat_FromDouble) -// strings are a special case +// string template <> struct py_tt { typedef char* tag; static PyObject *new_object(const std::string &s) { return CStringToPyString(s.c_str()); } }; -// TODO -- special case for teca_metadata +// teca_metadata +template <> struct py_tt +{ + typedef teca_metadata tag; + + static PyObject *new_object(const teca_metadata &md) + { + return SWIG_NewPointerObj(new teca_metadata(md), + SWIGTYPE_p_teca_metadata, SWIG_POINTER_OWN); + } +}; + // dispatch macro. // OBJ -- PyObject* instance @@ -106,22 +141,27 @@ template <> struct py_tt CODE \ } -#define TECA_PY_OBJECT_DISPATCH(PY_OBJ, CODE) \ - TECA_PY_OBJECT_DISPATCH_CASE(int, PY_OBJ, CODE) \ - else TECA_PY_OBJECT_DISPATCH_CASE(float, PY_OBJ, CODE) \ - else TECA_PY_OBJECT_DISPATCH_CASE(char*, PY_OBJ, CODE) \ - else TECA_PY_OBJECT_DISPATCH_CASE(long, PY_OBJ, CODE) +// all +#define TECA_PY_OBJECT_DISPATCH(PY_OBJ, CODE) \ + TECA_PY_OBJECT_DISPATCH_CASE(int, PY_OBJ, CODE) \ + else TECA_PY_OBJECT_DISPATCH_CASE(float, PY_OBJ, CODE) \ + else TECA_PY_OBJECT_DISPATCH_CASE(char*, PY_OBJ, CODE) \ + else TECA_PY_OBJECT_DISPATCH_CASE(long, PY_OBJ, CODE) \ + else TECA_PY_OBJECT_DISPATCH_CASE(teca_metadata, PY_OBJ, CODE) -// without string +// just numeric/POD #define TECA_PY_OBJECT_DISPATCH_NUM(PY_OBJ, CODE) \ TECA_PY_OBJECT_DISPATCH_CASE(int, PY_OBJ, CODE) \ else TECA_PY_OBJECT_DISPATCH_CASE(float, PY_OBJ, CODE) \ else TECA_PY_OBJECT_DISPATCH_CASE(long, PY_OBJ, CODE) -// just string -#define TECA_PY_OBJECT_DISPATCH_STR(PY_OBJ, CODE) \ +// just special cases +#define TECA_PY_OBJECT_DISPATCH_STR(PY_OBJ, CODE) \ TECA_PY_OBJECT_DISPATCH_CASE(char*, PY_OBJ, CODE) +#define TECA_PY_OBJECT_DISPATCH_MD(PY_OBJ, CODE) \ + TECA_PY_OBJECT_DISPATCH_CASE(teca_metadata, PY_OBJ, CODE) + // **************************************************************************** p_teca_variant_array new_variant_array(PyObject *obj) { @@ -158,6 +198,15 @@ bool copy(teca_variant_array *varr, PyObject *obj) return true; ) ) + else TEMPLATE_DISPATCH_CASE(teca_variant_array_impl, + teca_metadata, varr, + TT *varrt = static_cast(varr); + TECA_PY_OBJECT_DISPATCH_MD(obj, + varrt->resize(1); + varrt->set(0, cpp_tt::value(obj)); + return true; + ) + ) return false; } @@ -180,6 +229,14 @@ bool set(teca_variant_array *varr, unsigned long i, PyObject *obj) return true; ) ) + else TEMPLATE_DISPATCH_CASE(teca_variant_array_impl, + teca_metadata, varr, + TT *varrt = static_cast(varr); + TECA_PY_OBJECT_DISPATCH_MD(obj, + varrt->set(i, cpp_tt::value(obj)); + return true; + ) + ) return false; } @@ -202,6 +259,14 @@ bool append(teca_variant_array *varr, PyObject *obj) return true; ) ) + else TEMPLATE_DISPATCH_CASE(teca_variant_array_impl, + teca_metadata, varr, + TT *varrt = static_cast(varr); + TECA_PY_OBJECT_DISPATCH_MD(obj, + varrt->append(static_cast(cpp_tt::value(obj))); + return true; + ) + ) return false; } diff --git a/test/python/test_metadata.py b/test/python/test_metadata.py index ab91d7cbe..ebf4b9086 100644 --- a/test/python/test_metadata.py +++ b/test/python/test_metadata.py @@ -23,6 +23,9 @@ def end_sec(s): md['float'] = 1.0 md['string'] = 'other string' md['bool'] = True +nested = teca_metadata() +nested['foo'] = 42 +md['nested'] = nested end_sec('set from object') start_sec('set from list') @@ -48,6 +51,7 @@ def end_sec(s): 'uint64 array', 'float32 array', 'float64 array'] for md_key in md_keys: sys.stderr.write('%s = %s\n'%(md_key, str(md[md_key]))) +sys.stderr.write('%s = {%s}\n'%('nested', str(md['nested']))) end_sec('get') start_sec('empty') From 4686c75a57ce55c2c781f4952cdce66637bd8184 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Fri, 17 Apr 2020 03:26:46 -0700 Subject: [PATCH 028/385] Added ar_probability attributes for teca_deeplab --- .travis.yml | 2 +- alg/teca_deeplabv3p_ar_detect.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cc9bacec2..85092dc4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - BUILD_TYPE=Debug - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - - TECA_DATA_REVISION=54 + - TECA_DATA_REVISION=56 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31 diff --git a/alg/teca_deeplabv3p_ar_detect.py b/alg/teca_deeplabv3p_ar_detect.py index d2594c8f4..26b7a977a 100644 --- a/alg/teca_deeplabv3p_ar_detect.py +++ b/alg/teca_deeplabv3p_ar_detect.py @@ -472,5 +472,18 @@ def execute(port, data_in, req): out_mesh = super_execute(port, data_in, req) + out_mesh.get_point_arrays().remove(self.variable_name) + + ar_probability_atts = teca_py.teca_metadata() + ar_probability_atts["long_name"] = "posterior AR flag" + ar_probability_atts["units"] = "probability" + + out_md = out_mesh.get_metadata() + attributes = out_md["attributes"] + attributes["ar_probability"] = ar_probability_atts + out_md["attributes"] = attributes + + out_mesh.set_metadata(out_md) + return out_mesh return execute From a854fdb916bb8fc359716ac6c99f89b738fac30c Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Fri, 17 Apr 2020 05:48:28 -0700 Subject: [PATCH 029/385] fix default value in the deeplab ar detect app --- apps/teca_deeplabv3p_ar_detect.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/teca_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in index 1f451e52c..56723c073 100755 --- a/apps/teca_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -26,7 +26,7 @@ def main(): parser.add_argument( '--output_file', type=str, required=False, - default=r'deeplabv3p_ar_detect_%%t%%.nc', + default=r'deeplabv3p_ar_detect_%t%.nc', help=r'file pattern for output netcdf files (%%t%% is the time index)' ) From fc781a9d85121ea13c6bb9a1b961251ba6ab6494 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Tue, 5 May 2020 16:39:17 -0700 Subject: [PATCH 030/385] Added teca_bayesian_ar_detect to cmakefile --- apps/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 325f3788e..8b3e2181e 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -27,6 +27,9 @@ teca_add_app(teca_tc_trajectory LIBS ${teca_app_link} teca_add_app(teca_metadata_probe LIBS ${teca_app_link} FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) +teca_add_app(teca_bayesian_ar_detect LIBS ${teca_app_link} + FEATURES ${TECA_HAS_BOOST} ${TECA_HAS_NETCDF} ${TECA_HAS_UDUNITS}) + teca_add_python_app(teca_convert_table) teca_add_python_app(teca_dataset_metadata From 22d32e274f6e321a1233abcd2541e36f4ce3cb52 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Thu, 7 May 2020 15:20:02 -0700 Subject: [PATCH 031/385] Removing teca_tc_storm_size --- apps/teca_tc_storm_size.cpp | 252 ------------------------------------ 1 file changed, 252 deletions(-) delete mode 100644 apps/teca_tc_storm_size.cpp diff --git a/apps/teca_tc_storm_size.cpp b/apps/teca_tc_storm_size.cpp deleted file mode 100644 index a5cabb81b..000000000 --- a/apps/teca_tc_storm_size.cpp +++ /dev/null @@ -1,252 +0,0 @@ -#include "teca_config.h" -#include "teca_table_reader.h" -#include "teca_table_remove_rows.h" -#include "teca_table_sort.h" -#include "teca_cf_reader.h" -#include "teca_normalize_coordinates.h" -#include "teca_tc_storm_size.h" -#include "teca_table_reduce.h" -#include "teca_table_to_stream.h" -#include "teca_table_writer.h" -#include "teca_dataset_diff.h" -#include "teca_index_executive.h" -#include "teca_file_util.h" -#include "teca_mpi_manager.h" - -#include -#include -#include - -#include - -using std::cerr; -using std::endl; - -using boost::program_options::value; - - -int main(int argc, char **argv) -{ - // initialize MPI - teca_mpi_manager mpi_man(argc, argv); - - // initialize command line options description - // set up some common options to simplify use for most - // common scenarios - options_description basic_opt_defs( - "Basic usage:\n\n" - "The following options are the most commonly used. Information\n" - "on advanced options can be displayed using --advanced_help\n\n" - "Basic command line options", 120, -1 - ); - basic_opt_defs.add_options() - ("track_file", value(), "file path to read the cyclone from (tracks.bin)") - ("wind_files", value(), "regex matching simulation files containing wind fields ()") - ("track_file_out", value(), "file path to write cyclone tracks with size (tracks_size.bin)") - ("wind_u_var", value(), "name of variable with wind x-component (UBOT)") - ("wind_v_var", value(), "name of variable with wind y-component (VBOT)") - ("track_mask", value(), "expression to filter tracks by ()") - ("number_of_bins", value(), "number of bins in the radial wind decomposition (32)") - ("profile_type", value(), "radial wind profile type. max or avg (avg)") - ("search_radius", value(), "size of search window in deg lat (6)") - ("first_track", value(), "first track to process") - ("last_track", value(), "last track to process") - ("n_threads", value(), "thread pool size. default is 1. -1 for all") - ("help", "display the basic options help") - ("advanced_help", "display the advanced options help") - ("full_help", "display entire help message") - ; - - // add all options from each pipeline stage for more advanced use - options_description advanced_opt_defs( - "Advanced usage:\n\n" - "The following list contains the full set options giving one full\n" - "control over all runtime modifiable parameters. The basic options\n" - "(see" "--help) map to these, and will override them if both are\n" - "specified.\n\n" - "tc storm size pipeline:\n\n" - " (track reader)--(track filter)\n" - " \\\n" - " (wind reader)--(storm size)\n" - " \\\n" - " (map reduce)--(table sort)\n" - " \\\n" - " (track writer)\n\n" - "Advanced command line options", -1, 1 - ); - - // create the pipeline stages here, they contain the - // documentation and parse command line. - // objects report all of their properties directly - // set default options here so that command line options override - // them. while we are at it connect the pipeline - p_teca_table_reader track_reader = teca_table_reader::New(); - track_reader->get_properties_description("track_reader", advanced_opt_defs); - track_reader->set_file_name("tracks.bin"); - - p_teca_table_remove_rows track_filter = teca_table_remove_rows::New(); - track_filter->get_properties_description("track_filter", advanced_opt_defs); - - p_teca_cf_reader wind_reader = teca_cf_reader::New(); - wind_reader->get_properties_description("wind_reader", advanced_opt_defs); - - p_teca_normalize_coordinates wind_coords = teca_normalize_coordinates::New(); - wind_coords->set_input_connection(wind_reader->get_output_port()); - - p_teca_tc_storm_size storm_size = teca_tc_storm_size::New(); - storm_size->get_properties_description("storm_size", advanced_opt_defs); - storm_size->set_input_connection(1, wind_coords->get_output_port()); - - p_teca_table_reduce map_reduce = teca_table_reduce::New(); - map_reduce->get_properties_description("map_reduce", advanced_opt_defs); - map_reduce->set_input_connection(storm_size->get_output_port()); - - p_teca_table_sort sort = teca_table_sort::New(); - sort->get_properties_description("table_sort", advanced_opt_defs); - sort->set_input_connection(map_reduce->get_output_port()); - sort->set_index_column("track_id"); - sort->enable_stable_sort(); - - p_teca_table_writer track_writer = teca_table_writer::New(); - track_writer->get_properties_description("track_writer", advanced_opt_defs); - track_writer->set_input_connection(sort->get_output_port()); - track_writer->set_file_name("tracks_size.bin"); - - // package basic and advanced options for display - options_description all_opt_defs(-1, -1); - all_opt_defs.add(basic_opt_defs).add(advanced_opt_defs); - - // parse the command line - variables_map opt_vals; - try - { - boost::program_options::store( - boost::program_options::command_line_parser(argc, argv).options(all_opt_defs).run(), - opt_vals); - - if (mpi_man.get_comm_rank() == 0) - { - if (opt_vals.count("help")) - { - cerr << endl - << "usage: teca_tc_storm_size [options]" << endl - << endl - << basic_opt_defs << endl - << endl; - return -1; - } - if (opt_vals.count("advanced_help")) - { - cerr << endl - << "usage: teca_tc_detect [options]" << endl - << endl - << advanced_opt_defs << endl - << endl; - return -1; - } - if (opt_vals.count("full_help")) - { - cerr << endl - << "usage: teca_tc_detect [options]" << endl - << endl - << all_opt_defs << endl - << endl; - return -1; - } - } - - boost::program_options::notify(opt_vals); - } - catch (std::exception &e) - { - TECA_ERROR("Error parsing command line options. See --help " - "for a list of supported options. " << e.what()) - return -1; - } - - // pass command line arguments into the pipeline objects - // advanced options are processed first, so that the basic - // options will override them - track_reader->set_properties("track_reader", opt_vals); - track_filter->set_properties("track_filter", opt_vals); - wind_reader->set_properties("wind_reader", opt_vals); - storm_size->set_properties("storm_size", opt_vals); - map_reduce->set_properties("map_reduce", opt_vals); - sort->set_properties("table_sort", opt_vals); - track_writer->set_properties("track_writer", opt_vals); - - // now pass in the basic options, these are processed - // last so that they will take precedence - if (opt_vals.count("track_file")) - track_reader->set_file_name(opt_vals["track_file"].as()); - - if (opt_vals.count("wind_files")) - { - wind_reader->set_files_regex(opt_vals["wind_files"].as()); - } - else - { - TECA_ERROR("--wind_files is a required option") - return -1; - } - - if (opt_vals.count("track_file_out")) - track_writer->set_file_name(opt_vals["track_file_out"].as()); - - if (opt_vals.count("track_mask")) - { - track_filter->set_input_connection(track_reader->get_output_port()); - track_filter->set_mask_expression(opt_vals["track_mask"].as()); - storm_size->set_input_connection(0, track_filter->get_output_port()); - } - else - { - storm_size->set_input_connection(0, track_reader->get_output_port()); - } - - if (opt_vals.count("wind_u_var")) - storm_size->set_wind_u_variable(opt_vals["wind_u_var"].as()); - - if (opt_vals.count("wind_v_var")) - storm_size->set_wind_v_variable(opt_vals["wind_v_var"].as()); - - if (opt_vals.count("n_radial_bins")) - storm_size->set_number_of_radial_bins(opt_vals["n_radial_bins"].as()); - - if (opt_vals.count("profile_type")) - { - std::string profile_type = opt_vals["profile_type"].as(); - if (profile_type == "avg") - { - storm_size->set_profile_type(1); - } - else if (profile_type == "max") - { - storm_size->set_profile_type(0); - } - else - { - TECA_ERROR("invalid profile_type " << profile_type) - return -1; - } - } - - if (opt_vals.count("search_radius")) - storm_size->set_search_radius(opt_vals["search_radius"].as()); - - if (opt_vals.count("first_track")) - map_reduce->set_start_index(opt_vals["first_track"].as()); - - if (opt_vals.count("last_track")) - map_reduce->set_end_index(opt_vals["last_track"].as()); - - if (opt_vals.count("n_threads")) - map_reduce->set_thread_pool_size(opt_vals["n_threads"].as()); - else - map_reduce->set_thread_pool_size(-1); - - // run the pipeline - track_writer->update(); - - return 0; -} From 55c4330cf03d7bc492a891fffacae2082bb6f183 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Thu, 7 May 2020 15:14:00 -0700 Subject: [PATCH 032/385] Supporting thread num specification from the app --- apps/teca_deeplabv3p_ar_detect.in | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/teca_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in index 56723c073..ad6258637 100755 --- a/apps/teca_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -35,6 +35,11 @@ def main(): help='name of variable with integrated vapor transport (IVT)' ) + parser.add_argument( + '--num_threads', type=int, default=-1, + help='number of threads used in torch for parallelizing CPU operations' + ) + parser.add_argument( '--pytorch_deeplab_model', type=str, required=False, help='the pretrained deeplabv3plus model file' @@ -145,6 +150,11 @@ def main(): args.ivt ) + if args.num_threads: + deeplabv3p_ar_detect.set_num_threads( + args.num_threads + ) + if args.pytorch_deeplab_model and args.pytorch_resnet_model: deeplabv3p_ar_detect.build_model( args.pytorch_deeplab_model, From d229bda95f9a7257d9d0685a201ca2e93747f84f Mon Sep 17 00:00:00 2001 From: elbashandy Date: Thu, 7 May 2020 16:24:14 -0700 Subject: [PATCH 033/385] Cleaning up formatting --- apps/teca_deeplabv3p_ar_detect.in | 148 +++++++++++------------------- 1 file changed, 56 insertions(+), 92 deletions(-) diff --git a/apps/teca_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in index ad6258637..d25ecba74 100755 --- a/apps/teca_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -14,96 +14,60 @@ except ImportError: def main(): # parse the command line parser = argparse.ArgumentParser() - parser.add_argument( - '--input_file', type=str, required=False, - help='file path to the simulation to search for atmospheric rivers' - ) + parser.add_argument('--input_file', type=str, required=False, + help='file path to the simulation to search for atmospheric rivers') - parser.add_argument( - '--input_regex', type=str, required=False, - help='regex matching simulation files to search for atmospheric rivers' - ) + parser.add_argument('--input_regex', type=str, required=False, + help='regex matching simulation files to search for atmospheric rivers') - parser.add_argument( - '--output_file', type=str, required=False, + parser.add_argument('--output_file', type=str, required=False, default=r'deeplabv3p_ar_detect_%t%.nc', - help=r'file pattern for output netcdf files (%%t%% is the time index)' - ) + help=r'file pattern for output netcdf files (%%t%% is the time index)') - parser.add_argument( - '--ivt', type=str, default='IVT', - help='name of variable with integrated vapor transport (IVT)' - ) + parser.add_argument('--ivt', type=str, default='IVT', + help='name of variable with integrated vapor transport (IVT)') - parser.add_argument( - '--num_threads', type=int, default=-1, - help='number of threads used in torch for parallelizing CPU operations' - ) + parser.add_argument('--num_threads', type=int, default=-1, + help='number of threads used in torch for parallelizing CPU operations') - parser.add_argument( - '--pytorch_deeplab_model', type=str, required=False, - help='the pretrained deeplabv3plus model file' - ) + parser.add_argument('--pytorch_deeplab_model', type=str, required=False, + help='the pretrained deeplabv3plus model file') - parser.add_argument( - '--pytorch_resnet_model', type=str, required=False, - help='the pretrained resnet model file' - ) + parser.add_argument('--pytorch_resnet_model', type=str, required=False, + help='the pretrained resnet model file') - parser.add_argument( - '--t_axis_variable', type=str, required=False, - help='time dimension name' - ) + parser.add_argument('--t_axis_variable', type=str, required=False, + help='time dimension name') - parser.add_argument( - '--t_calendar', type=str, required=False, - help='time calendar' - ) + parser.add_argument('--t_calendar', type=str, required=False, + help='time calendar') - parser.add_argument( - '--t_units', type=str, required=False, - help='time unit' - ) + parser.add_argument('--t_units', type=str, required=False, + help='time unit') - parser.add_argument( - '--filename_time_template', type=str, required=False, - help='filename time template' - ) + parser.add_argument('--filename_time_template', type=str, required=False, + help='filename time template') - parser.add_argument( - '--compression_level', type=int, required=False, - help='the compression level used for each variable' - ) + parser.add_argument('--compression_level', type=int, required=False, + help='the compression level used for each variable') - parser.add_argument( - '--date_format', type=str, required=False, - help='the format for the date to write in the filename' - ) + parser.add_argument('--date_format', type=str, required=False, + help='the format for the date to write in the filename') - parser.add_argument( - '--first_step', type=int, required=False, - help='first time step to process' - ) + parser.add_argument('--first_step', type=int, required=False, + help='first time step to process') - parser.add_argument( - '--last_step', type=int, required=False, - help='last time step to process' - ) + parser.add_argument('--last_step', type=int, required=False, + help='last time step to process') - parser.add_argument( - '--steps_per_file', type=int, required=False, - help='number of time steps per output file' - ) + parser.add_argument('--steps_per_file', type=int, required=False, + help='number of time steps per output file') - parser.add_argument( - '--start_date', type=str, required=False, - help='first time to proces in YYYY-MM-DD hh:mm:ss format' - ) + parser.add_argument('--start_date', type=str, required=False, + help='first time to proces in YYYY-MM-DD hh:mm:ss format') - parser.add_argument( - '--end_date', type=str, required=False, - help='end time to proces in YYYY-MM-DD hh:mm:ss format' - ) + parser.add_argument('--end_date', type=str, required=False, + help='end time to proces in YYYY-MM-DD hh:mm:ss format') args = parser.parse_args() @@ -112,7 +76,7 @@ def main(): raise parser.error( "missing file name or regex for simulation reader. " "See --help for a list of command line options." - ) + ) cf_reader = teca_cf_reader.New() @@ -133,80 +97,80 @@ def main(): if args.input_file: cf_reader.append_file_name( args.input_file - ) + ) if args.input_regex: cf_reader.set_files_regex( args.input_regex - ) + ) if args.output_file: cf_writer.set_file_name( args.output_file - ) + ) if args.ivt: deeplabv3p_ar_detect.set_variable_name( args.ivt - ) + ) if args.num_threads: deeplabv3p_ar_detect.set_num_threads( args.num_threads - ) + ) if args.pytorch_deeplab_model and args.pytorch_resnet_model: deeplabv3p_ar_detect.build_model( args.pytorch_deeplab_model, args.pytorch_resnet_model - ) + ) else: deeplabv3p_ar_detect.build_model() if args.t_axis_variable is not None: cf_reader.set_t_axis_variable( args.t_axis_variable - ) + ) if args.t_calendar: cf_reader.set_t_calendar( args.t_calendar - ) + ) if args.t_units: cf_reader.set_t_units( args.t_units - ) + ) if args.filename_time_template: cf_reader.set_filename_time_template( args.filename_time_template - ) + ) if args.compression_level: cf_writer.set_compression_level( args.compression_level - ) + ) if args.date_format: cf_writer.set_date_format( args.date_format - ) + ) if args.first_step: cf_writer.set_first_step( args.first_step - ) + ) if args.last_step: cf_writer.set_last_step( args.last_step - ) + ) if args.steps_per_file: cf_writer.set_steps_per_file( args.steps_per_file - ) + ) # some minimal check for missing options if (cf_reader.get_number_of_file_names() == 0 and @@ -215,14 +179,14 @@ def main(): raise ValueError( "missing file name or regex for simulation reader. " "See --help for a list of command line options." - ) + ) if not cf_writer.get_file_name(): if rank == 0: raise ValueError( "missing file name pattern for netcdf writer. " "See --help for a list of command line options." - ) + ) if args.start_date or args.end_date: @@ -237,14 +201,14 @@ def main(): if args.start_date: first_step = teca_coordinate.time_step_of( time, True, calendar, units, args.start_date - ) + ) cf_writer.set_first_step(first_step) # and end date if args.end_date: last_step = teca_coordinate.time_step_of( time, False, calendar, units, args.end_date - ) + ) cf_writer.set_last_step(last_step) # run the pipeline From e864651186d6a30aa77e600969bfdc1244dea8f6 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Wed, 13 May 2020 18:44:53 -0700 Subject: [PATCH 034/385] Added ar_binary_tag to the deeplab algorithm --- alg/teca_model_segmentation.py | 8 +++++++- apps/teca_deeplabv3p_ar_detect.in | 28 +++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/alg/teca_model_segmentation.py b/alg/teca_model_segmentation.py index 033b4106d..429fdeff8 100644 --- a/alg/teca_model_segmentation.py +++ b/alg/teca_model_segmentation.py @@ -162,7 +162,13 @@ def request(port, md_in, req_in): arrays = [] if req.has('arrays'): - arrays = req['arrays'] + if isinstance(req['arrays'], list): + arrays.extend(req['arrays']) + else: + arrays.append(req['arrays']) + + if self.pred_name in arrays: + arrays.remove(self.pred_name) arrays.append(self.variable_name) req['arrays'] = arrays diff --git a/apps/teca_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in index d25ecba74..edb111d94 100755 --- a/apps/teca_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -27,6 +27,10 @@ def main(): parser.add_argument('--ivt', type=str, default='IVT', help='name of variable with integrated vapor transport (IVT)') + parser.add_argument('--binary_ar_threshold', type=float, + default=0.6666666667, help='probability threshold for segmenting' + 'ar_probability to produce ar_binary_tag') + parser.add_argument('--num_threads', type=int, default=-1, help='number of threads used in torch for parallelizing CPU operations') @@ -86,12 +90,16 @@ def main(): deeplabv3p_ar_detect = teca_deeplabv3p_ar_detect.New() deeplabv3p_ar_detect.set_input_connection(coords.get_output_port()) + ar_tag = teca_binary_segmentation.New() + ar_tag.set_input_connection(deeplabv3p_ar_detect.get_output_port()) + ar_tag.set_threshold_mode(ar_tag.BY_VALUE) + ar_tag.set_threshold_variable("ar_probability") + ar_tag.set_segmentation_variable("ar_binary_tag") + teca_exec = teca_index_executive.New() cf_writer = teca_cf_writer.New() - cf_writer.set_input_connection( - deeplabv3p_ar_detect.get_output_port() - ) + cf_writer.set_input_connection(ar_tag.get_output_port()) cf_writer.set_thread_pool_size(1) if args.input_file: @@ -211,6 +219,20 @@ def main(): ) cf_writer.set_last_step(last_step) + ar_tag_threshold = args.binary_ar_threshold + # set the threshold for calculating ar_binary_tag + ar_tag.set_low_threshold_value(ar_tag_threshold) + # add metadata for ar_binary_tag + seg_atts = teca_metadata() + seg_atts["long_name"] = "binary indicator of atmospheric river" + seg_atts["description"] = "binary indicator of atmospheric river" + seg_atts["scheme"] = "cascade_bard" + seg_atts["version"] = "1.0" + seg_atts["note"] = "derived by thresholding ar_probability >= %f" \ + % ar_tag_threshold + + ar_tag.set_segmentation_variable_atts(seg_atts) + # run the pipeline cf_writer.set_executive(teca_exec) cf_writer.update() From 934f0a5c30c306c54d3a91aa4a3c74e39ebde868 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Wed, 13 May 2020 19:05:16 -0700 Subject: [PATCH 035/385] Removed resnet model dependence as the deeplab model was overwriting the resnet parameters --- alg/teca_deeplabv3p_ar_detect.py | 40 +++++------------------- apps/teca_deeplabv3p_ar_detect.in | 8 ++--- test/python/CMakeLists.txt | 2 -- test/python/test_deeplabv3p_ar_detect.py | 18 +++++------ 4 files changed, 16 insertions(+), 52 deletions(-) diff --git a/alg/teca_deeplabv3p_ar_detect.py b/alg/teca_deeplabv3p_ar_detect.py index 26b7a977a..74a6c9cf6 100644 --- a/alg/teca_deeplabv3p_ar_detect.py +++ b/alg/teca_deeplabv3p_ar_detect.py @@ -50,8 +50,7 @@ def forward(self, x): class ResNet(nn.Module): - def __init__(self, nInputChannels, block, layers, os=16, - pretrained=False, state_dict_resnet=None): + def __init__(self, nInputChannels, block, layers, os=16): self.inplanes = 64 super(ResNet, self).__init__() if os == 16: @@ -65,8 +64,6 @@ def __init__(self, nInputChannels, block, layers, os=16, else: raise NotImplementedError - self.state_dict_resnet = state_dict_resnet - # Modules self.conv1 = nn.Conv2d(nInputChannels, 64, kernel_size=7, stride=2, padding=3, bias=False) @@ -85,9 +82,6 @@ def __init__(self, nInputChannels, block, layers, os=16, self._init_weight() - if pretrained: - self._load_pretrained_model() - def _make_layer(self, block, planes, blocks, stride=1, rate=1): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: @@ -152,16 +146,10 @@ def _init_weight(self): m.weight.data.fill_(1) m.bias.data.zero_() - def _load_pretrained_model(self): - self.load_state_dict(self.state_dict_resnet) - -def ResNet101(nInputChannels=3, os=16, pretrained=False, - state_dict_resnet=None): +def ResNet101(nInputChannels=3, os=16): model = ResNet( - nInputChannels, Bottleneck, [3, 4, 23, 3], os, - pretrained=pretrained, state_dict_resnet=state_dict_resnet - ) + nInputChannels, Bottleneck, [3, 4, 23, 3], os) return model @@ -200,7 +188,7 @@ def _init_weight(self): class DeepLabv3_plus(nn.Module): def __init__(self, nInputChannels=3, n_classes=21, os=16, - pretrained=False, state_dict_resnet=None, _print=True): + _print=True): if _print: sys.stdout.write("Constructing DeepLabv3+ model...\n") sys.stdout.write("Number of classes: {}\n".format(n_classes)) @@ -211,10 +199,7 @@ def __init__(self, nInputChannels=3, n_classes=21, os=16, super(DeepLabv3_plus, self).__init__() # Atrous Conv - self.resnet_features = ResNet101( - nInputChannels, os, pretrained=pretrained, - state_dict_resnet=state_dict_resnet - ) + self.resnet_features = ResNet101(nInputChannels, os) # ASPP if os == 16: @@ -402,8 +387,7 @@ def output_postprocess(self, ouput_data): return ouput_data.ravel() - def build_model(self, state_dict_deeplab_file=None, - state_dict_resnet_file=None): + def build_model(self, state_dict_deeplab_file=None): """ Load model from file system. If multi-threading is used rank 0 loads the model file and broadcasts it to the other ranks @@ -416,21 +400,11 @@ def build_model(self, state_dict_deeplab_file=None, deeplab_sd_path ) - if not state_dict_resnet_file: - resnet_sd_path = \ - "resnet101-5d3b4d8f-state_dict.pth" - state_dict_resnet_file = os.path.join( - teca_py.get_teca_data_root(), - resnet_sd_path - ) - comm = self.get_communicator() state_dict_deeplab = self.load_state_dict(state_dict_deeplab_file) - state_dict_resnet = self.load_state_dict(state_dict_resnet_file) - model = DeepLabv3_plus(n_classes=1, pretrained=True, _print=False, - state_dict_resnet=state_dict_resnet) + model = DeepLabv3_plus(n_classes=1, _print=False) model.load_state_dict(state_dict_deeplab) self.set_model(model) diff --git a/apps/teca_deeplabv3p_ar_detect.in b/apps/teca_deeplabv3p_ar_detect.in index edb111d94..06dc6b318 100755 --- a/apps/teca_deeplabv3p_ar_detect.in +++ b/apps/teca_deeplabv3p_ar_detect.in @@ -37,9 +37,6 @@ def main(): parser.add_argument('--pytorch_deeplab_model', type=str, required=False, help='the pretrained deeplabv3plus model file') - parser.add_argument('--pytorch_resnet_model', type=str, required=False, - help='the pretrained resnet model file') - parser.add_argument('--t_axis_variable', type=str, required=False, help='time dimension name') @@ -127,10 +124,9 @@ def main(): args.num_threads ) - if args.pytorch_deeplab_model and args.pytorch_resnet_model: + if args.pytorch_deeplab_model: deeplabv3p_ar_detect.build_model( - args.pytorch_deeplab_model, - args.pytorch_resnet_model + args.pytorch_deeplab_model ) else: deeplabv3p_ar_detect.build_model() diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index c9a487f86..db22b399b 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -233,7 +233,6 @@ teca_add_test(py_test_bayesian_ar_detect_mpi_threads teca_add_test(py_test_deeplabv3p_ar_detect COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" - "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1 FEATURES ${TECA_HAS_NETCDF} @@ -243,7 +242,6 @@ teca_add_test(py_test_deeplabv3p_ar_detect_mpi COMMAND ${MPIEXEC} -n ${HALF_TEST_CORES} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_deeplabv3p_ar_detect.py "${TECA_DATA_ROOT}/cascade_deeplab_IVT.pt" - "${TECA_DATA_ROOT}/resnet101-5d3b4d8f-state_dict.pth" "${TECA_DATA_ROOT}/ARTMIP_MERRA_2D.*\.nc$" "${TECA_DATA_ROOT}/test_deeplabv3p_ar_detect.bin" IVT 1 FEATURES ${TECA_HAS_NETCDF} ${TECA_HAS_MPI} ${MPI4Py_FOUND} diff --git a/test/python/test_deeplabv3p_ar_detect.py b/test/python/test_deeplabv3p_ar_detect.py index 88e9d4fd0..a78615498 100644 --- a/test/python/test_deeplabv3p_ar_detect.py +++ b/test/python/test_deeplabv3p_ar_detect.py @@ -15,10 +15,10 @@ def main(): set_stack_trace_on_error() set_stack_trace_on_mpi_error() - if (len(sys.argv) != 7): + if (len(sys.argv) != 6): sys.stderr.write( "\n\nUsage error:\n" - "test_deeplabv3p_ar_detect [deeplab model] [resnet model]" + "test_deeplabv3p_ar_detect [deeplab model]" "[mesh data regex] [baseline mesh]" "[water vapor var] [num threads]\n\n" ) @@ -26,11 +26,10 @@ def main(): # parse command line deeplab_model = sys.argv[1] - resnet_model = sys.argv[2] - mesh_data_regex = sys.argv[3] - baseline_mesh = sys.argv[4] - water_vapor_var = sys.argv[5] - n_threads = int(sys.argv[6]) + mesh_data_regex = sys.argv[2] + baseline_mesh = sys.argv[3] + water_vapor_var = sys.argv[4] + n_threads = int(sys.argv[5]) cf_reader = teca_cf_reader.New() cf_reader.set_files_regex(mesh_data_regex) @@ -44,10 +43,7 @@ def main(): deeplabv3p_ar_detect.set_variable_name(water_vapor_var) deeplabv3p_ar_detect.set_num_threads(n_threads) - deeplabv3p_ar_detect.build_model( - deeplab_model, - resnet_model - ) + deeplabv3p_ar_detect.build_model(deeplab_model) if os.path.exists(baseline_mesh): # run the test From 4f13c5d16042a9fef84f8771456737d7585fdc76 Mon Sep 17 00:00:00 2001 From: elbashandy Date: Thu, 14 May 2020 14:39:35 -0700 Subject: [PATCH 036/385] Updating svn revision in travis-ci yml file for tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 85092dc4a..692bf9744 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ env: - BUILD_TYPE=Debug - TECA_DIR=/travis_teca_dir - TECA_PYTHON_VERSION=3 - - TECA_DATA_REVISION=56 + - TECA_DATA_REVISION=58 jobs: - DOCKER_IMAGE=ubuntu IMAGE_VERSION=18.04 IMAGE_NAME=ubuntu_18_04 - DOCKER_IMAGE=fedora IMAGE_VERSION=31 IMAGE_NAME=fedora_31 From e84563a91acecc571d2c3af1259d79d1524bb8e4 Mon Sep 17 00:00:00 2001 From: Burlen Loring Date: Mon, 14 Oct 2019 09:35:15 -0700 Subject: [PATCH 037/385] add thread parallel streaming mode this lets results from the teca_threaded_algorithm be processed as they become ready rather than all at once when all are ready. results are processed by the thread driving the pipeline. A new streaming flag passed to ::execute and virtual method indicates if it is safe to use MPI or other non-threadsafe operations. The flag is set to 0 when it is safe to use MPI etc. Includes a regression test. --- alg/teca_bayesian_ar_detect.cxx | 30 ++--------- alg/teca_table_reduce.cxx | 13 ++--- alg/teca_table_reduce.h | 9 ++-- core/teca_index_executive.cxx | 1 + core/teca_index_reduce.cxx | 47 +++++++++-------- core/teca_index_reduce.h | 3 +- core/teca_thread_pool.h | 67 +++++++++++++++++++++++- core/teca_threaded_algorithm.cxx | 64 +++++++++++++++++----- core/teca_threaded_algorithm.h | 28 +++++++++- io/teca_cf_reader.cxx | 2 +- python/teca_py_algorithm.h | 35 ++++++++++--- test/CMakeLists.txt | 18 +++++++ test/test_array/array_temporal_stats.cxx | 3 +- test/test_array/array_temporal_stats.h | 9 ++-- test/test_bayesian_ar_detect.cpp | 6 +-- test/test_descriptive_statistics.cpp | 16 +++--- 16 files changed, 247 insertions(+), 104 deletions(-) diff --git a/alg/teca_bayesian_ar_detect.cxx b/alg/teca_bayesian_ar_detect.cxx index 4285f4c87..aa17ea5a4 100644 --- a/alg/teca_bayesian_ar_detect.cxx +++ b/alg/teca_bayesian_ar_detect.cxx @@ -31,6 +31,7 @@ #include #endif +namespace { // This routine appends the contents of dataset_0.get_metadata.get(property_name) // onto that from dataset_0 and overwrites the contents `property_name' in the @@ -44,7 +45,7 @@ // dataset_1 : (p_teca_dataset) the RHS dataset in the reduction // // mesh_out : (p_teca_cartesian_mesh) the output of the reduction -void property_reduce(std::string property_name, +/*void property_reduce(std::string property_name, p_teca_dataset dataset_0, p_teca_dataset dataset_1, p_teca_cartesian_mesh mesh_out) { @@ -63,12 +64,7 @@ void property_reduce(std::string property_name, // Overwrite the concatenated property vector in the output dataset mesh_out->get_metadata().set(property_name, property_vector); -} - - - - -namespace { +}*/ // drive the pipeline execution once for each parameter table row // injects the parameter values into the upstream requests @@ -195,7 +191,7 @@ class parameter_table_reduction return 0; } - // this reducion computes the probability from each parameter table run + // this reduction computes the probability from each parameter table run // if the inputs have the probability array this is used, if not the // array is computed from the filtered connected components. after the // reduction runs, the result will need to be normalized. @@ -413,7 +409,7 @@ class parameter_table_reduction if (prob) { - // probability has already been comnputed, pass it through + // probability has already been computed, pass it through prob_out = prob; n_wvcc_out = n_wvcc; pt_row_out = pt_row; @@ -486,22 +482,6 @@ class parameter_table_reduction else if (dataset_1) mesh_out->copy_metadata(dataset_1); - // TODO -- this essentally copies the parameter table into the metadata - // instead could we access the parameter table directly. and use the - // information arrays ar_count, and parameter_table_row? - - // Do property reduction on AR detector parameters and output that - // are stored in the metadata. This operation overwrites the metadata - // in mesh_out with the combined metadata from the LHS and RHS datasets. - /*property_reduce("low_threshold_value", dataset_0, dataset_1, mesh_out); - property_reduce("high_threshold_value", dataset_0, dataset_1, mesh_out); - property_reduce("low_area_threshold_km", dataset_0, dataset_1, mesh_out); - property_reduce("high_area_threshold_km", dataset_0, dataset_1, mesh_out); - property_reduce("gaussian_filter_center_lat", dataset_0, dataset_1, mesh_out); - property_reduce("gaussian_filter_hwhm", dataset_0, dataset_1, mesh_out); - property_reduce("number_of_components", dataset_0, dataset_1, mesh_out); - property_reduce("component_area", dataset_0, dataset_1, mesh_out);*/ - mesh_out->get_point_arrays()->append(this->probability_array_name, prob_out); mesh_out->get_information_arrays()->append("ar_count", n_wvcc_out); diff --git a/alg/teca_table_reduce.cxx b/alg/teca_table_reduce.cxx index 720a5cb9c..e243fc7a0 100644 --- a/alg/teca_table_reduce.cxx +++ b/alg/teca_table_reduce.cxx @@ -48,14 +48,14 @@ teca_metadata teca_table_reduce::initialize_output_metadata( } // -------------------------------------------------------------------------- -p_teca_dataset teca_table_reduce::reduce( - const const_p_teca_dataset &left_ds, +p_teca_dataset teca_table_reduce::reduce(const const_p_teca_dataset &left_ds, const const_p_teca_dataset &right_ds) { #ifdef TECA_DEBUG cerr << teca_parallel_id() << "teca_table_reduce::reduce" << endl; #endif + const_p_teca_table left_table = std::dynamic_pointer_cast(left_ds); @@ -64,10 +64,7 @@ p_teca_dataset teca_table_reduce::reduce( p_teca_table output_table; - bool left = left_table && *left_table; - bool right = right_table && *right_table; - - if (left && right) + if (left_table && right_table) { output_table = std::dynamic_pointer_cast(left_table->new_copy()); @@ -75,13 +72,13 @@ p_teca_dataset teca_table_reduce::reduce( output_table->concatenate_rows(right_table); } else - if (left) + if (left_table) { output_table = std::dynamic_pointer_cast(left_table->new_copy()); } else - if (right) + if (right_table) { output_table = std::dynamic_pointer_cast(right_table->new_copy()); diff --git a/alg/teca_table_reduce.h b/alg/teca_table_reduce.h index 17c25fcc0..2740494d6 100644 --- a/alg/teca_table_reduce.h +++ b/alg/teca_table_reduce.h @@ -29,17 +29,14 @@ class teca_table_reduce : public teca_index_reduce teca_table_reduce(); // overrides - p_teca_dataset reduce( - const const_p_teca_dataset &left, + p_teca_dataset reduce(const const_p_teca_dataset &left, const const_p_teca_dataset &right) override; std::vector initialize_upstream_request( - unsigned int port, - const std::vector &input_md, + unsigned int port, const std::vector &input_md, const teca_metadata &request) override; - teca_metadata initialize_output_metadata( - unsigned int port, + teca_metadata initialize_output_metadata(unsigned int port, const std::vector &input_md) override; }; diff --git a/core/teca_index_executive.cxx b/core/teca_index_executive.cxx index 9540d6c40..d864cec5a 100644 --- a/core/teca_index_executive.cxx +++ b/core/teca_index_executive.cxx @@ -75,6 +75,7 @@ int teca_index_executive::initialize(MPI_Comm comm, const teca_metadata &md) #if !defined(TECA_HAS_MPI) (void)comm; #endif + this->requests.clear(); // locate the keys that enable us to know how many diff --git a/core/teca_index_reduce.cxx b/core/teca_index_reduce.cxx index 3c450e21c..9f87a24c1 100644 --- a/core/teca_index_reduce.cxx +++ b/core/teca_index_reduce.cxx @@ -119,7 +119,9 @@ void block_decompose(MPI_Comm comm, unsigned long n_indices, unsigned long n_ran // -------------------------------------------------------------------------- teca_index_reduce::teca_index_reduce() : start_index(0), end_index(-1) -{} +{ + this->set_stream_size(2); +} #if defined(TECA_HAS_BOOST) // -------------------------------------------------------------------------- @@ -262,40 +264,39 @@ teca_metadata teca_index_reduce::get_output_metadata( // -------------------------------------------------------------------------- const_p_teca_dataset teca_index_reduce::reduce_local( - std::vector input_data) // pass by value is intentional + std::vector input_data) // pass by value is necessary { unsigned long n_in = input_data.size(); if (n_in == 0) return p_teca_dataset(); - if (n_in == 1) - return input_data[0]; - - while (n_in > 1) + do { if (n_in % 2) TECA_PROFILE_METHOD(128, this, "reduce", - input_data[0] = this->reduce(input_data[0], input_data[n_in-1]); + input_data[0] = this->reduce(input_data[0], + (n_in > 1 ? input_data[n_in-1] : nullptr)); ) - unsigned long n = n_in/2; - for (unsigned long i = 0; i < n; ++i) + n_in /= 2; + for (unsigned long i = 0; i < n_in; ++i) { unsigned long ii = 2*i; TECA_PROFILE_METHOD(128, this, "reduce", - input_data[i] = this->reduce(input_data[ii], input_data[ii+1]); + input_data[i] = this->reduce(input_data[ii], + input_data[ii+1]); ) } - - n_in = n; } + while (n_in > 1); + return input_data[0]; } // -------------------------------------------------------------------------- const_p_teca_dataset teca_index_reduce::reduce_remote( - const_p_teca_dataset local_data) // pass by value is intentional + const_p_teca_dataset local_data) { #if defined(TECA_HAS_MPI) int is_init = 0; @@ -389,21 +390,25 @@ const_p_teca_dataset teca_index_reduce::reduce_remote( } // -------------------------------------------------------------------------- -const_p_teca_dataset teca_index_reduce::execute( - unsigned int port, +const_p_teca_dataset teca_index_reduce::execute(unsigned int port, const std::vector &input_data, - const teca_metadata &request) + const teca_metadata &request, int streaming) { (void)port; (void)request; - // note: it is not an error to have no input data. - // this can occur if there are fewer indices - // to process than there are MPI ranks. + // note: it is not an error to have no input data. this can occur if there + // are fewer indices to process than there are MPI ranks. + + const_p_teca_dataset tmp = this->reduce_local(input_data); - const_p_teca_dataset tmp = - this->reduce_remote(this->reduce_local(input_data)); + // when streaming execute will be called multiple times with 1 or more + // input datasets. When all the data has been passed streaming is 0. Only + // then do we reduce remote data and finalize the reduction. + if (streaming) + return tmp; + tmp = this->reduce_remote(tmp); if (!tmp) return nullptr; diff --git a/core/teca_index_reduce.h b/core/teca_index_reduce.h index 1f8916e4f..fea368534 100644 --- a/core/teca_index_reduce.h +++ b/core/teca_index_reduce.h @@ -75,7 +75,6 @@ class teca_index_reduce : public teca_threaded_algorithm virtual teca_metadata initialize_output_metadata(unsigned int port, const std::vector &input_md) = 0; - protected: // customized pipeline behavior and parallel code. // most derived classes won't need to override these. @@ -94,7 +93,7 @@ class teca_index_reduce : public teca_threaded_algorithm // dataset, which is returned. const_p_teca_dataset execute(unsigned int port, const std::vector &input_data, - const teca_metadata &request) override; + const teca_metadata &request, int streaming) override; // consumes time metadata, partitions time's across // MPI ranks. diff --git a/core/teca_thread_pool.h b/core/teca_thread_pool.h index a08251ca0..ba66f9352 100644 --- a/core/teca_thread_pool.h +++ b/core/teca_thread_pool.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #if defined(_GNU_SOURCE) #include @@ -65,7 +66,19 @@ class teca_thread_pool // datasets in the order that corresponding requests // were added to the queue. template