From 4ec7b776880858841df852e109cb9836bd9cd214 Mon Sep 17 00:00:00 2001 From: blychs Date: Wed, 2 Oct 2024 16:45:10 -0600 Subject: [PATCH 1/9] Check to avoid looking for columns if invalid Quick and dirty fix. A more thorough refactor will be performed after the tutorial. --- melodies_monet/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 109125cd..6d73173c 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -1756,7 +1756,7 @@ def plotting(self): if filter_criteria and 'altitude' in filter_criteria: vmin_y2, vmax_y2 = filter_criteria['altitude']['value'] elif filter_criteria is None: - if 'altitude' in pairdf.columns: + if isinstance(pairdf, pd.DataFrame) and 'altitude' in pairdf.columns: vmin_y2 = pairdf['altitude'].min() vmax_y2 = pairdf['altitude'].max() else: From c140897d088fa8e4473669b0de00d81fb78ba1c4 Mon Sep 17 00:00:00 2001 From: blychs Date: Wed, 2 Oct 2024 16:48:11 -0600 Subject: [PATCH 2/9] fix savefig in satplots --- melodies_monet/plots/satplots.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/melodies_monet/plots/satplots.py b/melodies_monet/plots/satplots.py index daf61d04..c2be2040 100644 --- a/melodies_monet/plots/satplots.py +++ b/melodies_monet/plots/satplots.py @@ -524,7 +524,7 @@ def make_spatial_overlay(df, vmodel, column_o=None, label_o=None, column_m=None, cax.tick_params(labelsize=text_kwargs['fontsize']*0.8,length=10.0,width=2.0,grid_linewidth=2.0) #plt.tight_layout(pad=0) - savefig(outname + '.png',loc=4, height=100, decorate=True, bbox_inches='tight', dpi=150) + savefig(outname + '.png',loc=4, logo_height=100, decorate=True, bbox_inches='tight', dpi=150) return ax def calculate_boxplot(df, df_reg=None,column=None, label=None, plot_dict=None, comb_bx = None, label_bx = None): @@ -671,7 +671,7 @@ def make_boxplot(comb_bx, label_bx, ylabel = None, vmin = None, vmax = None, out ax.set_ylim(ymin = vmin, ymax = vmax) plt.tight_layout() - savefig(outname + '.png',loc=4, height=100, decorate=True, bbox_inches='tight', dpi=200) + savefig(outname + '.png',loc=4, logo_height=100, decorate=True, bbox_inches='tight', dpi=200) def make_spatial_bias_gridded(df, column_o=None, label_o=None, column_m=None, label_m=None, ylabel = None, vmin=None, @@ -777,5 +777,5 @@ def make_spatial_bias_gridded(df, column_o=None, label_o=None, column_m=None, cax.tick_params(labelsize=text_kwargs['fontsize']*0.8,length=10.0,width=2.0,grid_linewidth=2.0) #plt.tight_layout(pad=0) - savefig(outname + '.png',loc=4, height=100, decorate=True, bbox_inches='tight', dpi=150) + savefig(outname + '.png',loc=4, logo_height=100, decorate=True, bbox_inches='tight', dpi=150) return ax From 91ad0c3111b8b042e9c92efb359b42bec41e15e5 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Thu, 3 Oct 2024 20:02:07 +0000 Subject: [PATCH 3/9] small fixes in OMPS level 3 pairing --- melodies_monet/util/satellite_utilities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/melodies_monet/util/satellite_utilities.py b/melodies_monet/util/satellite_utilities.py index 002bd773..6c93e7a7 100644 --- a/melodies_monet/util/satellite_utilities.py +++ b/melodies_monet/util/satellite_utilities.py @@ -140,11 +140,12 @@ def omps_l3_daily_o3_pairing(model_data,obs_data,ozone_ppbv_varname): grid_adjust = xe.Regridder(model_data[['latitude','longitude']],obs_data[['latitude','longitude']],'bilinear') mod_col_obsgrid = grid_adjust(column) # Aggregate time-step to daily means - daily_mean = mod_col_obsgrid.groupby('time.date').mean(numeric_only=True).compute() - + #daily_mean = mod_col_obsgrid.groupby('time.date').mean().compute() + daily_mean = mod_col_obsgrid.resample(time='1D').mean() # change dimension name for date to time - daily_mean = daily_mean.rename({'date':'time'}) + #daily_mean = daily_mean.rename({'date':'time'}) daily_mean = daily_mean.rename(ozone_ppbv_varname) + return xr.merge([daily_mean,obs_data]) def space_and_time_pairing(model_data,obs_data,pair_variables): From 13b03a33077801606f1e0687a69e265a5a611cc5 Mon Sep 17 00:00:00 2001 From: blychs Date: Thu, 3 Oct 2024 15:13:33 -0600 Subject: [PATCH 4/9] make 'altitude' check for ds and df --- melodies_monet/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 6d73173c..d63caa9e 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -1756,7 +1756,7 @@ def plotting(self): if filter_criteria and 'altitude' in filter_criteria: vmin_y2, vmax_y2 = filter_criteria['altitude']['value'] elif filter_criteria is None: - if isinstance(pairdf, pd.DataFrame) and 'altitude' in pairdf.columns: + if 'altitude' in pairdf.keys(): vmin_y2 = pairdf['altitude'].min() vmax_y2 = pairdf['altitude'].max() else: From 6ffc0315f46b36a67adf32c7e342ab8a5974970f Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Thu, 3 Oct 2024 22:25:01 +0000 Subject: [PATCH 5/9] update to OMPS_l2 file subsetter to work for NOAA-20 files (and others in series) --- melodies_monet/util/time_interval_subset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/melodies_monet/util/time_interval_subset.py b/melodies_monet/util/time_interval_subset.py index 3e671f39..4009cdca 100644 --- a/melodies_monet/util/time_interval_subset.py +++ b/melodies_monet/util/time_interval_subset.py @@ -20,6 +20,7 @@ def subset_model_filelist(all_files,timeformat,timestep,timeinterval): def subset_OMPS_l2(file_path,timeinterval): '''Dependent on filenaming convention + OMPS-satelliteid_NMTO3-L2_version_startingtimestamp_orbitnumber_endingtimestamp.h5 ''' import pandas as pd from glob import glob @@ -29,7 +30,7 @@ def subset_OMPS_l2(file_path,timeinterval): subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq='D',inclusive='left') for i in subset_interval: - fst = fnmatch.filter(all_files,'*OMPS-NPP_NMTO3-L2_v*_{}*_o*'.format(i.strftime('%Ym%m%d'))) + fst = fnmatch.filter(all_files,'*OMPS-*_NMTO3-L2_v*_{}*_o*'.format(i.strftime('%Ym%m%d'))) fst.sort() for j in fst: interval_files.append(j) From 27b3a85a4f66516bfc7239bbf59f1e81a60fbeb5 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 16:39:24 +0000 Subject: [PATCH 6/9] time chunking implementation for RAQMS and MOPITT --- melodies_monet/driver.py | 24 +++++++++++++++------ melodies_monet/util/time_interval_subset.py | 24 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 6d73173c..5dd7392f 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -271,7 +271,7 @@ def open_sat_obs(self, time_interval=None, control_dict=None): else: flst = self.file self.obj = mio.sat._omps_nadir_mm.read_OMPS_nm(flst) - + # couple of changes to move to reader self.obj = self.obj.swap_dims({'x':'time'}) # indexing needs self.obj = self.obj.sortby('time') # enforce time in order. @@ -279,10 +279,14 @@ def open_sat_obs(self, time_interval=None, control_dict=None): # additional development to deal with files crossing intervals needed (eg situtations where orbit start at 23hrs, ends next day). if time_interval is not None: self.obj = self.obj.sel(time=slice(time_interval[0],time_interval[-1])) - + elif self.sat_type == 'mopitt_l3': print('Reading MOPITT') - self.obj = mio.sat._mopitt_l3_mm.open_dataset(self.file, ['column','pressure_surf','apriori_col', + if time_interval is not None: + flst = tsub.subset_mopitt_l3(self.file,time_interval) + else: flst = self.file + print(flst) + self.obj = mio.sat._mopitt_l3_mm.open_dataset(flst, ['column','pressure_surf','apriori_col', 'apriori_surf','apriori_prof','ak_col']) elif self.sat_type == 'modis_l2': # from monetio import modis_l2 @@ -601,10 +605,18 @@ def open_model_files(self, time_interval=None, control_dict=None): self.mod_kwargs.update({"var_list": list_input_var}) self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs) elif 'raqms' in self.model.lower(): - if len(self.files) > 1: - self.obj = mio.raqms.open_mfdataset(self.files,**self.mod_kwargs) + if time_interval is not None: + # fill filelist with subset + print('subsetting model files to interval') + file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval) + else: + file_list = self.files + #print(file_list) + if len(file_list) > 1: + self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs) else: - self.obj = mio.raqms.open_dataset(self.files,**self.mod_kwargs) + self.obj = mio.models.raqms.open_dataset(file_list) + print('RAQMS files loaded') else: print('**** Reading Unspecified model output. Take Caution...') if len(self.files) > 1: diff --git a/melodies_monet/util/time_interval_subset.py b/melodies_monet/util/time_interval_subset.py index 4009cdca..d431eab1 100644 --- a/melodies_monet/util/time_interval_subset.py +++ b/melodies_monet/util/time_interval_subset.py @@ -36,6 +36,30 @@ def subset_OMPS_l2(file_path,timeinterval): interval_files.append(j) return interval_files +def subset_mopitt_l3(file_path,timeinterval): + '''Dependent on filenaming conventions + MOP03J-YYYYMMDD- + MOP03JM-201909- + ''' + import pandas as pd + from glob import glob + import fnmatch + all_files = glob(file_path) + interval_files = [] + if 'MOP03JM-' in all_files[0]: + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq='M') + strfmt = '%Y%m' + else: + subset_interval = pd.date_range(start=timeinterval[0],end=timeinterval[-1],freq='D') + strfmt = '%Y%m%d' + for i in subset_interval: + fst = fnmatch.filter(all_files,'*MOP*-{}*'.format(i.strftime(strfmt))) + fst.sort() + for j in fst: + interval_files.append(j) + print(interval_files) + return interval_files + def subset_MODIS_l2(file_path,timeinterval): '''Dependent on filenaming convention MOD04_L2.AYYYYDDD.HHMM.collection.timestamp.hdf From 43b316bfaac4f03864c29e3eef79ef59d35f0a63 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Fri, 4 Oct 2024 16:49:42 +0000 Subject: [PATCH 7/9] remove unnecessary print from RAQMS and MOPITT reading --- melodies_monet/driver.py | 4 +--- melodies_monet/util/time_interval_subset.py | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 5dd7392f..193638ee 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -285,7 +285,6 @@ def open_sat_obs(self, time_interval=None, control_dict=None): if time_interval is not None: flst = tsub.subset_mopitt_l3(self.file,time_interval) else: flst = self.file - print(flst) self.obj = mio.sat._mopitt_l3_mm.open_dataset(flst, ['column','pressure_surf','apriori_col', 'apriori_surf','apriori_prof','ak_col']) elif self.sat_type == 'modis_l2': @@ -611,12 +610,11 @@ def open_model_files(self, time_interval=None, control_dict=None): file_list = tsub.subset_model_filelist(self.files,'%m_%d_%Y_%HZ','6H',time_interval) else: file_list = self.files - #print(file_list) if len(file_list) > 1: self.obj = mio.models.raqms.open_mfdataset(file_list,**self.mod_kwargs) else: self.obj = mio.models.raqms.open_dataset(file_list) - print('RAQMS files loaded') + else: print('**** Reading Unspecified model output. Take Caution...') if len(self.files) > 1: diff --git a/melodies_monet/util/time_interval_subset.py b/melodies_monet/util/time_interval_subset.py index d431eab1..f942005a 100644 --- a/melodies_monet/util/time_interval_subset.py +++ b/melodies_monet/util/time_interval_subset.py @@ -57,7 +57,6 @@ def subset_mopitt_l3(file_path,timeinterval): fst.sort() for j in fst: interval_files.append(j) - print(interval_files) return interval_files def subset_MODIS_l2(file_path,timeinterval): From 2bfa6f21e80cf14cc7f9dea62aa6b2728417116f Mon Sep 17 00:00:00 2001 From: blychs Date: Fri, 4 Oct 2024 14:46:48 -0600 Subject: [PATCH 8/9] Add reading CAMx 3D variables --- melodies_monet/driver.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/melodies_monet/driver.py b/melodies_monet/driver.py index 30f2b2ac..e70ea474 100644 --- a/melodies_monet/driver.py +++ b/melodies_monet/driver.py @@ -602,6 +602,9 @@ def open_model_files(self, time_interval=None, control_dict=None): #self.obj.monet.scrip = self.obj_scrip elif "camx" in self.model.lower(): self.mod_kwargs.update({"var_list": list_input_var}) + self.mod_kwargs.update({"surf_only": control_dict['model'][self.label].get('surf_only', False)}) + self.mod_kwargs.update({"fname_met_3D": control_dict['model'][self.label].get('files_vert', None)}) + self.mod_kwargs.update({"fname_met_2D": control_dict['model'][self.label].get('files_met_surf', None)}) self.obj = mio.models._camx_mm.open_mfdataset(self.files, **self.mod_kwargs) elif 'raqms' in self.model.lower(): if time_interval is not None: From 8cc250d12e2734e751d36078a86f2e7a9200fdb8 Mon Sep 17 00:00:00 2001 From: Maggie Bruckner Date: Mon, 7 Oct 2024 16:10:04 -0600 Subject: [PATCH 9/9] remove old code in satellite_utilities.py --- melodies_monet/util/satellite_utilities.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/melodies_monet/util/satellite_utilities.py b/melodies_monet/util/satellite_utilities.py index 6c93e7a7..9fe00fd2 100644 --- a/melodies_monet/util/satellite_utilities.py +++ b/melodies_monet/util/satellite_utilities.py @@ -140,10 +140,8 @@ def omps_l3_daily_o3_pairing(model_data,obs_data,ozone_ppbv_varname): grid_adjust = xe.Regridder(model_data[['latitude','longitude']],obs_data[['latitude','longitude']],'bilinear') mod_col_obsgrid = grid_adjust(column) # Aggregate time-step to daily means - #daily_mean = mod_col_obsgrid.groupby('time.date').mean().compute() daily_mean = mod_col_obsgrid.resample(time='1D').mean() # change dimension name for date to time - #daily_mean = daily_mean.rename({'date':'time'}) daily_mean = daily_mean.rename(ozone_ppbv_varname) return xr.merge([daily_mean,obs_data])