diff --git a/rs_tools/_src/geoprocessing/interp.py b/rs_tools/_src/geoprocessing/interp.py index 3c83736..77f5253 100644 --- a/rs_tools/_src/geoprocessing/interp.py +++ b/rs_tools/_src/geoprocessing/interp.py @@ -29,6 +29,17 @@ def create_goes16_interp_mesh(ds: xr.Dataset, variable: str="Rad"): + """ + Create an interpolation mesh using the given dataset and variable for GOES 16 + dataset. + + Parameters: + ds (xr.Dataset): The dataset containing the data. + variable (str, optional): The variable to interpolate. Defaults to "Rad". + + Returns: + pyinterp.RTree: The interpolation mesh. + """ mesh = pyinterp.RTree() @@ -46,6 +57,16 @@ def create_goes16_interp_mesh(ds: xr.Dataset, variable: str="Rad"): def create_goes16_coords(scale: float=1.0): + """ + Create coordinate vector for GOES-16 data. Uses previous knowledge of the coordinate + system bounds and resolutions to generate a coordinate vector for the X/Y coordss + + Args: + scale (float): Scaling factor for the coordinate vector. Default is 1.0. + + Returns: + numpy.ndarray: Coordinate vector for GOES-16 data. + """ DX = interp1d(RES_GOES16, DX_GOES16, kind='linear', bounds_error=False, fill_value="extrapolate")(scale) NX = bounds_and_step_to_points(X0, X1, DX) @@ -57,27 +78,34 @@ def create_goes16_coords(scale: float=1.0): def resample_goes16(ds, scale: float=1.0): + """ + Resamples a GOES-16 dataset to a specified scale using inverse distance weighting. + #TODO: add more options, e.g., RBF, Kriging. + + Parameters: + ds (xarray.Dataset): The input GOES-16 dataset. + scale (float): The scale factor for resampling. Default is 1.0. + + Returns: + xarray.Dataset: The resampled dataset. + """ # create interpolation mesh - # print("Creating interp mesh...") mesh = create_goes16_interp_mesh(ds, variable="Rad") # create query coordinates - # print("Creating coords...") x_coords = create_goes16_coords(scale=scale) # create meshgrid - # print("Creating meshgrid") X, Y = np.meshgrid(x_coords, x_coords) # Inverse Distance Weighting - # print("doing distance calc...") idw_eta, neighbors = mesh.inverse_distance_weighting( np.vstack((X.ravel(), Y.ravel())).T, - within=True, # Extrapolation is forbidden - radius=5500, # In a radius of 5.5 Km - k=8, # We are looking for at most 8 neighbours - num_threads=0, # Parallel processing + within=True, + radius=5500, + k=8, + num_threads=0, ) idw_eta = idw_eta.reshape(X.shape) @@ -88,7 +116,6 @@ def resample_goes16(ds, scale: float=1.0): "band": ds.band_id.values} ) - ds_new.Rad.attrs = ds.Rad.attrs dx = interp1d(RES_GOES16, DX_GOES16, kind='linear', bounds_error=False, fill_value="extrapolate")(scale) ds_new.Rad.attrs["resolution"] = f"y: {dx} rad x: {dx} rad" diff --git a/rs_tools/_src/geoprocessing/reproject.py b/rs_tools/_src/geoprocessing/reproject.py index cc87f48..f6ae370 100644 --- a/rs_tools/_src/geoprocessing/reproject.py +++ b/rs_tools/_src/geoprocessing/reproject.py @@ -4,6 +4,16 @@ def reproject_goes16(ds: xr.Dataset, crs_projection: str="EPSG:4326") -> xr.Dataset: + """ + Reprojects a GOES-16 dataset to a desired coordinate system. + + Parameters: + ds (xr.Dataset): The input dataset to be reprojected. + crs_projection (str): The desired coordinate system for reprojection. Default is "EPSG:4326". + + Returns: + xr.Dataset: The reprojected dataset. + """ # transpose data try: diff --git a/scripts/goes-download.py b/scripts/goes-download.py index 15ab825..dae94a3 100644 --- a/scripts/goes-download.py +++ b/scripts/goes-download.py @@ -22,6 +22,7 @@ 'M': 1 } + def goes_download( start_date: str, end_date: Optional[str]=None, @@ -39,7 +40,44 @@ def goes_download( bands: str = "all", check_bands_downloaded: bool = False, ): - # TODO: Add docstrings + """ + Downloads GOES satellite data for a specified time period and set of bands. + + Args: + start_date (str): The start date of the data download in the format 'YYYY-MM-DD'. + end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date. + start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'. + end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'. + daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required. + daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required. + time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour. + satellite_number (int, optional): The satellite number. Default is 16. + save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory. + instrument (str, optional): The instrument name. Default is 'ABI'. + processing_level (str, optional): The processing level of the data. Default is 'L1b'. + data_product (str, optional): The data product to download. Default is 'Rad'. + domain (str, optional): The domain of the data. Default is 'F' - Full Disk. + bands (str, optional): The bands to download. Default is 'all'. + check_bands_downloaded (bool, optional): Whether to check if all bands were successfully downloaded for each time step. Default is False. + + Returns: + list: A list of file paths for the downloaded files. + + Examples: + # custom day + python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 + # custom day + end points + python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 + # custom day + end points + time window + python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 + # custom day + end points + time window + timestep + python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00 + # ==================== + # FAILURE TEST CASES + # ==================== + python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ + python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded + """ # run checks _check_input_processing_level(processing_level=processing_level) @@ -282,10 +320,6 @@ def convert_str2time(time: str): return hours, minutes, seconds -def _check_if_file_exists(file_path: str) -> bool: - return os.path.isfile(file_path) - - def delete_list_of_files(file_list: List[str]) -> None: for file_path in file_list: try: @@ -293,18 +327,6 @@ def delete_list_of_files(file_list: List[str]) -> None: except OSError as e: print(f"Error: {file_path} : {e.strerror}") -# Usage: -# delete_files(["file1.txt", "file2.txt", "file3.txt"]) - - # def check_file_exists(file_path): - - - # file_path = "/path/to/your/file" - # if check_file_exists(file_path): - # print("File exists.") - # else: - # print("File does not exist.") - def main(input: str): print(input) @@ -319,7 +341,7 @@ def main(input: str): python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 # custom day + end points + time window python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 - # custom day + end points + time window + timestep + # custom day + end points + time window + time step python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00 # ==================== # FAILURE TEST CASES diff --git a/scripts/modis-download.py b/scripts/modis-download.py index e69de29..882130c 100644 --- a/scripts/modis-download.py +++ b/scripts/modis-download.py @@ -0,0 +1,145 @@ +""" +Anna Tips 4 MODIS +- all bands are in a single file +- Format: hdf5 file +- resolution dependent [0.5km, 0.25km, 1km] +- downloader has the correct Level! +- time will be painful + * multiple files 4 multiple SWATHS + * very little revisit time during the day +- Filtering Locations / Bounding Boxes / Tiles +- Day & Night Flag: file sizes, filler value +- Level 1B - SWATH Product + + + +potentially useful packages: +- modis-tools: https://github.com/fraymio/modis-tools +- pyMODIS: http://www.pymodis.org and https://github.com/lucadelu/pyModis + + + +------------------------------------------------------------------------- +Summary: The download script to interact directly with the goes2go package. +We only want to specify what is necessary and compatible with the goes2go package. +In general, we want to satellite, the spatial domain, and the period. + +Args: + satellite_number: + +**Input Parameters** + +Downloading: +- satellite number: int --> (16,17,18) +- spatial extent: str --> full disk (F), CONUS (C), Mesoscale domains (M, M1, M2) +- goes instrument: str --> e.g. ABI radiance (or SUVI for helio) +- preprocessing level: str --> e.g. level-1b +- directory: str +- return xarray dataset or list of files --> return as file list works better? +- band specifications: list[int] --> download all or subset only +- start time (of range of times to be downloaded): str +- end time: str +- timesteps/number of files: str +- day vs. night mode: --> e.g. for only downloading day mode images + +---------- + +--- +Basic Processing: +- resolution: --> downscale all bands to common resolution (e.g. 2 km) +- coordinate system transformations +- etc. + + +================= +INPUT PARAMETERS +================= + +# LIST OF DATES | START DATE, END DATE, STEP +np.arange, np.linspace +t0, t1, dt | num_files +timestamps = [t0, t1, t2] +# create list of dates +list_of_dates: list = ["2020-10-19 12:00:00", "2020-10-12 12:00:00", ...] + +# SATELLITE INFORMATION +satellite_number: int = (16, 17, 18)i +instrument: str = (ABI, ...) +processing_level: str = (level-1b,): str = (level-1b, ...) +data_product: str = (radiances, ...) + +# LIST OF BANDS +list_of_bands: list = [1, 2, ..., 15, 16] + +# TARGET-GRID +target_grid: xr.Dataset = ... + +% =============== +HOW DO WE CHECK DAYTIME HOURS? +* Get Centroid for SATELLITE FOV - FIXED +* Get Radius points for SATELLITE FOV - FIXED +* Check if centroid and/or radius points for FOV is within daytime hours +* Add warning if chosen date is before GOES orbit was stabilized +* True: + download that date +False: + Skippppp / Continue + +@dataclass +class SatelliteFOV: + lon_min: float + lon_max: float + lat_min: float + lat_max: float + viewing_angle: ... # [0.15, -0.15] [rad] + + @property + def get_radius(self): + ... + +class GOES16(SatelliteFOV): + ... + +================= +ALGORITHMS +================= + +for itime in timestamps: + for iband in list_of_bands: + # ------------------------------------------------- + # download to folder - use GOES2GO loader + # ------------------------------------------------- + + # open data in folder + + # ------------------------------------------------- + # quality check 1 - did it download + # ------------------------------------------------- + if download_criteria: + continue if allow_missing else break + + # quality check 2 - day and/or night specification + if day_night_criteria: + continue if allow_missing else break + + # ------------------------------------------------- + # CRS Transformation (Optional, preferred) + # ------------------------------------------------- + # load dataset + ds: xr.Dataset = xr.load_dataset(...) + # coordinate transformation + ds: xr.Dataset = crs_transform(ds, target_crs, *args, **kwargs) + # resave + ds.to_netcdf(...) + + # ------------------------------------------------- + # downsample/upscale/lower-res (optional, preferred) + # ------------------------------------------------- + # load dataset + ds: xr.Dataset = xr.load_dataset(...) + # resample + ds: xr.Dataset = downsample(ds, target_grid, *args, **kwargs) + ds: xr.Dataset = transform_coords(ds, target_coords) + # resave + ds.to_netcdf(...) +""" \ No newline at end of file