Skip to content

Commit

Permalink
Added comments.
Browse files Browse the repository at this point in the history
  • Loading branch information
jejjohnson committed Jan 15, 2024
1 parent 2f0c4a7 commit 8ec7431
Show file tree
Hide file tree
Showing 4 changed files with 231 additions and 27 deletions.
45 changes: 36 additions & 9 deletions rs_tools/_src/geoprocessing/interp.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,17 @@


def create_goes16_interp_mesh(ds: xr.Dataset, variable: str="Rad"):
"""
Create an interpolation mesh using the given dataset and variable for GOES 16
dataset.
Parameters:
ds (xr.Dataset): The dataset containing the data.
variable (str, optional): The variable to interpolate. Defaults to "Rad".
Returns:
pyinterp.RTree: The interpolation mesh.
"""

mesh = pyinterp.RTree()

Expand All @@ -46,6 +57,16 @@ def create_goes16_interp_mesh(ds: xr.Dataset, variable: str="Rad"):


def create_goes16_coords(scale: float=1.0):
"""
Create coordinate vector for GOES-16 data. Uses previous knowledge of the coordinate
system bounds and resolutions to generate a coordinate vector for the X/Y coordss
Args:
scale (float): Scaling factor for the coordinate vector. Default is 1.0.
Returns:
numpy.ndarray: Coordinate vector for GOES-16 data.
"""

DX = interp1d(RES_GOES16, DX_GOES16, kind='linear', bounds_error=False, fill_value="extrapolate")(scale)
NX = bounds_and_step_to_points(X0, X1, DX)
Expand All @@ -57,27 +78,34 @@ def create_goes16_coords(scale: float=1.0):


def resample_goes16(ds, scale: float=1.0):
"""
Resamples a GOES-16 dataset to a specified scale using inverse distance weighting.
#TODO: add more options, e.g., RBF, Kriging.
Parameters:
ds (xarray.Dataset): The input GOES-16 dataset.
scale (float): The scale factor for resampling. Default is 1.0.
Returns:
xarray.Dataset: The resampled dataset.
"""
# create interpolation mesh
# print("Creating interp mesh...")
mesh = create_goes16_interp_mesh(ds, variable="Rad")

# create query coordinates
# print("Creating coords...")
x_coords = create_goes16_coords(scale=scale)

# create meshgrid
# print("Creating meshgrid")
X, Y = np.meshgrid(x_coords, x_coords)

# Inverse Distance Weighting
# print("doing distance calc...")
idw_eta, neighbors = mesh.inverse_distance_weighting(
np.vstack((X.ravel(), Y.ravel())).T,
within=True, # Extrapolation is forbidden
radius=5500, # In a radius of 5.5 Km
k=8, # We are looking for at most 8 neighbours
num_threads=0, # Parallel processing
within=True,
radius=5500,
k=8,
num_threads=0,
)
idw_eta = idw_eta.reshape(X.shape)

Expand All @@ -88,7 +116,6 @@ def resample_goes16(ds, scale: float=1.0):
"band": ds.band_id.values}
)


ds_new.Rad.attrs = ds.Rad.attrs
dx = interp1d(RES_GOES16, DX_GOES16, kind='linear', bounds_error=False, fill_value="extrapolate")(scale)
ds_new.Rad.attrs["resolution"] = f"y: {dx} rad x: {dx} rad"
Expand Down
10 changes: 10 additions & 0 deletions rs_tools/_src/geoprocessing/reproject.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,16 @@


def reproject_goes16(ds: xr.Dataset, crs_projection: str="EPSG:4326") -> xr.Dataset:
"""
Reprojects a GOES-16 dataset to a desired coordinate system.
Parameters:
ds (xr.Dataset): The input dataset to be reprojected.
crs_projection (str): The desired coordinate system for reprojection. Default is "EPSG:4326".
Returns:
xr.Dataset: The reprojected dataset.
"""

# transpose data
try:
Expand Down
58 changes: 40 additions & 18 deletions scripts/goes-download.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
'M': 1
}


def goes_download(
start_date: str,
end_date: Optional[str]=None,
Expand All @@ -39,7 +40,44 @@ def goes_download(
bands: str = "all",
check_bands_downloaded: bool = False,
):
# TODO: Add docstrings
"""
Downloads GOES satellite data for a specified time period and set of bands.
Args:
start_date (str): The start date of the data download in the format 'YYYY-MM-DD'.
end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date.
start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'.
end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'.
daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required.
daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required.
time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour.
satellite_number (int, optional): The satellite number. Default is 16.
save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory.
instrument (str, optional): The instrument name. Default is 'ABI'.
processing_level (str, optional): The processing level of the data. Default is 'L1b'.
data_product (str, optional): The data product to download. Default is 'Rad'.
domain (str, optional): The domain of the data. Default is 'F' - Full Disk.
bands (str, optional): The bands to download. Default is 'all'.
check_bands_downloaded (bool, optional): Whether to check if all bands were successfully downloaded for each time step. Default is False.
Returns:
list: A list of file paths for the downloaded files.
Examples:
# custom day
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01
# custom day + end points
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00
# custom day + end points + time window
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00
# custom day + end points + time window + timestep
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00
# ====================
# FAILURE TEST CASES
# ====================
python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/
python scripts/goes-download.py 2018-10-01 --end-date 2018-10-01 --daily-window-t0 17:00:00 --daily-window-t1 17:14:00 --time-step 00:15:00 --save-dir /home/juanjohn/data/ --check-bands-downloaded
"""

# run checks
_check_input_processing_level(processing_level=processing_level)
Expand Down Expand Up @@ -282,29 +320,13 @@ def convert_str2time(time: str):

return hours, minutes, seconds

def _check_if_file_exists(file_path: str) -> bool:
return os.path.isfile(file_path)


def delete_list_of_files(file_list: List[str]) -> None:
for file_path in file_list:
try:
os.remove(file_path)
except OSError as e:
print(f"Error: {file_path} : {e.strerror}")

# Usage:
# delete_files(["file1.txt", "file2.txt", "file3.txt"])

# def check_file_exists(file_path):


# file_path = "/path/to/your/file"
# if check_file_exists(file_path):
# print("File exists.")
# else:
# print("File does not exist.")

def main(input: str):

print(input)
Expand All @@ -319,7 +341,7 @@ def main(input: str):
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00
# custom day + end points + time window
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00
# custom day + end points + time window + timestep
# custom day + end points + time window + time step
python rs_tools/scripts/goes-download.py 2020-10-01 --end-date 2020-10-01 --start-time 00:00:00 --end-time 23:00:00 --daily-window-t0 08:30:00 --daily-window-t1 21:30:00 --time-step 06:00:00
# ====================
# FAILURE TEST CASES
Expand Down
145 changes: 145 additions & 0 deletions scripts/modis-download.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
"""
Anna Tips 4 MODIS
- all bands are in a single file
- Format: hdf5 file
- resolution dependent [0.5km, 0.25km, 1km]
- downloader has the correct Level!
- time will be painful
* multiple files 4 multiple SWATHS
* very little revisit time during the day
- Filtering Locations / Bounding Boxes / Tiles
- Day & Night Flag: file sizes, filler value
- Level 1B - SWATH Product
potentially useful packages:
- modis-tools: https://github.com/fraymio/modis-tools
- pyMODIS: http://www.pymodis.org and https://github.com/lucadelu/pyModis
-------------------------------------------------------------------------
Summary: The download script to interact directly with the goes2go package.
We only want to specify what is necessary and compatible with the goes2go package.
In general, we want to satellite, the spatial domain, and the period.
Args:
satellite_number:
**Input Parameters**
Downloading:
- satellite number: int --> (16,17,18)
- spatial extent: str --> full disk (F), CONUS (C), Mesoscale domains (M, M1, M2)
- goes instrument: str --> e.g. ABI radiance (or SUVI for helio)
- preprocessing level: str --> e.g. level-1b
- directory: str
- return xarray dataset or list of files --> return as file list works better?
- band specifications: list[int] --> download all or subset only
- start time (of range of times to be downloaded): str
- end time: str
- timesteps/number of files: str
- day vs. night mode: --> e.g. for only downloading day mode images
----------
---
Basic Processing:
- resolution: --> downscale all bands to common resolution (e.g. 2 km)
- coordinate system transformations
- etc.
=================
INPUT PARAMETERS
=================
# LIST OF DATES | START DATE, END DATE, STEP
np.arange, np.linspace
t0, t1, dt | num_files
timestamps = [t0, t1, t2]
# create list of dates
list_of_dates: list = ["2020-10-19 12:00:00", "2020-10-12 12:00:00", ...]
# SATELLITE INFORMATION
satellite_number: int = (16, 17, 18)i
instrument: str = (ABI, ...)
processing_level: str = (level-1b,): str = (level-1b, ...)
data_product: str = (radiances, ...)
# LIST OF BANDS
list_of_bands: list = [1, 2, ..., 15, 16]
# TARGET-GRID
target_grid: xr.Dataset = ...
% ===============
HOW DO WE CHECK DAYTIME HOURS?
* Get Centroid for SATELLITE FOV - FIXED
* Get Radius points for SATELLITE FOV - FIXED
* Check if centroid and/or radius points for FOV is within daytime hours
* Add warning if chosen date is before GOES orbit was stabilized
* True:
download that date
False:
Skippppp / Continue
@dataclass
class SatelliteFOV:
lon_min: float
lon_max: float
lat_min: float
lat_max: float
viewing_angle: ... # [0.15, -0.15] [rad]
@property
def get_radius(self):
...
class GOES16(SatelliteFOV):
...
=================
ALGORITHMS
=================
for itime in timestamps:
for iband in list_of_bands:
# -------------------------------------------------
# download to folder - use GOES2GO loader
# -------------------------------------------------
# open data in folder
# -------------------------------------------------
# quality check 1 - did it download
# -------------------------------------------------
if download_criteria:
continue if allow_missing else break
# quality check 2 - day and/or night specification
if day_night_criteria:
continue if allow_missing else break
# -------------------------------------------------
# CRS Transformation (Optional, preferred)
# -------------------------------------------------
# load dataset
ds: xr.Dataset = xr.load_dataset(...)
# coordinate transformation
ds: xr.Dataset = crs_transform(ds, target_crs, *args, **kwargs)
# resave
ds.to_netcdf(...)
# -------------------------------------------------
# downsample/upscale/lower-res (optional, preferred)
# -------------------------------------------------
# load dataset
ds: xr.Dataset = xr.load_dataset(...)
# resample
ds: xr.Dataset = downsample(ds, target_grid, *args, **kwargs)
ds: xr.Dataset = transform_coords(ds, target_coords)
# resave
ds.to_netcdf(...)
"""

0 comments on commit 8ec7431

Please sign in to comment.