From c7517769dd0295e2823c62f9c596b5f1b4c941da Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 15 Jun 2024 13:12:01 +0000 Subject: [PATCH 01/18] added functionality to pass specific dates to downloaders --- rs_tools/_src/data/goes/download.py | 145 ++++++++++++++++++---------- rs_tools/_src/data/msg/download.py | 133 ++++++++++++++++--------- 2 files changed, 182 insertions(+), 96 deletions(-) diff --git a/rs_tools/_src/data/goes/download.py b/rs_tools/_src/data/goes/download.py index 6d4bb7c..2e0b594 100644 --- a/rs_tools/_src/data/goes/download.py +++ b/rs_tools/_src/data/goes/download.py @@ -1,5 +1,6 @@ from typing import Optional, List, Union import os +import warnings import xarray as xr import matplotlib.pyplot as plt import pandas as pd @@ -22,15 +23,20 @@ 'M': 1 } +# Ignore FutureWarning +warnings.simplefilter(action='ignore', category=FutureWarning) + +# Your code here def goes_download( - start_date: str, + start_date: Optional[str]='2018-10-01', # TODO: Wrap date/time parameters in dict to reduce arguments? end_date: Optional[str]=None, start_time: Optional[str]='00:00:00', end_time: Optional[str]='23:59:00', daily_window_t0: Optional[str]='00:00:00', daily_window_t1: Optional[str]='23:59:00', time_step: Optional[str]=None, + predefined_timestamps: Optional[List[str]]=None, satellite_number: int=16, save_dir: Optional[str] = ".", instrument: str = "ABI", @@ -44,13 +50,14 @@ def goes_download( Downloads GOES satellite data for a specified time period and set of bands. Args: - start_date (str): The start date of the data download in the format 'YYYY-MM-DD'. + start_date (str, optional): The start date of the data download in the format 'YYYY-MM-DD'. end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date. start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'. end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'. daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required. daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required. time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour. + predefined_timestamps (list, optional): A list of timestamps to download. Expected format is 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. satellite_number (int, optional): The satellite number. Default is 16. save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory. instrument (str, optional): The instrument name. Default is 'ABI'. @@ -106,54 +113,20 @@ def goes_download( logger.info(f"Data Product: {data_product}") _check_data_product_name(data_product=data_product) - # check start/end dates/times - if end_date is None: - end_date = start_date - - # combine date and time information - start_datetime_str = start_date + ' ' + start_time - end_datetime_str = end_date + ' ' + end_time - _check_datetime_format(start_datetime_str, end_datetime_str) - # datetime conversion - start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S") - end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S") - _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) - - # define time step for data query - if time_step is None: - time_step = '1:00:00' - logger.info("No timedelta specified. Default is 1 hour.") - _check_timedelta_format(time_delta=time_step) - - # convert str to datetime object - hours, minutes, seconds = convert_str2time(time=time_step) - time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds) - - _check_timedelta(time_delta=time_delta, domain=domain) - - # Compile list of dates/times - list_of_dates = np.arange(start_datetime, end_datetime + time_delta, time_delta).astype(datetime) - print('Times to check: ',list_of_dates[0], list_of_dates[-1]) - - window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object - start_datetime_window_str = window_date + ' ' + str(daily_window_t0) - end_datetime_window_str = window_date + ' ' + str(daily_window_t1) - _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) - # datetime conversion - daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S") - daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S") - _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime) - - # filter function - check that query times fall within desired time window - def is_in_between(date): - return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time() - - # compile new list of dates within desired time window - list_of_dates = list(filter(is_in_between, list_of_dates)) - if list_of_dates == []: - msg = f"No times within the specified time window {daily_window_t0_datetime.time()} - {daily_window_t1_datetime.time()} for time step {time_step}" - msg += f"\n adjust daily window times or time step" - raise ValueError(msg) + # TODO: Allow passing as argument? + timestamp_dict = { + 'start_date': start_date, + 'end_date': end_date, + 'start_time': start_time, + 'end_time': end_time, + 'daily_window_t0': daily_window_t0, + 'daily_window_t1': daily_window_t1, + 'time_step': time_step, + 'domain': domain, + } + + # compile list of dates + list_of_dates = _compile_list_of_dates(timestamp_dict=timestamp_dict, predefined_timestamps=predefined_timestamps) # check if save_dir is valid before attempting to download _check_save_dir(save_dir=save_dir) @@ -256,6 +229,78 @@ def _goes_level1_download(time, return sub_files_list +def _compile_list_of_dates(timestamp_dict: dict, predefined_timestamps: List[str]) -> List[datetime]: + if predefined_timestamps is not None: + _check_predefined_timestamps(predefined_timestamps=predefined_timestamps) + list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] + logger.info(f"Using predefined timestamps.") + elif timestamp_dict['start_date'] is not None: + # check start/end dates/times + if timestamp_dict['end_date'] is None: + end_date = timestamp_dict['start_date'] + else: + end_date = timestamp_dict['end_date'] + # combine date and time information + start_datetime_str = timestamp_dict['start_date'] + ' ' + timestamp_dict['start_time'] + end_datetime_str = end_date + ' ' + timestamp_dict['end_time'] + _check_datetime_format(start_datetime_str, end_datetime_str) + # datetime conversion + start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S") + end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S") + _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) + + # define time step for data query + if timestamp_dict['time_step'] is None: + time_step = '1:00:00' + logger.info("No timedelta specified. Default is 1 hour.") + _check_timedelta_format(time_delta=time_step) + + # convert str to datetime object + hours, minutes, seconds = convert_str2time(time=time_step) + time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds) + + _check_timedelta(time_delta=time_delta, domain=timestamp_dict['domain']) + + # Compile list of dates/times + list_of_dates = np.arange(start_datetime, end_datetime, time_delta).astype(datetime) + print('Times to check: ',list_of_dates[0], list_of_dates[-1]) + + window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object + start_datetime_window_str = window_date + ' ' + timestamp_dict['daily_window_t0'] + end_datetime_window_str = window_date + ' ' + timestamp_dict['daily_window_t1'] + _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) + # datetime conversion + daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S") + daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S") + _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime) + + # filter function - check that query times fall within desired time window + def is_in_between(date): + return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time() + + # compile new list of dates within desired time window + list_of_dates = list(filter(is_in_between, list_of_dates)) + logger.info("Compiling timestamps from specific parameters.") + + else: + msg = "Please provide either predefined timestamps or start date" + raise ValueError(msg) + return list_of_dates + +def _check_predefined_timestamps(predefined_timestamps: List[str]) -> bool: + if type(predefined_timestamps) is not list: + msg = "Please provide predefined timestamps as a list" + raise ValueError(msg) + try: + for x in predefined_timestamps: + datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + return True + except Exception as e: + msg = "Please check predefined timestamps" + msg += "\nExpected date format: %Y-%m-%d" + msg += "\nExpected time format: %H:%M:%S" + raise SyntaxError(msg) + def _check_datetime_format(start_datetime_str: str, end_datetime_str: str) -> bool: try: diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index a96c26c..e8af513 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -12,13 +12,14 @@ from datetime import datetime, timedelta def msg_download( - start_date: str, + start_date: Optional[str]=None, # TODO: Wrap date/time parameters in dict to reduce arguments? end_date: Optional[str]=None, start_time: Optional[str]='00:00:00', # EUMDAC did not find any data for 00:00:00 end_time: Optional[str]='23:59:00', # EUMDAC did not find any data for 23:59:00 daily_window_t0: Optional[str]='00:00:00', daily_window_t1: Optional[str]='23:59:00', time_step: Optional[str]=None, + predefined_timestamps: Optional[List[str]]=None, satellite: str="MSG", instrument: str ="HRSEVIRI", processing_level: Optional[str] = "L1", @@ -30,13 +31,14 @@ def msg_download( Downloads MSG satellite data for a specified time period and set of bands. Args: - start_date (str): The start date of the data download in the format 'YYYY-MM-DD'. + start_date (str, optional): The start date of the data download in the format 'YYYY-MM-DD'. end_date (str, optional): The end date of the data download in the format 'YYYY-MM-DD'. If not provided, the end date will be the same as the start date. start_time (str, optional): The start time of the data download in the format 'HH:MM:SS'. Default is '00:00:00'. end_time (str, optional): The end time of the data download in the format 'HH:MM:SS'. Default is '23:59:00'. daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required. daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required. time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour. + predefined_timestamps (list, optional): A list of timestamps to download. Expected format is 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. satellite (str): The satellite. Default is MSG. instrument (str): The data product to download. Default is 'HRSEVIRI', also implemented for Cloud Mask (CLM). processing_level (str, optional): The processing level of the data. Default is 'L1'. @@ -95,50 +97,19 @@ def msg_download( logger.info(f"Data Product: {data_product}") _check_data_product_name(data_product=data_product) - # check start/end dates/times - if end_date is None: - end_date = start_date - - # combine date and time information - start_datetime_str = start_date + ' ' + start_time - end_datetime_str = end_date + ' ' + end_time - _check_datetime_format(start_datetime_str, end_datetime_str) - # datetime conversion - start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S") - end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S") - _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) - - # define time step for data query - if time_step is None: - time_step = '1:00:00' - logger.info("No timedelta specified. Default is 1 hour.") - _check_timedelta_format(time_delta=time_step) - - # convert str to datetime object - hours, minutes, seconds = convert_str2time(time=time_step) - time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds) - - _check_timedelta(time_delta=time_delta) - - # Compile list of dates/times - list_of_dates = np.arange(start_datetime, end_datetime, time_delta).astype(datetime) - print('Times to check: ',list_of_dates[0], list_of_dates[-1]) - - window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object - start_datetime_window_str = window_date + ' ' + daily_window_t0 - end_datetime_window_str = window_date + ' ' + daily_window_t1 - _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) - # datetime conversion - daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S") - daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S") - _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime) - - # filter function - check that query times fall within desired time window - def is_in_between(date): - return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time() - - # compile new list of dates within desired time window - list_of_dates = list(filter(is_in_between, list_of_dates)) + # TODO: Allow passing as argument? + timestamp_dict = { + 'start_date': start_date, + 'end_date': end_date, + 'start_time': start_time, + 'end_time': end_time, + 'daily_window_t0': daily_window_t0, + 'daily_window_t1': daily_window_t1, + 'time_step': time_step, + } + + # compile list of dates + list_of_dates = _compile_list_of_dates(timestamp_dict=timestamp_dict, predefined_timestamps=predefined_timestamps) # check if save_dir is valid before attempting to download _check_save_dir(save_dir=save_dir) @@ -239,7 +210,77 @@ def _check_netcdf4_backend() -> bool: msg += "\npip install netCDF4" msg += "\nconda install -c conda-forge netCDF4" raise ValueError(msg) + +def _compile_list_of_dates(timestamp_dict: dict, predefined_timestamps: List[str]) -> List[datetime]: + if predefined_timestamps is not None: + _check_predefined_timestamps(predefined_timestamps=predefined_timestamps) + list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] + logger.info(f"Using predefined timestamps.") + elif timestamp_dict['start_date'] is not None: + # check start/end dates/times + if timestamp_dict['end_date'] is None: + end_date = timestamp_dict['start_date'] + else: + end_date = timestamp_dict['end_date'] + # combine date and time information + start_datetime_str = timestamp_dict['start_date'] + ' ' + timestamp_dict['start_time'] + end_datetime_str = end_date + ' ' + timestamp_dict['end_time'] + _check_datetime_format(start_datetime_str, end_datetime_str) + # datetime conversion + start_datetime = datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S") + end_datetime = datetime.strptime(end_datetime_str, "%Y-%m-%d %H:%M:%S") + _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) + + # define time step for data query + if timestamp_dict['time_step'] is None: + time_step = '1:00:00' + logger.info("No timedelta specified. Default is 1 hour.") + _check_timedelta_format(time_delta=time_step) + + # convert str to datetime object + hours, minutes, seconds = convert_str2time(time=time_step) + time_delta = timedelta(hours=hours, minutes=minutes, seconds=seconds) + + _check_timedelta(time_delta=time_delta) + + # Compile list of dates/times + list_of_dates = np.arange(start_datetime, end_datetime, time_delta).astype(datetime) + print('Times to check: ',list_of_dates[0], list_of_dates[-1]) + + window_date = '1991-10-19' # Add arbitrary date to convert into proper datetime object + start_datetime_window_str = window_date + ' ' + timestamp_dict['daily_window_t0'] + end_datetime_window_str = window_date + ' ' + timestamp_dict['daily_window_t1'] + _check_start_end_times(start_datetime=start_datetime, end_datetime=end_datetime) + # datetime conversion + daily_window_t0_datetime = datetime.strptime(start_datetime_window_str, "%Y-%m-%d %H:%M:%S") + daily_window_t1_datetime = datetime.strptime(end_datetime_window_str, "%Y-%m-%d %H:%M:%S") + _check_start_end_times(start_datetime=daily_window_t0_datetime, end_datetime=daily_window_t1_datetime) + + # filter function - check that query times fall within desired time window + def is_in_between(date): + return daily_window_t0_datetime.time() <= date.time() <= daily_window_t1_datetime.time() + + # compile new list of dates within desired time window + list_of_dates = list(filter(is_in_between, list_of_dates)) + logger.info("Compiling timestamps from specific parameters.") + else: + msg = "Please provide either predefined timestamps or start date" + raise ValueError(msg) + return list_of_dates +def _check_predefined_timestamps(predefined_timestamps: List[str]) -> bool: + if type(predefined_timestamps) is not list: + msg = "Please provide predefined timestamps as a list" + raise ValueError(msg) + try: + for x in predefined_timestamps: + datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + return True + except Exception as e: + msg = "Please check predefined timestamps" + msg += "\nExpected date format: %Y-%m-%d" + msg += "\nExpected time format: %H:%M:%S" + raise SyntaxError(msg) def _check_datetime_format(start_datetime_str: str, end_datetime_str: str) -> bool: try: From 80bbcb120eb779be72227fcd4c6eb286a1af6351 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 15 Jun 2024 14:11:03 +0000 Subject: [PATCH 02/18] modified scripts to download overpasses --- rs_tools/_src/data/goes/download.py | 27 +++-- rs_tools/_src/data/msg/download.py | 32 ++++-- .../msg/downloader_msg_(modis_overpass).py | 100 ++++++------------ 3 files changed, 73 insertions(+), 86 deletions(-) diff --git a/rs_tools/_src/data/goes/download.py b/rs_tools/_src/data/goes/download.py index 2e0b594..99b61a9 100644 --- a/rs_tools/_src/data/goes/download.py +++ b/rs_tools/_src/data/goes/download.py @@ -57,7 +57,7 @@ def goes_download( daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required. daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required. time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour. - predefined_timestamps (list, optional): A list of timestamps to download. Expected format is 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. + predefined_timestamps (list, optional): A list of timestamps to download. Expected format is datetime or str following 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. satellite_number (int, optional): The satellite number. Default is 16. save_dir (str, optional): The directory where the downloaded files will be saved. Default is the current directory. instrument (str, optional): The instrument name. Default is 'ABI'. @@ -232,7 +232,10 @@ def _goes_level1_download(time, def _compile_list_of_dates(timestamp_dict: dict, predefined_timestamps: List[str]) -> List[datetime]: if predefined_timestamps is not None: _check_predefined_timestamps(predefined_timestamps=predefined_timestamps) - list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] + if type(predefined_timestamps[0]) is datetime: + list_of_dates = predefined_timestamps + elif type(predefined_timestamps[0]) is str: + list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] logger.info(f"Using predefined timestamps.") elif timestamp_dict['start_date'] is not None: # check start/end dates/times @@ -291,17 +294,23 @@ def _check_predefined_timestamps(predefined_timestamps: List[str]) -> bool: if type(predefined_timestamps) is not list: msg = "Please provide predefined timestamps as a list" raise ValueError(msg) - try: - for x in predefined_timestamps: - datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + if type(predefined_timestamps[0]) is str: # Check type of first element + try: + for x in predefined_timestamps: + datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + return True + except Exception as e: + msg = "Please check predefined timestamps" + msg += "\nExpected date format: %Y-%m-%d" + msg += "\nExpected time format: %H:%M:%S" + raise SyntaxError(msg) + elif type(predefined_timestamps[0]) is datetime: # Check type of first element return True - except Exception as e: + else: msg = "Please check predefined timestamps" - msg += "\nExpected date format: %Y-%m-%d" - msg += "\nExpected time format: %H:%M:%S" + msg += "\nExpected either datetime objects or strings in the format %Y-%m-%d %H:%M:%S" raise SyntaxError(msg) - def _check_datetime_format(start_datetime_str: str, end_datetime_str: str) -> bool: try: datetime.strptime(start_datetime_str, "%Y-%m-%d %H:%M:%S") diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index e8af513..c427724 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -19,7 +19,7 @@ def msg_download( daily_window_t0: Optional[str]='00:00:00', daily_window_t1: Optional[str]='23:59:00', time_step: Optional[str]=None, - predefined_timestamps: Optional[List[str]]=None, + predefined_timestamps: Optional[List]=None, satellite: str="MSG", instrument: str ="HRSEVIRI", processing_level: Optional[str] = "L1", @@ -38,7 +38,7 @@ def msg_download( daily_window_t0 (str, optional): The start time of the daily window in the format 'HH:MM:SS'. Default is '00:00:00'. Used if e.g., only day/night measurements are required. daily_window_t1 (str, optional): The end time of the daily window in the format 'HH:MM:SS'. Default is '23:59:00'. Used if e.g., only day/night measurements are required. time_step (str, optional): The time step between each data download in the format 'HH:MM:SS'. If not provided, the default is 1 hour. - predefined_timestamps (list, optional): A list of timestamps to download. Expected format is 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. + predefined_timestamps (list, optional): A list of timestamps to download. Expected format is datetime or str following 'YYYY-MM-DD HH:MM:SS'. If provided, start/end dates/times will be ignored. satellite (str): The satellite. Default is MSG. instrument (str): The data product to download. Default is 'HRSEVIRI', also implemented for Cloud Mask (CLM). processing_level (str, optional): The processing level of the data. Default is 'L1'. @@ -114,6 +114,7 @@ def msg_download( # check if save_dir is valid before attempting to download _check_save_dir(save_dir=save_dir) + successful_queries = [] files = [] # create progress bars for dates and bands @@ -139,8 +140,9 @@ def msg_download( logger.info(f"Could not find data for time {itime}. Skipping to next time.") else: files += sub_files_list + successful_queries.append(itime) - return files + return (files, successful_queries) def _download(time: datetime, data_product: str, save_dir: str, datastore): products = _compile_msg_products(data_product=data_product, time=time, datastore=datastore) @@ -214,7 +216,10 @@ def _check_netcdf4_backend() -> bool: def _compile_list_of_dates(timestamp_dict: dict, predefined_timestamps: List[str]) -> List[datetime]: if predefined_timestamps is not None: _check_predefined_timestamps(predefined_timestamps=predefined_timestamps) - list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] + if type(predefined_timestamps[0]) is datetime: + list_of_dates = predefined_timestamps + elif type(predefined_timestamps[0]) is str: + list_of_dates = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in predefined_timestamps] logger.info(f"Using predefined timestamps.") elif timestamp_dict['start_date'] is not None: # check start/end dates/times @@ -272,14 +277,21 @@ def _check_predefined_timestamps(predefined_timestamps: List[str]) -> bool: if type(predefined_timestamps) is not list: msg = "Please provide predefined timestamps as a list" raise ValueError(msg) - try: - for x in predefined_timestamps: - datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + if type(predefined_timestamps[0]) is str: # Check type of first element + try: + for x in predefined_timestamps: + datetime.strptime(x, "%Y-%m-%d %H:%M:%S") + return True + except Exception as e: + msg = "Please check predefined timestamps" + msg += "\nExpected date format: %Y-%m-%d" + msg += "\nExpected time format: %H:%M:%S" + raise SyntaxError(msg) + elif type(predefined_timestamps[0]) is datetime: # Check type of first element return True - except Exception as e: + else: msg = "Please check predefined timestamps" - msg += "\nExpected date format: %Y-%m-%d" - msg += "\nExpected time format: %H:%M:%S" + msg += "\nExpected either datetime objects or strings in the format %Y-%m-%d %H:%M:%S" raise SyntaxError(msg) def _check_datetime_format(start_datetime_str: str, end_datetime_str: str) -> bool: diff --git a/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py b/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py index 2555f59..744699f 100644 --- a/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py +++ b/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py @@ -17,24 +17,12 @@ @dataclass class MSGDownload: """Downloading class for MSG data and cloud mask""" - start_date: str - end_date: str - start_time: str - end_time: str - daily_window_t0: str - daily_window_t1: str - time_step: str + predefined_timestamps: List[str] save_dir: str def download(self) -> List[str]: msg_files = msg_download( - start_date=self.start_date, - end_date=self.end_date, - start_time=self.start_time, - end_time=self.end_time, - daily_window_t0=self.daily_window_t0, - daily_window_t1=self.daily_window_t1, - time_step=self.time_step, + predefined_timestamps=self.predefined_timestamps, satellite="MSG", save_dir=Path(self.save_dir).joinpath("L1b"), instrument="HRSEVIRI", @@ -44,13 +32,7 @@ def download(self) -> List[str]: def download_cloud_mask(self) -> List[str]: msg_files = msg_download( - start_date=self.start_date, - end_date=self.end_date, - start_time=self.start_time, - end_time=self.end_time, - daily_window_t0=self.daily_window_t0, - daily_window_t1=self.daily_window_t1, - time_step=self.time_step, + predefined_timestamps=self.predefined_timestamps, satellite="MSG", save_dir=Path(self.save_dir).joinpath("CM"), instrument="CLM", @@ -61,10 +43,10 @@ def download_cloud_mask(self) -> List[str]: def download( modis_product: str="MYD021KM", # TODO: Add advanced data product mapping - start_date: str="2020-10-02", - end_date: str="2020-10-02", + start_date: str="2010-01-01", + end_date: str="2010-01-01", start_time: str="00:00:00", - end_time: str="00:15:00", + end_time: str="00:00:01", save_dir: str='./data/', cloud_mask: bool = True ): @@ -83,9 +65,6 @@ def download( Returns: List[str]: List of downloaded file names """ - list_modis_times = [] - list_msg_times = [] - list_msg_cm_times = [] logger.info("Querying MODIS overpasses for MSG field-of-view and specified time period...") # Check EartData login @@ -103,54 +82,41 @@ def download( # Extract MODIS timestamps modis_timestamps = [modis_granule_to_datetime(x) for x in modis_results] - # create progress bar for dates - pbar_time = tqdm.tqdm(modis_timestamps) - # Initialize MSG Downloader logger.info("Initializing MSG Downloader...") - for itime in pbar_time: - pbar_time.set_description(f"Time - {itime}") - end_itime = itime + timedelta(minutes=10) + dc_msg_download = MSGDownload( + predefined_timestamps=modis_timestamps, + save_dir=Path(save_dir).joinpath("msg"), + ) + logger.info("Downloading MSG Data...") + msg_filenames, msg_queries = dc_msg_download.download() + logger.info("Done!") + if cloud_mask: + logger.info("Downloading MSG Cloud Mask...") + msg_cm_filenames, msg_cm_queries = dc_msg_download.download_cloud_mask() + logger.info("Done!") - dc_msg_download = MSGDownload( - start_date=str(itime.date()), - end_date=str(end_itime.date()), - start_time=str(itime.time()), - end_time=str(end_itime.time()), - daily_window_t0="00:00:00", - daily_window_t1="23:59:00", - time_step="00:15:00", - save_dir=Path(save_dir).joinpath("msg"), - ) + logger.info("Finished Data Download...") - msg_filenames = dc_msg_download.download() + if len(msg_filenames) > 0: # Check if that files were downloade= + assert len(msg_filenames) == len(msg_queries), "Mismatch between queries and downloaded files" + list_modis_times = msg_queries + list_msg_times = [str(MSGFileName.from_filename(msg_filenames[x]).datetime_acquisition) for x in range(len(msg_filenames))] + + # Compile summary file + logger.info("Compiling Summary File...") + df = pd.DataFrame() + df['MODIS'] = list_modis_times + df['MSG'] = list_msg_times if cloud_mask: - msg_cm_filenames = dc_msg_download.download_cloud_mask() - - if len(msg_filenames) > 0: # Check if any files were downloaded - assert len(msg_filenames) == 1, "More than one MSG file was downloaded" - list_modis_times.append(str(itime)) - - msg_filename = str(MSGFileName.from_filename(msg_filenames[0]).datetime_acquisition) - list_msg_times.append(msg_filename) - - if cloud_mask: - assert len(msg_filenames) == len(msg_cm_filenames), "Different number of MSG and cloud mask files downloaded" - msg_cm_filename = str(MSGFileName.from_filename(msg_cm_filenames[0]).datetime_acquisition) - list_msg_cm_times.append(msg_cm_filename) - - logger.info("Finished Data Download...") - # Compile summary file - logger.info("Compiling summary file...") - df = pd.DataFrame() - df['MODIS'] = list_modis_times - df['MSG'] = list_msg_times - if cloud_mask: - df['MSG_cloudmask'] = list_msg_cm_times - df.to_csv(Path(save_dir).joinpath("msg-modis-timestamps.csv"), index=False) + assert len(msg_filenames) == len(msg_cm_filenames), "Different number of MSG and cloud mask files downloaded" + list_msg_cm_times = [str(MSGFileName.from_filename(msg_cm_filenames[x]).datetime_acquisition) for x in range(len(msg_cm_filenames))] + df['MSG_cloudmask'] = list_msg_cm_times + + df.to_csv(Path(save_dir).joinpath(f"msg-modis-timestamps_{start_date}_{end_date}.csv"), index=False) logger.info("Done!") logger.info("Finished MSG Downloading Script...") From 19ed59bd5ad266e19686e1d5506ab9841b0c4c34 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 15 Jun 2024 14:35:34 +0000 Subject: [PATCH 03/18] small bug fix and setting up parallel downloading --- config/esl-3dclouds/satellite/msg.yaml | 2 +- rs_tools/_src/data/msg/download.py | 5 +++++ ..._overpass).py => downloader_msg_modis_overpass.py} | 4 ++-- scripts/download/download_msg_feb.sh | 11 +++++++++++ scripts/download/download_msg_jan.sh | 11 +++++++++++ scripts/download/download_msg_mar copy.sh | 11 +++++++++++ scripts/download/download_msg_mar.sh | 11 +++++++++++ 7 files changed, 52 insertions(+), 3 deletions(-) rename rs_tools/_src/data/msg/{downloader_msg_(modis_overpass).py => downloader_msg_modis_overpass.py} (98%) create mode 100644 scripts/download/download_msg_feb.sh create mode 100644 scripts/download/download_msg_jan.sh create mode 100644 scripts/download/download_msg_mar copy.sh create mode 100644 scripts/download/download_msg_mar.sh diff --git a/config/esl-3dclouds/satellite/msg.yaml b/config/esl-3dclouds/satellite/msg.yaml index 618bb40..55741c2 100644 --- a/config/esl-3dclouds/satellite/msg.yaml +++ b/config/esl-3dclouds/satellite/msg.yaml @@ -1,5 +1,5 @@ download: - _target_: rs_tools._src.data.msg.downloader_msg_(modis_overpass).download + _target_: rs_tools._src.data.msg.downloader_msg_modis_overpass.download save_dir: ${save_dir} start_date: ${period.start_date} start_time: ${period.start_time} diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index c427724..aa7f349 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -1,5 +1,6 @@ from typing import Optional, List, Union import os +import warnings import xarray as xr import matplotlib.pyplot as plt import pandas as pd @@ -11,6 +12,10 @@ from loguru import logger from datetime import datetime, timedelta +# Ignore FutureWarning +warnings.simplefilter(action='ignore', category=FutureWarning) + + def msg_download( start_date: Optional[str]=None, # TODO: Wrap date/time parameters in dict to reduce arguments? end_date: Optional[str]=None, diff --git a/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py similarity index 98% rename from rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py rename to rs_tools/_src/data/msg/downloader_msg_modis_overpass.py index 744699f..9043983 100644 --- a/rs_tools/_src/data/msg/downloader_msg_(modis_overpass).py +++ b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py @@ -46,8 +46,8 @@ def download( start_date: str="2010-01-01", end_date: str="2010-01-01", start_time: str="00:00:00", - end_time: str="00:00:01", - save_dir: str='./data/', + end_time: str="23:59:00", + save_dir: str='.', cloud_mask: bool = True ): """ diff --git a/scripts/download/download_msg_feb.sh b/scripts/download/download_msg_feb.sh new file mode 100644 index 0000000..4cb65e1 --- /dev/null +++ b/scripts/download/download_msg_feb.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-02-01" +END_DATE="2010-02-28" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_jan.sh b/scripts/download/download_msg_jan.sh new file mode 100644 index 0000000..7451872 --- /dev/null +++ b/scripts/download/download_msg_jan.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-01-01" +END_DATE="2010-01-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_mar copy.sh b/scripts/download/download_msg_mar copy.sh new file mode 100644 index 0000000..95c481d --- /dev/null +++ b/scripts/download/download_msg_mar copy.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-04-01" +END_DATE="2010-04-30" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_mar.sh b/scripts/download/download_msg_mar.sh new file mode 100644 index 0000000..1e3326a --- /dev/null +++ b/scripts/download/download_msg_mar.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-03-01" +END_DATE="2010-03-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + From a1233807fce0a6db104c8ed45becd60b6dc43b94 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 15 Jun 2024 16:29:40 +0000 Subject: [PATCH 04/18] added script to run download --- .../download/{download_msg_mar copy.sh => download_msg_apr.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/download/{download_msg_mar copy.sh => download_msg_apr.sh} (100%) diff --git a/scripts/download/download_msg_mar copy.sh b/scripts/download/download_msg_apr.sh similarity index 100% rename from scripts/download/download_msg_mar copy.sh rename to scripts/download/download_msg_apr.sh From 0319bb8210db05f3767a79ea2d5063fe9ae5c726 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Mon, 17 Jun 2024 08:38:04 +0000 Subject: [PATCH 05/18] modified try except --- rs_tools/_src/data/msg/download.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index aa7f349..4396273 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -173,8 +173,9 @@ def _msg_data_download(products, save_dir: str): shutil.copyfileobj(fsrc, fdst) print(f"Successfully downloaded {entry}.") return [save_path] - except eumdac.product.ProductError as error: - print(f"Could not download entry {entry} from product '{product}': '{error.msg}'") + except Exception as error: + print(f"Could not download entry {entry} from product '{product}': '{error}'") + pass def _check_eumdac_login(eumdac_key: str, eumdac_secret: str) -> bool: """check if eumdac login is available in environment variables / as input arguments""" From 916ad429d1d406a68d6ff8f44640c50a2b0273de Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Mon, 17 Jun 2024 08:46:41 +0000 Subject: [PATCH 06/18] restarted download --- scripts/download/download_msg_feb.sh | 3 ++- scripts/download/download_msg_jan.sh | 3 ++- scripts/download/download_msg_mar.sh | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/download/download_msg_feb.sh b/scripts/download/download_msg_feb.sh index 4cb65e1..84af7c8 100644 --- a/scripts/download/download_msg_feb.sh +++ b/scripts/download/download_msg_feb.sh @@ -4,7 +4,8 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -START_DATE="2010-02-01" +# START_DATE="2010-02-01" # repeat download for cloudmask!!! +START_DATE="2010-02-28" END_DATE="2010-02-28" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jan.sh b/scripts/download/download_msg_jan.sh index 7451872..869fd39 100644 --- a/scripts/download/download_msg_jan.sh +++ b/scripts/download/download_msg_jan.sh @@ -4,7 +4,8 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -START_DATE="2010-01-01" +# START_DATE="2010-01-01" # repeat download for cloudmask!!! +START_DATE="2010-01-31" END_DATE="2010-01-31" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_mar.sh b/scripts/download/download_msg_mar.sh index 1e3326a..69d97f8 100644 --- a/scripts/download/download_msg_mar.sh +++ b/scripts/download/download_msg_mar.sh @@ -4,7 +4,8 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -START_DATE="2010-03-01" +# START_DATE="2010-03-01" # repeat download for cloudmask!!! +START_DATE="2010-03-31" END_DATE="2010-03-31" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE From c06649dc4abf263410a756d5a991e904449f2605 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Mon, 17 Jun 2024 13:30:00 +0000 Subject: [PATCH 07/18] tried to fix error --- rs_tools/_src/data/msg/download.py | 14 +++++++------- scripts/download/download_msg_feb.sh | 3 +-- scripts/download/download_msg_jan.sh | 3 +-- scripts/download/download_msg_mar.sh | 3 +-- scripts/setup.sh | 6 ++++++ 5 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 scripts/setup.sh diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index 4396273..ce4f7be 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -162,10 +162,10 @@ def _compile_msg_products(data_product: str, time: datetime, datastore): return products def _msg_data_download(products, save_dir: str): - for product in products: - for entry in product.entries: - if entry.endswith(".nat") or entry.endswith(".grb"): - try: + try: + for product in products: + for entry in product.entries: + if entry.endswith(".nat") or entry.endswith(".grb"): with product.open(entry=entry) as fsrc: # Create a full file path for saving the file save_path = os.path.join(save_dir, os.path.basename(fsrc.name)) @@ -173,9 +173,9 @@ def _msg_data_download(products, save_dir: str): shutil.copyfileobj(fsrc, fdst) print(f"Successfully downloaded {entry}.") return [save_path] - except Exception as error: - print(f"Could not download entry {entry} from product '{product}': '{error}'") - pass + except Exception as error: + print(f"Error downloading product': '{error}'") + pass def _check_eumdac_login(eumdac_key: str, eumdac_secret: str) -> bool: """check if eumdac login is available in environment variables / as input arguments""" diff --git a/scripts/download/download_msg_feb.sh b/scripts/download/download_msg_feb.sh index 84af7c8..45bfdd6 100644 --- a/scripts/download/download_msg_feb.sh +++ b/scripts/download/download_msg_feb.sh @@ -4,8 +4,7 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -# START_DATE="2010-02-01" # repeat download for cloudmask!!! -START_DATE="2010-02-28" +START_DATE="2010-02-01" # repeat download for cloudmask!!! END_DATE="2010-02-28" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jan.sh b/scripts/download/download_msg_jan.sh index 869fd39..1020a9f 100644 --- a/scripts/download/download_msg_jan.sh +++ b/scripts/download/download_msg_jan.sh @@ -4,8 +4,7 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -# START_DATE="2010-01-01" # repeat download for cloudmask!!! -START_DATE="2010-01-31" +START_DATE="2010-01-01" # repeat download for cloudmask!!! END_DATE="2010-01-31" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_mar.sh b/scripts/download/download_msg_mar.sh index 69d97f8..2c1a1ed 100644 --- a/scripts/download/download_msg_mar.sh +++ b/scripts/download/download_msg_mar.sh @@ -4,8 +4,7 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -# START_DATE="2010-03-01" # repeat download for cloudmask!!! -START_DATE="2010-03-31" +START_DATE="2010-03-01" # repeat download for cloudmask!!! END_DATE="2010-03-31" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100644 index 0000000..6ed29be --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +export EARTHDATA_USERNAME=anna.jungbluth +export EARTHDATA_PASSWORD=xuvve1-nyrSuj-pegnar +export EUMDAC_KEY=OqP8jMXE_Xv9VsnG7tahrSD8f2Ya +export EUMDAC_SECRET=Z5Gt3gpvdlUiNrtHt0VRxR103P4a \ No newline at end of file From a62157ce816986fae3d3c154e653d392463000c1 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Mon, 17 Jun 2024 13:31:37 +0000 Subject: [PATCH 08/18] added scripts --- scripts/download/download_msg_apr.sh | 2 +- scripts/download/download_msg_jun.sh | 11 +++++++++++ scripts/download/download_msg_may.sh | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 scripts/download/download_msg_jun.sh create mode 100644 scripts/download/download_msg_may.sh diff --git a/scripts/download/download_msg_apr.sh b/scripts/download/download_msg_apr.sh index 95c481d..4abb110 100644 --- a/scripts/download/download_msg_apr.sh +++ b/scripts/download/download_msg_apr.sh @@ -4,7 +4,7 @@ set -o xtrace SAVE_DIR="/mnt/disks/data/msg/2010/" -START_DATE="2010-04-01" +START_DATE="2010-04-01" # repeat download for cloudmask!!! END_DATE="2010-04-30" python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jun.sh b/scripts/download/download_msg_jun.sh new file mode 100644 index 0000000..2ddfff8 --- /dev/null +++ b/scripts/download/download_msg_jun.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-06-01" # repeat download for cloudmask!!! +END_DATE="2010-06-30" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_may.sh b/scripts/download/download_msg_may.sh new file mode 100644 index 0000000..c52b9c1 --- /dev/null +++ b/scripts/download/download_msg_may.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-05-01" # repeat download for cloudmask!!! +END_DATE="2010-05-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + From 5da28cae202e7f34feeb581c4b23f37d4705b104 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Mon, 17 Jun 2024 13:33:28 +0000 Subject: [PATCH 09/18] removed setup script --- scripts/setup.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 scripts/setup.sh diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100644 index 6ed29be..0000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -export EARTHDATA_USERNAME=anna.jungbluth -export EARTHDATA_PASSWORD=xuvve1-nyrSuj-pegnar -export EUMDAC_KEY=OqP8jMXE_Xv9VsnG7tahrSD8f2Ya -export EUMDAC_SECRET=Z5Gt3gpvdlUiNrtHt0VRxR103P4a \ No newline at end of file From 622c3ce4e5db5097af5ba3fff6130fe35ac6a919 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Tue, 18 Jun 2024 07:26:57 +0000 Subject: [PATCH 10/18] reverted changes for downloader --- rs_tools/_src/data/msg/downloader_msg_modis_overpass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py index 9043983..279eb36 100644 --- a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py +++ b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py @@ -112,8 +112,9 @@ def download( df['MSG'] = list_msg_times if cloud_mask: - assert len(msg_filenames) == len(msg_cm_filenames), "Different number of MSG and cloud mask files downloaded" + assert len(msg_cm_filenames) == len(msg_cm_queries), "Mismatch between queries and downloaded files" list_msg_cm_times = [str(MSGFileName.from_filename(msg_cm_filenames[x]).datetime_acquisition) for x in range(len(msg_cm_filenames))] + df['MODIS_cloudmask'] = msg_cm_queries df['MSG_cloudmask'] = list_msg_cm_times df.to_csv(Path(save_dir).joinpath(f"msg-modis-timestamps_{start_date}_{end_date}.csv"), index=False) From d682da0a60aa1a8759c632d0b1d8f1f2dee50884 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Tue, 18 Jun 2024 09:14:53 +0000 Subject: [PATCH 11/18] added scripts and changed downloader --- .../_src/data/msg/downloader_msg_modis_overpass.py | 8 ++++++-- scripts/download/download_msg_aug.sh | 11 +++++++++++ scripts/download/download_msg_jul.sh | 11 +++++++++++ scripts/download/download_msg_sep.sh | 11 +++++++++++ 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 scripts/download/download_msg_aug.sh create mode 100644 scripts/download/download_msg_jul.sh create mode 100644 scripts/download/download_msg_sep.sh diff --git a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py index 279eb36..487f3f1 100644 --- a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py +++ b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py @@ -114,8 +114,12 @@ def download( if cloud_mask: assert len(msg_cm_filenames) == len(msg_cm_queries), "Mismatch between queries and downloaded files" list_msg_cm_times = [str(MSGFileName.from_filename(msg_cm_filenames[x]).datetime_acquisition) for x in range(len(msg_cm_filenames))] - df['MODIS_cloudmask'] = msg_cm_queries - df['MSG_cloudmask'] = list_msg_cm_times + try: # TODO: Add fix is length of cloud mask timestamps is not equal to MSG timestamps + df['MODIS_cloudmask'] = msg_cm_queries + df['MSG_cloudmask'] = list_msg_cm_times + except Exception as e: + logger.error(f"Error: {e}") + logger.error("Could add cloud mask timestamps to summary file...") df.to_csv(Path(save_dir).joinpath(f"msg-modis-timestamps_{start_date}_{end_date}.csv"), index=False) diff --git a/scripts/download/download_msg_aug.sh b/scripts/download/download_msg_aug.sh new file mode 100644 index 0000000..b676373 --- /dev/null +++ b/scripts/download/download_msg_aug.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-08-01" # repeat download for cloudmask!!! +END_DATE="2010-08-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_jul.sh b/scripts/download/download_msg_jul.sh new file mode 100644 index 0000000..3016802 --- /dev/null +++ b/scripts/download/download_msg_jul.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-07-01" # repeat download for cloudmask!!! +END_DATE="2010-07-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_sep.sh b/scripts/download/download_msg_sep.sh new file mode 100644 index 0000000..525386f --- /dev/null +++ b/scripts/download/download_msg_sep.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-09-01" # repeat download for cloudmask!!! +END_DATE="2010-09-30" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + From ee7a70c2d46040d997d1b2a74755c1dd51481dff Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 22 Jun 2024 15:17:55 +0000 Subject: [PATCH 12/18] fixed typo in logger --- rs_tools/_src/data/msg/downloader_msg_modis_overpass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py index 487f3f1..f6569c6 100644 --- a/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py +++ b/rs_tools/_src/data/msg/downloader_msg_modis_overpass.py @@ -119,7 +119,7 @@ def download( df['MSG_cloudmask'] = list_msg_cm_times except Exception as e: logger.error(f"Error: {e}") - logger.error("Could add cloud mask timestamps to summary file...") + logger.error("Could not add cloud mask timestamps to summary file...") df.to_csv(Path(save_dir).joinpath(f"msg-modis-timestamps_{start_date}_{end_date}.csv"), index=False) From a687198c4ffbb6711016ef7d99b5592b2bee569f Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 22 Jun 2024 15:18:51 +0000 Subject: [PATCH 13/18] added additional download scripts --- scripts/download/download_msg_dec.sh | 11 +++++++++++ scripts/download/download_msg_nov.sh | 11 +++++++++++ scripts/download/download_msg_oct.sh | 11 +++++++++++ 3 files changed, 33 insertions(+) create mode 100644 scripts/download/download_msg_dec.sh create mode 100644 scripts/download/download_msg_nov.sh create mode 100644 scripts/download/download_msg_oct.sh diff --git a/scripts/download/download_msg_dec.sh b/scripts/download/download_msg_dec.sh new file mode 100644 index 0000000..ab7107a --- /dev/null +++ b/scripts/download/download_msg_dec.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-12-01" # repeat download for cloudmask!!! +END_DATE="2010-12-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_nov.sh b/scripts/download/download_msg_nov.sh new file mode 100644 index 0000000..c015450 --- /dev/null +++ b/scripts/download/download_msg_nov.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-11-01" # repeat download for cloudmask!!! +END_DATE="2010-11-30" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + diff --git a/scripts/download/download_msg_oct.sh b/scripts/download/download_msg_oct.sh new file mode 100644 index 0000000..bb9995d --- /dev/null +++ b/scripts/download/download_msg_oct.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o xtrace + +SAVE_DIR="/mnt/disks/data/msg/2010/" + +START_DATE="2010-10-01" # repeat download for cloudmask!!! +END_DATE="2010-10-31" + +python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE + From b758e658c49cd8b218817efb2812195f409d9bd3 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Sat, 22 Jun 2024 15:51:07 +0000 Subject: [PATCH 14/18] removed print statement --- rs_tools/_src/data/msg/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rs_tools/_src/data/msg/__init__.py b/rs_tools/_src/data/msg/__init__.py index ff57ecd..d6aa526 100644 --- a/rs_tools/_src/data/msg/__init__.py +++ b/rs_tools/_src/data/msg/__init__.py @@ -34,7 +34,6 @@ def from_filename(cls, file_name: str): """ file_name = Path(file_name) - print(file_name) components = file_name.name.split('-') save_path = str(file_name.parents[0]) From 4049235481adac695bc8cba10acc68f6cd6a4a0d Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Wed, 3 Jul 2024 20:01:38 +0000 Subject: [PATCH 15/18] added float32 conversion and modified raml --- config/esl-3dclouds/geoprocess.yaml | 4 ++-- config/esl-3dclouds/main.yaml | 6 +++--- config/esl-3dclouds/satellite/msg.yaml | 6 +++--- rs_tools/_src/geoprocessing/msg/geoprocessor_msg.py | 3 +++ 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/config/esl-3dclouds/geoprocess.yaml b/config/esl-3dclouds/geoprocess.yaml index c9e4e09..99aa1b7 100644 --- a/config/esl-3dclouds/geoprocess.yaml +++ b/config/esl-3dclouds/geoprocess.yaml @@ -1,8 +1,8 @@ # PATH WITH RAW DATA -read_path: /mnt/disks/data/miniset/ +read_path: /home/anna.jungbluth/data/ # PATH FOR SAVING GEOPROCESSED DATA -save_path: /mnt/disks/data/miniset-full-disk/ +save_path: /home/anna.jungbluth/data/geoprocessing/ defaults: - _self_ diff --git a/config/esl-3dclouds/main.yaml b/config/esl-3dclouds/main.yaml index fba8caa..9e5f3e1 100644 --- a/config/esl-3dclouds/main.yaml +++ b/config/esl-3dclouds/main.yaml @@ -1,7 +1,7 @@ defaults: - download - geoprocess - - patch - - satellite: aqua + # - patch + - satellite: msg -stage: download \ No newline at end of file +stage: geoprocess \ No newline at end of file diff --git a/config/esl-3dclouds/satellite/msg.yaml b/config/esl-3dclouds/satellite/msg.yaml index 55741c2..2712219 100644 --- a/config/esl-3dclouds/satellite/msg.yaml +++ b/config/esl-3dclouds/satellite/msg.yaml @@ -8,10 +8,10 @@ download: geoprocess: _target_: rs_tools._src.geoprocessing.msg.geoprocessor_msg.geoprocess - read_path: ${read_path}/msg/raw - save_path: ${save_path}/msg/geoprocessed + read_path: ${read_path} + save_path: ${save_path} resolution: null - region: "-70 -15 20 5" + region: null resample_method: bilinear # preprocess: diff --git a/rs_tools/_src/geoprocessing/msg/geoprocessor_msg.py b/rs_tools/_src/geoprocessing/msg/geoprocessor_msg.py index 4efdf96..5ad19d7 100644 --- a/rs_tools/_src/geoprocessing/msg/geoprocessor_msg.py +++ b/rs_tools/_src/geoprocessing/msg/geoprocessor_msg.py @@ -180,6 +180,7 @@ def preprocess_fn_radiances(self, file: List[str], cloud_mask: np.array) -> xr.D # concatinate in new band dimension # NOTE: Concatination overwrites attrs of bands. ds_subset = ds_subset.assign(Rad=xr.concat(list(map(lambda x: ds_subset[x], channels)), dim="band")) + ds_subset['Rad'] = ds_subset['Rad'].astype('float32') # rename band dimensions ds_subset = ds_subset.assign_coords(band=list(map(lambda x: x, channels))) @@ -206,6 +207,8 @@ def preprocess_fn_radiances(self, file: List[str], cloud_mask: np.array) -> xr.D ) # assign band wavelengths ds_subset = ds_subset.assign_coords({"band_wavelength": list(MSG_WAVELENGTHS.values())}) + # change to float32 to reduce file size + ds_subset['Rad'] = ds_subset['Rad'].astype('float32') return ds_subset From 1eb8f55968a5400e528d42f3a588f9257ca2e382 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Wed, 3 Jul 2024 20:05:56 +0000 Subject: [PATCH 16/18] added check to see if files is already downloaded --- rs_tools/_src/data/msg/download.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index ce4f7be..ef1262a 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -169,10 +169,15 @@ def _msg_data_download(products, save_dir: str): with product.open(entry=entry) as fsrc: # Create a full file path for saving the file save_path = os.path.join(save_dir, os.path.basename(fsrc.name)) - with open(save_path, mode='wb') as fdst: - shutil.copyfileobj(fsrc, fdst) - print(f"Successfully downloaded {entry}.") - return [save_path] + # Check if file already exists + if os.path.exists(save_path): + print(f"File {save_path} already exists. Skipping download.") + return [save_path] + else: + with open(save_path, mode='wb') as fdst: + shutil.copyfileobj(fsrc, fdst) + print(f"Successfully downloaded {entry}.") + return [save_path] except Exception as error: print(f"Error downloading product': '{error}'") pass From 112af7405c7924ea2dd40d71ac44e74c82ee0198 Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Wed, 3 Jul 2024 20:29:27 +0000 Subject: [PATCH 17/18] started download of missing time steps --- rs_tools/_src/data/msg/download.py | 2 ++ rs_tools/_src/data/msg/downloader_msg.py | 8 ++++---- scripts/download/download_msg_feb.sh | 3 ++- scripts/download/download_msg_jan.sh | 3 ++- scripts/download/download_msg_mar.sh | 3 ++- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/rs_tools/_src/data/msg/download.py b/rs_tools/_src/data/msg/download.py index ef1262a..aae75fe 100644 --- a/rs_tools/_src/data/msg/download.py +++ b/rs_tools/_src/data/msg/download.py @@ -251,6 +251,8 @@ def _compile_list_of_dates(timestamp_dict: dict, predefined_timestamps: List[str if timestamp_dict['time_step'] is None: time_step = '1:00:00' logger.info("No timedelta specified. Default is 1 hour.") + else: + time_step = timestamp_dict['time_step'] _check_timedelta_format(time_delta=time_step) # convert str to datetime object diff --git a/rs_tools/_src/data/msg/downloader_msg.py b/rs_tools/_src/data/msg/downloader_msg.py index 2c15dc1..ec81c5a 100644 --- a/rs_tools/_src/data/msg/downloader_msg.py +++ b/rs_tools/_src/data/msg/downloader_msg.py @@ -57,10 +57,10 @@ def download_cloud_mask(self) -> List[str]: def download( start_date: str="2020-10-02", end_date: str="2020-10-02", - start_time: str="14:00:00", - end_time: str="20:00:00", - daily_window_t0: str="14:00:00", - daily_window_t1: str="14:30:00", + start_time: str="00:00:00", + end_time: str="23:59:59", + daily_window_t0: str="00:00:00", + daily_window_t1: str="23:59:59", time_step: str="00:15:00", save_dir: str='./data/', cloud_mask: bool = True diff --git a/scripts/download/download_msg_feb.sh b/scripts/download/download_msg_feb.sh index 45bfdd6..867bbdc 100644 --- a/scripts/download/download_msg_feb.sh +++ b/scripts/download/download_msg_feb.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-02-01" # repeat download for cloudmask!!! END_DATE="2010-02-28" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jan.sh b/scripts/download/download_msg_jan.sh index 1020a9f..a646efb 100644 --- a/scripts/download/download_msg_jan.sh +++ b/scripts/download/download_msg_jan.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-01-01" # repeat download for cloudmask!!! END_DATE="2010-01-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_mar.sh b/scripts/download/download_msg_mar.sh index 2c1a1ed..7dee204 100644 --- a/scripts/download/download_msg_mar.sh +++ b/scripts/download/download_msg_mar.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-03-01" # repeat download for cloudmask!!! END_DATE="2010-03-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE From d4400df5e5eaf9e96c8d31290c5f24d9d057a55a Mon Sep 17 00:00:00 2001 From: annajungbluth Date: Wed, 3 Jul 2024 20:42:38 +0000 Subject: [PATCH 18/18] modified scripts for download --- scripts/download/download_msg_apr.sh | 3 ++- scripts/download/download_msg_aug.sh | 3 ++- scripts/download/download_msg_dec.sh | 3 ++- scripts/download/download_msg_jul.sh | 3 ++- scripts/download/download_msg_jun.sh | 3 ++- scripts/download/download_msg_may.sh | 3 ++- scripts/download/download_msg_nov.sh | 3 ++- scripts/download/download_msg_oct.sh | 3 ++- scripts/download/download_msg_sep.sh | 3 ++- 9 files changed, 18 insertions(+), 9 deletions(-) diff --git a/scripts/download/download_msg_apr.sh b/scripts/download/download_msg_apr.sh index 4abb110..2da1001 100644 --- a/scripts/download/download_msg_apr.sh +++ b/scripts/download/download_msg_apr.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-04-01" # repeat download for cloudmask!!! END_DATE="2010-04-30" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_aug.sh b/scripts/download/download_msg_aug.sh index b676373..0f98294 100644 --- a/scripts/download/download_msg_aug.sh +++ b/scripts/download/download_msg_aug.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-08-01" # repeat download for cloudmask!!! END_DATE="2010-08-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_dec.sh b/scripts/download/download_msg_dec.sh index ab7107a..a09ec7e 100644 --- a/scripts/download/download_msg_dec.sh +++ b/scripts/download/download_msg_dec.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-12-01" # repeat download for cloudmask!!! END_DATE="2010-12-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jul.sh b/scripts/download/download_msg_jul.sh index 3016802..2e86495 100644 --- a/scripts/download/download_msg_jul.sh +++ b/scripts/download/download_msg_jul.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-07-01" # repeat download for cloudmask!!! END_DATE="2010-07-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_jun.sh b/scripts/download/download_msg_jun.sh index 2ddfff8..a1cf2fb 100644 --- a/scripts/download/download_msg_jun.sh +++ b/scripts/download/download_msg_jun.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-06-01" # repeat download for cloudmask!!! END_DATE="2010-06-30" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_may.sh b/scripts/download/download_msg_may.sh index c52b9c1..e83d321 100644 --- a/scripts/download/download_msg_may.sh +++ b/scripts/download/download_msg_may.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-05-01" # repeat download for cloudmask!!! END_DATE="2010-05-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_nov.sh b/scripts/download/download_msg_nov.sh index c015450..9c164f4 100644 --- a/scripts/download/download_msg_nov.sh +++ b/scripts/download/download_msg_nov.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-11-01" # repeat download for cloudmask!!! END_DATE="2010-11-30" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_oct.sh b/scripts/download/download_msg_oct.sh index bb9995d..94af0ad 100644 --- a/scripts/download/download_msg_oct.sh +++ b/scripts/download/download_msg_oct.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-10-01" # repeat download for cloudmask!!! END_DATE="2010-10-31" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE diff --git a/scripts/download/download_msg_sep.sh b/scripts/download/download_msg_sep.sh index 525386f..4500e08 100644 --- a/scripts/download/download_msg_sep.sh +++ b/scripts/download/download_msg_sep.sh @@ -7,5 +7,6 @@ SAVE_DIR="/mnt/disks/data/msg/2010/" START_DATE="2010-09-01" # repeat download for cloudmask!!! END_DATE="2010-09-30" -python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +# python rs_tools/_src/data/msg/downloader_msg_modis_overpass.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE +python rs_tools/_src/data/msg/downloader_msg.py --save-dir $SAVE_DIR --start-date $START_DATE --end-date $END_DATE