First Merge to Hydra-Integration

config/example/download.yaml

@@ -0,0 +1,26 @@
+# input
+
+# REGION
+region:
+  lat_min: -130
+  lat_max: -90
+  lon_min: -15
+  lon_max: 5
+
+# PERIOD
+period:
+  start_date: '2018-10-01'
+  start_time: ...
+  end_date: '2018-10-31'
+  end_time: ...
+
+# EXTRAS
+cloud_mask: True
+
+# PATH
+save_dir: ./goes
+
+defaults:
+  - _self_
+  - satellite: goes.yaml
+

config/example/satellite/goes.yaml

@@ -0,0 +1,15 @@
+__target__: rs_tools.OTHER_WRAPPER_SCRIPT
+start_date: ${download.start_date}
+end_date: ${download.end_date}
+save_dir: ${download.save_dir}
+cloud_mask: ${download.cloud_mask}
+daily_window_t0: '14:00:00'
+daily_window_t1: '20:00:00'
+time_step: '4:00:00'
+satellite_number: 16
+instrument: 'ABI'
+processing_level: 'L1b'
+data_product: 'Rad'
+domain: 'F'
+bands: 'all'
+check_bands_downloaded: False

rs_tools/__main__.py

@@ -0,0 +1,40 @@
+import hydra
+from loguru import logger
+from omegaconf import OmegaConf, DictConfig
+
+
+@hydra.main(config_path="../config", config_name="main", version_base="1.3")
+def main(cfg):
+    lib = hydra.utils.instantiate(cfg...)
+
+    logger.debug(f"Choosing stage...")
+    if cfg.stage == "download":
+        for satellite in cfg.download.keys():
+            # instantiate download function
+            # pick parameters
+            # download
+        logger.debug(f"starting download script...")
+        raise NotImplementedError()
+    elif cfg.stage == "preprocess":
+        logger.debug(f"starting preprocessing script...")
+        raise NotImplementedError()
+    # elif cfg.stage == "train":
+    #     logger.debug(f"starting training script...")
+    #     raise NotImplementedError()
+    # elif cfg.stage == "inference":
+    #     logger.debug(f"starting inference script...")
+    #     raise NotImplementedError()
+    # elif cfg.stage == "evaluation":
+    #     logger.debug(f"starting evaluation script...")
+    #     raise NotImplementedError()
+    else:
+        raise ValueError(f"Unrecognized stage: {cfg.stage}")
+
+
+
+def main(config: DictConfig):
+    pass
+
+if __name__ == "__main__":
+    main()
+

scripts/pipeline/goes/download_goes.py

@@ -115,7 +115,6 @@ def download(
     logger.info("Done!")
 
     # TODO: Create DataFrame
-    # TODO: save MODIS-TERRA filenames to CSV?
     # TODO: save GOES-16 filenames to CSV?
     logger.info("Finished Script...")
 

scripts/pipeline/modis/download_modis.py

@@ -125,5 +125,27 @@ def download(
     logger.info("Finished Script...")
 
 
+# ALOT OF HYDRA META INFORMATION
+# Create a configuration associated with the above function (cf next cell)
+import hydra_zen
+import hydra
+
+main_config = hydra_zen.builds(download, populate_full_signature=True)
+
+# Wrap the function to accept the configuration as input
+zen_endpoint = hydra_zen.zen(download)
+
+#Store the config
+store = hydra_zen.ZenStore()
+store(main_config, name='download_modis')
+store.add_to_hydra_store(overwrite_ok=True)
+
+# Create CLI endpoint
+api_endpoint = hydra.main(
+    config_name='download_modis', version_base="1.3", config_path=None
+)(zen_endpoint)
+
 if __name__ == '__main__':
-    typer.run(download)
+    # HYDRA ENDPOINT
+    # typer.run(download)
+    api_endpoint()

scripts/pipeline/modis/preprocess_modis.py

@@ -34,8 +34,6 @@ class GeoProcessingParams:
 class MODISGeoProcessing:
     resolution: float = 1_000 # km
     read_path: str = "./"
-    patch_size: int = 256
-    stride_size: int = 256
     save_path: str = "./"
 
     @property
@@ -103,11 +101,8 @@ def load_modis_files(self, satellite: str="aqua"):
 
 
 def preprocess_modis(
-        modis_save_dir: str = "./",
         resolution: float = 1000,
         read_path: str = "./",
-        patch_size: int = 256,
-        stride_size: int = 256,
         save_path: str = "./"
 ):
     """