added msg components and cleaned up repo

Co-authored-by: Juan Emmanuel Johnson <[email protected]>
Co-authored-by: lillif <[email protected]>
Co-authored-by: Lilli Freischem <[email protected]>

scripts/pipeline/preprocess_goes.py → scripts/pipeline/goes/preprocess_goes.py

@@ -35,7 +35,7 @@ class GOES16GeoProcessing:
     @property
     def goes_files(self) -> List[str]:
         # get a list of all filenames within the path
-        # get all modis files
+        # get all GOES files
         files = get_list_filenames(self.read_path, ".nc")
         return files
 
@@ -202,11 +202,11 @@ def preprocess_goes16(
     logger.info(f"Starting Script...")
 
     logger.info(f"Initializing GeoProcessor...")
-    modis_geoprocessor = GOES16GeoProcessing(
+    goes_geoprocessor = GOES16GeoProcessing(
         resolution=resolution, read_path=read_path, save_path=save_path
         )
     logger.info(f"Saving Files...")
-    modis_geoprocessor.preprocess_files()
+    goes_geoprocessor.preprocess_files()
 
     logger.info(f"Finished Script...!")
 

scripts/pipeline/download_modis.py → scripts/pipeline/modis/download_modis.py

@@ -54,6 +54,8 @@ def download(self) -> List[str]:
         #     day_night_flag="day",
         #     identifier= "02"
         # )
+
+        # TODO: Use our download function instead of earthaccess
         pbar = tqdm(pd.date_range(start=self.start_date, end=self.end_date, freq="D"))
         for idate in pbar:
             pbar.set_description(f"Downloading Terra - Date: {idate}")

scripts/pipeline/prepatch.py

@@ -32,25 +32,46 @@ class GeoProcessingParams:
 
 @dataclass(frozen=True)
 class PrePatcher:
+    """
+    A class for preprocessing and saving patches from NetCDF files.
+
+    Attributes:
+        read_path (str): The path to the directory containing the NetCDF files.
+        patch_size (int): The size of each patch.
+        stride_size (int): The stride size for generating patches.
+        save_path (str): The path to save the patches.
+
+    Methods:
+        nc_files(self) -> List[str]: Returns a list of all NetCDF filenames in the read_path directory.
+        save_patches(self): Preprocesses and saves patches from the NetCDF files.
+    """
+
     read_path: str = "./"
     patch_size: int = 256
     stride_size: int = 256
     save_path: str = "./"
 
     @property
     def nc_files(self) -> List[str]:
+        """
+        Returns a list of all NetCDF filenames in the read_path directory.
+
+        Returns:
+            List[str]: A list of NetCDF filenames.
+        """
         # get a list of all filenames within the path
         # get all modis files
         modis_files = get_list_filenames(self.read_path, ".nc")
         return modis_files
 
     def save_patches(self):
-
+        """
+        Preprocesses and saves patches from the NetCDF files.
+        """
         pbar = tqdm(self.nc_files)
 
         save_path = str(Path(self.save_path))
 
-
         for ifile in pbar:
             # open dataset
             itime = str(Path(ifile).name).split("_")[0]
@@ -65,29 +86,26 @@ def save_patches(self):
                 np.save(Path(save_path).joinpath(f"{itime}_longitude_patch_{i}"), ipatch.longitude.values)
                 np.save(Path(save_path).joinpath(f"{itime}_cloudmask_patch_{i}"), ipatch.cloud_mask.values)
 
-
-
-
-def preprocess_modis(
+def prepatch(
         read_path: str = "./",
         patch_size: int = 256,
         stride_size: int = 256,
         save_path: str = "./"
 ):
     """
-    Downloads MODIS TERRA and GOES 16 files for the specified period, region, and save path.
-
+    Patches satellite data into smaller patches for training.
     Args:
-        period (List[str], optional): The period of time to download files for. Defaults to ["2020-10-01", "2020-10-31"].
-        region (Tuple[str], optional): The geographic region to download files for. Defaults to (-180, -90, 180, 90).
-        save_path (str, optional): The path to save the downloaded files. Defaults to "./".
+        read_path (str, optional): The path to read the input files from. Defaults to "./".
+        patch_size (int, optional): The size of each patch. Defaults to 256.
+        stride_size (int, optional): The stride size for patch extraction. Defaults to 256.
+        save_path (str, optional): The path to save the extracted patches. Defaults to "./".
 
     Returns:
         None
     """
     logger.info(f"Starting Script...")
 
-    logger.info(f"Initializing GeoProcessor...")
+    logger.info(f"Initializing Prepatcher...")
     prepatcher = PrePatcher(
         read_path=read_path, save_path=save_path
         )
@@ -103,10 +121,8 @@ def preprocess_modis(
     logger.info(f"Finished Script...!")
 
 
-
-
 if __name__ == '__main__':
     """
-    python scripts/pipeline/preprocess_modis.py --read-path "/home/juanjohn/data/rs/modis/raw" --save-path /home/juanjohn/data/rs/modis/analysis
+    python scripts/pipeline/prepatch.py --read-path "/path/to/netcdf/file" --save-path /path/to/save/patches
     """
-    typer.run(preprocess_modis)
+    typer.run(prepatch)