started developing prepatcher

notebooks/1.1-pipeline-goes16.ipynb

@@ -18,10 +18,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "5d7511e4-80b3-4053-8c5c-bb430d8a0c14",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'autoroot'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mautoroot\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mrs_tools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_src\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mio\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_list_filenames\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrioxarray\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'autoroot'"
+     ]
+    }
+   ],
    "source": [
     "import autoroot\n",
     "from rs_tools._src.utils.io import get_list_filenames\n",
@@ -5913,9 +5925,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:rs_tools]",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "conda-env-rs_tools-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -5927,7 +5939,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.9.6"
   }
  },
  "nbformat": 4,

rs_tools/_src/preprocessing/prepatcher.py

@@ -0,0 +1,137 @@
+import autoroot
+import numpy as np
+from xrpatcher._src.base import XRDAPatcher
+import rioxarray
+import os
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Optional, List, Union, Tuple
+from tqdm import tqdm
+from rs_tools import goes_download, modis_download, MODIS_VARIABLES, get_modis_channel_numbers
+from rs_tools._src.utils.io import get_list_filenames
+from rs_tools._src.geoprocessing.grid import create_latlon_grid
+import typer
+from loguru import logger
+import xarray as xr
+from satpy import Scene
+import datetime
+from rs_tools._src.data.modis import MODISFileName, MODIS_ID_TO_NAME, MODIS_NAME_TO_ID, get_modis_paired_files
+import pandas as pd
+from datetime import datetime
+
+def _check_filetype(file_type: str) -> bool:
+    """checks instrument for GOES data."""
+    if file_type in ["nc", "np"]:
+        return True
+    else:
+        msg = "Unrecognized file type"
+        msg += f"\nNeeds to be 'nc' or 'np'. Others are not yet tested"
+        raise ValueError(msg)
+
+@dataclass(frozen=True)
+class PrePatcher:
+    read_path: str
+    save_path: str 
+    patch_size: int
+    stride_size: int 
+    save_filetype: str
+    """
+    A class for preprocessing and saving patches from NetCDF files.
+
+    Attributes:
+        read_path (str): The path to the directory containing the NetCDF files.
+        save_path (str): The path to save the patches.
+        patch_size (int): The size of each patch.
+        stride_size (int): The stride size for generating patches.
+        save_filetype (str, optional): The file type to save patches as. Options are [nc, np]
+
+    Methods:
+        nc_files(self) -> List[str]: Returns a list of all NetCDF filenames in the read_path directory.
+        save_patches(self): Preprocesses and saves patches from the NetCDF files.
+    """
+    @property
+    def nc_files(self) -> List[str]:
+        """
+        Returns a list of all NetCDF filenames in the read_path directory.
+
+        Returns:
+            List[str]: A list of NetCDF filenames.
+        """
+        # get list of all filenames within the path
+        files = get_list_filenames(self.read_path, ".nc")
+        return files
+
+    def save_patches(self):
+        """
+        Preprocesses and saves patches from the NetCDF files.
+        """
+        pbar = tqdm(self.nc_files)
+
+        for ifile in pbar:
+            # extract & log timestamp
+            itime = str(Path(ifile).name).split("_")[0]
+            pbar.set_description(f"Processing: {itime}")
+            # open dataset
+            ds = xr.open_dataarray(ifile, engine="netcdf4")
+            # define patch parameters
+            patches = dict(x=self.patch_size, y=self.patch_size)
+            strides = dict(x=self.stride_size, y=self.stride_size)
+            # start patching
+            patcher = XRDAPatcher(da=ds, patches=patches, strides=strides)
+
+            # check if save path exists, and create if not
+            if not os.path.exists(self.save_path):
+                os.makedirs(self.save_path)
+
+            for i, ipatch in tqdm(enumerate(patcher), total=len(patcher)):
+                if self.save_filetype == "nc":
+                    ipatch.to_netcdf(Path(self.save_path).joinpath(f"{itime}_patch_{i}.nc"), engine="netcdf4")
+                elif self.save_filetype == "np":
+                    # save as numpy files
+                    # NOTE: This saves all variables, including time and band_wavelength
+                    # TODO: Simplify saving?
+                    np.save(Path(self.save_path).joinpath(f"{itime}_radiance_patch_{i}"), ipatch.values)
+                    np.save(Path(self.save_path).joinpath(f"{itime}_latitude_patch_{i}"), ipatch.latitude.values)
+                    np.save(Path(self.save_path).joinpath(f"{itime}_longitude_patch_{i}"), ipatch.longitude.values)
+                    np.save(Path(self.save_path).joinpath(f"{itime}_cloudmask_patch_{i}"), ipatch.cloud_mask.values)
+
+def prepatch(
+        read_path: str = "/Users/anna.jungbluth/Desktop/git/rs_tools/data/goes16/geoprocessed",
+        save_path: str = "/Users/anna.jungbluth/Desktop/git/rs_tools/data/goes16/analysis",
+        patch_size: int = 256,
+        stride_size: int = 256,
+        save_filetype: str = 'nc'
+):
+    """
+    Patches satellite data into smaller patches for training.
+    Args:
+        read_path (str, optional): The path to read the input files from. Defaults to "./".
+        save_path (str, optional): The path to save the extracted patches. Defaults to "./".
+        patch_size (int, optional): The size of each patch. Defaults to 256.
+        stride_size (int, optional): The stride size for patch extraction. Defaults to 256.
+        save_filetype (str, optional): The file type to save patches as. Options are [nc, np]
+
+    Returns:
+        None
+    """
+    _check_filetype(file_type=save_filetype)
+
+    # Initialize Prepatcher
+    logger.info(f"Initializing Prepatcher...")
+    prepatcher = PrePatcher(
+        read_path=read_path, 
+        save_path=save_path,
+        patch_size=patch_size,
+        stride_size=stride_size,
+        save_filetype=save_filetype
+        )
+    logger.info(f"Patching Files...: {save_path}")
+    prepatcher.save_patches()
+
+    logger.info(f"Finished Prepatching Script...!")
+
+if __name__ == '__main__':
+    """
+    python scripts/pipeline/prepatch.py --read-path "/path/to/netcdf/file" --save-path /path/to/save/patches
+    """
+    typer.run(prepatch)