Hydra integration

config/example/download.yaml

@@ -0,0 +1,16 @@
+# PERIOD
+period:
+  start_date: '2020-10-01'
+  start_time: '00:00:00'
+  end_date: '2020-10-31'
+  end_time: '23:59:00'
+
+# CLOUD MASK
+cloud_mask: True
+
+# PATH FOR SAVING DATA
+save_dir: data
+
+defaults:
+  - _self_
+

config/example/geoprocess.yaml

@@ -0,0 +1,8 @@
+# PATH WITH RAW DATA
+read_path: data
+
+# PATH FOR SAVING GEOPROCESSED DATA
+save_path: data
+
+defaults:
+  - _self_

config/example/main.yaml

@@ -0,0 +1,7 @@
+defaults:
+  - download
+  - geoprocess
+  - patch
+  - satellite: terra
+
+stage: patch

config/example/patch.yaml

@@ -0,0 +1,18 @@
+# PATH WITH GEOPROCESSED DATA
+read_path: data
+
+# PATH FOR SAVING PATCHES
+save_path: data
+
+# PATCH PARAMETERS
+patch_size: 256
+stride_size: 256
+
+# NAN CUTOFF
+nan_cutoff: 0.5
+
+# FILETYPE TO SAVE [nc = netcdf, np = numpy]
+save_filetype: nc
+
+defaults:
+  - _self_

config/example/satellite/aqua.yaml

@@ -0,0 +1,25 @@
+download:
+  _target_: rs_tools._src.data.modis.downloader_aqua.download
+  save_dir: ${save_dir}/aqua/
+  start_date: ${period.start_date}
+  start_time: ${period.start_time}
+  end_date: ${period.end_date}
+  end_time: ${period.end_time}
+  region: "-130 -15 -90 5" # "lon_min lat_min lon_max lat_max"
+
+geoprocess:
+  _target_: rs_tools._src.geoprocessing.modis.geoprocessor_modis.geoprocess
+  read_path: ${read_path}/aqua/raw
+  save_path: ${save_path}/aqua/geoprocessed
+  satellite: aqua
+
+# preprocess:
+
+patch:
+  _target_: rs_tools._src.preprocessing.prepatcher.prepatch
+  read_path: ${read_path}/aqua/geoprocessed
+  save_path: ${save_path}/aqua/analysis
+  patch_size: ${patch_size}
+  stride_size: ${stride_size}
+  nan_cutoff: ${nan_cutoff}
+  save_filetype: ${save_filetype}

config/example/satellite/goes.yaml

@@ -0,0 +1,29 @@
+download:
+  _target_: rs_tools._src.data.goes.downloader_goes16.download
+  save_dir: ${save_dir}/goes16/
+  start_date: ${period.start_date}
+  start_time: ${period.start_time}
+  end_date: ${period.end_date}
+  end_time: ${period.end_time}
+  daily_window_t0: "14:00:00"
+  daily_window_t1: "20:00:00"
+  time_step: "1:00:00"
+
+geoprocess:
+  _target_: rs_tools._src.geoprocessing.goes.geoprocessor_goes16.geoprocess
+  read_path: ${read_path}/goes16/raw
+  save_path: ${save_path}/goes16/geoprocessed
+  resolution: null
+  region: "-130 -15 -90 5"
+  resample_method: bilinear
+
+# preprocess:
+
+patch:
+  _target_: rs_tools._src.preprocessing.prepatcher.prepatch
+  read_path: ${read_path}/goes16/geoprocessed
+  save_path: ${save_path}/goes16/analysis
+  patch_size: ${patch_size}
+  stride_size: ${stride_size}
+  nan_cutoff: ${nan_cutoff}
+  save_filetype: ${save_filetype}

config/example/satellite/msg.yaml

@@ -0,0 +1,29 @@
+download:
+  _target_: rs_tools._src.data.msg.downloader_msg.download
+  save_dir: ${save_dir}/msg/
+  start_date: ${period.start_date}
+  start_time: ${period.start_time}
+  end_date: ${period.end_date}
+  end_time: ${period.end_time}
+  daily_window_t0: "9:00:00"
+  daily_window_t1: "17:00:00"
+  time_step: "1:00:00"
+
+geoprocess:
+  _target_: rs_tools._src.geoprocessing.msg.geoprocessor_msg.geoprocess
+  read_path: ${read_path}/msg/raw
+  save_path: ${save_path}/msg/geoprocessed
+  resolution: null
+  region: "-70 -15 20 5"
+  resample_method: bilinear
+
+# preprocess:
+
+patch:
+  _target_: rs_tools._src.preprocessing.prepatcher.prepatch
+  read_path: ${read_path}/msg/geoprocessed
+  save_path: ${save_path}/msg/analysis
+  patch_size: ${patch_size}
+  stride_size: ${stride_size}
+  nan_cutoff: ${nan_cutoff}
+  save_filetype: ${save_filetype}

config/example/satellite/terra.yaml

@@ -0,0 +1,25 @@
+download:
+  _target_: rs_tools._src.data.modis.downloader_terra.download
+  save_dir: ${save_dir}/terra/
+  start_date: ${period.start_date}
+  start_time: ${period.start_time}
+  end_date: ${period.end_date}
+  end_time: ${period.end_time}
+  region: "-130 -15 -90 5" # "lon_min lat_min lon_max lat_max"
+
+geoprocess:
+  _target_: rs_tools._src.geoprocessing.modis.geoprocessor_modis.geoprocess
+  read_path: ${read_path}/terra/raw
+  save_path: ${save_path}/terra/geoprocessed
+  satellite: terra
+
+# preprocess:
+
+patch:
+  _target_: rs_tools._src.preprocessing.prepatcher.prepatch
+  read_path: ${read_path}/terra/geoprocessed
+  save_path: ${save_path}/terra/analysis
+  patch_size: ${patch_size}
+  stride_size: ${stride_size}
+  nan_cutoff: ${nan_cutoff}
+  save_filetype: ${save_filetype}

notebooks/1.1-pipeline-goes16.ipynb

@@ -13,15 +13,27 @@
    "id": "1e79e0fb-2f8d-48ea-b2cc-9f86db6ca4e3",
    "metadata": {},
    "source": [
-    "In this demo, we will do a full walkthrough of a minimal MODIS pipeline to get some data for training"
+    "In this demo, we will do a full walkthrough of a minimal GOES pipeline to get some data for training"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "5d7511e4-80b3-4053-8c5c-bb430d8a0c14",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ModuleNotFoundError",
+     "evalue": "No module named 'autoroot'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mautoroot\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mrs_tools\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m_src\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mio\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_list_filenames\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mrioxarray\u001b[39;00m\n",
+      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'autoroot'"
+     ]
+    }
+   ],
    "source": [
     "import autoroot\n",
     "from rs_tools._src.utils.io import get_list_filenames\n",
@@ -172,14 +184,10 @@
    "id": "06c9b06e-2eb1-4be0-88f1-e2b3e0c2c622",
    "metadata": {},
    "source": [
-    "For this case, we are going to download MODIS data.\n",
-    "\n",
-    "* AQUA - \"MYDO21KM\"\n",
-    "* TERRA - \"MODO21KM\"\n",
-    "* Geolocation - \"MYDO3\"\n",
+    "For this case, we are going to download GOES data.\n",
     "\n",
     "**Resources**:\n",
-    "* NASA LAADS - [Webpage](https://ladsweb.modaps.eosdis.nasa.gov/missions-and-measurements/science-domain/modis-L0L1/)"
+    "* GOES2GO"
    ]
   },
   {
@@ -205,14 +213,6 @@
     "goes_files[0], len(goes_files)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "915384f2-3f98-49f3-907c-a760166179ba",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
   {
    "cell_type": "code",
    "execution_count": 10,
@@ -245,28 +245,10 @@
    "metadata": {},
    "source": [
     "1. Get All Files\n",
-    "2. Create MODISFileName Identifiers\n",
+    "2. Create GOESFileName Identifiers\n",
     "3. Filter Based on criteria"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f19010d4-a37a-4784-ac5b-18ef86a0ea0f",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "bd61b03a-ff48-49ac-b24c-b971c9c30f3b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import rioxarray"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 12,
@@ -5943,9 +5925,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python [conda env:rs_tools]",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "conda-env-rs_tools-py"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {
@@ -5957,7 +5939,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.9.6"
   }
  },
  "nbformat": 4,