fix cluster_mode option2

aodn · Jun 26, 2024 · 77d8f17 · 77d8f17
1 parent 447191e
commit 77d8f17
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 3 deletions.
diff --git a/aodn_cloud_optimised/bin/srs_l3s_1d_dn_to_zarr.py b/aodn_cloud_optimised/bin/srs_l3s_1d_dn_to_zarr.py
@@ -14,7 +14,7 @@
 
 def main():
     BUCKET_RAW_DEFAULT = load_variable_from_config("BUCKET_RAW_DEFAULT")
-    nc_obj_ls = s3_ls(BUCKET_RAW_DEFAULT, "IMOS/SRS/SST/ghrsst/L3S-1d/dn/2024")
+    nc_obj_ls = s3_ls(BUCKET_RAW_DEFAULT, "IMOS/SRS/SST/ghrsst/L3S-1d/dn/2023")
 
     dataset_config = load_dataset_config(
         str(
@@ -23,7 +23,17 @@ def main():
             )
         )
     )
+    # reprocess one file! it works!!
+    # TODO: rename cloud_optimised_creation_loop as this can handle (for ZARR at least) an array of one file or many files
+    #       and for zarr cloud_optimised_creation_loop bypasses cloud_optimised_creation
+    # cloud_optimised_creation_loop(
+    #     [nc_obj_ls[0]],
+    #     dataset_config=dataset_config,
+    #     # reprocess=True,
+    #     cluster_mode="remote",
+    # )
 
+    # TODO: rename the reprocess argument! it's not clear. something such as clear_existing_data
     cloud_optimised_creation_loop(
         nc_obj_ls,
         dataset_config=dataset_config,

diff --git a/aodn_cloud_optimised/lib/CommonHandler.py b/aodn_cloud_optimised/lib/CommonHandler.py
@@ -52,7 +52,7 @@ def __init__(self, **kwargs):
 
         # Cluster options
         valid_clusters = ["remote", "local", None]
-        self.cluster_mode = kwargs.get("cluster", "remote")
+        self.cluster_mode = kwargs.get("cluster_mode", "local")
 
         if self.cluster_mode not in valid_clusters:
             raise ValueError(

diff --git a/aodn_cloud_optimised/lib/GenericZarrHandler.py b/aodn_cloud_optimised/lib/GenericZarrHandler.py
@@ -220,7 +220,7 @@ def create_cluster(self):
                     f"Could not create Coiled cluster: {e}. Falling back to local cluster."
                 )
                 # Create a local Dask cluster as a fallback
-                self.cluster = LocalCluster()
+                self.cluster = LocalCluster(memory_limit="4GB")
                 self.client = Client(self.cluster)
         elif self.cluster_mode == "local":
             self.logger.info("Creating a local cluster")