diff --git a/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py b/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py index 1722275..3050eb8 100755 --- a/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py +++ b/aodn_cloud_optimised/bin/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.py @@ -9,8 +9,7 @@ def main(): - # for i, year in enumerate(range(2007, 2025)): - for i, year in enumerate(range(2017, 2025)): + for i, year in enumerate(range(2007, 2025)): command = [ "generic_cloud_optimised_creation", "--paths", @@ -24,8 +23,8 @@ def main(): ] # Add --clear-existing-data for the first iteration only - # if i == 0: - # command.append("--clear-existing-data") + if i == 0: + command.append("--clear-existing-data") # Run the command subprocess.run(command, check=True) diff --git a/aodn_cloud_optimised/config/dataset/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.json b/aodn_cloud_optimised/config/dataset/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.json index d07e5df..be96aa9 100644 --- a/aodn_cloud_optimised/config/dataset/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.json +++ b/aodn_cloud_optimised/config/dataset/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.json @@ -1,5 +1,5 @@ { - "dataset_name": "radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc", + "dataset_name": "radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc_test", "logger_name": "radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc", "parent_config": "radar_velocity_hourly_averaged_delayed_qc_no_I_J_version_main.json", "metadata_uuid": "cb2e22b5-ebb9-460b-8cff-b446fe14ea2f", diff --git a/notebooks/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.ipynb b/notebooks/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.ipynb index 0d52a01..79f4758 100644 --- a/notebooks/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.ipynb +++ b/notebooks/radar_SouthAustraliaGulfs_velocity_hourly_averaged_delayed_qc.ipynb @@ -60,7 +60,7 @@ "Using CPython 3.12.6 interpreter at: \u001b[36m/home/lbesnard/miniforge3/envs/AodnCloudOptimised/bin/python\u001b[39m\n", "Creating virtual environment at: \u001b[36m.venv\u001b[39m\n", "Activate with: \u001b[32msource .venv/bin/activate\u001b[39m\n", - "\u001b[2mAudited \u001b[1m232 packages\u001b[0m \u001b[2min 29ms\u001b[0m\u001b[0m\n" + "\u001b[2mAudited \u001b[1m234 packages\u001b[0m \u001b[2min 27ms\u001b[0m\u001b[0m\n" ] } ], @@ -503,37 +503,37 @@ " stroke: currentColor;\n", " fill: currentColor;\n", "}\n", - "
<xarray.Dataset> Size: 14GB\n",
-       "Dimensions:               (TIME: 25027, LATITUDE: 74, LONGITUDE: 102)\n",
+       "
<xarray.Dataset> Size: 41GB\n",
+       "Dimensions:               (TIME: 74523, LATITUDE: 74, LONGITUDE: 102)\n",
        "Coordinates:\n",
        "  * LATITUDE              (LATITUDE) float64 592B -37.46 -37.42 ... -34.82\n",
        "  * LONGITUDE             (LONGITUDE) float64 816B 133.0 133.0 ... 137.4 137.5\n",
-       "  * TIME                  (TIME) datetime64[ns] 200kB 2009-11-23T17:29:59.999...\n",
+       "  * TIME                  (TIME) datetime64[ns] 596kB 2009-11-23T17:29:59.999...\n",
        "Data variables:\n",
-       "    GDOP                  (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    NOBS1                 (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    NOBS2                 (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    UCUR                  (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    UCUR_quality_control  (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    UCUR_sd               (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    VCUR                  (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    VCUR_quality_control  (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    VCUR_sd               (TIME, LATITUDE, LONGITUDE) float64 2GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
-       "    filename              (TIME) <U53 5MB dask.array<chunksize=(100,), meta=np.ndarray>\n",
+       "    GDOP                  (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    NOBS1                 (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    NOBS2                 (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    UCUR                  (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    UCUR_quality_control  (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    UCUR_sd               (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    VCUR                  (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    VCUR_quality_control  (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    VCUR_sd               (TIME, LATITUDE, LONGITUDE) float64 4GB dask.array<chunksize=(100, 74, 102), meta=np.ndarray>\n",
+       "    filename              (TIME) <U53 16MB dask.array<chunksize=(100,), meta=np.ndarray>\n",
        "Attributes: (12/40)\n",
        "    Conventions:                   CF-1.6,IMOS-1.4\n",
-       "    abstract:                      The ACORN facility is producing NetCDF fil...\n",
+       "    abstract:                      The IMOS Ocean Radar Facility (previously ...\n",
        "    acknowledgement:               Any users (including re-packagers) of IMOS...\n",
-       "    author:                        Cosoli, Simone\n",
-       "    author_email:                  simone.cosoli@uwa.edu.au\n",
+       "    author:                        Cosoli, Simone; Hetzel, Yasha\n",
+       "    author_email:                  simone.cosoli@uwa.edu.au; yasha.hetzel@uwa...\n",
        "    citation:                      The citation in a list of references is: I...\n",
        "    ...                            ...\n",
        "    source:                        Terrestrial HF radar\n",
        "    ssr_Stations:                  Cape Wiles (CWI), Cape Spencer (CSP)\n",
        "    standard_name_vocabulary:      NetCDF Climate and Forecast (CF) Metadata ...\n",
-       "    time_coverage_end:             2013-11-09T00:30:00Z\n",
-       "    time_coverage_start:           2013-11-09T00:30:00Z\n",
-       "    title:                         IMOS ACORN South Australian Gulf (SAG), on...