Merge pull request #20 from aodn/UpdatePackage

(Feat) Update package dependencies and improve README

.pre-commit-config.yaml

@@ -1,13 +1,34 @@
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+  - pre-push
+
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
     hooks:
-    -   id: check-yaml
-    -   id: check-json
-        exclude : aodn_cloud_optimised/config/dataset/dataset_template.json
-    -   id: end-of-file-fixer
-    -   id: trailing-whitespace
--   repo: https://github.com/psf/black
+      - id: check-yaml
+        files: \.ya?ml$
+      - id: check-json
+        files: \.json$
+        exclude: ^aodn_cloud_optimised/config/dataset/dataset_template.json$
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-toml
+
+  - repo: https://github.com/psf/black
     rev: 22.10.0
     hooks:
-    -   id: black
+      - id: black
+
+  - repo: https://github.com/python-poetry/poetry
+    rev: '1.8.3'
+    hooks:
+      - id: poetry-export
+        args: ["-f", "requirements.txt"]
+        verbose: true
+      - id: poetry-check
+        args: ["--lock"]
+        verbose: true
+      - id: poetry-lock
+      - id: poetry-install

README.md

@@ -21,8 +21,18 @@ Requirements:
 mamba env create --file=environment.yml
 mamba activate CloudOptimisedParquet
 
-poetry install
+poetry install --with dev
 ```
+### Update dependencies
+
+1. Update manually the pyproject.toml file with the required package versions
+2. run
+```bash
+poetry update
+```
+to update the poetry.lock file. Commit the changes to poetry.lock
+
+
 # Requirements
 AWS SSO to push files to S3
 

aodn_cloud_optimised/config/dataset/argo_core.json

@@ -395,9 +395,6 @@
     },
     "polygon": {
       "type": "string"
-    },
-    "DATA_CENTRE": {
-      "type": "string"
     }
   },
   "dataset_gattrs": {

pyproject.toml

@@ -44,9 +44,15 @@ shapely = ">=2.0.4"
 xarray = { version = ">=2024.3.0", extras = ["complete"] }
 zarr = ">=2.18.0"
 
-[tool.poetry.extras]
-testing = ["pytest", "ipython", "ipdb"]
-interactive = ["pytest", "ipython", "ipdb"]
+[tool.poetry.dev-dependencies]
+pytest = "^8.2.1"
+ipdb = "^0.13"
+ipython = "^7.5.3"
+
+
+#[tool.poetry.extras]
+#testing = ["pytest", "ipython", "ipdb"]
+#interactive = ["pytest", "ipython", "ipdb"]
 
 
 [tool.poetry.scripts]
@@ -63,13 +69,13 @@ soop_xbt_nrt_to_parquet = "aodn_cloud_optimised.bin.soop_xbt_nrt_to_parquet:main
 srs_oc_ljco_to_parquet = "aodn_cloud_optimised.bin.srs_oc_ljco_to_parquet:main"
 srs_l3s_1d_dn_to_zarr = "aodn_cloud_optimised.bin.srs_l3s_1d_dn_to_zarr:main"
 
-
 #[tool.poetry.include]
 #data = ["aodn_cloud_optimised/config/*.json", "aodn_cloud_optimised/config/dataset/*.json"]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.2.1"
 coverage = "^7.5.3"
+pre-commit = "^3.7.1"
 
 [tool.pre_commit]
 version = "2.3.0"

requirements.txt

@@ -8,16 +8,16 @@ attrs==23.2.0 ; python_full_version == "3.10.14"
 bokeh==3.4.1 ; python_full_version == "3.10.14"
 boto3==1.34.106 ; python_full_version == "3.10.14"
 botocore==1.34.106 ; python_full_version == "3.10.14"
-certifi==2024.2.2 ; python_full_version == "3.10.14"
+certifi==2024.6.2 ; python_full_version == "3.10.14"
 cftime==1.6.3 ; python_full_version == "3.10.14"
 click==8.1.7 ; python_full_version == "3.10.14"
 cloudpickle==3.0.0 ; python_full_version == "3.10.14"
 colorama==0.4.6 ; python_full_version == "3.10.14" and platform_system == "Windows"
 contourpy==1.2.1 ; python_full_version == "3.10.14"
-dask[array,diagnostics]==2024.5.1 ; python_full_version == "3.10.14"
+dask[array,diagnostics]==2024.5.2 ; python_full_version == "3.10.14"
 fasteners==0.19 ; sys_platform != "emscripten" and python_full_version == "3.10.14"
 frozenlist==1.4.1 ; python_full_version == "3.10.14"
-fsspec==2024.5.0 ; python_full_version == "3.10.14"
+fsspec==2024.6.0 ; python_full_version == "3.10.14"
 h5netcdf==1.3.0 ; python_full_version == "3.10.14"
 h5py==3.11.0 ; python_full_version == "3.10.14"
 idna==3.7 ; python_full_version == "3.10.14"
@@ -44,7 +44,7 @@ pyyaml==6.0.1 ; python_full_version == "3.10.14"
 rechunker==0.5.2 ; python_full_version == "3.10.14"
 referencing==0.35.1 ; python_full_version == "3.10.14"
 rpds-py==0.18.1 ; python_full_version == "3.10.14"
-s3fs==2024.5.0 ; python_full_version == "3.10.14"
+s3fs==2024.6.0 ; python_full_version == "3.10.14"
 s3transfer==0.10.1 ; python_full_version == "3.10.14"
 scipy==1.13.1 ; python_full_version == "3.10.14"
 shapely==2.0.4 ; python_full_version == "3.10.14"
@@ -58,4 +58,4 @@ xarray[complete]==2024.5.0 ; python_full_version == "3.10.14"
 xyzservices==2024.4.0 ; python_full_version == "3.10.14"
 yarl==1.9.4 ; python_full_version == "3.10.14"
 zarr==2.18.2 ; python_full_version == "3.10.14"
-zipp==3.19.0 ; python_full_version == "3.10.14"
+zipp==3.19.1 ; python_full_version == "3.10.14"

test_aodn_cloud_optimised/resources/argo_core.json

@@ -384,9 +384,6 @@
     },
     "polygon": {
       "type": "string"
-    },
-    "DATA_CENTRE": {
-      "type": "string"
     }
   },
   "dataset_gattrs": {