From 4409030e464d586c564d94aa5a4d728528060b67 Mon Sep 17 00:00:00 2001 From: lbesnard Date: Tue, 30 Apr 2024 12:28:40 +1000 Subject: [PATCH] (Fix) Github actions --- .github/workflows/test-mamba.yml | 50 +++++++++++++++++++++++++++++ .github/workflows/test-setup.yml | 51 ++++++++++++++++++++++++++++++ .github/workflows/test.yml | 40 ----------------------- aodn_cloud_optimised/lib/config.py | 8 ++--- environment.yml | 1 - setup.py | 5 ++- 6 files changed, 109 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/test-mamba.yml create mode 100644 .github/workflows/test-setup.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test-mamba.yml b/.github/workflows/test-mamba.yml new file mode 100644 index 0000000..95b40e5 --- /dev/null +++ b/.github/workflows/test-mamba.yml @@ -0,0 +1,50 @@ +name: aodn_cloud_optimised mamba + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build-linux: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: AodnCloudOptimised + environment-file: environment.yml + + - name: Install dependencies + shell: bash -el {0} + run: | + conda install --yes mamba + mamba env update --name AodnCloudOptimised --file environment.yml + + - name: Activate Conda environment + shell: bash -el {0} + run: | + conda init + conda activate AodnCloudOptimised + + - name: Run pytest and coverage + run: | + pytest + coverage run -m pytest + coverage report --fail-under=60 + coverage xml -o coverage.xml + + - name: Upload coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage-report + path: coverage.xml diff --git a/.github/workflows/test-setup.yml b/.github/workflows/test-setup.yml new file mode 100644 index 0000000..9008364 --- /dev/null +++ b/.github/workflows/test-setup.yml @@ -0,0 +1,51 @@ +name: aodn_cloud_optimised package build + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build-linux: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -e {0} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install package dependencies + run: | + pip install . + + - name: Install additional dependencies + run: | + pip install pytest coverage + + - name: Run setup.py install + run: | + python setup.py install + + - name: Run tests with coverage + run: | + pytest + coverage run -m pytest + coverage report --fail-under=60 + coverage xml -o coverage.xml + + - name: Upload coverage report + uses: actions/upload-artifact@v2 + with: + name: coverage-report + path: coverage.xml + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index be513b4..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: aodn_cloud_optimised - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build-linux: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - submodules: 'true' - - - name: Install Miniconda - uses: conda-incubator/setup-miniconda@v2 - with: - installer-url: https://repo.anaconda.com/miniconda/Miniconda3-py38_4.11.0-Linux-x86_64.sh - auto-update-conda: false # Disable auto-update to use mamba - activate-environment: AodbCloudOptimised - - - name: Install mamba - run: | - conda install -n base -c conda-forge mamba - - - name: Create Conda environment with mamba - run: | - mamba env update -n AodbCloudOptimised -f environment.yml - - - name: Activate Conda environment - run: | - conda activate AodbCloudOptimised - - - name: Run unittests for AODN CLOUD OPTIMISED module - run: | - pytest . - diff --git a/aodn_cloud_optimised/lib/config.py b/aodn_cloud_optimised/lib/config.py index fd372fd..5b1a0b6 100644 --- a/aodn_cloud_optimised/lib/config.py +++ b/aodn_cloud_optimised/lib/config.py @@ -3,6 +3,7 @@ import os import importlib.resources from collections import OrderedDict +from importlib.resources import path def load_variable_from_file(file_path, variable_name) -> str: @@ -24,10 +25,9 @@ def load_variable_from_file(file_path, variable_name) -> str: def load_variable_from_config(variable_name) -> str: - # Construct the path to common.py relative to the current directory - common_config_path = str(importlib.resources.path("aodn_cloud_optimised.config", "common.json")) - - return load_variable_from_file(common_config_path, variable_name) + # Obtain the file path using the context manager + with path("aodn_cloud_optimised.config", "common.json") as common_config_path: + return load_variable_from_file(str(common_config_path), variable_name) def load_dataset_config(config_path) -> dict: diff --git a/environment.yml b/environment.yml index 6eff58f..acdf42a 100755 --- a/environment.yml +++ b/environment.yml @@ -23,6 +23,5 @@ dependencies: - pyarrow==15.0.0 - rechunker - s3fs - - yaml - xarray[complete] - zarr diff --git a/setup.py b/setup.py index 8198e7f..6351803 100644 --- a/setup.py +++ b/setup.py @@ -15,13 +15,16 @@ 'numpy', 'pandas', 'python-dateutil', + 'jsonschema', 'h5py', 'scipy', 'boto3', 'pyarrow==15.0.0', + 'rechunker', 's3fs', 'shapely', 'xarray[complete]', + 'zarr' ] PACKAGE_DATA = { @@ -79,7 +82,7 @@ tests_require=TESTS_REQUIRE, extras_require=EXTRAS_REQUIRE, zip_safe=False, - python_requires='>=3.8', + python_requires='==3.10', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers',