Merge pull request #25 from ecmwf/develop

Develop

.github/ci-hpc-config.yml

@@ -0,0 +1,3 @@
+build:
+  python: 3.10
+  parallel: 1

.github/workflows/ci.yaml

@@ -1,153 +1,40 @@
 name: ci
+
 on:
   # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
       - 'main'
       - 'develop'
+    tags-ignore:
+      - '**'
+
   # Trigger the workflow on pull request
   pull_request: ~
+
   # Trigger the workflow manually
   workflow_dispatch: ~
+
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
-  release:
-    types: [created]
-jobs:
-  qa:
-    name: qa
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ inputs.repository }}
-          ref: ${{ inputs.ref }}
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
 
-      - name: Install Python Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install black flake8 isort
-      - name: Check isort
-        run: isort --check .
-
-      - name: Check black
-        run: black --check .
-
-      - name: Check flake8
-        run: flake8 .
-  setup:
-    name: setup
-    runs-on: ubuntu-20.04
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-      inputs: ${{ steps.prepare-inputs.outputs.inputs }}
-      inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }}
-    steps:
-      - name: Set Matrix
-        id: set-matrix
-        shell: bash -eux {0}
-        run: |
-          MATRIX=$(cat << 'EOS'
-          name:
-          - [email protected]
-          - [email protected]
-          include:
-          - name: [email protected]
-            os: ubuntu-22.04
-            compiler: gnu-11
-            compiler_cc: gcc-11
-            compiler_cxx: g++-11
-            compiler_fc: gfortran-11
-          - name: [email protected]
-            os: ubuntu-22.04
-            compiler: clang-14
-            compiler_cc: clang-14
-            compiler_cxx: clang++-14
-            compiler_fc: gfortran-11
-          # Xcode compiler requires empty environment variables, so we pass null (~) here
-          EOS
-          )
-          SKIP_MATRIX_JOBS=$(cat << 'EOS'
-          ${{ inputs.skip_matrix_jobs }}
-          EOS
-          )
-          SELECT_NAME_COND="1 != 1"
-          SELECT_INCLUDE_COND="1 != 1"
-          for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done
-          echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-      - name: Prepare build-package Inputs
-        id: prepare-inputs
-        shell: bash -eux {0}
-        run: |
-          echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-          echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-  test:
-    name: test
-    needs:
-      - qa
-      - setup
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
-
-      - name: Install Python Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade pip setuptools wheel
-          python -m pip install pytest pytest-cov
-          python -m pip install -r requirements.txt 
-          python -m pip install -r ./tests/requirements_test.txt
-
-      - name: Verify Source Distribution
-        shell: bash -eux {0}
-        run: |
-          python setup.py sdist
-          python -m pip install dist/*
-      - name: Run Tests with Repository Code
-        env:
-          LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }}
-        shell: bash -eux {0}
-        run: |
-          DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not data" tests --cov=./ --cov-report=xml
-          python -m coverage report
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      covjsonkit: ecmwf/covjsonkit@${{ github.event.pull_request.head.sha || github.sha }}
+      python_qa: true
+    secrets: inherit
 
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          files: coverage.xml
-  deploy:
-    needs: test
-    if: ${{ github.event_name == 'release' }}
-    name: Upload to Pypi
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: "__token__"
-          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-        run: |
-          python setup.py sdist
-          twine upload dist/*
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      covjsonkit: ecmwf/covjsonkit@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

README.md

@@ -1,17 +1,89 @@
 | :warning: This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change. DO NOT use this software in any project/software that is operational. |
 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 
+<h3 align="center">
+<img src="./docs/images/ECMWF_logo.svg.png" width=60%>
+</br>
+
 # covjsonkit
 
-ECMWF library for encoding and decoding coverageJSON files/objects of meteorlogical features such as vertical profiles and time series.
+<p align="center">
+  <a href="https://github.com/ecmwf/covjsonkit/actions/workflows/ci.yaml">
+  <img src="https://github.com/ecmwf/covjsonkit/actions/workflows/ci.yaml/badge.svg" alt="ci">
+</a>
+  <a href="https://codecov.io/gh/ecmwf/covjsonkit"><img src="https://codecov.io/gh/ecmwf/covjsonkit/branch/develop/graph/badge.svg"></a>
+  <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg"></a>
+  <a href="https://github.com/ecmwf/covjsonkit/releases"><img src="https://img.shields.io/github/v/release/ecmwf/covjsonkit?color=blue&label=Release&style=flat-square"></a>
+</p>
+<p align="center">
+  <a href="#concept">Concept</a> •
+  <a href="#installation">Installation</a> •
+  <a href="#example">Example</a> •
+  <a href="#testing">Testing</a>
+</p>
+
+## Concept
+
+Covjsonkit is an ECMWF library for encoding and decoding coverageJSON files/objects of meteorlogical features such as vertical profiles and time series.
 
 * Encodes and decodes CoverageJSON objects
 * Convert CoverageJSON files to and from xarray
+* Works in conjunction with ECMWFs Polytope feature extraction library
 
 Current features implemented:
 
 * Time Series
 * Vertical Profile
 * Bounding Box
 * Frame
-* Path
+* Path
+* Wkt Polygons
+* Shapefiles
+
+## Installation
+
+Install the covjsonkit with Python 3 (>=3.7) from GitHub directly with the command
+
+    python3 -m pip install git+ssh://[email protected]/ecmwf/covjsonkit.git@develop
+
+or from PyPI with the command
+
+    python3 -m pip install covjsonkit
+
+## Example
+
+The library consists of an encoder and a decoder element. The decoder can be used to decode valid coverageJSON files that can be then be edited and accessed via the api. It can also be used to convert to ther formats such as xarray.
+
+### Decoder
+
+```Python
+from covjsonkit.api import Covjsonkit
+
+decoder = Covjsonkit().decode(coverage.covjson)
+
+print(decoder.type)
+print(decoder.parameters)
+print(decoder.get_referencing())
+
+ds = decoder.to_xarray()
+```
+
+
+### Encoder
+
+The following example encodes data output from the polytope feature extraction library assuming polytope_output is a valid output from polytope.
+
+```Python
+from covjsonkit.api import Covjsonkit
+
+encoder = Covjsonkit().encode("CoverageCollection", "BoundingBox")
+res = encoder.from_polytope(polytope_output)
+```
+
+## Testing
+
+Python unit tests can be run with pytest:
+
+    python -m pytest 
+
+When a pull request is merged into develop or main a github actions CI pipeline is triggered to test formatting and unit tests.

covjsonkit/decoder/BoundingBox.py

@@ -39,7 +39,7 @@ def to_geopandas(self):
         pass
 
     def to_xarray(self):
-        dims = ["points"]
+        dims = ["number", "steps", "points"]
         dataarraydict = {}
 
         # Get coordinates
@@ -49,22 +49,60 @@ def to_xarray(self):
             x.append(float(coord[0]))
             y.append(float(coord[1]))
 
+        """
         # Get values
         for parameter in self.parameters:
             dataarray = xr.DataArray(self.get_values()[parameter][0], dims=dims)
             dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
             dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
             dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
             dataarraydict[dataarray.attrs["long_name"]] = dataarray
+        """
+
+        values = {}
+        for parameter in self.parameters:
+            values[parameter] = []
+
+        numbers = []
+        steps = []
+        for coverage in self.coverages:
+            numbers.append(coverage["mars:metadata"]["number"])
+            steps.append(coverage["mars:metadata"]["step"])
+            for parameter in self.parameters:
+                values[parameter].append(coverage["ranges"][parameter]["values"])
+
+        numbers = list(set(numbers))
+        steps = list(set(steps))
+
+        new_values = {}
+        for parameter in self.parameters:
+            new_values[parameter] = []
+            for i, num in enumerate(numbers):
+                new_values[parameter].append([])
+                for j, step in enumerate(steps):
+                    new_values[parameter][i].append(values[parameter][i * len(steps) + j])
+
+        for parameter in self.parameters:
+            dataarray = xr.DataArray(new_values[parameter], dims=dims)
+            dataarray.attrs["type"] = self.get_parameter_metadata(parameter)["type"]
+            dataarray.attrs["units"] = self.get_parameter_metadata(parameter)["unit"]["symbol"]
+            dataarray.attrs["long_name"] = self.get_parameter_metadata(parameter)["observedProperty"]["id"]
+            dataarraydict[dataarray.attrs["long_name"]] = dataarray
 
         ds = xr.Dataset(
             dataarraydict,
-            coords=dict(points=(["points"], list(range(0, len(x)))), x=(["points"], x), y=(["points"], y)),
+            coords=dict(
+                number=(["number"], numbers),
+                steps=(["steps"], steps),
+                points=(["points"], list(range(0, len(x)))),
+                x=(["points"], x),
+                y=(["points"], y),
+            ),
         )
         for mars_metadata in self.mars_metadata[0]:
             ds.attrs[mars_metadata] = self.mars_metadata[0][mars_metadata]
 
         # Add date attribute
-        ds.attrs["date"] = self.get_coordinates()["t"]["values"]
+        ds.attrs["date"] = self.get_coordinates()["t"]["values"][0]
 
         return ds

covjsonkit/decoder/decoder.py

@@ -28,6 +28,7 @@ def __init__(self, covjson):
         self.parameters = self.get_parameters()
         self.coordinates = self.get_referencing()
         self.mars_metadata = self.get_mars_metadata()
+        self.coverages = self.coverage.coverages
 
     def get_type(self):
         return self.covjson["type"]