Update Benchmark branch to use NDSL v2024.09.00 (#96)

* working baroclinic test case example (#81)

* fixing dockerfile overriding netcdf, mpi4py, and numpy versions with mutually incompatible versions (#84)

* Update notebooks for eta file creations  (#85)

* Update pace to use python 3.11.9 (#82)

* Updated container for Github Action and reusable workflow (#87)

* Updated container for Github Action and reusable workflow - Redo (#89)

* Matching packages between main and develop branches and removal of outdated pull request template (#86)

* Changed workflows to use python 3.11.7, added pytest-subtests to requirments_dev.txt

* Updating constraints file

* Updating Dockerfile to use 3.11-slim-bookworm base python image

* reusable workflow CI - checkout corrrect repository (#90)

* pace/tests/main/grid/test_eta.py moved to NDSL/tests/grid/test_eta.py (#92)

* Updated README to include directions for generating eta files (#99)

* Updated submodules to most recent versions as of 3 Jan 2025 15:35 EST

* Updated c5 build script to match new methods

---------

Co-authored-by: Oliver Elbert <[email protected]>
Co-authored-by: MiKyung Lee <[email protected]>
Co-authored-by: mlee03 <[email protected]>
Co-authored-by: Xingqiu Yuan <[email protected]>

.github/workflows/lint.yml → .github/workflows/lint.yaml

@@ -1,7 +1,6 @@
 name: "Lint"
 on:
   pull_request:
-      types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
 
 jobs:
   lint:
@@ -11,10 +10,10 @@ jobs:
           uses: actions/[email protected]
           with:
             submodules: 'recursive'
-        - name: Step Python 3.8.12
+        - name: Step Python 3.11.7
           uses: actions/[email protected]
           with:
-            python-version: '3.8.12'
+            python-version: '3.11.7'
         - name: Install OpenMPI for gt4py
           run: |
             sudo apt-get install libopenmpi-dev

.github/workflows/main_unit_tests.yaml

@@ -0,0 +1,61 @@
+name: "pace main unit tests"
+on:
+  workflow_call:
+    inputs:
+      component_trigger:
+        type: boolean
+        default: false
+        required: false
+      component_name:
+        type: string
+        default: ''
+        required: false
+  pull_request:
+
+# cancel running jobs if theres a newer push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pace_main_unit_tests:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/noaa-gfdl/miniforge:mpich
+    steps:
+        - name: external trigger Checkout pace/develop
+          if:  ${{ inputs.component_trigger }}
+          uses: actions/checkout@v4
+          with:
+            submodules: 'recursive'
+            repository: NOAA-GFDL/pace
+            path: pace
+            ref: develop
+
+        - name: external trigger Remove existing component in pace/develop
+          if:  ${{ inputs.component_trigger }}
+          run: rm -rf ${GITHUB_WORKSPACE}/pace/${{inputs.component_name}}
+
+        - name: Checkout out hash that triggered CI
+          uses: actions/checkout@v4
+          with:
+            submodules: 'recursive'
+            path: pace/${{inputs.component_name}}
+
+        - name: install packages
+          run: |
+            cd ${GITHUB_WORKSPACE}/pace
+            pip3 install --upgrade pip setuptools wheel
+            pip3 install -r requirements_dev.txt -c constraints.txt
+
+        - name: prepare input files
+          run: |
+            cd ${GITHUB_WORKSPACE}/pace
+            mkdir tests/main/input
+            python3 examples/generate_eta_files.py
+            mv *eta*.nc tests/main/input
+
+        - name: run tests
+          run: |
+            cd ${GITHUB_WORKSPACE}/pace
+            pytest -x tests/main

.pre-commit-config.yaml

@@ -15,9 +15,10 @@ repos:
       args: ["--profile", "black"]
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v1.4.1
     hooks:
     - id: mypy
+      additional_dependencies: [types-pyyaml]
       name: mypy-pace
       files: pace
       args: [--config-file, setup.cfg]

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.13-bullseye@sha256:2a01d88a1684e6d7f08030cf5ae73b536926c64076cab197e9e3d9f699255283
+FROM python:3.11-slim-bookworm@sha256:7cd3fa11d619688317226bc93dc59bc8966e9aec6bc2a6abb847e8ab7d656706
 
 RUN apt-get update && apt-get install -y make \
     software-properties-common \
@@ -13,14 +13,11 @@ RUN apt-get update && apt-get install -y make \
     proj-bin \
     proj-data \
     python3 \
-    python3-pip
+    python3-pip \
+    git
 
 RUN pip3 install --upgrade setuptools wheel
 
-COPY constraints.txt /pace/constraints.txt
-
-RUN pip3 install -r /pace/constraints.txt
-
 COPY . /pace
 
 RUN cd /pace && \
@@ -32,14 +29,11 @@ RUN cd / && \
 ENV CFLAGS="-I/usr/include -DACCEPT_USE_OF_DEPRECATED_PROJ_API_H=1"
 
 RUN python3 -m pip install \
-    numpy==1.21.2 \
-    netCDF4==1.5.7 \
-    mpi4py==3.1.1 \
     matplotlib==3.5.2 \
     ipyparallel==8.4.1 \
     jupyterlab==3.4.4 \
     shapely==1.8.5 \
-    cartopy==0.18.0 \
+    cartopy==0.23.0 \
     jupyterlab_code_formatter==1.5.2 \
     isort==5.10.1 \
     black==22.3.0 \

README.md

@@ -74,7 +74,16 @@ Shell scripts to install Pace on specific machines such as Gaea can be found in
 
 ### Run
 
-With the environment activated, you can run an example baroclinic test case with the following command:
+Located in the directory `examples/configs/` there are several example configurations to exhibit the current functionality of Pace. We suggest a new user start with the `baroclinic_c12.yaml` configuration.
+
+Before starting any run, including unit tests, the user must ensure that the proper input files are available. For the `baroclinic_c12.yaml` configuration a script to generate these files and place them in the location referenced in the configuration file is available in the `examples/` directory; `generate_eta_files.py`. To generate the files use the following commands from the top level of the clone of Pace:
+
+```shell
+mkdir tests/main/input
+python3 examples/generate_eta_files.py
+mv *eta*.nc tests/main/input
+```
+These commands will generate the files necessary and place them in the `tests/main/input` directory. Once the files are generated the `baroclinic_c12.yaml` configuration can be used to generate a run:
 
 ```shell
 mpirun -n 6 python3 -m pace.run examples/configs/baroclinic_c12.yaml
@@ -116,7 +125,7 @@ make build
 
 ```shell
 make dev
-mpirun --mca btl_vader_single_copy_mechanism none -n 6 python3 -m pace.run /examples/configs/baroclinic_c12.yaml
+mpirun --mca btl_vader_single_copy_mechanism none -n 6 python -m pace.run /examples/configs/baroclinic_c12.yaml
 ```
 
 ## History