Skip to content

Commit

Permalink
Merge pull request #127 from ACCESS-NRI/126-spack-repo-name-update
Browse files Browse the repository at this point in the history
Replaced all `spack_*` repos with `spack-*`
  • Loading branch information
CodeGat authored Nov 27, 2023
2 parents a4e5d8c + 7ab9122 commit b5beac9
Show file tree
Hide file tree
Showing 6 changed files with 37 additions and 37 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/dep-image-1-start.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
required: true
type: string
default: "main"
description: "Either a git tag or branch name for the ACCESS-NRI/spack_packages repository, which defaults to the main branch"
description: "Either a git tag or branch name for the ACCESS-NRI/spack-packages repository, which defaults to the main branch"
model:
required: true
type: string
Expand Down Expand Up @@ -39,7 +39,7 @@ jobs:
- name: Determine compilers to build
id: get-compiler
run: echo "compilers=$(jq -c . config/compilers.json)" >> $GITHUB_OUTPUT

dependency-image-workflow:
name: Create Dependency Image
needs:
Expand All @@ -48,14 +48,14 @@ jobs:
fail-fast: false
matrix:
compiler: ${{ fromJson(needs.generate-matrix.outputs.compilers) }}
uses: access-nri/build-ci/.github/workflows/dep-image-2-build-base.yml@main
uses: access-nri/build-ci/.github/workflows/dep-image-2-build.yml@main
with:
compiler-name: ${{ matrix.compiler.name }}
compiler-package: ${{ matrix.compiler.package }}
compiler-version: ${{ matrix.compiler.version }}
spack-packages-version: ${{ inputs.spack-packages-version }}
models: ${{ needs.generate-matrix.outputs.models }}
permissions:
permissions:
packages: write
contents: read
secrets: inherit
2 changes: 1 addition & 1 deletion .github/workflows/dep-image-2-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ on:
workflow_call:
inputs:
spack-packages-version:
description: the tag/branch of the access-nri/spack_packages repo to use
description: the tag/branch of the access-nri/spack-packages repo to use
type: string
compiler-name:
description: the short name of the compiler
Expand Down
26 changes: 13 additions & 13 deletions .github/workflows/model-1-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,24 @@ on:

jobs:
setup-spack-packages:
name: Info from spack_packages
name: Info from spack-packages
runs-on: ubuntu-latest
outputs:
version: ${{ steps.get-version.outputs.version }}
steps:
- uses: actions/checkout@v3
with:
repository: access-nri/spack_packages
repository: access-nri/spack-packages
ref: main
fetch-tags: true
- name: Fetch history for main
# By default, actions/checkout only checks out `--depth=1` (the tip of the main branch).
# By default, actions/checkout only checks out `--depth=1` (the tip of the main branch).
# Even when we fetch tags the history is fragmented and usually isn't traversable from HEAD with `git describe --tags`.
# We fetch the entirety of the main branch history to be able to do the next step.
# We fetch the entirety of the main branch history to be able to do the next step.
run: git fetch --unshallow
- name: Get latest spack_packages tags
- name: Get latest spack-packages tags
id: get-version
# This command traverses `main` to find the last `git tag` and suppresses the 'long form' of tags (that have a bunch of commit metadata).
# This command traverses `main` to find the last `git tag` and suppresses the 'long form' of tags (that have a bunch of commit metadata).
run: echo "version=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT

setup-model:
Expand All @@ -35,7 +35,7 @@ jobs:
id: get-package-name
# for the cases where the repo name is in uppercase but the package name is lowercase (eg. access-nri/MOM5)
run: echo "package=$(echo ${{ github.event.repository.name }} | tr [:upper:] [:lower:])" >> $GITHUB_OUTPUT

setup-build-ci:
name: Info from build-ci
runs-on: ubuntu-latest
Expand All @@ -56,7 +56,7 @@ jobs:
# model-components are associated with an overarching model (for example, cice5 is associated with access-om2), this uses models.json to find the associated model
run: |
model=$(jq -cr 'to_entries[] | select(.value | contains(["${{ needs.setup-model.outputs.package-name }}"])) | .key' config/models.json)
if [ -z "$model" ]; then
if [ -z "$model" ]; then
exit 1
else
echo "model=$model" >> $GITHUB_OUTPUT
Expand Down Expand Up @@ -89,13 +89,13 @@ jobs:
else
echo "GH_REF=${{ github.ref_name }})" >> $GITHUB_ENV
fi
- name: Activate environment
run: spack env activate ${{ needs.setup-model.outputs.package-name }}

- name: Build package via spack
run: spack -d install --add --only package --fail-fast --no-checksum ${{ needs.setup-model.outputs.package-name }}@=$GH_REF%${{ matrix.compiler.name }}@${{ matrix.compiler.version }}

- name: Generate lockfile
run: |
cd $SPACK_ROOT/var/spack/environments/${{ needs.setup-model.outputs.package-name }}
Expand All @@ -110,10 +110,10 @@ jobs:
- name: Upload lockfiles
uses: actions/upload-artifact@v3
with:
with:
name: lockfile-output
path: $SPACK_ROOT/var/spack/environments/${{ needs.setup-model.outputs.package-name }}/${{ needs.setup-model.outputs.package-name }}.*.spack.lock


# Install and launch interactive debugging on build failure
- name: Install tmate on failure
Expand Down
20 changes: 10 additions & 10 deletions README-DEV.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ How they are used can be found in the [CI Run Through section](#ci-workflow-run-

This pipeline has explicit inputs, defined in the [`on.workflow_call.inputs` section](https://github.com/ACCESS-NRI/build-ci/blob/1b998192946c364fb75040e6807e7143c9123527/.github/workflows/dep-image-1-start.yml#L5-L15):

* `spack-packages-version`: A tag or branch of the `access-nri/spack_packages` [repository](https://github.com/ACCESS-NRI/spack_packages). This allows provenance of the build process of models.
* `spack-packages-version`: A tag or branch of the `access-nri/spack-packages` [repository](https://github.com/ACCESS-NRI/spack-packages). This allows provenance of the build process of models.
* `model`: a coupled model name (such as `access-om2` or `access-om3`) or `all` if we want to build dependency images for all coupled models defined in `config/models.json`.

It also indirectly uses:
Expand All @@ -31,8 +31,8 @@ It also indirectly uses:

This pipeline creates two docker image outputs:

* `base-spack` Docker image: Of the form `base-spack-<compiler name><compiler version>-<spack_packages version>:latest`. A docker image that contains a `spack` install, `access-nri/spack_packages` repo at the specified version, and a site-wide compiler for spack to use to build dependencies. An example of this package is [`base-spack-intel2021.2.0-main`](https://github.com/ACCESS-NRI/build-ci/pkgs/container/base-spack-intel2021.2.0-main).
* `dependency` Docker image: Of the form `build-<coupled model>-<compiler name><compiler version>-<spack_packages version>:latest`: A docker image based on the above `base-spack` image, that contains all the model components dependencies (separated by `spack env`s), but not the models themselves. The models are added on top of the install in a different pipeline, negating the need for a costly install of the dependencies again (in most cases). An example of this package is [`build-access-om3-intel2021.2.0-main`](https://github.com/orgs/ACCESS-NRI/packages/container/package/build-access-om3-intel2021.2.0-main).
* `base-spack` Docker image: Of the form `base-spack-<compiler name><compiler version>-<spack-packages version>:latest`. A docker image that contains a `spack` install, `access-nri/spack-packages` repo at the specified version, and a site-wide compiler for spack to use to build dependencies. An example of this package is [`base-spack-intel2021.2.0-main`](https://github.com/ACCESS-NRI/build-ci/pkgs/container/base-spack-intel2021.2.0-main).
* `dependency` Docker image: Of the form `build-<coupled model>-<compiler name><compiler version>-<spack-packages version>:latest`: A docker image based on the above `base-spack` image, that contains all the model components dependencies (separated by `spack env`s), but not the models themselves. The models are added on top of the install in a different pipeline, negating the need for a costly install of the dependencies again (in most cases). An example of this package is [`build-access-om3-intel2021.2.0-main`](https://github.com/orgs/ACCESS-NRI/packages/container/package/build-access-om3-intel2021.2.0-main).

### Model Test Pipeline

Expand All @@ -42,7 +42,7 @@ There are no explicit inputs to this workflow. The information required is infer

However, there are indirect inputs into this pipeline:

* Appropriate `dependency` Docker images of the form: `ghcr.io/access-nri/build-<coupled model>-<compiler name><compiler version>-<spack_packages version>:latest`.
* Appropriate `dependency` Docker images of the form: `ghcr.io/access-nri/build-<coupled model>-<compiler name><compiler version>-<spack-packages version>:latest`.
* [`config/compilers.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/compilers.json): This is a data structure containing all the compilers we want to test against.
* [`config/models.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/models.json): This is a data structure containing all the coupled models (and their associated model components) that we want to test against.

Expand Down Expand Up @@ -74,10 +74,10 @@ There are no specific outputs from this pipeline. Only the normal output to the

The rationale for this pipeline is the creation of a model-dependency docker image. This image contains spack, the `spack env`s, and the dependencies for the install of a model, but not the model itself. This allows modified models/model components to be 'dropped in' to the dependency image and installed quickly, without needing to install the dependencies again.

As an overview, this workflow, given a `access-nri/spack_packages` repo version and coupled model(s):
As an overview, this workflow, given a `access-nri/spack-packages` repo version and coupled model(s):

* Generates a staggered `compiler x model` matrix based on the [`compilers.json`](https://github.com/ACCESS-NRI/build-ci/blob/maine/config/compilers.json) and [`models.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/models.json). This allows generation and testing of multiple different compiler and model image combinations in parallel.
* Uses an existing `base-spack` docker image (or creates it if it doesn't exist) that contains an install of spack, `access-nri/spack_packages` and a given compiler.
* Uses an existing `base-spack` docker image (or creates it if it doesn't exist) that contains an install of spack, `access-nri/spack-packages` and a given compiler.
* Using the above `base-spack` image, creates a spack-based model-dependency docker image that separates each model (and it's components) into `spack env`s. This has all the dependencies of the model installed, but not the model itself.

#### Pipeline Overview
Expand Down Expand Up @@ -137,9 +137,9 @@ dep-image-1-start.yml [compilers c1 c2]

##### Creation of `base-spack` and `dependency` images (dep-image-2-build.yml)

In this workflow, given the specs for a given compiler, a `spack_packages` version, and a list of models for a future `model` matrix strategy, we seek to:
In this workflow, given the specs for a given compiler, a `spack-packages` version, and a list of models for a future `model` matrix strategy, we seek to:

* Check that a suitable `base-spack` image doesn't already exists. This would be one that has the same compiler and same version of `spack_packages`.
* Check that a suitable `base-spack` image doesn't already exists. This would be one that has the same compiler and same version of `spack-packages`.
* If it doesn't exist, create and push it using the [`access-nri/actions docker-build-push` action](https://github.com/ACCESS-NRI/actions/tree/main/.github/actions/docker-build-push).
* After those steps, create the aforementioned `model` matrix strategy for each of the models. At this point, the pipeline looks like this:

Expand Down Expand Up @@ -182,11 +182,11 @@ This workflow seeks to build upon the Dependency Image Pipeline as explained abo

Model repositories that implement the [`model-build-test-ci.yml`](https://github.com/ACCESS-NRI/.github/blob/main/workflow-templates/model-build-test-ci.yml) starter workflow (such as the [access-nri/MOM5](https://github.com/ACCESS-NRI/MOM5/blob/master/.github/workflows/model-build-test-ci.yml) repo) will call `build-ci`s [`model-1-build.yml`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml) workflow.

This workflow begins by inferring the 'appropriate dependency image' based on a number of factors, mostly coming from the name of the dependency image (which is of the form `build-<coupled model>-<compiler name><compiler version>-<spack_packages version>:latest`).
This workflow begins by inferring the 'appropriate dependency image' based on a number of factors, mostly coming from the name of the dependency image (which is of the form `build-<coupled model>-<compiler name><compiler version>-<spack-packages version>:latest`).

In order to find:

* `spack_packages version`: In the [`setup-spack-packages`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml#L8-L26) job, we take the latest tagged version of the `access-nri/spack_packages` repo.
* `spack-packages version`: In the [`setup-spack-packages`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml#L8-L26) job, we take the latest tagged version of the `access-nri/spack-packages` repo.
* `coupled model`: In the [`setup-model`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml#L28-L37) and [`setup-build-ci`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml#L39-L63) jobs, we use the name of the calling repository (eg. `cice5`) and `build-ci`s [`config/models.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/models.json) to infer the overarching `coupled model` name.
* `compiler name`/`compiler version`: In the [`setup-build-ci`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/model-1-build.yml#L39-L63) job, we use all compilers from the [`config/compilers.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/compilers.json) file. This would mean that another matrix strategy would be in order.

Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ This repository contains three overarching CI pipelines:

### Dependency Image Pipeline (dep-image-*)

This pipeline creates Docker images that contain an install of `spack`, a version of the `access-nri/spack_packages` [repository](https://github.com/ACCESS-NRI/spack_packages), and a set of independent `spack env`s that contain all the dependencies for all the model components of a coupled model.
This pipeline creates Docker images that contain an install of `spack`, a version of the `access-nri/spack-packages` [repository](https://github.com/ACCESS-NRI/spack-packages), and a set of independent `spack env`s that contain all the dependencies for all the model components of a coupled model.

This allows the install of modified models (and model components) for quick CI testing, rather than having to install an entire dependency tree every time a PR is opened.

Expand All @@ -36,12 +36,12 @@ If you want to use the Model Test Pipeline go to the repo, then the `Actions` ta

Model must meet these requirements:

- Be [available as a spack package](https://github.com/ACCESS-NRI/spack_packages/tree/main/packages) in the [`access-nri/spack_packages` repo](https://github.com/ACCESS-NRI/spack_packages)
- Be [available as a spack package](https://github.com/ACCESS-NRI/spack-packages/tree/main/packages) in the [`access-nri/spack-packages` repo](https://github.com/ACCESS-NRI/spack-packages)
- Have an entry in [`config/models.json`](https://github.com/ACCESS-NRI/build-ci/blob/main/config/models.json) in this repo

### Create your own Dependency Images

There is an associated `workflow_dispatch` trigger on [`dep-image-1-start.yml`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/dep-image-1-start.yml) that allows the creation of your own `base-spack` and `dependency` images. Just make sure that the `spack_packages version` tag exists in the `access-nri/spack_packages` repo.
There is an associated `workflow_dispatch` trigger on [`dep-image-1-start.yml`](https://github.com/ACCESS-NRI/build-ci/blob/main/.github/workflows/dep-image-1-start.yml) that allows the creation of your own `base-spack` and `dependency` images. Just make sure that the `spack-packages version` tag exists in the `access-nri/spack-packages` repo.

A [Web UI trigger](https://github.com/ACCESS-NRI/build-ci/actions/workflows/build-and-push-image-build.yml) is available.

Expand Down
12 changes: 6 additions & 6 deletions containers/Dockerfile.base-spack
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ ARG COMPILER_NAME=intel
ENV SPACK_ROOT=/opt/spack
ENV GNUPGHOME=${SPACK_ROOT}/opt/spack/gpg
ENV SPACK_ENV_ARCH=${SPACK_ARCH}
ENV SPACK_PACKAGES_REPO_ROOT=/opt/spack_packages
ENV SPACK_CONFIG_REPO_ROOT=/opt/spack_config
ENV SPACK_PACKAGES_REPO_ROOT=/opt/spack-packages
ENV SPACK_CONFIG_REPO_ROOT=/opt/spack-config
ENV SPACK_CONFIG_DIR=${SPACK_CONFIG_REPO_ROOT}/${SPACK_VERSION}/ci
ENV SPACK_ENV_COMPILER_PACKAGE=${COMPILER_PACKAGE}
ENV SPACK_ENV_COMPILER_VERSION=${COMPILER_VERSION}
Expand All @@ -81,11 +81,11 @@ SHELL ["/bin/bash", "-c"]
# Install spack
RUN git clone -c feature.manyFiles=true https://github.com/spack/spack.git ${SPACK_ROOT} --branch ${SPACK_REPO_VERSION} --single-branch --depth=1

# Install ACCESS-NRI's spack_packages repo
RUN git clone https://github.com/ACCESS-NRI/spack_packages.git ${SPACK_PACKAGES_REPO_ROOT} --branch ${SPACK_PACKAGES_REPO_VERSION}
# Install ACCESS-NRI's spack-packages repo
RUN git clone https://github.com/ACCESS-NRI/spack-packages.git ${SPACK_PACKAGES_REPO_ROOT} --branch ${SPACK_PACKAGES_REPO_VERSION}

# Install ACCESS-NRI's spack_config repo
RUN git clone https://github.com/ACCESS-NRI/spack_config.git ${SPACK_CONFIG_REPO_ROOT} --branch ${SPACK_CONFIG_REPO_VERSION}
# Install ACCESS-NRI's spack-config repo
RUN git clone https://github.com/ACCESS-NRI/spack-config.git ${SPACK_CONFIG_REPO_ROOT} --branch ${SPACK_CONFIG_REPO_VERSION}

RUN ln -s -r -v ${SPACK_CONFIG_DIR}/* ${SPACK_ROOT}/etc/spack/

Expand Down

0 comments on commit b5beac9

Please sign in to comment.