diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6a80d50..d9c64287 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,12 +4,34 @@ on: push: branches: "main" tags: ["*"] + paths: + - '.github/workflows/ci.yml' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'benchmarks/reframe_config.py' + - 'benchmarks/examples/**' + - 'benchmarks/modules/**' + - 'post-processing/**' pull_request: + paths: + - '.github/workflows/ci.yml' + - 'MANIFEST.in' + - 'pyproject.toml' + - 'reframe_config.py' + - 'benchmarks/examples/**' + - 'benchmarks/modules/**' + - 'post-processing/**' release: env: RFM_CONFIG_FILES: "${{ github.workspace }}/benchmarks/reframe_config.py" +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: always. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: name: Tests on ${{ matrix.os }} - Spack ${{ matrix.spack_version }} - Spack spec ${{ matrix.spack_spec }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..936c17e4 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,138 @@ +name: Publish docs via GitHub Pages +on: + push: + branches: + - main + paths: + - 'mkdocs.yml' + - '.github/workflows/docs.yml' + - 'docs/**' + - 'benchmarks/**/*.md' + - 'post-processing/**/*.md' + pull_request: + branches: + - main + paths: + - 'mkdocs.yml' + - '.github/workflows/docs.yml' + - 'docs/**' + - 'benchmarks/**/*.md' + - 'post-processing/**/*.md' + +concurrency: + # Same group concurrency as the `docs_cleanup.yml` workflow, because they both + # git-push to the same branch, so we want to avoid clashes. NOTE: this is + # different from the concurrency group below, which is to cancel successive + # jobs from within the branch/PR. + group: docs-pushing + +jobs: + build: + timeout-minutes: 10 + name: Deploy docs + runs-on: ubuntu-latest + concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: always. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: + # See + # . + # We need `contents: write` to push the docs to the `gh-pages` branch, and + # `statuses: write` to create the custom status. + contents: write + statuses: write + steps: + - name: Checkout main + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.10.6' + cache: 'pip' + + - name: Install Python dependencies + run: pip install mkdocs-material github3.py + + - name: Checkout gh-pages + uses: actions/checkout@v4 + with: + ref: gh-pages + path: gh-pages + + - name: Cleanup unrelated files + run: | + # Under `apps` we want to keep only the `README.md` files. All other files + # (reframe tests, input files, etc...) should be ignored by the docs. + find benchmarks/apps -type f \! \( -name 'README.md' \) -print -delete + + - name: Rewrite URLs in Markdown files for previews + # Run only if this is a PR for which we're going to deploy the preview. + if: ${{ github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork }} + run: | + BASE_URL="https://ukri-excalibur.github.io/excalibur-tests" + echo "BASE_URL=${BASE_URL}" >> "${GITHUB_ENV}" + PREVIEW_SUBDIR="preview/PR${{ github.event.number }}" + echo "PREVIEW_SUBDIR=${PREVIEW_SUBDIR}" >> "${GITHUB_ENV}" + find . -name '*.md' -print -exec sed -i "s|${BASE_URL}|${BASE_URL}/${PREVIEW_SUBDIR}|g" '{}' \; + + - name: Build docs + run: | + if [[ ${{ github.event_name }} == 'pull_request' ]]; then + export MKDOCS_SITE_DIR="site/${PREVIEW_SUBDIR}" + export MKDOCS_SITE_URL="${BASE_URL}/${PREVIEW_SUBDIR}" + fi + mkdocs build + + - name: Deploy docs + # Run only if push is to `main`, or if it's a PR not from a fork. + if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork) }} + working-directory: gh-pages + run: | + git config user.name ${{github.actor}} + git config user.email "${{github.actor_id}}+${{github.actor}}@users.noreply.github.com" + # Before copying the new files, delete the old ones, so that we can + # cleanly update the website. + if [[ ${{ github.event_name }} == 'pull_request' ]]; then + COMMIT_MESSAGE="Update docs preview from ${{ github.sha }}" + # Only delete files in the preview subdir, if any + if [[ -d "${PREVIEW_SUBDIR}" ]]; then + git rm -rf "${PREVIEW_SUBDIR}" + fi + else + COMMIT_MESSAGE="Update docs from ${{ github.sha }}" + for file in $(git ls-files); do + if [[ x"${file}" != xpreview/* ]]; then + # Do not delete files in the `preview/` subdir + git rm ${file} + fi + done + fi + cp -fr ../site/* . + git add . + git commit -m "${COMMIT_MESSAGE}" + git push origin gh-pages + + - name: Create custom status for pull requests + # Similar condition as previous step, but only for PRs + if: ${{ github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork }} + shell: python {0} + run: | + import os + from github3 import login + + token = "${{ github.token }}" + gh = login(token=token) + + owner, repo_name = "${{ github.repository }}".split('/') + repo = gh.repository(owner, repo_name) + + sha = "${{ github.event.pull_request.head.sha }}" + state = "success" + base_url = os.getenv("BASE_URL") + preview_subdir = os.getenv("PREVIEW_SUBDIR") + target_url = f"{base_url}/{preview_subdir}/" + description = "Documentation deployed" + context = "${{ github.workflow }} / Preview" + repo.create_status(sha, state, target_url, description, context) diff --git a/.github/workflows/docs_cleanup.yml b/.github/workflows/docs_cleanup.yml new file mode 100644 index 00000000..f00153c7 --- /dev/null +++ b/.github/workflows/docs_cleanup.yml @@ -0,0 +1,27 @@ +name: Doc Preview Cleanup + +on: + pull_request: + types: [closed] + +concurrency: + # Same group concurrency as the `docs.yml` workflow, because they both + # git-push to the same branch, so we want to avoid clashes. + group: docs-pushing + +jobs: + doc-preview-cleanup: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Checkout gh-pages branch + uses: actions/checkout@v4 + with: + ref: gh-pages + - name: Delete preview and push changes + run: | + git config user.name "${{github.actor}}" + git config user.email "${{github.actor_id}}+${{github.actor}}@users.noreply.github.com" + git rm -rf preview/PR${{ github.event.number }} + git commit -m 'Cleanup docs for PR #${{ github.event.number }}' + git push origin gh-pages diff --git a/.gitignore b/.gitignore index 7e268480..78fdda97 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ outdated/ #ignore virtual environment myvenv/ perflogs/ + +# docs +site/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 0a9dc621..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,117 +0,0 @@ -# How to contribute to ExCALIBUR tests - -You are welcome to contribute new application benchmarks and new systems part of -the ExCALIBUR benchmarking effort. - -## Adding new systems - -We need the [ReFrame](https://reframe-hpc.readthedocs.io/en/stable/) -configuration and a Spack environment for the system. - -### ReFrame configuration - -Add a new system to the ReFrame configuration in -[`benchmarks/reframe_config.py`](./benchmarks/reframe_config.py). Read [ReFrame documentation about -configuration](https://reframe-hpc.readthedocs.io/en/stable/configure.html) for -more details, or see the examples of the existing systems. Note: you likely do -not need to customise the programming environment in ReFrame, as we will use -Spack as build system, which will deal with that. - -If available, the command `lscpu`, run on a compute node, is typically useful to -get information about the CPUs, to be used in the `processor` item of the system -configuration. The numbers you need to watch out for are: - -* "CPU(s)", (`num_cpus` in ReFrame configuration), -* "Thread(s) per core", (`num_cpus_per_core`), -* "Socket(s)", (`num_sockets`), -* "Core(s) per socket", (`num_cpus_per_socket`). - -### Spack configuration - -When using Spack as build system, ReFrame needs a [Spack -environment](https://spack.readthedocs.io/en/latest/environments.html) to run -its tests. We provide already configured Spack environments in the -[`benchmarks/spack`](./benchmarks/spack) directory, for each system in the -ReFrame configuration. - -Here are the steps to create a Spack environment for a new system: - -* create the environment: - ``` - spack env create --without-view -d spack/SYSTEM-NAME - ``` - where `SYSTEM-NAME` is the same name as the ReFrame system name. Remember to - [disable - views](https://spack.readthedocs.io/en/latest/environments.html#filesystem-views) - with `--without-view` to avoid conflicts when installing incompatible packages - in the same environment -* activate it: - ``` - spack env activate -d spack/SYSTEM-NAME - ``` -* set the - [`install_tree`](https://spack.readthedocs.io/en/latest/config_yaml.html#install-tree): - ``` - spack config add 'config:install_tree:root:$env/opt/spack' - ``` -* make sure the compilers you want to add are in the `PATH` (e.g., load the - relevant modules), then add them to the Spack environment with: - ``` - spack compiler find - ``` -* add other packages (e.g., MPI): make sure the package you want to add are - "visible" (e.g., load the relevant modules) and add them to the environment - with - ``` - spack external find PACKAGE-NAME ... - ``` - where `PACKAGE-NAME ...` are the names of the corresponding Spack packages -* manually tweak the environment as necessary. For example, if there is already - a global Spack available in the system, you can [include its configuration - files](https://spack.readthedocs.io/en/latest/environments.html#included-configurations), - or [add its install trees as - upstreams](https://spack.readthedocs.io/en/latest/chain.html). - -* If you are using a custom repo for spack package recipes (see *Spack package* below), add - it to the spack environment with - ``` - spack -e /path/to/environment repo add /path/to/repo - ``` - -## Adding new benchmarks - -### Spack package - -Before adding a new benchmark, make sure the application is available in -[Spack](https://spack.readthedocs.io/en/latest/package_list.html). If it is -not, you can read the [Spack Package Creation -Tutorial](https://spack-tutorial.readthedocs.io/en/latest/tutorial_packaging.html) -to contribute a new recipe to build the application. - -While we encourage users to contribute all Spack recipes upstream, we have a custom -repo for packages not yet ready to be contributed to the main Spack repository. -This is in the `spack/repo` directory, create a subdirectory inside `spack/repo/packages` -with the name of the package you want to add, and place into it the `package.py` Spack recipe. -On supported HPC systems, this repo is automatically added to the provided Spack environments. - -### ReFrame benchmark - -New benchmarks should be added in the `apps/` directory, under the specific -application subdirectory. Please, add also a `README.md` file explaining what -the application does and how to run the benchmarks. - -For writing ReFrame benchmarks you can read the documentation, in particular - -* [ReFrame Tutorials](https://reframe-hpc.readthedocs.io/en/stable/tutorials.html) -* [Regression Test API](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html) - -but you can also have a look at the sample file in -[`benchmarks/examples/sombrero/sombrero.py`](./benchmarks/examples/sombrero/sombrero.py). - -For GPU benchmarks you need to - -* set [`valid_systems = ['+gpu']`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.valid_systems) -* set the [`num_gpus_per_node`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.num_gpus_per_node) attribute, -* and add the key `gpu` to the [`extra_resources`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.extra_resources) dictionary to request the appropriate number of GPUs. - -For an example of a GPU benchmark take a look at [OpenMM](benchmarks/apps/openmm/openmm_rfm.py). diff --git a/MANIFEST.in b/MANIFEST.in index ec58fdbf..51ab81e0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,5 @@ recursive-include benchmarks/modules * recursive-include benchmarks/spack *.yaml recursive-include benchmarks/spack/repo * recursive-include post-processing * +prune docs prune __pycache__ diff --git a/README.md b/README.md index 683a0372..809fdb0f 100644 --- a/README.md +++ b/README.md @@ -5,224 +5,33 @@ Performance benchmarks and regression tests for the ExCALIBUR project. These benchmarks are based on a similar project by [StackHPC](https://github.com/stackhpc/hpc-tests). -_**Note**: at the moment the ExCALIBUR benchmarks are a work-in-progress._ - -## Installation - -We require Python version 3.7 or later. Install the **excalibur-tests** package with `pip` by - -```sh -pip install -e . -``` - -The `-e/--editable` flag is recommended for two reasons. -- Spack installs packages in a `opt` directory under the spack environment. With `-e` the spack -environment remains in your local directory and `pip` creates symlinks to it. Without `-e` spack -will install packages inside your python environment. -- For [development](https://setuptools.pypa.io/en/latest/userguide/development_mode.html), -the `-e` flag to `pip` links the installed package to the files in the local -directory, instead of copying, to allow making changes to the installed package. - -Note that to use `-e` with a project configured with a `pyproject.toml` you need `pip` version 22 or later. - -On most systems, it is recommended to install the package in a virtual environment. -For example, using the python3 [built-in virtual environment tool `venv`](https://docs.python.org/3/library/venv.html), -create an environment called `my_environment` with - -```sh -python3 -m venv ./my_environment -``` - -and activate it with - -```sh -source ./my_environment/bin/activate -``` - -### Spack - -The `pip install` command will install a compatible version of **ReFrame** from -[PyPi](https://pypi.org/project/ReFrame-HPC/). However, you will have to -manually provide an installation of **Spack**. - -[Spack](https://spack.io/) is a package manager specifically designed for HPC -facilities. In some HPC facilities there may be already a central Spack installation available. -However, the version installed is most likely too old to support all the features -used by this package. Therefore we recommend you install the latest version locally, -following the instructions below. - -_**Note**: if you have already installed spack locally and you want to upgrade to -a newer version, you might first have to clear the cache to avoid conflicts: -`spack clean -m`_ - -Follow the [official instructions](https://spack.readthedocs.io/en/latest/getting_started.html) -to install the latest version of Spack (summarised here for convenience, but not guaranteed to be -up-to-date): -- git clone spack: -`git clone -c feature.manyFiles=true https://github.com/spack/spack.git` -- run spack setup script: `source ./spack/share/spack/setup-env.sh` -- check spack is in `$PATH`, for example `spack --version` - -In order to use Spack in ReFrame, the framework we use to run the benchmarks -(see below), the directory where the `spack` program is installed needs to be in -the `PATH` environment variable. This is taken care of by the `setup-env.sh` -script as above, and you can have your shell init script (e.g. `.bashrc`) -do that automatically in every session, by adding the following lines to it: -```sh -export SPACK_ROOT="/path/to/spack" -if [ -f "${SPACK_ROOT}/share/spack/setup-env.sh" ]; then - source "${SPACK_ROOT}/share/spack/setup-env.sh" -fi -``` -replacing `/path/to/spack` with the actual path to your Spack installation. - -ReFrame also requires a [Spack -Environment](https://spack.readthedocs.io/en/latest/environments.html). We -provide Spack environments for some of the systems that are part of the -ExCALIBUR and DiRAC projects in -[https://github.com/ukri-excalibur/excalibur-tests/tree/main/benchmarks/spack/](https://github.com/ukri-excalibur/excalibur-tests/tree/main/benchmarks/spack). -If you want to use a different Spack environment, -set the environment variable `EXCALIBUR_SPACK_ENV` to the path of the directory -where the environment is. If this is not set, ReFrame will try to use the -environment for the current system if known, otherwise it will automatically -create a very basic environment (see "Usage on unsupported systems" section below). - -## Configuration - -### ReFrame - -[ReFrame](https://reframe-hpc.readthedocs.io/en/stable/) is a high-level -framework for writing regression tests for HPC systems. For our tests we -require ReFrame v4.1.3. - -We provide a ReFrame configuration file with the settings of some systems that -are part of the ExCALIBUR or DiRAC projects. You can point ReFrame to this file by -setting the -[`RFM_CONFIG_FILES`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#envvar-RFM_CONFIG_FILES) -environment variable: - -```sh -export RFM_CONFIG_FILES="${PWD}/benchmarks/reframe_config.py" -``` - -If you want to use a different ReFrame configuration file, for example because -you use a different system, you can set this environment variable to the path of -that file. - -**Note**: in order to use the Spack build system in ReFrame, the `spack` -executable must be in the `PATH` also on the compute nodes of a cluster, if -you want to run your benchmarks on them. This is taken care of by adding it -to your init file (see spack section above). +Feel free to add new benchmark applications or support new systems that are part of the +ExCALIBUR benchmarking collaboration. -However, you will also need to set the -[`RFM_USE_LOGIN_SHELL`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#envvar-RFM_USE_LOGIN_SHELL) -environment variable (`export RFM_USE_LOGIN_SHELL="true"`) in order to make ReFrame use - -```sh -!#/bin/bash -l -``` - -as [shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) line, which would load -the user's init script. - -## Usage - -Once you have set up Spack and ReFrame, you can execute a benchmark with - -```sh -reframe -c benchmarks/apps/BENCH_NAME -r --performance-report -``` - -where `benchmarks/apps/BENCH_NAME` is the directory where the benchmark is. The command -above assumes you have the program `reframe` in your PATH. If you have followed the instructions -to install using `pip` into the default directory, it should have been automatically added. -If it is not the case, call `reframe` with its relative or absolute path. - -For example, to run the Sombrero benchmark in the `benchmarks/apps/sombrero` directory you can -use - -```sh -reframe -c benchmarks/apps/sombrero -r --performance-report -``` - -For benchmarks that use the Spack build system, the tests define a default Spack specification -to be installed in the environment, but users can change it when invoking ReFrame on the -command line with the -[`-S`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#cmdoption-S) option to set -the `spack_spec` variable: - -``` -reframe -c benchmarks/apps/sombrero -r --performance-report -S spack_spec='sombrero@2021-08-16%intel' -``` - -### Setting environment variables - -All the built-in fields of ReFrame regression classes can be set on a per-job basis using the -`-S` command-line option. One useful such field is -[`env_vars`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.env_vars), -which controls the environment variables used in a job. -The syntax to set dictionary items, like for `env_vars`, is a comma-separated list of `key:value` pairs: `-S dict=key_1:value_1,key_2:value_2`. -For example - -``` -reframe -c benchmarks/apps/sombrero -r --performance-report -S env_vars=OMP_PLACES:threads -``` - -runs the `benchmarks/apps/sombrero` benchmark setting the environment variable `OMP_PLACES` -to `threads`. - -### Selecting system and queue access options - -The provided ReFrame configuration file contains the settings for multiple systems. If you -use it, the automatic detection of the system may fail, as some systems may use clashing -hostnames. To avoid this, you can use the flag [`--system -NAME:PARTITION`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#cmdoption-system) -to specify the system (and optionally the partition) to use. - -Additionally, if submitting jobs to the compute nodes requires additional options, like for -example the resource group you belong to (for example `--account=...` for Slurm), you have -to pass the command line flag -[`--job-option=...`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#cmdoption-J) -to `reframe` (e.g., `--job-option='--account=...'`). - -### Usage on unsupported systems - -The configuration provided in [`reframe_config.py`](./reframe_config.py) lets you run the -benchmarks on pre-configured HPC systems. However you -can use this framework on any system by choosing the "default" system with `--system -default`, or by using your own ReFrame configuration. You can use the "default" system to run -benchmarks in ReFrame without using a queue manager or an MPI launcher (e.g. on a personal workstation). - -If you choose the "default" system and a benchmark using the Spack build system, -a new empty Spack environment will be automatically created in -`benchmarks/spack/default` when ReFrame is launched for the first time. -You should populate the environment with the packages already installed on your system -before running Spack to avoid excessively rebuilding system packages. See the -*Spack configuration* section of [`CONTRIBUTING.md`](./CONTRIBUTING.md) for instructions on how -to set up a Spack environment. -In particular, make sure that at least a compiler and an MPI library are added into the environment. -After the Spack environment is set up, tell ReFrame to use it by setting the environment -variable `EXCALIBUR_SPACK_ENV`, as described above. - - -### System-specific flags - -While the aim is to automate as much system-specific configuration as possible, there are some options that have to be provided by the user, such as accounting details, and unfortunately the syntax can vary. -The file [`SYSTEMS.md`](./SYSTEMS.md) contains information about the use of this framework on specific systems. +_**Note**: at the moment the ExCALIBUR benchmarks are a work-in-progress._ -### Selecting multiple benchmarks +## Documentation -ReFrame tests may contain tags that allow the user to select which tests to run. These can be leveraged to defined sets of benchmarks. To run all tests in a directory, pass the [`-R` flag](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#cmdoption-R) to ReFrame. Then filter down to a specific tag by passing the [`-t` flag](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#cmdoption-0). +- [Installation](https://ukri-excalibur.github.io/excalibur-tests/install/) +- [Configuration](https://ukri-excalibur.github.io/excalibur-tests/setup/) +- [Usage](https://ukri-excalibur.github.io/excalibur-tests/use/) +- [Post-processing](https://ukri-excalibur.github.io/excalibur-tests/post-processing/) +- [Contributing](https://ukri-excalibur.github.io/excalibur-tests/contributing/) +- [Supported benchmarks](https://ukri-excalibur.github.io/excalibur-tests/apps/) +- [Supported systems](https://ukri-excalibur.github.io/excalibur-tests/systems/) +- [ARCHER2 tutorial](https://ukri-excalibur.github.io/excalibur-tests/tutorial/tutorial/) -For example, the [tag "example" is defined](https://github.com/ukri-excalibur/excalibur-tests/blob/1a7377e885977833c150569c32eb1db478f63087/benchmarks/examples/sombrero/sombrero.py#L113) in the sombrero example. To select the sombrero example out of all benchmarks, run +## Acknowledgements -```bash -reframe -c benchmarks/ -R -r -t example -``` +This work was supported by the Engineering and Physical Sciences +Research Council [EP/X031829/1]. -Tests can contain multiple tags. To create a custom set of benchmarks, add a new tag to the tests you want to include in the set. +This work used the DiRAC@Durham facility managed by the Institute for Computational +Cosmology on behalf of the STFC DiRAC HPC Facility (www.dirac.ac.uk). The equipment +was funded by BEIS capital funding via STFC capital grants ST/P002293/1, ST/R002371/1 +and ST/S002502/1, Durham University and STFC operations grant ST/R000832/1. +DiRAC is part of the National e-Infrastructure. -## Contributing new systems or benchmarks +The main outcomes of this work were published in a [paper](https://dl.acm.org/doi/10.1145/3624062.3624133) in the HPCTESTS workshop in SC23. -Feel free to add new benchmark apps or support new systems that are part of the -ExCALIBUR benchmarking collaboration. Read [`CONTRIBUTING.md`](./CONTRIBUTING.md) for more details. +This work was [presented in RSECon23](https://virtual.oxfordabstracts.com/#/event/4430/submission/74). A [recording of the talk](https://youtu.be/vpTD_tJqWOA?si=zl9sWvPEQYyPhJTV) is available. diff --git a/benchmarks/apps/README.md b/benchmarks/apps/README.md new file mode 100644 index 00000000..c2e4e61d --- /dev/null +++ b/benchmarks/apps/README.md @@ -0,0 +1,3 @@ +# Supported benchmarks + +This directory contains the benchmarks currently supported by the project. More can be added by opening a Pull Request following the guidance in [contributing](../contributing.md). diff --git a/benchmarks/apps/hpgmg/README.md b/benchmarks/apps/hpgmg/README.md new file mode 100644 index 00000000..0a756cf2 --- /dev/null +++ b/benchmarks/apps/hpgmg/README.md @@ -0,0 +1,3 @@ +# HPGMG + +See [hpgmg](https://hpgmg.org/) diff --git a/benchmarks/apps/hpl/README.md b/benchmarks/apps/hpl/README.md index 7714b6ae..75794be1 100644 --- a/benchmarks/apps/hpl/README.md +++ b/benchmarks/apps/hpl/README.md @@ -21,6 +21,7 @@ Appropriate `HPL.dat` configuration files must be generated and placed in `` command line option of `reframe`: + - `mini`: A debug run, on a very small lattice, on 2 processes. - `ITT-sn`: A run on a single node, using all the cores in each node (as described [here](https://github.com/sa2c/sombrero/wiki/Dirac-ITT-2020-Benchmarks)). @@ -34,6 +35,7 @@ using the `--tag=` command line option of `reframe`: In all these cases, the benchmark for each theory is launched. The following performance variables are captured: + - 'flops' : the computing performance (Gflop/second) - 'time' : time spent in the CG algorithm (seconds) - 'communicated': number of bytes communicated via MPI (bytes) diff --git a/benchmarks/apps/swift/README.md b/benchmarks/apps/swift/README.md new file mode 100644 index 00000000..ae14ae4e --- /dev/null +++ b/benchmarks/apps/swift/README.md @@ -0,0 +1,3 @@ +# Swift + +See [swift](https://swift.strw.leidenuniv.nl/) diff --git a/benchmarks/modules/utils.py b/benchmarks/modules/utils.py index d47784fc..2e4c6eb7 100644 --- a/benchmarks/modules/utils.py +++ b/benchmarks/modules/utils.py @@ -290,6 +290,14 @@ def set_sge_num_slots(self): # Set the total number of CPUs to be requested for the SGE scheduler. self.extra_resources['mpi'] = {'num_slots': num_tasks * num_cpus_per_task} + @run_before('compile') + def set_isambard_memory(self): + # We define the `memory` extra resource, so that we + # can use the corresponding resources in the config file. + system, partition = self.current_partition.fullname.split(':') + if system == "isambard-phase3": + self.extra_resources['memory'] = {} + @run_after('setup') def setup_build_job_num_cpus(self): # When running a build on a compute node, ReFrame by default uses only a diff --git a/benchmarks/reframe_config.py b/benchmarks/reframe_config.py index dafcea4f..9d2319b7 100644 --- a/benchmarks/reframe_config.py +++ b/benchmarks/reframe_config.py @@ -434,9 +434,9 @@ def spack_root_to_path(): }, 'resources': [ { - 'name': 'cpu', - # TODO: memory should be a separate resource. - 'options': ['ncpus={num_cpus}:mem=100g'], + 'name': 'memory', + # TODO: memory should be a more general resource. + 'options': ['mem=100g'], }, ], }, @@ -455,6 +455,13 @@ def spack_root_to_path(): 'num_sockets': 2, 'num_cpus_per_socket': 64, }, + 'resources': [ + { + 'name': 'memory', + # TODO: memory should be a more general resource. + 'options': ['mem=100g'], + }, + ], }, ] }, # end Isambard Phase3 @@ -770,6 +777,7 @@ def spack_root_to_path(): '%(check_display_name)s|' '%(check_system)s|' '%(check_partition)s|' + '%(check_job_nodelist)s|' '%(check_environ)s|' '%(check_extra_resources)s|' '%(check_env_vars)s|' diff --git a/docs/apps b/docs/apps new file mode 120000 index 00000000..b6b55e9a --- /dev/null +++ b/docs/apps @@ -0,0 +1 @@ +../benchmarks/apps/ \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 00000000..bcca808d --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,58 @@ +# How to contribute to ExCALIBUR tests + +You are welcome to contribute new application benchmarks and new systems part of +the ExCALIBUR benchmarking effort. The easiest way to contribute is to open an +issue or a pull request to the repository. + +## Adding new benchmarks + +In particular, adding new benchmarks that are useful to the scientific community is +welcome! + +### Spack package + +Before adding a new benchmark, make sure the application is available in +[Spack](https://spack.readthedocs.io/en/latest/package_list.html). If it is +not, you can read the [Spack Package Creation +Tutorial](https://spack-tutorial.readthedocs.io/en/latest/tutorial_packaging.html) +to contribute a new recipe to build the application. + +While we encourage users to contribute all Spack recipes upstream, we have a custom +repo for packages not yet ready to be contributed to the main Spack repository. +This is in the `spack/repo` directory, create a subdirectory inside `spack/repo/packages` +with the name of the package you want to add, and place into it the `package.py` Spack recipe. +On supported HPC systems, this repo is automatically added to the provided Spack environments. + +### ReFrame benchmark + +New benchmarks should be added in the `apps/` directory, under the specific +application subdirectory. Please, add also a `README.md` file explaining what +the application does and how to run the benchmarks in the same directory. Then, +link the `README.md` file under `nav: Supported Benchmarks:` +in the [mkdocs documentation config](https://github.com/ukri-excalibur/excalibur-tests/blob/main/mkdocs.yml). + +For writing ReFrame benchmarks you can read the documentation, in particular + +* [ReFrame Tutorials](https://reframe-hpc.readthedocs.io/en/stable/tutorials.html) +* [Regression Test API](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html) + +but you can also have a look at the +[sombrero example](https://github.com/ukri-excalibur/excalibur-tests/blob/main/benchmarks/examples/sombrero). + +For GPU benchmarks you need to + +* set [`valid_systems = ['+gpu']`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.valid_systems) +* set the [`num_gpus_per_node`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.num_gpus_per_node) attribute, +* and add the key `gpu` to the [`extra_resources`](https://reframe-hpc.readthedocs.io/en/stable/regression_test_api.html#reframe.core.pipeline.RegressionTest.extra_resources) dictionary to request the appropriate number of GPUs. + +For an example of a GPU benchmark take a look at [OpenMM](https://github.com/ukri-excalibur/excalibur-tests/tree/main/benchmarks/apps/openmm). + + + +## Adding new systems + +If you [configure the framework](./setup.md) on a HPC system that is not included in [supported systems](./systems.md), please upen a pull request to upload the configuration so that other users can benefit from it. To add a new system, consider the following items + +* [ReFrame configuration](./setup.md#reframe_1) +* [Spack configuration](./setup.md#spack_1) +* [Documentation](./systems.md) diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 00000000..a4da1a66 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,89 @@ +# Installation + +## Excalibur-tests + +### Requirements + +**Python version 3.7** or later is required. + +### Virtual environments + +On most systems, it is recommended to install +the package in a virtual environment. For example, using the python3 +[built-in virtual environment tool `venv`](https://docs.python.org/3/library/venv.html), +create an environment called `my_environment` with + +```sh +python3 -m venv ./my_environment +``` + +and activate it with + +```sh +source ./my_environment/bin/activate +``` + +### Installation + +First, clone the git repository + +```sh +git clone https://github.com/ukri-excalibur/excalibur-tests.git +``` + +Install the **excalibur-tests** package and the necessary dependencies with `pip` by + +```sh +pip install -e ./excalibur-tests +``` + +### Notes + +The `-e/--editable` flag is recommended for two reasons. + +- Spack installs packages in a `opt` directory under the spack environment. With `-e` the spack +environment remains in your local directory and `pip` creates symlinks to it. Without `-e` spack +will install packages inside your python environment. +- For [development](https://setuptools.pypa.io/en/latest/userguide/development_mode.html), +the `-e` flag to `pip` links the installed package to the files in the local +directory, instead of copying, to allow making changes to the installed package. + +Note that to use `-e` with a project configured with a `pyproject.toml` you need `pip` version 22 or later. + +## Spack + +The `pip install` command will install a compatible version of **ReFrame** from +[PyPi](https://pypi.org/project/ReFrame-HPC/). However, you will have to +manually provide an installation of **Spack**. + +[Spack](https://spack.io/) is a package manager specifically designed for HPC +facilities. In some HPC facilities there may be already a central Spack installation available. +However, the version installed is most likely too old to support all the features +used by this package. Therefore we recommend you install the latest version locally, +following the instructions below. + +Follow the [official instructions](https://spack.readthedocs.io/en/latest/getting_started.html) +to install the latest version of Spack (summarised here for convenience, but not guaranteed to be +up-to-date): + +### Installation + +Git clone the spack repository +```sh +git clone -c feature.manyFiles=true https://github.com/spack/spack.git +``` +Run spack setup script +```sh +source ./spack/share/spack/setup-env.sh +``` +Check spack is in `$PATH`, for example +```sh +spack --version +``` + +### Notes + +_**Note**: if you have already installed spack locally and you want to upgrade to +a newer version, you might first have to clear the cache to avoid conflicts: +`spack clean -m`_ + diff --git a/docs/post-processing/README.md b/docs/post-processing/README.md new file mode 120000 index 00000000..74ec9f7d --- /dev/null +++ b/docs/post-processing/README.md @@ -0,0 +1 @@ +../../post-processing/README.md \ No newline at end of file diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 00000000..a90b5b9a --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,150 @@ +# Configuration + +## Pre-configured systems + +A number of UK-based HPC systems that are part of the DiRAC and ExCALIBUR programs +have been pre-configured. See [systems](systems.md), or [https://github.com/ukri-excalibur/excalibur-tests/tree/main/benchmarks/spack/](https://github.com/ukri-excalibur/excalibur-tests/tree/main/benchmarks/spack) for a list. +On these systems the ReFrame configuration and Spack environment are included in the +`excalibur-tests` repository and all you need to do is point the framework to them. + +### ReFrame + +You can point ReFrame to the provided config file by setting the +[`RFM_CONFIG_FILES`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#envvar-RFM_CONFIG_FILES) +environment variable: + +```sh +export RFM_CONFIG_FILES="/benchmarks/reframe_config.py" +``` + +If you want to use a different ReFrame configuration file, for example because +you use a different system, you can set this environment variable to the path of +that file. + +**Note**: in order to use the Spack build system in ReFrame, the `spack` +executable must be in the `PATH` also on the compute nodes of a cluster, if +you want to run your benchmarks on them. This is taken care of by adding it +to your init file (see spack section below). + +However, you will also need to set the +[`RFM_USE_LOGIN_SHELL`](https://reframe-hpc.readthedocs.io/en/stable/manpage.html#envvar-RFM_USE_LOGIN_SHELL) +environment variable +```sh +export RFM_USE_LOGIN_SHELL="true" +``` +in order to make ReFrame use `!#/bin/bash -l` as +[shebang](https://en.wikipedia.org/wiki/Shebang_(Unix)) line, which would load +the user's init script. + +### Spack + +In order to use Spack in ReFrame, +the directory where the `spack` program is installed needs to be in +the `PATH` environment variable. This is taken care of by the `setup-env.sh` +script run in [install](install.md#installation_2). To have spack available in every session, +you can have your shell init script (e.g. `.bashrc`) +re-run it automatically, by adding the following lines to it: +```sh +export SPACK_ROOT="/path/to/spack" +if [ -f "${SPACK_ROOT}/share/spack/setup-env.sh" ]; then + source "${SPACK_ROOT}/share/spack/setup-env.sh" +fi +``` +replacing `/path/to/spack` with the actual path to your Spack installation. + +## New systems + +We need the [ReFrame](https://reframe-hpc.readthedocs.io/en/stable/) +configuration and a Spack environment for a new system. + +### ReFrame + +Add a new system to the ReFrame configuration in +`benchmarks/reframe_config.py`. Read [ReFrame documentation about +configuration](https://reframe-hpc.readthedocs.io/en/stable/configure.html) for +more details, or see the examples of the existing systems. + +**Note**: you likely do +not need to customise the programming environment in ReFrame, as we will use +Spack as build system, which will deal with that. + +If available, the command `lscpu`, run on a compute node, is typically useful to +get information about the CPUs, to be used in the `processor` item of the system +configuration. The numbers you need to watch out for are: + +* "CPU(s)", (`num_cpus` in ReFrame configuration), +* "Thread(s) per core", (`num_cpus_per_core`), +* "Socket(s)", (`num_sockets`), +* "Core(s) per socket", (`num_cpus_per_socket`). + +### Spack + +When using Spack as build system, ReFrame needs a [Spack +environment](https://spack.readthedocs.io/en/latest/environments.html) to run +its tests. Follow these steps to create a Spack environment for a new system: + +#### Create the environment +```sh +spack env create --without-view -d /path/to/environment +``` +Remember to +[disable views](https://spack.readthedocs.io/en/latest/environments.html#filesystem-views) +with `--without-view` to avoid conflicts when installing incompatible packages +in the same environment + +#### Activate the environment +```sh +spack env activate -d /path/to/environment +``` + +#### Set [`install_tree`](https://spack.readthedocs.io/en/latest/config_yaml.html#install-tree) +```sh +spack config add 'config:install_tree:root:$env/opt/spack' +``` + +#### Add compilers + +Make sure the compilers you want to add are in the `PATH` (e.g., load the +relevant modules), then add them to the Spack environment with: +```sh +spack compiler find +``` + +#### Add external packages + +Add other packages (e.g., MPI): make sure the package you want to add are +"visible" (e.g., load the relevant modules) and add them to the environment +with +```sh +spack external find PACKAGE-NAME ... +``` +where `PACKAGE-NAME ...` are the names of the corresponding Spack packages + +#### Set `EXCALIBUR_SPACK_ENV` variable + +To use the new Spack environment in ReFrame, +set the environment variable `EXCALIBUR_SPACK_ENV` to the path of the directory +where the environment is, i.e. +```sh +export EXCALIBUR_SPACK_ENV=/path/to/environment +``` +If this is not set, ReFrame will try to use the +environment for the current system if known, otherwise it will automatically +create a very basic environment (see [Usage on unsupported systems](use.md#usage-on-unsupported-systems). + +#### (optional) Make manual tweaks + +If necessary, manually tweak the environment. For example, if there is already +a global Spack available in the system, you can [include its configuration +files](https://spack.readthedocs.io/en/latest/environments.html#included-configurations), +or [add its install trees as +upstreams](https://spack.readthedocs.io/en/latest/chain.html). + + +#### (optional) Add spack repositories + +If you are using a custom repo for spack package recipes (see *Spack package* below), add +it to the spack environment with +```sh +spack -e /path/to/environment repo add /path/to/repo +``` diff --git a/SYSTEMS.md b/docs/systems.md similarity index 98% rename from SYSTEMS.md rename to docs/systems.md index c5fec37c..807ea7c5 100644 --- a/SYSTEMS.md +++ b/docs/systems.md @@ -26,7 +26,7 @@ reframe -c benchmarks/examples/sombrero -r --performance-report --system archer2 ARCHER2 allows [choosing the CPU frequency](https://docs.archer2.ac.uk/user-guide/energy/#controlling-cpu-frequency) during jobs by setting the environment variable `SLURM_CPU_FREQ_REQ` to specific values. In ReFrame v3 the list of environment variables set by the framework is held by the dictionary attribute called `env_vars`, and you can initialise it on the command line when running a benchmark with `-S`/`--setvar`. -For more details, see Setting environment variables in [`README.md`](./README.md). +For more details, see Setting environment variables in [usage](use.md). For example, to submit a benchmark using the lowest CPU frequency (1.5 GHz) you can use ``` @@ -38,7 +38,7 @@ reframe -c benchmarks/examples/sombrero -r --performance-report --system archer2 ARCHER2 is a Cray system, and they [recommend using a cray optimised python version](https://docs.archer2.ac.uk/user-guide/python/). The HPE Cray Python distribution can be loaded using `module load cray-python`. -This is necessary to pip install excalibur-tests following the instructions in [README.md](./README.md). +This is necessary to pip install excalibur-tests following the instructions in [install](install.md). ### Spack install path diff --git a/tutorial/Performance_vs_number_of_tasks_and_CPUs_per_task.html b/docs/tutorial/Performance_vs_number_of_tasks_and_CPUs_per_task.html similarity index 100% rename from tutorial/Performance_vs_number_of_tasks_and_CPUs_per_task.html rename to docs/tutorial/Performance_vs_number_of_tasks_and_CPUs_per_task.html diff --git a/tutorial/SombreroBenchmark.log b/docs/tutorial/SombreroBenchmark.log similarity index 100% rename from tutorial/SombreroBenchmark.log rename to docs/tutorial/SombreroBenchmark.log diff --git a/tutorial/post_processing_config.yaml b/docs/tutorial/post_processing_config.yaml similarity index 100% rename from tutorial/post_processing_config.yaml rename to docs/tutorial/post_processing_config.yaml diff --git a/tutorial/tutorial.md b/docs/tutorial/tutorial.md similarity index 95% rename from tutorial/tutorial.md rename to docs/tutorial/tutorial.md index fcd0b3bf..8f8752bf 100644 --- a/tutorial/tutorial.md +++ b/docs/tutorial/tutorial.md @@ -1,6 +1,4 @@ --- -#type: slide ----