From 11dc47d49772e2e5f4f9f381544e9aef684dc512 Mon Sep 17 00:00:00 2001 From: Martin Yeo <40734014+trexfeathers@users.noreply.github.com> Date: Wed, 16 Jun 2021 14:42:00 +0100 Subject: [PATCH] CI Benchmarking (#79) * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * WIP. * Benchmarking CI integration. * Added curly brackets in Cirrus clone_script. --- .cirrus.yml | 23 +++++++++++++++++++---- benchmarks/asv.conf.json | 3 ++- noxfile.py | 40 ++++++++++++++++++++++++++-------------- pyproject.toml | 2 +- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index a867245a..ff4a4227 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -18,9 +18,7 @@ env: # Skip specific tasks by name. Set to a non-empty string to skip. SKIP_LINT_TASK: "" SKIP_TEST_TASK: "" - # Currently disabled: - # https://github.com/SciTools-incubator/iris-esmf-regrid/pull/76#issuecomment-850378861 - SKIP_BENCHMARK_TASK: "." + SKIP_BENCHMARK_TASK: "" # Maximum cache period (in weeks) before forcing a new cache upload. CACHE_PERIOD: "2" # Increment the build number to force new conda cache upload. @@ -118,6 +116,17 @@ benchmark_task: only_if: ${SKIP_BENCHMARK_TASK} == "" auto_cancellation: true name: "${CIRRUS_OS}: performance benchmarking" + # Custom clone behaviour to enable ASV to access the PR base branch (if on a + # PR). + clone_script: | + if [ -z "$CIRRUS_PR" ]; then + git clone --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git reset --hard $CIRRUS_CHANGE_IN_REPO + else + git clone --recursive https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git fetch origin pull/${CIRRUS_PR}/head:pull/${CIRRUS_PR} + git reset --hard $CIRRUS_CHANGE_IN_REPO + fi << : *LINUX_CONDA_TEMPLATE asv_cache: folder: ${CIRRUS_WORKING_DIR}/benchmarks/.asv-env @@ -127,4 +136,10 @@ benchmark_task: - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi benchmarks_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" - - nox --session "benchmarks(ci_mode=True)" + - nox --session="benchmarks(ci compare)" + # If running on the main branch (i.e. post-merge): + # Run the full benchmark sequence then publish (``asv gh-pages``). + - | + if [ "$CIRRUS_BRANCH" == "$CIRRUS_DEFAULT_BRANCH" ]; then + nox --session="benchmarks(full then publish)" + fi diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index ed6caceb..2b261b52 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -4,11 +4,12 @@ "repo": "..", "environment_type": "nox-conda", "pythons": [], + "branches": ["main"], "benchmark_dir": "benchmarks", "env_dir": ".asv-env", "results_dir": ".asv-results", "html_dir": ".asv-html", "project_url": "https://github.com/SciTools-incubator/iris-esmf-regrid", "show_commit_url": "https://github.com/SciTools-incubator/iris-esmf-regrid/commit/", - "plugins": ["nox_asv_plugin"], + "plugins": [".nox_asv_plugin"], } diff --git a/noxfile.py b/noxfile.py index dd8d4a7e..eefe4e6b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,6 @@ import os from pathlib import Path import shutil -from typing import List from urllib.request import urlopen import nox @@ -338,10 +337,12 @@ def tests(session: nox.sessions.Session): @nox.session -# CI_MODE=TRUE IS CURRENTLY DISABLED: -# https://github.com/SciTools-incubator/iris-esmf-regrid/pull/76#issuecomment-850378861 -@nox.parametrize("ci_mode", [False]) -def benchmarks(session: nox.sessions.Session, ci_mode: bool): +@nox.parametrize( + ["ci_mode", "gh_pages"], + [(True, False), (False, False), (False, True)], + ids=["ci compare", "full", "full then publish"], +) +def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool): """ Perform esmf-regrid performance benchmarks (using Airspeed Velocity). @@ -352,6 +353,8 @@ def benchmarks(session: nox.sessions.Session, ci_mode: bool): ci_mode: bool Run a cut-down selection of benchmarks, comparing the current commit to the last commit for performance regressions. + gh_pages: bool + Run ``asv gh-pages --rewrite`` once finished. Notes ----- @@ -364,16 +367,25 @@ def benchmarks(session: nox.sessions.Session, ci_mode: bool): # Skip over setup questions for a new machine. session.run("asv", "machine", "--yes") - def asv_exec(sub_command: List[str]) -> None: - session.run( - "asv", - *sub_command, - f"--python={PY_VER[-1]}", - ) + def asv_exec(*sub_args: str) -> None: + run_args = ["asv", *sub_args] + help_output = session.run(*run_args, "--help", silent=True) + if "--python" in help_output: + # Not all asv commands accept the --python kwarg. + run_args.append(f"--python={PY_VER[-1]}") + session.run(*run_args) if ci_mode: - asv_exec(["continuous", "HEAD^1", "HEAD", "--bench=ci"]) - asv_exec(["compare", "HEAD^1", "HEAD"]) + # If on a PR: compare to the base (target) branch. + # Else: compare to previous commit. + previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1") + try: + asv_exec("continuous", previous_commit, "HEAD", "--bench=ci") + finally: + asv_exec("compare", previous_commit, "HEAD") else: # f32f23a5 = first supporting commit for nox_asv_plugin.py . - asv_exec(["run", "f32f23a5..HEAD"]) + asv_exec("run", "f32f23a5..HEAD") + + if gh_pages: + asv_exec("gh-pages", "--rewrite") diff --git a/pyproject.toml b/pyproject.toml index 971596d9..6024add1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,8 @@ exclude = ''' | \.nox | \.tox | \.venv - | \benchmarks\.asv* | _build + | benchmarks\/\.asv.* | buck-out | build | dist