CI Benchmarking (#79)

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* WIP.

* Benchmarking CI integration.

* Added curly brackets in Cirrus clone_script.

.cirrus.yml

@@ -18,9 +18,7 @@ env:
   # Skip specific tasks by name. Set to a non-empty string to skip.
   SKIP_LINT_TASK: ""
   SKIP_TEST_TASK: ""
-  # Currently disabled:
-  #  https://github.com/SciTools-incubator/iris-esmf-regrid/pull/76#issuecomment-850378861
-  SKIP_BENCHMARK_TASK: "."
+  SKIP_BENCHMARK_TASK: ""
   # Maximum cache period (in weeks) before forcing a new cache upload.
   CACHE_PERIOD: "2"
   # Increment the build number to force new conda cache upload.
@@ -118,6 +116,17 @@ benchmark_task:
   only_if: ${SKIP_BENCHMARK_TASK} == ""
   auto_cancellation: true
   name: "${CIRRUS_OS}: performance benchmarking"
+  # Custom clone behaviour to enable ASV to access the PR base branch (if on a
+  #  PR).
+  clone_script: |
+    if [ -z "$CIRRUS_PR" ]; then
+      git clone --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
+      git reset --hard $CIRRUS_CHANGE_IN_REPO
+    else
+      git clone --recursive https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
+      git fetch origin pull/${CIRRUS_PR}/head:pull/${CIRRUS_PR}
+      git reset --hard $CIRRUS_CHANGE_IN_REPO
+    fi
   << : *LINUX_CONDA_TEMPLATE
   asv_cache:
     folder: ${CIRRUS_WORKING_DIR}/benchmarks/.asv-env
@@ -127,4 +136,10 @@ benchmark_task:
       - if [ -n "${IRIS_SOURCE}" ]; then echo "${IRIS_SOURCE}"; fi
   benchmarks_script:
     - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)"
-    - nox --session "benchmarks(ci_mode=True)"
+    - nox --session="benchmarks(ci compare)"
+    # If running on the main branch (i.e. post-merge):
+    #  Run the full benchmark sequence then publish (``asv gh-pages``).
+    - |
+      if [ "$CIRRUS_BRANCH" == "$CIRRUS_DEFAULT_BRANCH" ]; then
+        nox --session="benchmarks(full then publish)"
+      fi

benchmarks/asv.conf.json

@@ -4,11 +4,12 @@
     "repo": "..",
     "environment_type": "nox-conda",
     "pythons": [],
+    "branches": ["main"],
     "benchmark_dir": "benchmarks",
     "env_dir": ".asv-env",
     "results_dir": ".asv-results",
     "html_dir": ".asv-html",
     "project_url": "https://github.com/SciTools-incubator/iris-esmf-regrid",
     "show_commit_url": "https://github.com/SciTools-incubator/iris-esmf-regrid/commit/",
-    "plugins": ["nox_asv_plugin"],
+    "plugins": [".nox_asv_plugin"],
 }

noxfile.py

@@ -8,7 +8,6 @@
 import os
 from pathlib import Path
 import shutil
-from typing import List
 from urllib.request import urlopen
 
 import nox
@@ -338,10 +337,12 @@ def tests(session: nox.sessions.Session):
 
 
 @nox.session
-# CI_MODE=TRUE IS CURRENTLY DISABLED:
-#  https://github.com/SciTools-incubator/iris-esmf-regrid/pull/76#issuecomment-850378861
-@nox.parametrize("ci_mode", [False])
-def benchmarks(session: nox.sessions.Session, ci_mode: bool):
+@nox.parametrize(
+    ["ci_mode", "gh_pages"],
+    [(True, False), (False, False), (False, True)],
+    ids=["ci compare", "full", "full then publish"],
+)
+def benchmarks(session: nox.sessions.Session, ci_mode: bool, gh_pages: bool):
     """
     Perform esmf-regrid performance benchmarks (using Airspeed Velocity).
 
@@ -352,6 +353,8 @@ def benchmarks(session: nox.sessions.Session, ci_mode: bool):
     ci_mode: bool
         Run a cut-down selection of benchmarks, comparing the current commit to
         the last commit for performance regressions.
+    gh_pages: bool
+        Run ``asv gh-pages --rewrite`` once finished.
 
     Notes
     -----
@@ -364,16 +367,25 @@ def benchmarks(session: nox.sessions.Session, ci_mode: bool):
     # Skip over setup questions for a new machine.
     session.run("asv", "machine", "--yes")
 
-    def asv_exec(sub_command: List[str]) -> None:
-        session.run(
-            "asv",
-            *sub_command,
-            f"--python={PY_VER[-1]}",
-        )
+    def asv_exec(*sub_args: str) -> None:
+        run_args = ["asv", *sub_args]
+        help_output = session.run(*run_args, "--help", silent=True)
+        if "--python" in help_output:
+            # Not all asv commands accept the --python kwarg.
+            run_args.append(f"--python={PY_VER[-1]}")
+        session.run(*run_args)
 
     if ci_mode:
-        asv_exec(["continuous", "HEAD^1", "HEAD", "--bench=ci"])
-        asv_exec(["compare", "HEAD^1", "HEAD"])
+        # If on a PR: compare to the base (target) branch.
+        #  Else: compare to previous commit.
+        previous_commit = os.environ.get("CIRRUS_BASE_SHA", "HEAD^1")
+        try:
+            asv_exec("continuous", previous_commit, "HEAD", "--bench=ci")
+        finally:
+            asv_exec("compare", previous_commit, "HEAD")
     else:
         # f32f23a5 = first supporting commit for nox_asv_plugin.py .
-        asv_exec(["run", "f32f23a5..HEAD"])
+        asv_exec("run", "f32f23a5..HEAD")
+
+    if gh_pages:
+        asv_exec("gh-pages", "--rewrite")

pyproject.toml

@@ -13,8 +13,8 @@ exclude = '''
     | \.nox
     | \.tox
     | \.venv
-    | \benchmarks\.asv*
     | _build
+    | benchmarks\/\.asv.*
     | buck-out
     | build
     | dist