Add canary for openjdk regression test (#47)

The canary will be run alongside the newest revision in regression test to identify unnoticed environment changes. Related PR: mmtk/mmtk-core#1209
mmtk · Oct 21, 2024 · d27c09b · d27c09b
1 parent b0b2c58
commit d27c09b
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 10 deletions.
diff --git a/configs/openjdk-plot.yml b/configs/openjdk-plot.yml
@@ -14,6 +14,11 @@ plans:
     baseline: ['jdk-g1', 'jdk-zgc']
   - plan: marksweep
     baseline: ['jdk-g1', 'jdk-zgc']
+  # "canary" is not a real plan.  It is for the canary build.
+  # We add it here to make history_report.py recognize its result
+  # and plot it in a separate block in parallel to other real plans in OpenJDK.
+  - plan: canary
+    baseline: ['jdk-g1', 'jdk-zgc']
 notes:
   - date: "20230116"
     time: "0000"

diff --git a/configs/running-openjdk-base.yml b/configs/running-openjdk-base.yml
@@ -97,3 +97,7 @@ runtimes:
     type: OpenJDK
     release: 11
     home: "./build/jdk-mmtk-branch/images/jdk"
+  jdk-mmtk-canary:
+    type: OpenJDK
+    release: 11
+    home: "./build/jdk-mmtk-canary/images/jdk"
diff --git a/configs/running-openjdk-canary-complete.yml b/configs/running-openjdk-canary-complete.yml
@@ -0,0 +1,6 @@
+includes:
+  - "./running-openjdk-base.yml"
+
+configs:
+  # Run GenImmix using the canary version.
+  - "jdk-mmtk-canary|genimmix|common_mmtk"
diff --git a/scripts/history_report.py b/scripts/history_report.py
@@ -26,7 +26,7 @@
 
 prefix = config['name']
 
-# all subfolders are plan names
+# all subfolders are plan names, or "canary" for the canary version
 plans = os.listdir(result_repo_vm_root)
 
 # check from date and to date

diff --git a/scripts/openjdk-history-run.sh b/scripts/openjdk-history-run.sh
@@ -3,31 +3,41 @@ set -ex
 # include common.sh
 . $(dirname "$0")/common.sh
 
-openjdk_binding=$(realpath $1)
-output_dir=$(realpath -m $2)
-openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD)
+openjdk_binding_latest=$(realpath $1)
+openjdk_binding_canary=$(realpath $2)
+output_dir=$(realpath -m $3)
+# openjdk_rev is used for the commit message.  We use the revision ID of the latest version.
+openjdk_rev=$(git -C $openjdk_binding_latest rev-parse HEAD)
 
 # OpenJDK root
-openjdk=$openjdk_binding/repos/openjdk
+openjdk=$openjdk_binding_latest/repos/openjdk
 
 ensure_empty_dir $kit_build
 ensure_empty_dir $kit_upload
 ensure_empty_dir $log_dir
 checkout_result_repo
 
 # Build
-build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk
+build_openjdk_with_mmtk $openjdk_binding_latest release jdk-mmtk
+build_openjdk_with_mmtk $openjdk_binding_canary release jdk-mmtk-canary
 
 run_exp() {
-    plan=$1
+    dir_name=$1
     config=$2
     heap_modifier=$3
+    invocations=$history_invocations
+
+    # Use this when testing the scripts so that tests run faster, albeit producing less accurate results.
+    if [ "$OPENJDK_HISTORY_RUN_TEST_FAST" = "1" ]; then
+        invocations=1
+    fi
 
     # Run
-    run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $history_invocations)
+    run_id=$(run_benchmarks $log_dir $kit_root/configs/$config $heap_modifier $invocations)
     # Save result
-    mkdir -p $result_repo_dir/openjdk/$plan
-    cp -r $log_dir/$run_id $result_repo_dir/openjdk/$plan
+    RESULT_DIR=$result_repo_dir/openjdk/$dir_name
+    mkdir -p $RESULT_DIR
+    cp -r $log_dir/$run_id $RESULT_DIR
 }
 
 # Build
@@ -54,6 +64,11 @@ run_exp stickyimmix running-openjdk-stickyimmix-complete.yml 6
 # MarkSweep
 run_exp marksweep running-openjdk-marksweep-complete.yml 6
 
+# GenImmix using the canary version.
+# If the performance of the canary version changed,
+# it means there is an environment change that impacts the performance.
+run_exp canary running-openjdk-canary-complete.yml 6
+
 # Commit result
 commit_result_repo 'OpenJDK Binding: '$openjdk_rev