Add notes on the history graph. Create a folder of builds for uploadi…

…ng (#38)

* Build OpenJDK with product-bundles.
* Keep two copies of builds: one for running benchmarks, one for uploading.
* Plot notes for a given date on the history graph.
* Clear log and build directories for each run.

configs/jikesrvm-plot.yml

@@ -6,3 +6,10 @@ plans:
     baseline: ['JavaMMTk_SemiSpace']
   - plan: marksweep
     baseline: []
+notes:
+  - date: "20230116"
+    note: "Move to Ubuntu 22.04"
+  - date: "20231015"
+    note: "Speculative RAS Overflow mitigation on Zen1/Zen2"
+  - date: "20231102"
+    note: "Move to running-ng"

configs/openjdk-plot.yml

@@ -14,3 +14,10 @@ plans:
     baseline: ['jdk-g1', 'jdk-zgc']
   - plan: marksweep
     baseline: ['jdk-g1', 'jdk-zgc']
+notes:
+  - date: "20230116"
+    note: "Move to Ubuntu 22.04"
+  - date: "20231015"
+    note: "Speculative RAS Overflow mitigation on Zen1/Zen2"
+  - date: "20231102"
+    note: "Move to running-ng. Use -Xcomp. Use image build."

scripts/common.sh

@@ -1,7 +1,9 @@
 # root dir of this perf kit
 kit_root=$(realpath $(dirname "$0")/..)
-# where we put all the builds
+# where we put all the builds. We run benchmarks from the build.
 kit_build=$kit_root/build
+# where we put all the builds for uploading. The builds here need to be small enough so we can upload them
+kit_upload=$kit_root/upload/
 # where we put all the scripts
 kit_script=$kit_root/scripts
 # where we put all the configs
@@ -41,8 +43,10 @@ build_jikesrvm_with_mmtk() {
     # build
     ./bin/buildit localhost $build_config -quick --answer-yes --use-third-party-heap=../.. --use-third-party-build-configs=../../jikesrvm/build/configs --use-external-source=../../jikesrvm/rvm/src --m32
 
-    # copy to build_path
-    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $build_path/
+    # Copy to build_path
+    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_build/$build_path
+    # Directly copy to upload. JikesRVM builds are small enough that we can directly upload.
+    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_upload/$build_path
 }
 
 # build_jikesrvm 'jikesrvm_path' 'plan' 'build_path'
@@ -60,7 +64,9 @@ build_jikesrvm() {
     bin/buildit localhost $build_config -quick --answer-yes --m32
 
     # copy to build_path
-    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $build_path/
+    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_build/$build_path
+    # Directly copy to upload. JikesRVM builds are small enough that we can directly upload.
+    cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_upload/$build_path
 }
 
 # openjdk_binding_use_local_mmtk 'binding_path'
@@ -90,10 +96,13 @@ build_openjdk_with_mmtk() {
     cd $openjdk_path
     export DEBUG_LEVEL=$debug_level
     sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL
-    make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk
+    make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk
 
     # copy to build_path
-    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path
+    # Copy bundles to upload
+    mkdir -p $kit_upload/$build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path
 }
 
 # build_openjdk ’binding_path' 'plan' 'debug_level' 'build_path'
@@ -109,10 +118,13 @@ build_openjdk_with_mmtk_plan() {
     export DEBUG_LEVEL=$debug_level
     export MMTK_PLAN=$plan
     sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL
-    make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk
+    make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk
 
     # copy to build_path
-    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path
+    # Copy bundles to upload
+    mkdir -p $kit_upload/$build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path
 }
 
 # build_openjdk 'openjdk_path' 'debug_level' 'build_path'
@@ -124,10 +136,13 @@ build_openjdk() {
     cd $openjdk_path
     export DEBUG_LEVEL=$debug_level
     sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL --with-jvm-features=zgc
-    make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL
+    make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL
 
     # copy to build_path
-    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path
+    # Copy bundles to upload
+    mkdir -p $kit_upload/$build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path
 }
 
 # build_openjdk_with_features 'openjdk_path' 'debug_level' 'build_path' 'features'
@@ -140,10 +155,13 @@ build_openjdk_with_features() {
     cd $openjdk_path
     export DEBUG_LEVEL=$debug_level
     sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL --with-jvm-features=$features
-    make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL
+    make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL
 
     # copy to build_path
-    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path
+    # Copy bundles to upload
+    mkdir -p $kit_upload/$build_path
+    cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path
 }
 
 # run_benchmarks 'log_dir' 'config' 'heap_modifier' 'invocations'

scripts/history_report.py

@@ -85,6 +85,6 @@
     pp.pprint(baseline)
 
     # plot
-    fig = plot.plot_history(runs, plan, benchmarks, from_date, to_date, "execution_times", baseline)
+    fig = plot.plot_history(runs, plan, benchmarks, from_date, to_date, "execution_times", baseline, config['notes'])
     path = os.path.join(output_dir, "%s_%s_history.html" % (prefix, plan))
     fig.write_html(path)

scripts/jikesrvm-compare.sh

@@ -26,6 +26,8 @@ fi
 
 # Build - JikesRVM buildit script requires current dir to be JikesRVM root dir
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 
 # Trunk
 rm -rf $jikesrvm_binding_trunk/repos/mmtk-core
@@ -59,10 +61,10 @@ run_exp() {
     heap_modifier=$4
 
     cd $jikesrvm_trunk
-    build_jikesrvm_with_mmtk $jikesrvm_binding_trunk $build_config $kit_build/$plan"_Trunk_x86_64_m32-linux"
+    build_jikesrvm_with_mmtk $jikesrvm_binding_trunk $build_config $plan"_Trunk_x86_64_m32-linux"
 
     cd $jikesrvm_branch
-    build_jikesrvm_with_mmtk $jikesrvm_binding_branch $build_config $kit_build/$plan"_Branch_x86_64_m32-linux"
+    build_jikesrvm_with_mmtk $jikesrvm_binding_branch $build_config $plan"_Branch_x86_64_m32-linux"
 
     run_id=$(run_benchmarks $log_dir $run_config $heap_modifier $compare_invocations)
     python $kit_root/scripts/compare_report.py $log_dir/$run_id $plan $plan"_Trunk" $plan"_Branch" $compare_invocations >> $output_file

scripts/jikesrvm-history-run.sh

@@ -11,6 +11,8 @@ jikesrvm_rev=$(git -C $jikesrvm_binding rev-parse HEAD)
 jikesrvm=$jikesrvm_binding/repos/jikesrvm
 
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 checkout_result_repo
 
 run_exp() {
@@ -23,7 +25,7 @@ run_exp() {
 
     # Build - JikesRVM buildit script requires current dir to be JikesRVM root dir
     cd $jikesrvm
-    build_jikesrvm_with_mmtk $jikesrvm_binding $build_config $kit_build/$plan"_x86_64_m32-linux"
+    build_jikesrvm_with_mmtk $jikesrvm_binding $build_config $plan"_x86_64_m32-linux"
     # Run
     run_id=$(run_benchmarks $log_dir $run_config $heap_modifier $history_invocations)
     # Save result

scripts/jikesrvm-stock.sh

@@ -7,13 +7,15 @@ jikesrvm_path=$(realpath $1)
 
 # Build
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 checkout_result_repo
 
 cd $jikesrvm_path
 
-build_jikesrvm $jikesrvm_path FastAdaptiveNoGC $kit_build/JavaMMTk_NoGC_x86_64_m32-linux
-build_jikesrvm $jikesrvm_path FastAdaptiveSemiSpace $kit_build/JavaMMTk_SemiSpace_x86_64_m32-linux
-build_jikesrvm $jikesrvm_path FastAdaptiveMarkSweep $kit_build/JavaMMTk_MarkSweep_x86_64_m32-linux
+build_jikesrvm $jikesrvm_path FastAdaptiveNoGC JavaMMTk_NoGC_x86_64_m32-linux
+build_jikesrvm $jikesrvm_path FastAdaptiveSemiSpace JavaMMTk_SemiSpace_x86_64_m32-linux
+build_jikesrvm $jikesrvm_path FastAdaptiveMarkSweep JavaMMTk_MarkSweep_x86_64_m32-linux
 
 # Run
 run1_id=$(run_benchmarks $log_dir $kit_root/configs/running-jikesrvm-stock-nogc.yml 0 $stock_invocations)

scripts/mutator-history-run.sh

@@ -11,6 +11,8 @@ openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD)
 openjdk=$openjdk_binding/repos/openjdk
 
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 checkout_result_repo
 
 # Build probes
@@ -24,14 +26,14 @@ make
 cd $openjdk
 
 # Normal MMTk build
-build_openjdk_with_mmtk $openjdk_binding release $kit_build/jdk-mmtk
+build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk
 
 # Special MMTk builds
-build_openjdk_with_mmtk_plan $openjdk_binding nogc_lock_free release $kit_build/jdk-mmtk-lock-free-nogc
-build_openjdk_with_mmtk_plan $openjdk_binding nogc_no_zeroing release $kit_build/jdk-mmtk-no-zeroing-nogc
+build_openjdk_with_mmtk_plan $openjdk_binding nogc_lock_free release jdk-mmtk-lock-free-nogc
+build_openjdk_with_mmtk_plan $openjdk_binding nogc_no_zeroing release jdk-mmtk-no-zeroing-nogc
 
 # Stock build
-build_openjdk $openjdk release $kit_build/jdk-stock
+build_openjdk $openjdk release jdk-stock
 
 # Run
 mu_run_id=$(run_benchmarks $log_dir $kit_root/configs/running-openjdk-mutator.yml 0 $history_invocations)

scripts/openjdk-compare.sh

@@ -26,16 +26,18 @@ fi
 
 # Build
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 
 # Build for trunk
 rm -rf $openjdk_binding_trunk/repos/mmtk-core
 ln -sfn $mmtk_core_trunk $openjdk_binding_trunk/repos/mmtk-core
-build_openjdk_with_mmtk $openjdk_binding_trunk release $kit_build/jdk-mmtk-trunk
+build_openjdk_with_mmtk $openjdk_binding_trunk release jdk-mmtk-trunk
 
 # Build for branch
 rm -rf $openjdk_binding_branch/repos/mmtk-core
 ln -sfn $mmtk_core_branch $openjdk_binding_branch/repos/mmtk-core
-build_openjdk_with_mmtk $openjdk_binding_branch release $kit_build/jdk-mmtk-branch
+build_openjdk_with_mmtk $openjdk_binding_branch release jdk-mmtk-branch
 
 # Run
 cd $kit_root

scripts/openjdk-history-run.sh

@@ -11,10 +11,12 @@ openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD)
 openjdk=$openjdk_binding/repos/openjdk
 
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 checkout_result_repo
 
 # Build
-build_openjdk_with_mmtk $openjdk_binding release $kit_build/jdk-mmtk
+build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk
 
 run_exp() {
     plan=$1

scripts/openjdk-micro-bm.sh

@@ -1,3 +1,5 @@
+# This is not used, and is not working.
+
 set -ex
 
 # include common.sh

scripts/openjdk-stock.sh

@@ -7,9 +7,11 @@ openjdk_path=$(realpath $1)
 
 # Build
 ensure_empty_dir $kit_build
+ensure_empty_dir $kit_upload
+ensure_empty_dir $log_dir
 checkout_result_repo
 
-build_openjdk_with_features $openjdk_path release $kit_build/jdk-stock zgc
+build_openjdk_with_features $openjdk_path release jdk-stock zgc
 
 ln -s $kit_build/jdk-stock $kit_build/jdk-epsilon
 ln -s $kit_build/jdk-stock $kit_build/jdk-g1

scripts/parse.py

@@ -89,6 +89,13 @@ def parse_run_date(run_id):
     if matcher:
         return datetime(int(matcher['year']), int(matcher['month']), int(matcher['day']), int(matcher['hour']), int(matcher['minute']), int(matcher['second']))
 
+# Given a note date, return the date object
+def parse_note_date(note_date):
+    from datetime import datetime
+    matcher = re.match("(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})", note_date)
+    if matcher:
+        return datetime(int(matcher['year']), int(matcher['month']), int(matcher['day']))
+
 # Given a yaml file path, return the file
 def parse_yaml(path):
     import yaml

scripts/plot.py

@@ -27,7 +27,8 @@
 # start_date, end_date: plot data between the given date range
 # data_key: the data to render
 # baseline: the baseline to plot as a dict {baseline: {benchmark: avg}}. None means no baseline, or no data for a certain benchmark.
-def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baseline):
+# notes: a list of [date, note]. date is YYYYMMDD
+def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baseline, notes=[]):
     layout = {
         "title": plan,
         # "margin": {"t": 80},
@@ -50,6 +51,9 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin
     y_range_upper = - float("inf")
     y_range_lower = float("inf")
 
+    benchmarks.sort()
+
+    aligned_notes = []
     for bm in benchmarks:
         # extract results
         print("Plotting %s %s..." % (plan, bm))
@@ -62,6 +66,25 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin
         # We have to sort by date, run_id includes the machine name, we cannot sort by alphabet
         x_labels.sort(key = lambda x: parse.parse_run_date(x))
 
+        # Only add notes for the first plot
+        if row == 1:
+            import datetime
+
+            # Sort notes
+            notes.sort(key = lambda x: parse.parse_note_date(x['date']))
+
+            # Align notes to logs/run_ids. Each note has a date, find the next log on or after the date.
+            def peek_next_note_date():
+                return parse.parse_note_date(notes[0]['date']) if len(notes) > 0 else datetime.datetime(9999, 1, 1) # end of the world. We will never find a log after this date.
+            next_note_date = peek_next_note_date()
+
+            for idx, run_id in enumerate(x_labels):
+                log_date = parse.parse_run_date(run_id)
+                if log_date >= next_note_date:
+                    note = notes.pop(0)
+                    aligned_notes.append({ 'run_id': run_id, 'x': x[idx], 'note': f"{note['date']}: {note['note']}" })
+                    next_note_date = peek_next_note_date()
+
         y_cur_aboslute = y[-1]
 
         # From now, all y's are normalized to this baseline
@@ -87,7 +110,6 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin
             if baseline_perf == 0:
                 baseline_perf = 1
             nonzero_y = [baseline_perf]
-            print(nonzero_y)
 
         y_baseline = min(nonzero_y)
         y_max = max(nonzero_y) / y_baseline
@@ -351,6 +373,23 @@ def keep_last(arr, f):
                     #     "text": "%s: %.2f" % (build, hline),
                     # }})
 
+        # Notes
+        # Somehow this line does not show. But it adds a hover text for all the plots.
+        for note in aligned_notes:
+            note_trace = {
+                "hoverinfo": "text",
+                "mode": "lines",
+                "line": {"width": 10, "color": "blue"},
+                "x": [note['x']],
+                "y": [0, 999],
+                "xaxis": x_axis,
+                "yaxis": y_axis,
+                "showlegend": False,
+                "opacity": 0,
+                "text": note['note']
+            }
+            traces.append(note_trace)
+
         row += 1
 
     # fix range for all the traces
@@ -364,6 +403,9 @@ def keep_last(arr, f):
         fig.add_annotation(anno)
     for line in baseline_hlines:
         fig.add_shape(line)
+    # This plots a vertical line for each note in the first subgraph.
+    for note in aligned_notes:
+        fig.add_vline(x = int(note['x']), line_color = 'blue', annotation = { "text": "📓", "hovertext": note['note'] })
 
     fig.update_layout(hovermode='x')