diff --git a/configs/jikesrvm-plot.yml b/configs/jikesrvm-plot.yml index 4c6a504..66bdc65 100644 --- a/configs/jikesrvm-plot.yml +++ b/configs/jikesrvm-plot.yml @@ -6,3 +6,10 @@ plans: baseline: ['JavaMMTk_SemiSpace'] - plan: marksweep baseline: [] +notes: + - date: "20230116" + note: "Move to Ubuntu 22.04" + - date: "20231015" + note: "Speculative RAS Overflow mitigation on Zen1/Zen2" + - date: "20231102" + note: "Move to running-ng" diff --git a/configs/openjdk-plot.yml b/configs/openjdk-plot.yml index 6382810..ceae4b9 100644 --- a/configs/openjdk-plot.yml +++ b/configs/openjdk-plot.yml @@ -14,3 +14,10 @@ plans: baseline: ['jdk-g1', 'jdk-zgc'] - plan: marksweep baseline: ['jdk-g1', 'jdk-zgc'] +notes: + - date: "20230116" + note: "Move to Ubuntu 22.04" + - date: "20231015" + note: "Speculative RAS Overflow mitigation on Zen1/Zen2" + - date: "20231102" + note: "Move to running-ng. Use -Xcomp. Use image build." diff --git a/scripts/common.sh b/scripts/common.sh index 17b4f7a..3cfbca5 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,7 +1,9 @@ # root dir of this perf kit kit_root=$(realpath $(dirname "$0")/..) -# where we put all the builds +# where we put all the builds. We run benchmarks from the build. kit_build=$kit_root/build +# where we put all the builds for uploading. The builds here need to be small enough so we can upload them +kit_upload=$kit_root/upload/ # where we put all the scripts kit_script=$kit_root/scripts # where we put all the configs @@ -41,8 +43,10 @@ build_jikesrvm_with_mmtk() { # build ./bin/buildit localhost $build_config -quick --answer-yes --use-third-party-heap=../.. --use-third-party-build-configs=../../jikesrvm/build/configs --use-external-source=../../jikesrvm/rvm/src --m32 - # copy to build_path - cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $build_path/ + # Copy to build_path + cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_build/$build_path + # Directly copy to upload. JikesRVM builds are small enough that we can directly upload. + cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_upload/$build_path } # build_jikesrvm 'jikesrvm_path' 'plan' 'build_path' @@ -60,7 +64,9 @@ build_jikesrvm() { bin/buildit localhost $build_config -quick --answer-yes --m32 # copy to build_path - cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $build_path/ + cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_build/$build_path + # Directly copy to upload. JikesRVM builds are small enough that we can directly upload. + cp -r $jikesrvm_path'/dist/'$build_config'_x86_64_m32-linux' $kit_upload/$build_path } # openjdk_binding_use_local_mmtk 'binding_path' @@ -90,10 +96,13 @@ build_openjdk_with_mmtk() { cd $openjdk_path export DEBUG_LEVEL=$debug_level sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL - make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk + make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk # copy to build_path - cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path + # Copy bundles to upload + mkdir -p $kit_upload/$build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path } # build_openjdk ’binding_path' 'plan' 'debug_level' 'build_path' @@ -109,10 +118,13 @@ build_openjdk_with_mmtk_plan() { export DEBUG_LEVEL=$debug_level export MMTK_PLAN=$plan sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL - make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk + make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$PWD/../../openjdk # copy to build_path - cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path + # Copy bundles to upload + mkdir -p $kit_upload/$build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path } # build_openjdk 'openjdk_path' 'debug_level' 'build_path' @@ -124,10 +136,13 @@ build_openjdk() { cd $openjdk_path export DEBUG_LEVEL=$debug_level sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL --with-jvm-features=zgc - make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL + make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL # copy to build_path - cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path + # Copy bundles to upload + mkdir -p $kit_upload/$build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path } # build_openjdk_with_features 'openjdk_path' 'debug_level' 'build_path' 'features' @@ -140,10 +155,13 @@ build_openjdk_with_features() { cd $openjdk_path export DEBUG_LEVEL=$debug_level sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL --with-jvm-features=$features - make images CONF=linux-x86_64-normal-server-$DEBUG_LEVEL + make product-bundles CONF=linux-x86_64-normal-server-$DEBUG_LEVEL # copy to build_path - cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL $kit_build/$build_path + # Copy bundles to upload + mkdir -p $kit_upload/$build_path + cp -r $openjdk_path/build/linux-x86_64-normal-server-$DEBUG_LEVEL/bundles/*_bin.tar.gz $kit_upload/$build_path } # run_benchmarks 'log_dir' 'config' 'heap_modifier' 'invocations' diff --git a/scripts/history_report.py b/scripts/history_report.py index 002730c..7cd2e80 100644 --- a/scripts/history_report.py +++ b/scripts/history_report.py @@ -85,6 +85,6 @@ pp.pprint(baseline) # plot - fig = plot.plot_history(runs, plan, benchmarks, from_date, to_date, "execution_times", baseline) + fig = plot.plot_history(runs, plan, benchmarks, from_date, to_date, "execution_times", baseline, config['notes']) path = os.path.join(output_dir, "%s_%s_history.html" % (prefix, plan)) fig.write_html(path) diff --git a/scripts/jikesrvm-compare.sh b/scripts/jikesrvm-compare.sh index 64e3cd4..f132c23 100755 --- a/scripts/jikesrvm-compare.sh +++ b/scripts/jikesrvm-compare.sh @@ -26,6 +26,8 @@ fi # Build - JikesRVM buildit script requires current dir to be JikesRVM root dir ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir # Trunk rm -rf $jikesrvm_binding_trunk/repos/mmtk-core @@ -59,10 +61,10 @@ run_exp() { heap_modifier=$4 cd $jikesrvm_trunk - build_jikesrvm_with_mmtk $jikesrvm_binding_trunk $build_config $kit_build/$plan"_Trunk_x86_64_m32-linux" + build_jikesrvm_with_mmtk $jikesrvm_binding_trunk $build_config $plan"_Trunk_x86_64_m32-linux" cd $jikesrvm_branch - build_jikesrvm_with_mmtk $jikesrvm_binding_branch $build_config $kit_build/$plan"_Branch_x86_64_m32-linux" + build_jikesrvm_with_mmtk $jikesrvm_binding_branch $build_config $plan"_Branch_x86_64_m32-linux" run_id=$(run_benchmarks $log_dir $run_config $heap_modifier $compare_invocations) python $kit_root/scripts/compare_report.py $log_dir/$run_id $plan $plan"_Trunk" $plan"_Branch" $compare_invocations >> $output_file diff --git a/scripts/jikesrvm-history-run.sh b/scripts/jikesrvm-history-run.sh index 5a717a7..9922899 100755 --- a/scripts/jikesrvm-history-run.sh +++ b/scripts/jikesrvm-history-run.sh @@ -11,6 +11,8 @@ jikesrvm_rev=$(git -C $jikesrvm_binding rev-parse HEAD) jikesrvm=$jikesrvm_binding/repos/jikesrvm ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir checkout_result_repo run_exp() { @@ -23,7 +25,7 @@ run_exp() { # Build - JikesRVM buildit script requires current dir to be JikesRVM root dir cd $jikesrvm - build_jikesrvm_with_mmtk $jikesrvm_binding $build_config $kit_build/$plan"_x86_64_m32-linux" + build_jikesrvm_with_mmtk $jikesrvm_binding $build_config $plan"_x86_64_m32-linux" # Run run_id=$(run_benchmarks $log_dir $run_config $heap_modifier $history_invocations) # Save result diff --git a/scripts/jikesrvm-stock.sh b/scripts/jikesrvm-stock.sh index 72bcef1..73b05bc 100755 --- a/scripts/jikesrvm-stock.sh +++ b/scripts/jikesrvm-stock.sh @@ -7,13 +7,15 @@ jikesrvm_path=$(realpath $1) # Build ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir checkout_result_repo cd $jikesrvm_path -build_jikesrvm $jikesrvm_path FastAdaptiveNoGC $kit_build/JavaMMTk_NoGC_x86_64_m32-linux -build_jikesrvm $jikesrvm_path FastAdaptiveSemiSpace $kit_build/JavaMMTk_SemiSpace_x86_64_m32-linux -build_jikesrvm $jikesrvm_path FastAdaptiveMarkSweep $kit_build/JavaMMTk_MarkSweep_x86_64_m32-linux +build_jikesrvm $jikesrvm_path FastAdaptiveNoGC JavaMMTk_NoGC_x86_64_m32-linux +build_jikesrvm $jikesrvm_path FastAdaptiveSemiSpace JavaMMTk_SemiSpace_x86_64_m32-linux +build_jikesrvm $jikesrvm_path FastAdaptiveMarkSweep JavaMMTk_MarkSweep_x86_64_m32-linux # Run run1_id=$(run_benchmarks $log_dir $kit_root/configs/running-jikesrvm-stock-nogc.yml 0 $stock_invocations) diff --git a/scripts/mutator-history-run.sh b/scripts/mutator-history-run.sh index 4dd948b..8cd7a50 100755 --- a/scripts/mutator-history-run.sh +++ b/scripts/mutator-history-run.sh @@ -11,6 +11,8 @@ openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD) openjdk=$openjdk_binding/repos/openjdk ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir checkout_result_repo # Build probes @@ -24,14 +26,14 @@ make cd $openjdk # Normal MMTk build -build_openjdk_with_mmtk $openjdk_binding release $kit_build/jdk-mmtk +build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk # Special MMTk builds -build_openjdk_with_mmtk_plan $openjdk_binding nogc_lock_free release $kit_build/jdk-mmtk-lock-free-nogc -build_openjdk_with_mmtk_plan $openjdk_binding nogc_no_zeroing release $kit_build/jdk-mmtk-no-zeroing-nogc +build_openjdk_with_mmtk_plan $openjdk_binding nogc_lock_free release jdk-mmtk-lock-free-nogc +build_openjdk_with_mmtk_plan $openjdk_binding nogc_no_zeroing release jdk-mmtk-no-zeroing-nogc # Stock build -build_openjdk $openjdk release $kit_build/jdk-stock +build_openjdk $openjdk release jdk-stock # Run mu_run_id=$(run_benchmarks $log_dir $kit_root/configs/running-openjdk-mutator.yml 0 $history_invocations) diff --git a/scripts/openjdk-compare.sh b/scripts/openjdk-compare.sh index 9f771a8..4a0dc08 100755 --- a/scripts/openjdk-compare.sh +++ b/scripts/openjdk-compare.sh @@ -26,16 +26,18 @@ fi # Build ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir # Build for trunk rm -rf $openjdk_binding_trunk/repos/mmtk-core ln -sfn $mmtk_core_trunk $openjdk_binding_trunk/repos/mmtk-core -build_openjdk_with_mmtk $openjdk_binding_trunk release $kit_build/jdk-mmtk-trunk +build_openjdk_with_mmtk $openjdk_binding_trunk release jdk-mmtk-trunk # Build for branch rm -rf $openjdk_binding_branch/repos/mmtk-core ln -sfn $mmtk_core_branch $openjdk_binding_branch/repos/mmtk-core -build_openjdk_with_mmtk $openjdk_binding_branch release $kit_build/jdk-mmtk-branch +build_openjdk_with_mmtk $openjdk_binding_branch release jdk-mmtk-branch # Run cd $kit_root diff --git a/scripts/openjdk-history-run.sh b/scripts/openjdk-history-run.sh index fa66985..1f1e7c3 100755 --- a/scripts/openjdk-history-run.sh +++ b/scripts/openjdk-history-run.sh @@ -11,10 +11,12 @@ openjdk_rev=$(git -C $openjdk_binding rev-parse HEAD) openjdk=$openjdk_binding/repos/openjdk ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir checkout_result_repo # Build -build_openjdk_with_mmtk $openjdk_binding release $kit_build/jdk-mmtk +build_openjdk_with_mmtk $openjdk_binding release jdk-mmtk run_exp() { plan=$1 diff --git a/scripts/openjdk-micro-bm.sh b/scripts/openjdk-micro-bm.sh index c899226..d651f4d 100755 --- a/scripts/openjdk-micro-bm.sh +++ b/scripts/openjdk-micro-bm.sh @@ -1,3 +1,5 @@ +# This is not used, and is not working. + set -ex # include common.sh diff --git a/scripts/openjdk-stock.sh b/scripts/openjdk-stock.sh index 47f357e..f64fb31 100755 --- a/scripts/openjdk-stock.sh +++ b/scripts/openjdk-stock.sh @@ -7,9 +7,11 @@ openjdk_path=$(realpath $1) # Build ensure_empty_dir $kit_build +ensure_empty_dir $kit_upload +ensure_empty_dir $log_dir checkout_result_repo -build_openjdk_with_features $openjdk_path release $kit_build/jdk-stock zgc +build_openjdk_with_features $openjdk_path release jdk-stock zgc ln -s $kit_build/jdk-stock $kit_build/jdk-epsilon ln -s $kit_build/jdk-stock $kit_build/jdk-g1 diff --git a/scripts/parse.py b/scripts/parse.py index acfdce8..aa8b5f4 100644 --- a/scripts/parse.py +++ b/scripts/parse.py @@ -89,6 +89,13 @@ def parse_run_date(run_id): if matcher: return datetime(int(matcher['year']), int(matcher['month']), int(matcher['day']), int(matcher['hour']), int(matcher['minute']), int(matcher['second'])) +# Given a note date, return the date object +def parse_note_date(note_date): + from datetime import datetime + matcher = re.match("(?P\d{4})(?P\d{2})(?P\d{2})", note_date) + if matcher: + return datetime(int(matcher['year']), int(matcher['month']), int(matcher['day'])) + # Given a yaml file path, return the file def parse_yaml(path): import yaml diff --git a/scripts/plot.py b/scripts/plot.py index ed62805..7c03e05 100644 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -27,7 +27,8 @@ # start_date, end_date: plot data between the given date range # data_key: the data to render # baseline: the baseline to plot as a dict {baseline: {benchmark: avg}}. None means no baseline, or no data for a certain benchmark. -def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baseline): +# notes: a list of [date, note]. date is YYYYMMDD +def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baseline, notes=[]): layout = { "title": plan, # "margin": {"t": 80}, @@ -50,6 +51,9 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin y_range_upper = - float("inf") y_range_lower = float("inf") + benchmarks.sort() + + aligned_notes = [] for bm in benchmarks: # extract results print("Plotting %s %s..." % (plan, bm)) @@ -62,6 +66,25 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin # We have to sort by date, run_id includes the machine name, we cannot sort by alphabet x_labels.sort(key = lambda x: parse.parse_run_date(x)) + # Only add notes for the first plot + if row == 1: + import datetime + + # Sort notes + notes.sort(key = lambda x: parse.parse_note_date(x['date'])) + + # Align notes to logs/run_ids. Each note has a date, find the next log on or after the date. + def peek_next_note_date(): + return parse.parse_note_date(notes[0]['date']) if len(notes) > 0 else datetime.datetime(9999, 1, 1) # end of the world. We will never find a log after this date. + next_note_date = peek_next_note_date() + + for idx, run_id in enumerate(x_labels): + log_date = parse.parse_run_date(run_id) + if log_date >= next_note_date: + note = notes.pop(0) + aligned_notes.append({ 'run_id': run_id, 'x': x[idx], 'note': f"{note['date']}: {note['note']}" }) + next_note_date = peek_next_note_date() + y_cur_aboslute = y[-1] # From now, all y's are normalized to this baseline @@ -87,7 +110,6 @@ def plot_history(runs, plan, benchmarks, start_date, end_date, data_key, baselin if baseline_perf == 0: baseline_perf = 1 nonzero_y = [baseline_perf] - print(nonzero_y) y_baseline = min(nonzero_y) y_max = max(nonzero_y) / y_baseline @@ -351,6 +373,23 @@ def keep_last(arr, f): # "text": "%s: %.2f" % (build, hline), # }}) + # Notes + # Somehow this line does not show. But it adds a hover text for all the plots. + for note in aligned_notes: + note_trace = { + "hoverinfo": "text", + "mode": "lines", + "line": {"width": 10, "color": "blue"}, + "x": [note['x']], + "y": [0, 999], + "xaxis": x_axis, + "yaxis": y_axis, + "showlegend": False, + "opacity": 0, + "text": note['note'] + } + traces.append(note_trace) + row += 1 # fix range for all the traces @@ -364,6 +403,9 @@ def keep_last(arr, f): fig.add_annotation(anno) for line in baseline_hlines: fig.add_shape(line) + # This plots a vertical line for each note in the first subgraph. + for note in aligned_notes: + fig.add_vline(x = int(note['x']), line_color = 'blue', annotation = { "text": "đŸ““", "hovertext": note['note'] }) fig.update_layout(hovermode='x')