From a5b8693c81b12d39fb590c7d3bf390f64c2e321f Mon Sep 17 00:00:00 2001 From: Stan Brubaker <120737309+stanbrub@users.noreply.github.com> Date: Thu, 19 Oct 2023 21:11:00 -0600 Subject: [PATCH] Some improvements to slack publish (#191) --- .../BenchmarkTables.md} | 0 docs/demo/NightlyDashboard.md | 22 ++++++++ .../benchmark/run/PublishNotification.java | 52 ++++++++++--------- .../run/profile/benchmark_tables.dh.py | 18 +++++-- .../benchmark/run/profile/queries/publish.py | 24 +++++---- 5 files changed, 77 insertions(+), 39 deletions(-) rename docs/{BenchmarkDemo.md => demo/BenchmarkTables.md} (100%) create mode 100644 docs/demo/NightlyDashboard.md diff --git a/docs/BenchmarkDemo.md b/docs/demo/BenchmarkTables.md similarity index 100% rename from docs/BenchmarkDemo.md rename to docs/demo/BenchmarkTables.md diff --git a/docs/demo/NightlyDashboard.md b/docs/demo/NightlyDashboard.md new file mode 100644 index 00000000..87ce7548 --- /dev/null +++ b/docs/demo/NightlyDashboard.md @@ -0,0 +1,22 @@ +# Nightly Dashboard + +Deephaven benchmarks run every night and summary tables (not show here) are published internally so that +developers can see how performance changed during the last nightly build. If more investigation is +needed, the logical next step is to search the benchmark data to get more details. + +The Nightly Dashboard allows a quick way to dig into the available data to get some clues to +performance issues or see more clearer why performance has improved. + +This Dashboard is built using Deephaven's [scripted UI](https://deephaven.io/core/docs/how-to-guides/plotting/category/) +using the Benchmark Tables snippet used in the previous notebook. + +```python +from urllib.request import urlopen; import os + +root = 'file:///nfs' if os.path.exists('/nfs/deephaven-benchmark') else 'https://storage.googleapis.com' +with urlopen(root + '/deephaven-benchmark/benchmark_tables.dh.py') as r: + benchmark_storage_uri_arg = root + '/deephaven-benchmark' + benchmark_category_arg = 'release' # release | nightly + benchmark_max_runs_arg = 5 # Latest X runs to include + exec(r.read().decode(), globals(), locals()) +``` diff --git a/src/main/java/io/deephaven/benchmark/run/PublishNotification.java b/src/main/java/io/deephaven/benchmark/run/PublishNotification.java index da5c74ed..8423be79 100644 --- a/src/main/java/io/deephaven/benchmark/run/PublishNotification.java +++ b/src/main/java/io/deephaven/benchmark/run/PublishNotification.java @@ -46,44 +46,46 @@ public void publish() { var query = Filer.getURLText(queryFile); var svgTemp = new String[] {Filer.getURLText(svgTemplate)}; Bench api = Bench.create("# Publish Notification"); - api.setName("# Publish"); - slackChannel = api.property("slack.channel", ""); - slackToken = api.property("slack.token", ""); - if (slackChannel.isBlank() || slackToken.isBlank()) { - api.close(); - System.out.println("-- Slack properties is not defined, skipping query notification --"); - return; - } - System.out.println("-- Running notification queries --"); - var aquery = api.query(query); - aquery.fetchAfter("platform_details", table -> { - svgTemp[0] = updatePlatformDetails(table, svgTemp[0]); - }); - for (String tableName : tables) { - aquery.fetchAfter(tableName + "_small", table -> { - generateCsv(table, outputDir, tableName + ".csv"); - }); - aquery.fetchAfter(tableName + "_large", table -> { - generateSvg(table, svgTemp[0], outputDir, tableName + ".svg"); + try { + api.setName("# Publish"); + slackChannel = api.property("slack.channel", ""); + slackToken = api.property("slack.token", ""); + if (slackChannel.isBlank() || slackToken.isBlank()) { + System.out.println("-- Slack properties are not defined. Skipping query notification --"); + return; + } + System.out.println("-- Running notification queries --"); + var aquery = api.query(query); + aquery.fetchAfter("platform_details", table -> { + svgTemp[0] = updatePlatformDetails(table, svgTemp[0]); }); + for (String tableName : tables) { + aquery.fetchAfter(tableName + "_small", table -> { + generateCsv(table, outputDir, tableName + ".csv"); + }); + aquery.fetchAfter(tableName + "_large", table -> { + generateSvg(table, svgTemp[0], outputDir, tableName + ".svg"); + }); + } + aquery.execute(); + } finally { + api.close(); } - aquery.execute(); - api.close(); publishToSlack(outputDir); } void publishToSlack(Path outDir) { var message = "Nightly Benchmark Changes " + - "\n"; - // "\n"; + "\n"; + // "\n"; + for (String table : tables) { message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```"; } var payload = """ - {"channel": "${channel}", "icon_emoji": ":horse_racing:", "unfurl_links": "false", - "unfurl_media": "false", "text": "${msg}"} + {"channel": "${channel}", "unfurl_links": "false", "unfurl_media": "false", "text": "${msg}"} """; payload = payload.replace("${channel}", slackChannel); payload = payload.replace("${msg}", message); diff --git a/src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py b/src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py index a1e85509..f724c005 100644 --- a/src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py +++ b/src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py @@ -104,6 +104,14 @@ def add_platform_value(pname, new_pname): import statistics def rstd(rates): return statistics.pstdev(rates) * 100.0 / statistics.mean(rates) + +def zscore(rate, rates): + return (rate - statistics.mean(rates)) / statistics.pstdev(rates) + +def zprob(zscore): + lower = -abs(zscore) + upper = abs(zscore) + return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower)) from array import array def rchange(rates) -> float: @@ -112,12 +120,16 @@ def rchange(rates) -> float: m = statistics.mean(rates[:-1]) return (rates[-1] - m) / m * 100.0 -def format_rates(rates): - return ' '.join("{:,}".format(r) for r in rates) - def gain(start:float, end:float) -> float: return (end - start) / start * 100.0 +def format_rates(rates): + return ' '.join("{:,}".format(r) for r in rates) + +def truncate(text, size): + if len(text) < size - 3: return text + return text[:size-3] + '...' + from deephaven.updateby import rolling_group_tick op_group = rolling_group_tick(cols=["op_group_rates = op_rate"], rev_ticks=history_runs, fwd_ticks=0) op_version = rolling_group_tick(cols=["op_group_versions = deephaven_version"], rev_ticks=history_runs, fwd_ticks=0) diff --git a/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py b/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py index df09a89d..7c9eadcf 100644 --- a/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py +++ b/src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py @@ -12,17 +12,15 @@ benchmark_max_runs_arg = 45 # Latest X runs to include exec(r.read().decode(), globals(), locals()) -import statistics -def zscore(rate, rates): - return (rate - statistics.mean(rates)) / statistics.pstdev(rates) - -def zprob(zscore): - lower = -abs(zscore) - upper = abs(zscore) - return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower)) - +# Used to provide platform (e.g. hardware, jvm version) for SVG footer during publish platform_details = bench_platforms.sort_descending(['run_id']).group_by(['run_id']).first_by().ungroup() +# Ensure that deleted benchmarks are not included in latest benchmarks +latest_benchmark_names = bench_results.view([ + 'epoch_day=(int)(timestamp/1000/60/60/24)','benchmark_name' +]).group_by(['epoch_day']).sort_descending(['epoch_day']).first_by().ungroup() +bench_results = bench_results.where_in(latest_benchmark_names,['benchmark_name=benchmark_name']) + nightly_worst_rate_change = bench_results.where([ 'benchmark_name.endsWith(`-Static`)' ]).exact_join( @@ -60,9 +58,13 @@ def zprob(zscore): ]) nightly_worst_rate_change_small = nightly_worst_rate_change.head_by(10).view([ - 'Static_Benchmark=Static_Benchmark.substring(0, Math.min(50,Static_Benchmark.length()))+`...`', - 'Chng5d=Change','Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob' + 'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change', + 'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob' ]).format_columns([ 'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)', 'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)' ]) + +bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None +bench_results_change = bench_results_diff = None +