Some improvements to slack publish (#191)

docs/demo/NightlyDashboard.md

@@ -0,0 +1,22 @@
+# Nightly Dashboard
+
+Deephaven benchmarks run every night and summary tables (not show here) are published internally so that
+developers can see how performance changed during the last nightly build. If more investigation is
+needed, the logical next step is to search the benchmark data to get more details.
+
+The Nightly Dashboard allows a quick way to dig into the available data to get some clues to
+performance issues or see more clearer why performance has improved.
+
+This Dashboard is built using Deephaven's [scripted UI](https://deephaven.io/core/docs/how-to-guides/plotting/category/)
+using the Benchmark Tables snippet used in the previous notebook.
+
+```python
+from urllib.request import urlopen; import os
+
+root = 'file:///nfs' if os.path.exists('/nfs/deephaven-benchmark') else 'https://storage.googleapis.com'
+with urlopen(root + '/deephaven-benchmark/benchmark_tables.dh.py') as r:
+    benchmark_storage_uri_arg = root + '/deephaven-benchmark'
+    benchmark_category_arg = 'release'  # release | nightly
+    benchmark_max_runs_arg = 5  # Latest X runs to include
+    exec(r.read().decode(), globals(), locals())
+```

src/main/java/io/deephaven/benchmark/run/PublishNotification.java

@@ -46,44 +46,46 @@ public void publish() {
         var query = Filer.getURLText(queryFile);
         var svgTemp = new String[] {Filer.getURLText(svgTemplate)};
         Bench api = Bench.create("# Publish Notification");
-        api.setName("# Publish");
-        slackChannel = api.property("slack.channel", "");
-        slackToken = api.property("slack.token", "");
-        if (slackChannel.isBlank() || slackToken.isBlank()) {
-            api.close();
-            System.out.println("-- Slack properties is not defined, skipping query notification --");
-            return;
-        }
-        System.out.println("-- Running notification queries --");
-        var aquery = api.query(query);
-        aquery.fetchAfter("platform_details", table -> {
-            svgTemp[0] = updatePlatformDetails(table, svgTemp[0]);
-        });
-        for (String tableName : tables) {
-            aquery.fetchAfter(tableName + "_small", table -> {
-                generateCsv(table, outputDir, tableName + ".csv");
-            });
-            aquery.fetchAfter(tableName + "_large", table -> {
-                generateSvg(table, svgTemp[0], outputDir, tableName + ".svg");
+        try {
+            api.setName("# Publish");
+            slackChannel = api.property("slack.channel", "");
+            slackToken = api.property("slack.token", "");
+            if (slackChannel.isBlank() || slackToken.isBlank()) {
+                System.out.println("-- Slack properties are not defined. Skipping query notification --");
+                return;
+            }
+            System.out.println("-- Running notification queries --");
+            var aquery = api.query(query);
+            aquery.fetchAfter("platform_details", table -> {
+                svgTemp[0] = updatePlatformDetails(table, svgTemp[0]);
             });
+            for (String tableName : tables) {
+                aquery.fetchAfter(tableName + "_small", table -> {
+                    generateCsv(table, outputDir, tableName + ".csv");
+                });
+                aquery.fetchAfter(tableName + "_large", table -> {
+                    generateSvg(table, svgTemp[0], outputDir, tableName + ".svg");
+                });
+            }
+            aquery.execute();
+        } finally {
+            api.close();
         }
-        aquery.execute();
-        api.close();
 
         publishToSlack(outputDir);
     }
 
     void publishToSlack(Path outDir) {
         var message = "Nightly Benchmark Changes " +
-                "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";
-        // "<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Dashboards)>\n";
+                "<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Tables)>\n";
+        // "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";
+
         for (String table : tables) {
             message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```";
         }
 
         var payload = """
-        {"channel": "${channel}", "icon_emoji": ":horse_racing:", "unfurl_links": "false", 
-         "unfurl_media": "false", "text": "${msg}"}
+        {"channel": "${channel}", "unfurl_links": "false", "unfurl_media": "false", "text": "${msg}"}
         """;
         payload = payload.replace("${channel}", slackChannel);
         payload = payload.replace("${msg}", message);

src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py

@@ -104,6 +104,14 @@ def add_platform_value(pname, new_pname):
 import statistics
 def rstd(rates):
     return statistics.pstdev(rates) * 100.0 / statistics.mean(rates)
+
+def zscore(rate, rates):
+    return (rate - statistics.mean(rates)) / statistics.pstdev(rates)
+
+def zprob(zscore):
+    lower = -abs(zscore)
+    upper = abs(zscore)
+    return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower))
 
 from array import array
 def rchange(rates) -> float:
@@ -112,12 +120,16 @@ def rchange(rates) -> float:
     m = statistics.mean(rates[:-1])
     return (rates[-1] - m) / m * 100.0
 
-def format_rates(rates):
-    return ' '.join("{:,}".format(r) for r in rates)
-
 def gain(start:float, end:float) -> float:
     return (end - start) / start * 100.0
 
+def format_rates(rates):
+    return ' '.join("{:,}".format(r) for r in rates)
+
+def truncate(text, size):
+    if len(text) < size - 3: return text
+    return text[:size-3] + '...'
+
 from deephaven.updateby import rolling_group_tick
 op_group = rolling_group_tick(cols=["op_group_rates = op_rate"], rev_ticks=history_runs, fwd_ticks=0)
 op_version = rolling_group_tick(cols=["op_group_versions = deephaven_version"], rev_ticks=history_runs, fwd_ticks=0)

src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py

@@ -12,17 +12,15 @@
     benchmark_max_runs_arg = 45  # Latest X runs to include   
     exec(r.read().decode(), globals(), locals())
 
-import statistics
-def zscore(rate, rates):
-    return (rate - statistics.mean(rates)) / statistics.pstdev(rates)
-
-def zprob(zscore):
-  lower = -abs(zscore)
-  upper = abs(zscore)
-  return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower))
-
+# Used to provide platform (e.g. hardware, jvm version) for SVG footer during publish
 platform_details = bench_platforms.sort_descending(['run_id']).group_by(['run_id']).first_by().ungroup()
 
+# Ensure that deleted benchmarks are not included in latest benchmarks
+latest_benchmark_names = bench_results.view([
+    'epoch_day=(int)(timestamp/1000/60/60/24)','benchmark_name'
+]).group_by(['epoch_day']).sort_descending(['epoch_day']).first_by().ungroup()
+bench_results = bench_results.where_in(latest_benchmark_names,['benchmark_name=benchmark_name'])
+
 nightly_worst_rate_change = bench_results.where([
     'benchmark_name.endsWith(`-Static`)'
 ]).exact_join(
@@ -60,9 +58,13 @@ def zprob(zscore):
 ])
 
 nightly_worst_rate_change_small = nightly_worst_rate_change.head_by(10).view([
-    'Static_Benchmark=Static_Benchmark.substring(0, Math.min(50,Static_Benchmark.length()))+`...`',
-    'Chng5d=Change','Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
+    'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
+    'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
 ]).format_columns([
     'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
     'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)'
 ])
+
+bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None
+bench_results_change = bench_results_diff = None
+