Merge branch 'deephaven:main' into select-update-test-mprovements

docs/demo/NightlyDashboard.md

@@ -0,0 +1,22 @@
+# Nightly Dashboard
+
+Deephaven benchmarks run every night and summary tables (not show here) are published internally so that
+developers can see how performance changed during the last nightly build. If more investigation is
+needed, the logical next step is to search the benchmark data to get more details.
+
+The Nightly Dashboard allows a quick way to dig into the available data to get some clues to
+performance issues or see more clearer why performance has improved.
+
+This Dashboard is built using Deephaven's [scripted UI](https://deephaven.io/core/docs/how-to-guides/plotting/category/)
+using the Benchmark Tables snippet used in the previous notebook.
+
+```python
+from urllib.request import urlopen; import os
+
+root = 'file:///nfs' if os.path.exists('/nfs/deephaven-benchmark') else 'https://storage.googleapis.com'
+with urlopen(root + '/deephaven-benchmark/benchmark_tables.dh.py') as r:
+    benchmark_storage_uri_arg = root + '/deephaven-benchmark'
+    benchmark_category_arg = 'release'  # release | nightly
+    benchmark_max_runs_arg = 5  # Latest X runs to include
+    exec(r.read().decode(), globals(), locals())
+```

pom.xml

@@ -9,7 +9,7 @@
 			<plugin>
 				<groupId>org.codehaus.mojo</groupId>
 				<artifactId>build-helper-maven-plugin</artifactId>
-				<version>3.3.0</version>
+				<version>3.4.0</version>
 				<executions>
 					<execution>
 						<id>add-test-source</id>
@@ -27,7 +27,7 @@
 			</plugin>
 			<plugin>
 				<artifactId>maven-compiler-plugin</artifactId>
-				<version>3.10.1</version>
+				<version>3.11.0</version>
 				<configuration>
 					<source>17</source>
 					<target>17</target>
@@ -36,7 +36,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
-				<version>3.4.1</version>
+				<version>3.5.1</version>
 				<executions>
 					<execution>
 						<phase>package</phase>
@@ -81,7 +81,7 @@
 				<!-- For spotless to work on Windows, Set git config global 'core.autocrlf' to true -->
 				<groupId>com.diffplug.spotless</groupId>
 				<artifactId>spotless-maven-plugin</artifactId>
-				<version>2.33.0</version>
+				<version>2.40.0</version>
 				<configuration>
 					<java>
 						<includes>
@@ -107,7 +107,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-surefire-plugin</artifactId>
-				<version>3.0.0</version>
+				<version>3.1.2</version>
 				<configuration>
 					<excludes>
 						<exclude>/io/deephaven/benchmark/tests/**/*Test</exclude>
@@ -124,7 +124,7 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-failsafe-plugin</artifactId>
-				<version>3.0.0</version>
+				<version>3.1.2</version>
 				<configuration>
 					<forkCount>0</forkCount>
 					<includes>
@@ -163,32 +163,32 @@
 			<artifactId>netty-all</artifactId>
 			<version>4.1.79.Final</version>
 		</dependency>
-		<!-- Added because of conflict with 1.49.1 -->
+		<!-- Added because of conflict with 1.49.1 --> 
 		<dependency>
 			<groupId>io.grpc</groupId>
 			<artifactId>grpc-all</artifactId>
-			<version>1.50.1</version>
+			<version>1.50.0</version> 
 		</dependency>
 		<!-- Added because of conflict with 3.17.3 -->
 		<dependency>
 			<groupId>com.google.protobuf</groupId>
 			<artifactId>protobuf-java</artifactId>
-			<version>3.21.7</version>
+			<version>3.21.7</version> 
 		</dependency>
 		<dependency>
 			<groupId>io.confluent</groupId>
 			<artifactId>kafka-avro-serializer</artifactId>
-			<version>7.3.1</version>
+			<version>7.5.1</version>
 		</dependency>
 		<dependency>
 			<groupId>io.deephaven</groupId>
 			<artifactId>deephaven-java-client-barrage-dagger</artifactId>
-			<version>0.28.0</version>
+			<version>0.29.0</version>
 		</dependency>
 		<dependency>
 			<groupId>io.deephaven</groupId>
 			<artifactId>deephaven-log-to-slf4j</artifactId>
-			<version>0.28.0</version>
+			<version>0.29.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.junit.platform</groupId>

src/main/java/io/deephaven/benchmark/connect/BarrageConnector.java

@@ -28,7 +28,6 @@
 import io.deephaven.engine.table.TableUpdate;
 import io.deephaven.engine.table.impl.InstrumentedTableUpdateListener;
 import io.deephaven.engine.updategraph.impl.PeriodicUpdateGraph;
-import io.deephaven.extensions.barrage.BarrageSnapshotOptions;
 import io.deephaven.extensions.barrage.BarrageSubscriptionOptions;
 import io.deephaven.extensions.barrage.table.BarrageTable;
 import io.deephaven.qst.TableCreationLogic;
@@ -53,7 +52,7 @@ public class BarrageConnector implements AutoCloseable {
     final private ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(4);
     final private BufferAllocator bufferAllocator = new RootAllocator();
     final private Map<String, Subscription> subscriptions = new LinkedHashMap<>();
-    final private Map<String, Snapshot> snapshots = new LinkedHashMap<>();
+    final private Map<String, Subscription> snapshots = new LinkedHashMap<>();
     final private Set<String> variableNames = new HashSet<>();
     final private AtomicBoolean isClosed = new AtomicBoolean(false);
     private Changes changes = null;
@@ -114,14 +113,16 @@ public Future<Metrics> fetchSnapshotData(String table, Consumer<ResultTable> tab
         MetricsFuture future = new MetricsFuture(metrics);
         snapshots.computeIfAbsent(table, s -> {
             try {
-                BarrageSnapshotOptions options = BarrageSnapshotOptions.builder().build();
+                BarrageSubscriptionOptions options = BarrageSubscriptionOptions.builder().build();
                 TableHandleManager snapshotManager = session.session().batch();
+
                 TableCreationLogic logic = findTable(table).ticket().ticketId().table().logic();
                 TableHandle handle = snapshotManager.executeLogic(logic);
-                BarrageSnapshot snapshot = session.snapshot(handle, options);
-                BarrageTable snapshotTable = snapshot.entireTable();
-                tableHandler.accept(CachedResultTable.create(snapshotTable));
-                return new Snapshot(handle, snapshot);
+                BarrageSubscription subscription = session.subscribe(handle, options);
+
+                BarrageTable snapTable = subscription.snapshotEntireTable();
+                tableHandler.accept(CachedResultTable.create(snapTable));
+                return new Subscription(handle, subscription);
             } catch (Exception ex) {
                 throw new RuntimeException("Failed to fetch snapshot table data: " + table, ex);
             } finally {

src/main/java/io/deephaven/benchmark/run/PublishNotification.java

@@ -46,44 +46,46 @@ public void publish() {
         var query = Filer.getURLText(queryFile);
         var svgTemp = new String[] {Filer.getURLText(svgTemplate)};
         Bench api = Bench.create("# Publish Notification");
-        api.setName("# Publish");
-        slackChannel = api.property("slack.channel", "");
-        slackToken = api.property("slack.token", "");
-        if (slackChannel.isBlank() || slackToken.isBlank()) {
-            api.close();
-            System.out.println("-- Slack properties is not defined, skipping query notification --");
-            return;
-        }
-        System.out.println("-- Running notification queries --");
-        var aquery = api.query(query);
-        aquery.fetchAfter("platform_details", table -> {
-            svgTemp[0] = updatePlatformDetails(table, svgTemp[0]);
-        });
-        for (String tableName : tables) {
-            aquery.fetchAfter(tableName + "_small", table -> {
-                generateCsv(table, outputDir, tableName + ".csv");
-            });
-            aquery.fetchAfter(tableName + "_large", table -> {
-                generateSvg(table, svgTemp[0], outputDir, tableName + ".svg");
+        try {
+            api.setName("# Publish");
+            slackChannel = api.property("slack.channel", "");
+            slackToken = api.property("slack.token", "");
+            if (slackChannel.isBlank() || slackToken.isBlank()) {
+                System.out.println("-- Slack properties are not defined. Skipping query notification --");
+                return;
+            }
+            System.out.println("-- Running notification queries --");
+            var aquery = api.query(query);
+            aquery.fetchAfter("platform_details", table -> {
+                svgTemp[0] = updatePlatformDetails(table, svgTemp[0]);
             });
+            for (String tableName : tables) {
+                aquery.fetchAfter(tableName + "_small", table -> {
+                    generateCsv(table, outputDir, tableName + ".csv");
+                });
+                aquery.fetchAfter(tableName + "_large", table -> {
+                    generateSvg(table, svgTemp[0], outputDir, tableName + ".svg");
+                });
+            }
+            aquery.execute();
+        } finally {
+            api.close();
         }
-        aquery.execute();
-        api.close();
 
         publishToSlack(outputDir);
     }
 
     void publishToSlack(Path outDir) {
         var message = "Nightly Benchmark Changes " +
-                "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";
-        // "<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Dashboards)>\n";
+                "<https://controller.try-dh.demo.community.deephaven.io/get_ide| (Benchmark Tables)>\n";
+        // "<https://github.com/deephaven/benchmark/blob/main/docs/PublishedResults.md| (Dig Deeper)>\n";
+
         for (String table : tables) {
             message += "```" + Filer.getFileText(outDir.resolve(table + ".csv")) + "```";
         }
 
         var payload = """
-        {"channel": "${channel}", "icon_emoji": ":horse_racing:", "unfurl_links": "false", 
-         "unfurl_media": "false", "text": "${msg}"}
+        {"channel": "${channel}", "unfurl_links": "false", "unfurl_media": "false", "text": "${msg}"}
         """;
         payload = payload.replace("${channel}", slackChannel);
         payload = payload.replace("${msg}", message);

src/main/resources/io/deephaven/benchmark/run/profile/benchmark_tables.dh.py

@@ -104,6 +104,14 @@ def add_platform_value(pname, new_pname):
 import statistics
 def rstd(rates):
     return statistics.pstdev(rates) * 100.0 / statistics.mean(rates)
+
+def zscore(rate, rates):
+    return (rate - statistics.mean(rates)) / statistics.pstdev(rates)
+
+def zprob(zscore):
+    lower = -abs(zscore)
+    upper = abs(zscore)
+    return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower))
 
 from array import array
 def rchange(rates) -> float:
@@ -112,12 +120,16 @@ def rchange(rates) -> float:
     m = statistics.mean(rates[:-1])
     return (rates[-1] - m) / m * 100.0
 
-def format_rates(rates):
-    return ' '.join("{:,}".format(r) for r in rates)
-
 def gain(start:float, end:float) -> float:
     return (end - start) / start * 100.0
 
+def format_rates(rates):
+    return ' '.join("{:,}".format(r) for r in rates)
+
+def truncate(text, size):
+    if len(text) < size - 3: return text
+    return text[:size-3] + '...'
+
 from deephaven.updateby import rolling_group_tick
 op_group = rolling_group_tick(cols=["op_group_rates = op_rate"], rev_ticks=history_runs, fwd_ticks=0)
 op_version = rolling_group_tick(cols=["op_group_versions = deephaven_version"], rev_ticks=history_runs, fwd_ticks=0)

src/main/resources/io/deephaven/benchmark/run/profile/queries/publish.py

@@ -12,17 +12,15 @@
     benchmark_max_runs_arg = 45  # Latest X runs to include   
     exec(r.read().decode(), globals(), locals())
 
-import statistics
-def zscore(rate, rates):
-    return (rate - statistics.mean(rates)) / statistics.pstdev(rates)
-
-def zprob(zscore):
-  lower = -abs(zscore)
-  upper = abs(zscore)
-  return 1 - (statistics.NormalDist().cdf(upper) - statistics.NormalDist().cdf(lower))
-
+# Used to provide platform (e.g. hardware, jvm version) for SVG footer during publish
 platform_details = bench_platforms.sort_descending(['run_id']).group_by(['run_id']).first_by().ungroup()
 
+# Ensure that deleted benchmarks are not included in latest benchmarks
+latest_benchmark_names = bench_results.view([
+    'epoch_day=(int)(timestamp/1000/60/60/24)','benchmark_name'
+]).group_by(['epoch_day']).sort_descending(['epoch_day']).first_by().ungroup()
+bench_results = bench_results.where_in(latest_benchmark_names,['benchmark_name=benchmark_name'])
+
 nightly_worst_rate_change = bench_results.where([
     'benchmark_name.endsWith(`-Static`)'
 ]).exact_join(
@@ -60,9 +58,13 @@ def zprob(zscore):
 ])
 
 nightly_worst_rate_change_small = nightly_worst_rate_change.head_by(10).view([
-    'Static_Benchmark=Static_Benchmark.substring(0, Math.min(50,Static_Benchmark.length()))+`...`',
-    'Chng5d=Change','Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
+    'Static_Benchmark=truncate(Static_Benchmark,50)','Chng5d=Change',
+    'Var5d=Variability','Rate','ChngRls=Since_Release','ScrProb=Score_Prob'
 ]).format_columns([
     'Rate=Decimal(`###,##0`)','Chng5d=Decimal(`0.0%`)','Var5d=Decimal(`0.0%`)',
     'ChngRls=Decimal(`0.0%`)','ScrProb=Decimal(`0.00%`)'
 ])
+
+bench_results = bench_metrics = bench_platforms = bench_metrics_diff = None
+bench_results_change = bench_results_diff = None
+