From f30ae69cb6276baa3187acf20b5d78d1966bdea1 Mon Sep 17 00:00:00 2001 From: stanbrub Date: Mon, 18 Nov 2024 20:24:08 -0700 Subject: [PATCH 01/13] Added optional warmups to StandardTestRunner --- .../tests/standard/StandardTestRunner.java | 61 ++++++++++++------- .../benchmark/run/profile/default.properties | 3 + 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index 0ba3983b..6f9e2cf6 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -173,22 +173,30 @@ public void test(String name, String operation, String... loadColumns) { */ public void test(String name, long maxExpectedRowCount, String operation, String... loadColumns) { if (staticFactor > 0) { - var read = getReadOperation(staticFactor, loadColumns); - var result = runStaticTest(name, operation, read, loadColumns); + var sname = name + " -Static"; + var warmup = getStaticQuery(sname, operation, getWarmupRowCount(), loadColumns); + var query = getStaticQuery(sname, operation, getGeneratedRowCount(), loadColumns); + var result = runTest(sname, warmup, query); var rcount = result.resultRowCount(); var ecount = getMaxExpectedRowCount(maxExpectedRowCount, staticFactor); assertTrue(rcount > 0 && rcount <= ecount, "Wrong result Static row count: " + rcount); } if (incFactor > 0) { - var read = getReadOperation(incFactor, loadColumns); - var result = runIncTest(name, operation, read, loadColumns); + var iname = name + " -Inc"; + var warmup = getIncQuery(iname, operation, getWarmupRowCount(), loadColumns); + var query = getIncQuery(iname, operation, getGeneratedRowCount(), loadColumns); + var result = runTest(iname, warmup, query); var rcount = result.resultRowCount(); var ecount = getMaxExpectedRowCount(maxExpectedRowCount, incFactor); assertTrue(rcount > 0 && rcount <= ecount, "Wrong result Inc row count: " + rcount); } } + long getWarmupRowCount() { + return (long) (api.propertyAsIntegral("warmup.row.count", "0") * rowCountFactor); + } + long getGeneratedRowCount() { return (long) (api.propertyAsIntegral("scale.row.count", "100000") * rowCountFactor); } @@ -197,30 +205,31 @@ long getMaxExpectedRowCount(long expectedRowCount, long scaleFactor) { return (expectedRowCount < 1) ? Long.MAX_VALUE : expectedRowCount; } - String getReadOperation(int scaleFactor, String... loadColumns) { + String getReadOperation(int scaleFactor, long rowCount, String... loadColumns) { if (scaleFactor > 1 && mainTable.equals("timed") && Arrays.asList(loadColumns).contains("timestamp")) { var read = """ merge([ read('/data/timed.parquet').view(formulas=[${loadColumns}]) ] * ${scaleFactor}).update_view([ 'timestamp=timestamp.plusMillis((long)(ii / ${rows}) * ${rows})' - ]).select() + ]).head(${rows}).select() """; - return read.replace("${scaleFactor}", "" + scaleFactor).replace("${rows}", "" + getGeneratedRowCount()); + return read.replace("${scaleFactor}", "" + scaleFactor).replace("${rows}", "" + rowCount); } - var read = "read('/data/${mainTable}.parquet').select(formulas=[${loadColumns}])"; - read = (loadColumns.length == 0) ? ("empty_table(" + getGeneratedRowCount() + ")") : read; + var read = "read('/data/${mainTable}.parquet').head(${rows}).select(formulas=[${loadColumns}])"; + read = (loadColumns.length == 0) ? ("empty_table(${rows})") : read; if (scaleFactor > 1) { read = "merge([${readTable}] * ${scaleFactor})".replace("${readTable}", read); read = read.replace("${scaleFactor}", "" + scaleFactor); } - return read; + return read.replace("${rows}", "" + rowCount); } - Result runStaticTest(String name, String operation, String read, String... loadColumns) { + String getStaticQuery(String name, String operation, long warmupRows, String... loadColumns) { var staticQuery = """ + source = right = timed = result = None ${loadSupportTables} ${mainTable} = ${readTable} loaded_tbl_size = ${mainTable}.size @@ -243,12 +252,13 @@ Result runStaticTest(String name, String operation, String read, String... loadC long_col("result_row_count", [result.size]), ]) """; - return runTest(name + " -Static", staticQuery, operation, read, loadColumns); + var read = getReadOperation(staticFactor, warmupRows, loadColumns); + return populateQuery(name, staticQuery, operation, read, loadColumns); } - Result runIncTest(String name, String operation, String read, String... loadColumns) { + String getIncQuery(String name, String operation, long warmupRows, String... loadColumns) { var incQuery = """ - source = right = timed = None + source = right = timed = result = source_filter = right_filter = autotune = None ${loadSupportTables} ${mainTable} = ${readTable} loaded_tbl_size = ${mainTable}.size @@ -287,15 +297,11 @@ Result runIncTest(String name, String operation, String read, String... loadColu long_col("result_row_count", [result.size]) ]) """; - return runTest(name + " -Inc", incQuery, operation, read, loadColumns); + var read = getReadOperation(staticFactor, warmupRows, loadColumns); + return populateQuery(name, incQuery, operation, read, loadColumns); } - Result runTest(String name, String query, String operation, String read, String... loadColumns) { - if (api.isClosed()) - initialize(testInst); - api.setName(name); - stopUnusedServices(requiredServices); - + String populateQuery(String name, String query, String operation, String read, String... loadColumns) { query = query.replace("${readTable}", read); query = query.replace("${mainTable}", mainTable); query = query.replace("${loadSupportTables}", loadSupportTables()); @@ -305,10 +311,21 @@ Result runTest(String name, String query, String operation, String read, String. query = query.replace("${operation}", operation); query = query.replace("${logOperationBegin}", getLogSnippet("Begin", name)); query = query.replace("${logOperationEnd}", getLogSnippet("End", name)); + return query; + } + + + Result runTest(String name, String warmupQuery, String mainQuery) { + if (api.isClosed()) + initialize(testInst); + api.setName(name); + stopUnusedServices(requiredServices); try { + if (getWarmupRowCount() > 0) + api.query(warmupQuery).execute(); var result = new AtomicReference(); - api.query(query).fetchAfter("stats", table -> { + api.query(mainQuery).fetchAfter("stats", table -> { long loadedRowCount = table.getSum("processed_row_count").longValue(); long resultRowCount = table.getSum("result_row_count").longValue(); long elapsedNanos = table.getSum("elapsed_nanos").longValue(); diff --git a/src/main/resources/io/deephaven/benchmark/run/profile/default.properties b/src/main/resources/io/deephaven/benchmark/run/profile/default.properties index bc4cab82..5432a2ec 100644 --- a/src/main/resources/io/deephaven/benchmark/run/profile/default.properties +++ b/src/main/resources/io/deephaven/benchmark/run/profile/default.properties @@ -29,6 +29,9 @@ record.compression=SNAPPY # Row count to scale tests (Tests can override but typically do not) scale.row.count=100000 +# Row count to scale warmups before tests +warmup.row.count=0 + # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run # Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch From d34dbcd1ca56a2b322ae961b25a6761fa776459d Mon Sep 17 00:00:00 2001 From: stanbrub Date: Mon, 18 Nov 2024 20:30:16 -0700 Subject: [PATCH 02/13] Change test runner metrics to use dot notation instead of underscores --- .../benchmark/tests/standard/StandardTestRunner.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index 6f9e2cf6..da14e086 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -334,9 +334,9 @@ Result runTest(String name, String warmupQuery, String mainQuery) { }).fetchAfter("standard_metrics", table -> { api.metrics().add(table); var metrics = new Metrics(Timer.now(), "test-runner", "setup.scale"); - metrics.set("static_scale_factor", staticFactor); - metrics.set("inc_scale_factor", incFactor); - metrics.set("row_count_factor", rowCountFactor); + metrics.set("static.factor", staticFactor); + metrics.set("inc.factor", incFactor); + metrics.set("row.factor", rowCountFactor); api.metrics().add(metrics); }).execute(); api.result().test("deephaven-engine", result.get().elapsedTime(), result.get().loadedRowCount()); From 3027b6a537c2faba7f26ea777d6908f085dd9fad Mon Sep 17 00:00:00 2001 From: stanbrub Date: Mon, 18 Nov 2024 21:11:00 -0700 Subject: [PATCH 03/13] Turned on warmups --- .github/resources/adhoc-scale-benchmark.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index bf5b6c06..39c1e166 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -29,6 +29,9 @@ record.compression=LZO # Row count to scale tests (Tests can override but typically do not) scale.row.count=${baseRowCount} +# Row count to scale warmups before tests +warmup.row.count=100000 + # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run # Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch From fe0e7500a90a3c79bc12a28c0c5b719f54c517eb Mon Sep 17 00:00:00 2001 From: stanbrub Date: Tue, 19 Nov 2024 12:52:52 -0700 Subject: [PATCH 04/13] Scaled up the warmup count --- .github/resources/adhoc-scale-benchmark.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index 39c1e166..886a9d48 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -30,7 +30,7 @@ record.compression=LZO scale.row.count=${baseRowCount} # Row count to scale warmups before tests -warmup.row.count=100000 +warmup.row.count=1000000 # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run From d1da68de27838d4e6d2899b365dee8016ded7b7b Mon Sep 17 00:00:00 2001 From: stanbrub Date: Tue, 19 Nov 2024 17:13:29 -0700 Subject: [PATCH 05/13] Fixed an issue where warmup metrics were being counted --- .../benchmark/tests/standard/StandardTestRunner.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index da14e086..b14519c7 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -235,6 +235,7 @@ String getStaticQuery(String name, String operation, long warmupRows, String... loaded_tbl_size = ${mainTable}.size ${setupQueries} ${preOpQueries} + bench_api_metrics_init() bench_api_metrics_start() print('${logOperationBegin}') @@ -272,6 +273,7 @@ String getIncQuery(String name, String operation, long warmupRows, String... loa right = right.where(right_filter) ${preOpQueries} + bench_api_metrics_init() bench_api_metrics_start() print('${logOperationBegin}') begin_time = time.perf_counter_ns() @@ -370,8 +372,6 @@ void initialize(Object testInst) { from numpy import typing as npt import numpy as np import numba as nb - - bench_api_metrics_init() """; this.api = Bench.create(testInst); From 00afa11aa105129f4f1ed8ae8e220073d4f98149 Mon Sep 17 00:00:00 2001 From: stanbrub Date: Tue, 19 Nov 2024 20:46:14 -0700 Subject: [PATCH 06/13] Try sleep after warmup --- .../benchmark/tests/standard/StandardTestRunner.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index b14519c7..7cbb29df 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -10,6 +10,7 @@ import io.deephaven.benchmark.controller.Controller; import io.deephaven.benchmark.controller.DeephavenDockerController; import io.deephaven.benchmark.metric.Metrics; +import io.deephaven.benchmark.util.Threads; import io.deephaven.benchmark.util.Timer; /** @@ -175,6 +176,7 @@ public void test(String name, long maxExpectedRowCount, String operation, String if (staticFactor > 0) { var sname = name + " -Static"; var warmup = getStaticQuery(sname, operation, getWarmupRowCount(), loadColumns); + Threads.sleep(5000); var query = getStaticQuery(sname, operation, getGeneratedRowCount(), loadColumns); var result = runTest(sname, warmup, query); var rcount = result.resultRowCount(); @@ -185,6 +187,7 @@ public void test(String name, long maxExpectedRowCount, String operation, String if (incFactor > 0) { var iname = name + " -Inc"; var warmup = getIncQuery(iname, operation, getWarmupRowCount(), loadColumns); + Threads.sleep(5000); var query = getIncQuery(iname, operation, getGeneratedRowCount(), loadColumns); var result = runTest(iname, warmup, query); var rcount = result.resultRowCount(); @@ -230,12 +233,12 @@ String getReadOperation(int scaleFactor, long rowCount, String... loadColumns) { String getStaticQuery(String name, String operation, long warmupRows, String... loadColumns) { var staticQuery = """ source = right = timed = result = None + bench_api_metrics_init() ${loadSupportTables} ${mainTable} = ${readTable} loaded_tbl_size = ${mainTable}.size ${setupQueries} ${preOpQueries} - bench_api_metrics_init() bench_api_metrics_start() print('${logOperationBegin}') @@ -260,6 +263,7 @@ String getStaticQuery(String name, String operation, long warmupRows, String... String getIncQuery(String name, String operation, long warmupRows, String... loadColumns) { var incQuery = """ source = right = timed = result = source_filter = right_filter = autotune = None + bench_api_metrics_init() ${loadSupportTables} ${mainTable} = ${readTable} loaded_tbl_size = ${mainTable}.size @@ -273,7 +277,6 @@ String getIncQuery(String name, String operation, long warmupRows, String... loa right = right.where(right_filter) ${preOpQueries} - bench_api_metrics_init() bench_api_metrics_start() print('${logOperationBegin}') begin_time = time.perf_counter_ns() From 77da3229c44dd088292e2d4f2134c671c6ecf86a Mon Sep 17 00:00:00 2001 From: stanbrub Date: Tue, 19 Nov 2024 21:44:25 -0700 Subject: [PATCH 07/13] Double the warmup rows --- .github/resources/adhoc-scale-benchmark.properties | 2 +- .../deephaven/benchmark/tests/standard/StandardTestRunner.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index 886a9d48..b4c3d40d 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -30,7 +30,7 @@ record.compression=LZO scale.row.count=${baseRowCount} # Row count to scale warmups before tests -warmup.row.count=1000000 +warmup.row.count=5000000 # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index 7cbb29df..6ebdea74 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -176,7 +176,6 @@ public void test(String name, long maxExpectedRowCount, String operation, String if (staticFactor > 0) { var sname = name + " -Static"; var warmup = getStaticQuery(sname, operation, getWarmupRowCount(), loadColumns); - Threads.sleep(5000); var query = getStaticQuery(sname, operation, getGeneratedRowCount(), loadColumns); var result = runTest(sname, warmup, query); var rcount = result.resultRowCount(); @@ -187,7 +186,6 @@ public void test(String name, long maxExpectedRowCount, String operation, String if (incFactor > 0) { var iname = name + " -Inc"; var warmup = getIncQuery(iname, operation, getWarmupRowCount(), loadColumns); - Threads.sleep(5000); var query = getIncQuery(iname, operation, getGeneratedRowCount(), loadColumns); var result = runTest(iname, warmup, query); var rcount = result.resultRowCount(); From 00ea3d4d3398f402e894ed383cd24b286aed5403 Mon Sep 17 00:00:00 2001 From: stanbrub Date: Wed, 20 Nov 2024 17:29:08 -0700 Subject: [PATCH 08/13] Set default warmup.row.count to 0 for adhoc --- .github/resources/adhoc-scale-benchmark.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index b4c3d40d..42825a65 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -30,7 +30,7 @@ record.compression=LZO scale.row.count=${baseRowCount} # Row count to scale warmups before tests -warmup.row.count=5000000 +warmup.row.count=0 # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run From 72a6614f14c4c372a4c8f207be2a0c3b78b7ee2a Mon Sep 17 00:00:00 2001 From: stanbrub Date: Wed, 20 Nov 2024 17:43:24 -0700 Subject: [PATCH 09/13] Updated all DH workflow benchmark properties files with warmup default --- .github/resources/compare-scale-benchmark.properties | 3 +++ .github/resources/nightly-scale-benchmark.properties | 3 +++ .github/resources/release-scale-benchmark.properties | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/resources/compare-scale-benchmark.properties b/.github/resources/compare-scale-benchmark.properties index f9662768..714de605 100644 --- a/.github/resources/compare-scale-benchmark.properties +++ b/.github/resources/compare-scale-benchmark.properties @@ -29,6 +29,9 @@ record.compression=SNAPPY # Row count to scale tests (Tests can override but typically do not) scale.row.count=70000000 +# Row count to scale warmups before tests +warmup.row.count=0 + # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run # Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch diff --git a/.github/resources/nightly-scale-benchmark.properties b/.github/resources/nightly-scale-benchmark.properties index a2c71760..aca9b1a6 100644 --- a/.github/resources/nightly-scale-benchmark.properties +++ b/.github/resources/nightly-scale-benchmark.properties @@ -29,6 +29,9 @@ record.compression=LZ4 # Row count to scale tests (Tests can override but typically do not) scale.row.count=${baseRowCount} +# Row count to scale warmups before tests +warmup.row.count=0 + # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run # Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch diff --git a/.github/resources/release-scale-benchmark.properties b/.github/resources/release-scale-benchmark.properties index f2bd0d6d..d72a43ef 100644 --- a/.github/resources/release-scale-benchmark.properties +++ b/.github/resources/release-scale-benchmark.properties @@ -29,6 +29,9 @@ record.compression=LZO # Row count to scale tests (Tests can override but typically do not) scale.row.count=${baseRowCount} +# Row count to scale warmups before tests +warmup.row.count=0 + # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run # Blank: Overwrite if JUnit launch, timestamp if Benchmark main launch From b02958a4a0fc7b8a9c41837b83a19e1aa87212b0 Mon Sep 17 00:00:00 2001 From: stanbrub Date: Wed, 20 Nov 2024 18:20:02 -0700 Subject: [PATCH 10/13] Removed unused import --- .../deephaven/benchmark/tests/standard/StandardTestRunner.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java index 6ebdea74..a495ec95 100644 --- a/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java +++ b/src/it/java/io/deephaven/benchmark/tests/standard/StandardTestRunner.java @@ -10,7 +10,6 @@ import io.deephaven.benchmark.controller.Controller; import io.deephaven.benchmark.controller.DeephavenDockerController; import io.deephaven.benchmark.metric.Metrics; -import io.deephaven.benchmark.util.Threads; import io.deephaven.benchmark.util.Timer; /** From 3a39df2dafe4d6c3078696d9aceb9595ebc258cc Mon Sep 17 00:00:00 2001 From: stanbrub Date: Thu, 21 Nov 2024 17:13:29 -0700 Subject: [PATCH 11/13] test with 1M warmup rows --- .github/resources/adhoc-scale-benchmark.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index 42825a65..886a9d48 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -30,7 +30,7 @@ record.compression=LZO scale.row.count=${baseRowCount} # Row count to scale warmups before tests -warmup.row.count=0 +warmup.row.count=1000000 # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run From 746792409b4c1a338a9f2189a2a22a4e362750dc Mon Sep 17 00:00:00 2001 From: stanbrub Date: Fri, 22 Nov 2024 14:10:31 -0700 Subject: [PATCH 12/13] Back to no warmups --- .github/resources/adhoc-scale-benchmark.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/resources/adhoc-scale-benchmark.properties b/.github/resources/adhoc-scale-benchmark.properties index 886a9d48..42825a65 100644 --- a/.github/resources/adhoc-scale-benchmark.properties +++ b/.github/resources/adhoc-scale-benchmark.properties @@ -30,7 +30,7 @@ record.compression=LZO scale.row.count=${baseRowCount} # Row count to scale warmups before tests -warmup.row.count=1000000 +warmup.row.count=0 # True: Use a timestamp for the parent directory of each test run # False: Overwrite previous test results for each test run From 06b6cecdd7a7335ad65268ddc8d5a585e68bc07a Mon Sep 17 00:00:00 2001 From: stanbrub Date: Fri, 22 Nov 2024 15:00:57 -0700 Subject: [PATCH 13/13] javadoc fix --- src/main/java/io/deephaven/benchmark/api/Snippets.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/deephaven/benchmark/api/Snippets.java b/src/main/java/io/deephaven/benchmark/api/Snippets.java index 67344ef6..07ad6f5e 100644 --- a/src/main/java/io/deephaven/benchmark/api/Snippets.java +++ b/src/main/java/io/deephaven/benchmark/api/Snippets.java @@ -139,7 +139,7 @@ def bench_api_metrics_start(): """; /** - * Get difference from bench_api_metrics_start values and add as collected metrics + * Get difference from bench_api_metrics_start values and add as collected metrics */ static String bench_api_metrics_end = """ def bench_api_metrics_end():