From c047444f96d4c8565260fae8cc8a2db1931601c4 Mon Sep 17 00:00:00 2001
From: Gregor <gregor.mitscha-baude@gmx.at>
Date: Wed, 6 Mar 2024 13:37:43 +0100
Subject: [PATCH] add benchmark runner and example

---
 benchmarks/benchmark.ts  | 104 +++++++++++++++++++++++++++++++++++++++
 benchmarks/ecdsa.ts      |  25 ++++++++++
 benchmarks/tsconfig.json |  12 +++++
 3 files changed, 141 insertions(+)
 create mode 100644 benchmarks/benchmark.ts
 create mode 100644 benchmarks/ecdsa.ts
 create mode 100644 benchmarks/tsconfig.json

diff --git a/benchmarks/benchmark.ts b/benchmarks/benchmark.ts
new file mode 100644
index 0000000000..859263a340
--- /dev/null
+++ b/benchmarks/benchmark.ts
@@ -0,0 +1,104 @@
+/**
+ * Benchmark runner
+ */
+export { BenchmarkResult, benchmark, printResults };
+
+type BenchmarkResult = {
+  label: string;
+  mean: number;
+  stdDev: number;
+  full: number[];
+};
+
+async function benchmark(
+  label: string,
+  run:
+    | ((
+        tic: (label?: string) => void,
+        toc: (label?: string) => void
+      ) => Promise<void>)
+    | ((tic: (label?: string) => void, toc: (label?: string) => void) => void),
+  options?: {
+    numberOfRuns?: number;
+    numberOfWarmups?: number;
+  }
+): Promise<BenchmarkResult[]> {
+  const { numberOfRuns = 5, numberOfWarmups = 0 } = options ?? {};
+
+  let lastStartKey: string;
+  let startTime: Record<string, number | undefined> = {}; // key: startTime
+  let runTimes: Record<string, number[]> = {}; // key: [(endTime - startTime)]
+
+  function reset() {
+    startTime = {};
+  }
+
+  function start(key?: string) {
+    lastStartKey = key ?? '';
+    key = getKey(label, key);
+    if (startTime[key] !== undefined)
+      throw Error('running `start(label)` with an already started label');
+    startTime[key] = performance.now();
+  }
+
+  function stop(key?: string) {
+    let end = performance.now();
+    key ??= lastStartKey;
+    if (key === undefined) {
+      throw Error('running `stop()` with no start defined');
+    }
+    key = getKey(label, key);
+    let start_ = startTime[key];
+    startTime[key] = undefined;
+    if (start_ === undefined)
+      throw Error('running `stop()` with no start defined');
+    let times = (runTimes[key] ??= []);
+    times.push(end - start_);
+  }
+
+  let noop = () => {};
+  for (let i = 0; i < numberOfWarmups; i++) {
+    reset();
+    await run(noop, noop);
+  }
+  for (let i = 0; i < numberOfRuns; i++) {
+    reset();
+    await run(start, stop);
+  }
+
+  const results: BenchmarkResult[] = [];
+
+  for (let key in runTimes) {
+    let times = runTimes[key];
+    results.push({ label: key, ...getStatistics(times) });
+  }
+  return results;
+}
+
+function getKey(label: string, key?: string) {
+  return key ? `${label} - ${key}` : label;
+}
+
+function getStatistics(numbers: number[]) {
+  let sum = 0;
+  let sumSquares = 0;
+  for (let i of numbers) {
+    sum += i;
+    sumSquares += i ** 2;
+  }
+  let n = numbers.length;
+  let mean = sum / n;
+  let stdDev = Math.sqrt((sumSquares - sum ** 2 / n) / (n - 1)) / mean;
+
+  return { mean, stdDev, full: numbers };
+}
+
+function printResults(results: BenchmarkResult[]) {
+  for (let result of results) {
+    console.log(`${result.label}: ${resultToString(result)}`);
+  }
+}
+
+function resultToString({ mean, stdDev }: BenchmarkResult) {
+  return `${mean.toFixed(2)}ms ± ${(stdDev * 100).toFixed(1)}%`;
+}
diff --git a/benchmarks/ecdsa.ts b/benchmarks/ecdsa.ts
new file mode 100644
index 0000000000..4399b0682d
--- /dev/null
+++ b/benchmarks/ecdsa.ts
@@ -0,0 +1,25 @@
+/**
+ * Benchmark runner example
+ *
+ * Run with
+ * ```
+ * ./run benchmarks/ecdsa.ts --bundle
+ * ```
+ */
+import { keccakAndEcdsa } from '../src/examples/crypto/ecdsa/ecdsa.js';
+import { benchmark, printResults } from './benchmark.js';
+
+let result = await benchmark(
+  'ecdsa',
+  async (tic, toc) => {
+    tic('compile (cached)');
+    await keccakAndEcdsa.compile();
+    toc();
+  },
+  // two warmups to ensure full caching
+  { numberOfWarmups: 2, numberOfRuns: 5 }
+);
+
+// just an example for how to log results
+console.log(result[0].full);
+printResults(result);
diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json
new file mode 100644
index 0000000000..3d5c7b6534
--- /dev/null
+++ b/benchmarks/tsconfig.json
@@ -0,0 +1,12 @@
+{
+  "extends": "../tsconfig.json",
+  "include": ["."],
+  "exclude": [],
+  "compilerOptions": {
+    "rootDir": "..",
+    "baseUrl": "..",
+    "paths": {
+      "o1js": ["."]
+    }
+  }
+}