Benchmarks in CI.

.github/workflows/benchmarks.yml

@@ -0,0 +1,42 @@
+name: Benchmark o1js
+on:
+  push:
+    branches:
+      - main
+      - berkeley
+      - develop
+  pull_request:
+  workflow_dispatch: {}
+
+jobs:
+  benchmarks:
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+      matrix:
+        node: [18, 20]
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Node.JS ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Build o1js and execute benchmarks on ${{ matrix.os }} and Node.JS ${{ matrix.node }}
+        env:
+          GIT_BRANCH: ${{ github.head_ref }}
+          INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }}
+          INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }}
+          INFLUXDB_BUCKET: ${{ secrets.INFLUXDB_BUCKET }}
+          INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }}
+          METRICS_SOURCE_ENVIRONMENT: 'o1js GitHub Actions'
+          METRICS_BASE_BRANCH_FOR_COMPARISON: 'main'
+        run: |
+          git submodule update --init --recursive
+          npm ci
+          npm run build
+          bash run-ci-benchmarks.sh
+          cat benchmarks.log >> $GITHUB_STEP_SUMMARY
+        shell: bash

benchmarks/benchmark.ts → benchmark/base-benchmark.ts

@@ -1,8 +1,16 @@
 /**
- * Benchmark runner
+ * Base benchmark harness
  */
+
 import jStat from 'jstat';
-export { BenchmarkResult, Benchmark, benchmark, printResult, pValue };
+export {
+  Benchmark,
+  BenchmarkResult,
+  benchmark,
+  calculateBounds,
+  logResult,
+  pValue,
+};
 
 type BenchmarkResult = {
   label: string;
@@ -100,14 +108,16 @@ function getStatistics(numbers: number[]) {
   return { mean, variance, size: n };
 }
 
-function printResult(
+function logResult(
   result: BenchmarkResult,
   previousResult?: BenchmarkResult
-) {
+): void {
   console.log(result.label + `\n`);
   console.log(`time: ${resultToString(result)}`);
 
-  if (previousResult === undefined) return;
+  if (!previousResult) {
+    return;
+  }
 
   let change = (result.mean - previousResult.mean) / previousResult.mean;
   let p = pValue(result, previousResult);
@@ -122,14 +132,13 @@ function printResult(
 
   if (p < 0.05) {
     if (result.mean < previousResult.mean) {
-      console.log('Performance has improved');
+      console.log('Performance has improved.');
     } else {
-      console.log('Performance has regressed');
+      console.log('Performance has regressed.');
     }
   } else {
     console.log('Change within noise threshold.');
   }
-  console.log('\n');
 }
 
 function resultToString({ mean, variance }: BenchmarkResult) {
@@ -156,3 +165,10 @@ function pValue(sample1: BenchmarkResult, sample2: BenchmarkResult): number {
   const pValue = 2 * (1 - jStat.studentt.cdf(Math.abs(tStatistic), df));
   return pValue;
 }
+
+function calculateBounds(result: BenchmarkResult) {
+  const percentage = (Math.sqrt(result.variance) / result.mean) * 100;
+  const upperBound = result.mean + (result.mean * percentage) / 100;
+  const lowerBound = result.mean - (result.mean * percentage) / 100;
+  return { upperBound, lowerBound };
+}

benchmark/runners/simple.ts

@@ -0,0 +1,21 @@
+/**
+ * Simple benchmarks runner
+ * Exercises benchmarks and logs the results
+ *
+ * Run with
+ * ```
+ * ./run benchmark/runners/simple.ts --bundle
+ * ```
+ */
+
+import { logResult } from '../base-benchmark.js';
+import EcdsaBenchmark from '../samples/ecdsa.js';
+
+// Run all benchmarks
+const results = [...(await EcdsaBenchmark.run())];
+
+// Process and log results
+for (const result of results) {
+  logResult(result);
+  console.log('\n');
+}

benchmark/runners/with-cloud-history.ts

@@ -0,0 +1,26 @@
+/**
+ * Benchmarks runner with historical data preservation in cloud storage (InfluxDB)
+ *
+ * Run with
+ * ```
+ * ./run benchmark/runners/with-cloud-history.ts --bundle
+ * ```
+ */
+
+import { logResult } from '../base-benchmark.js';
+import EcdsaBenchmark from '../samples/ecdsa.js';
+import {
+  readPreviousResultFromInfluxDb,
+  writeResultToInfluxDb,
+} from '../utils/influxdb-utils.js';
+
+// Run all benchmarks
+const results = [...(await EcdsaBenchmark.run())];
+
+// Process and log results
+for (const result of results) {
+  const previousResult = await readPreviousResultFromInfluxDb(result);
+  logResult(result, previousResult);
+  writeResultToInfluxDb(result);
+  console.log('\n');
+}

benchmark/samples/ecdsa.ts

@@ -0,0 +1,43 @@
+/**
+ * ECDSA benchmark sample
+ */
+
+import { Provable } from 'o1js';
+import {
+  Bytes32,
+  Ecdsa,
+  Secp256k1,
+  keccakAndEcdsa,
+} from '../../src/examples/crypto/ecdsa/ecdsa.js';
+import { benchmark } from '../base-benchmark.js';
+
+let privateKey = Secp256k1.Scalar.random();
+let publicKey = Secp256k1.generator.scale(privateKey);
+let message = Bytes32.fromString("what's up");
+let signature = Ecdsa.sign(message.toBytes(), privateKey.toBigInt());
+
+const EcdsaBenchmark = benchmark(
+  'ecdsa',
+  async (tic, toc) => {
+    tic('build constraint system');
+    await keccakAndEcdsa.analyzeMethods();
+    toc();
+
+    tic('witness generation');
+    await Provable.runAndCheck(async () => {
+      let message_ = Provable.witness(Bytes32.provable, () => message);
+      let signature_ = Provable.witness(Ecdsa.provable, () => signature);
+      let publicKey_ = Provable.witness(Secp256k1.provable, () => publicKey);
+      await keccakAndEcdsa.rawMethods.verifyEcdsa(
+        message_,
+        signature_,
+        publicKey_
+      );
+    });
+    toc();
+  },
+  // two warmups to ensure full caching
+  { numberOfWarmups: 2, numberOfRuns: 5 }
+);
+
+export default EcdsaBenchmark;

benchmark/utils/influxdb-utils.ts

@@ -0,0 +1,132 @@
+/**
+ * InfluxDB utils
+ */
+
+import { InfluxDB, Point } from '@influxdata/influxdb-client';
+import os from 'node:os';
+import { BenchmarkResult, calculateBounds } from '../base-benchmark.js';
+
+const INFLUXDB_CLIENT_OPTIONS = {
+  url: process.env.INFLUXDB_URL,
+  token: process.env.INFLUXDB_TOKEN,
+  org: process.env.INFLUXDB_ORG,
+  bucket: process.env.INFLUXDB_BUCKET,
+};
+const INFLUXDB_COMMON_POINT_TAGS = {
+  sourceEnvironment: process.env.METRICS_SOURCE_ENVIRONMENT ?? 'local',
+  operatingSystem: `${os.type()} ${os.release()} ${os.arch()}`,
+  hardware: `${os.cpus()[0].model}, ${os.cpus().length} cores, ${(
+    os.totalmem() / Math.pow(1024, 3)
+  ).toFixed(2)}Gb of RAM`,
+  gitBranch: process.env.GIT_BRANCH ?? 'unknown',
+};
+const influxDbClient = setupInfluxDbClient();
+
+function setupInfluxDbClient(): InfluxDB | undefined {
+  const { url, token } = INFLUXDB_CLIENT_OPTIONS;
+  if (url === undefined || token === undefined) {
+    return undefined;
+  }
+  return new InfluxDB({ url, token });
+}
+
+export function writeResultToInfluxDb(result: BenchmarkResult): void {
+  const { org, bucket } = INFLUXDB_CLIENT_OPTIONS;
+  if (influxDbClient && org && bucket) {
+    console.log('Writing result to InfluxDB.');
+    const influxDbWriteClient = influxDbClient.getWriteApi(org, bucket, 'ms');
+    try {
+      const sampleName = result.label.split('-')[1].trim();
+      const { upperBound, lowerBound } = calculateBounds(result);
+      const point = new Point(`${result.label} - ${result.size} samples`)
+        .tag('benchmarkName', result.label.trim())
+        .tag('sampledTimes', result.size.toString())
+        .floatField('mean', result.mean)
+        .floatField('variance', result.variance)
+        .floatField(`${sampleName} - upperBound`, upperBound)
+        .floatField(`${sampleName} - lowerBound`, lowerBound)
+        .intField('size', result.size);
+      for (const [key, value] of Object.entries(INFLUXDB_COMMON_POINT_TAGS)) {
+        point.tag(key, value.trim());
+      }
+      influxDbWriteClient.writePoint(point);
+    } catch (e) {
+      console.error('Error writing to InfluxDB: ', e);
+    } finally {
+      influxDbWriteClient.close();
+    }
+  } else {
+    console.info('Skipping writing to InfluxDB: client is not configured.');
+  }
+}
+
+export function readPreviousResultFromInfluxDb(
+  result: BenchmarkResult
+): Promise<BenchmarkResult | undefined> {
+  return new Promise((resolve) => {
+    const { org, bucket } = INFLUXDB_CLIENT_OPTIONS;
+    if (!influxDbClient || !org || !bucket) {
+      resolve(undefined);
+      return;
+    }
+    console.log('Querying InfluxDB for previous results.');
+    const influxDbPointTags = INFLUXDB_COMMON_POINT_TAGS;
+    const influxDbQueryClient = influxDbClient.getQueryApi(org);
+    const baseBranchForComparison =
+      process.env.METRICS_BASE_BRANCH_FOR_COMPARISON ?? 'main';
+    const fluxQuery = `
+      from(bucket: "${bucket}")
+        |> range(start: -90d)
+        |> filter(fn: (r) => r.benchmarkName == "${result.label}")
+        |> filter(fn: (r) => r.gitBranch == "${baseBranchForComparison}")
+        |> filter(fn: (r) => r.sampledTimes == "${result.size}")
+        |> filter(fn: (r) => r.sourceEnvironment == "${influxDbPointTags.sourceEnvironment}")
+        |> filter(fn: (r) => r.operatingSystem == "${influxDbPointTags.operatingSystem}")
+        |> filter(fn: (r) => r.hardware == "${influxDbPointTags.hardware}")
+        |> toFloat()
+        |> group()
+        |> pivot(
+          rowKey:["_measurement"],
+          columnKey: ["_field"],
+          valueColumn: "_value"
+        )
+        |> sort(desc: true)
+        |> limit(n:1)
+    `;
+    try {
+      let previousResult: BenchmarkResult | undefined = undefined;
+      influxDbQueryClient.queryRows(fluxQuery, {
+        next(row, tableMeta) {
+          const tableObject = tableMeta.toObject(row);
+          if (
+            !previousResult &&
+            tableObject._measurement &&
+            tableObject.mean &&
+            tableObject.variance &&
+            tableObject.size
+          ) {
+            const measurement = tableObject._measurement;
+            previousResult = {
+              label: measurement
+                .substring(0, measurement.lastIndexOf('-'))
+                .trim(),
+              mean: parseFloat(tableObject.mean),
+              variance: parseFloat(tableObject.variance),
+              size: parseInt(tableObject.size, 10),
+            };
+          }
+        },
+        error(e) {
+          console.error('Error querying InfluxDB: ', e);
+          resolve(undefined);
+        },
+        complete() {
+          resolve(previousResult);
+        },
+      });
+    } catch (e) {
+      console.error('Error querying InfluxDB: ', e);
+      resolve(undefined);
+    }
+  });
+}