diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..3edf1db7f1 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,42 @@ +name: Benchmark o1js +on: + push: + branches: + - main + - berkeley + - develop + pull_request: + workflow_dispatch: {} + +jobs: + benchmarks: + timeout-minutes: 30 + strategy: + fail-fast: true + matrix: + node: [18, 20] + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Node.JS ${{ matrix.node }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Build o1js and execute benchmarks on ${{ matrix.os }} and Node.JS ${{ matrix.node }} + env: + GIT_BRANCH: ${{ github.head_ref }} + INFLUXDB_URL: ${{ secrets.INFLUXDB_URL }} + INFLUXDB_ORG: ${{ secrets.INFLUXDB_ORG }} + INFLUXDB_BUCKET: ${{ secrets.INFLUXDB_BUCKET }} + INFLUXDB_TOKEN: ${{ secrets.INFLUXDB_TOKEN }} + METRICS_SOURCE_ENVIRONMENT: 'o1js GitHub Actions' + METRICS_BASE_BRANCH_FOR_COMPARISON: 'main' + run: | + git submodule update --init --recursive + npm ci + npm run build + bash run-ci-benchmarks.sh + cat benchmarks.log >> $GITHUB_STEP_SUMMARY + shell: bash diff --git a/benchmarks/benchmark.ts b/benchmark/benchmark.ts similarity index 89% rename from benchmarks/benchmark.ts rename to benchmark/benchmark.ts index e6d347b744..17ae0d9816 100644 --- a/benchmarks/benchmark.ts +++ b/benchmark/benchmark.ts @@ -1,8 +1,16 @@ /** - * Benchmark runner + * Base benchmark harness */ + import jStat from 'jstat'; -export { BenchmarkResult, Benchmark, benchmark, printResult, pValue }; +export { + Benchmark, + BenchmarkResult, + benchmark, + calculateBounds, + logResult, + pValue, +}; type BenchmarkResult = { label: string; @@ -100,10 +108,10 @@ function getStatistics(numbers: number[]) { return { mean, variance, size: n }; } -function printResult( +function logResult( result: BenchmarkResult, previousResult?: BenchmarkResult -) { +): void { console.log(result.label + `\n`); console.log(`time: ${resultToString(result)}`); @@ -122,14 +130,13 @@ function printResult( if (p < 0.05) { if (result.mean < previousResult.mean) { - console.log('Performance has improved'); + console.log('Performance has improved.'); } else { - console.log('Performance has regressed'); + console.log('Performance has regressed.'); } } else { console.log('Change within noise threshold.'); } - console.log('\n'); } function resultToString({ mean, variance }: BenchmarkResult) { @@ -156,3 +163,10 @@ function pValue(sample1: BenchmarkResult, sample2: BenchmarkResult): number { const pValue = 2 * (1 - jStat.studentt.cdf(Math.abs(tStatistic), df)); return pValue; } + +function calculateBounds(result: BenchmarkResult) { + const stdDev = Math.sqrt(result.variance); + const upperBound = result.mean + stdDev; + const lowerBound = result.mean - stdDev; + return { upperBound, lowerBound }; +} diff --git a/benchmark/benchmarks/ecdsa.ts b/benchmark/benchmarks/ecdsa.ts new file mode 100644 index 0000000000..28de1c4544 --- /dev/null +++ b/benchmark/benchmarks/ecdsa.ts @@ -0,0 +1,43 @@ +/** + * ECDSA benchmark + */ + +import { Provable } from 'o1js'; +import { + Bytes32, + Ecdsa, + Secp256k1, + keccakAndEcdsa, +} from '../../src/examples/crypto/ecdsa/ecdsa.js'; +import { benchmark } from '../benchmark.js'; + +let privateKey = Secp256k1.Scalar.random(); +let publicKey = Secp256k1.generator.scale(privateKey); +let message = Bytes32.fromString("what's up"); +let signature = Ecdsa.sign(message.toBytes(), privateKey.toBigInt()); + +const EcdsaBenchmark = benchmark( + 'ecdsa', + async (tic, toc) => { + tic('build constraint system'); + await keccakAndEcdsa.analyzeMethods(); + toc(); + + tic('witness generation'); + await Provable.runAndCheck(async () => { + let message_ = Provable.witness(Bytes32.provable, () => message); + let signature_ = Provable.witness(Ecdsa.provable, () => signature); + let publicKey_ = Provable.witness(Secp256k1.provable, () => publicKey); + await keccakAndEcdsa.rawMethods.verifyEcdsa( + message_, + signature_, + publicKey_ + ); + }); + toc(); + }, + // two warmups to ensure full caching + { numberOfWarmups: 2, numberOfRuns: 5 } +); + +export default EcdsaBenchmark; diff --git a/benchmark/runners/simple.ts b/benchmark/runners/simple.ts new file mode 100644 index 0000000000..4d447a4c81 --- /dev/null +++ b/benchmark/runners/simple.ts @@ -0,0 +1,21 @@ +/** + * Simple benchmarks runner + * Exercises benchmarks and logs the results + * + * Run with + * ``` + * ./run benchmark/runners/simple.ts --bundle + * ``` + */ + +import { logResult } from '../benchmark.js'; +import EcdsaBenchmark from '../benchmarks/ecdsa.js'; + +// Run all benchmarks +const results = [...(await EcdsaBenchmark.run())]; + +// Process and log results +for (const result of results) { + logResult(result); + console.log('\n'); +} diff --git a/benchmark/runners/with-cloud-history.ts b/benchmark/runners/with-cloud-history.ts new file mode 100644 index 0000000000..463a3c61b3 --- /dev/null +++ b/benchmark/runners/with-cloud-history.ts @@ -0,0 +1,26 @@ +/** + * Benchmarks runner with historical data preservation in cloud storage (InfluxDB) + * + * Run with + * ``` + * ./run benchmark/runners/with-cloud-history.ts --bundle + * ``` + */ + +import { logResult } from '../benchmark.js'; +import EcdsaBenchmark from '../benchmarks/ecdsa.js'; +import { + readPreviousResultFromInfluxDb, + writeResultToInfluxDb, +} from '../utils/influxdb-utils.js'; + +// Run all benchmarks +const results = [...(await EcdsaBenchmark.run())]; + +// Process and log results +for (const result of results) { + const previousResult = await readPreviousResultFromInfluxDb(result); + logResult(result, previousResult); + writeResultToInfluxDb(result); + console.log('\n'); +} diff --git a/benchmarks/tsconfig.json b/benchmark/tsconfig.json similarity index 100% rename from benchmarks/tsconfig.json rename to benchmark/tsconfig.json diff --git a/benchmarks/types.d.ts b/benchmark/types.d.ts similarity index 100% rename from benchmarks/types.d.ts rename to benchmark/types.d.ts diff --git a/benchmark/utils/influxdb-utils.ts b/benchmark/utils/influxdb-utils.ts new file mode 100644 index 0000000000..089d39a2dd --- /dev/null +++ b/benchmark/utils/influxdb-utils.ts @@ -0,0 +1,132 @@ +/** + * InfluxDB utils + */ + +import { InfluxDB, Point } from '@influxdata/influxdb-client'; +import os from 'node:os'; +import { BenchmarkResult, calculateBounds } from '../benchmark.js'; + +const INFLUXDB_CLIENT_OPTIONS = { + url: process.env.INFLUXDB_URL, + token: process.env.INFLUXDB_TOKEN, + org: process.env.INFLUXDB_ORG, + bucket: process.env.INFLUXDB_BUCKET, +}; +const INFLUXDB_COMMON_POINT_TAGS = { + sourceEnvironment: process.env.METRICS_SOURCE_ENVIRONMENT ?? 'local', + operatingSystem: `${os.type()} ${os.release()} ${os.arch()}`, + hardware: `${os.cpus()[0].model}, ${os.cpus().length} cores, ${( + os.totalmem() / Math.pow(1024, 3) + ).toFixed(2)}Gb of RAM`, + gitBranch: process.env.GIT_BRANCH ?? 'unknown', +}; +const influxDbClient = setupInfluxDbClient(); + +function setupInfluxDbClient(): InfluxDB | undefined { + const { url, token } = INFLUXDB_CLIENT_OPTIONS; + if (url === undefined || token === undefined) { + return undefined; + } + return new InfluxDB({ url, token }); +} + +export function writeResultToInfluxDb(result: BenchmarkResult): void { + const { org, bucket } = INFLUXDB_CLIENT_OPTIONS; + if (influxDbClient && org && bucket) { + console.log('Writing result to InfluxDB.'); + const influxDbWriteClient = influxDbClient.getWriteApi(org, bucket, 'ms'); + try { + const sampleName = result.label.split('-')[1].trim(); + const { upperBound, lowerBound } = calculateBounds(result); + const point = new Point(`${result.label} - ${result.size} samples`) + .tag('benchmarkName', result.label.trim()) + .tag('sampledTimes', result.size.toString()) + .floatField('mean', result.mean) + .floatField('variance', result.variance) + .floatField(`${sampleName} - upperBound`, upperBound) + .floatField(`${sampleName} - lowerBound`, lowerBound) + .intField('size', result.size); + for (const [key, value] of Object.entries(INFLUXDB_COMMON_POINT_TAGS)) { + point.tag(key, value.trim()); + } + influxDbWriteClient.writePoint(point); + } catch (e) { + console.error('Error writing to InfluxDB: ', e); + } finally { + influxDbWriteClient.close(); + } + } else { + console.info('Skipping writing to InfluxDB: client is not configured.'); + } +} + +export function readPreviousResultFromInfluxDb( + result: BenchmarkResult +): Promise<BenchmarkResult | undefined> { + return new Promise((resolve) => { + const { org, bucket } = INFLUXDB_CLIENT_OPTIONS; + if (!influxDbClient || !org || !bucket) { + resolve(undefined); + return; + } + console.log('Querying InfluxDB for previous results.'); + const influxDbPointTags = INFLUXDB_COMMON_POINT_TAGS; + const influxDbQueryClient = influxDbClient.getQueryApi(org); + const baseBranchForComparison = + process.env.METRICS_BASE_BRANCH_FOR_COMPARISON ?? 'main'; + const fluxQuery = ` + from(bucket: "${bucket}") + |> range(start: -90d) + |> filter(fn: (r) => r.benchmarkName == "${result.label}") + |> filter(fn: (r) => r.gitBranch == "${baseBranchForComparison}") + |> filter(fn: (r) => r.sampledTimes == "${result.size}") + |> filter(fn: (r) => r.sourceEnvironment == "${influxDbPointTags.sourceEnvironment}") + |> filter(fn: (r) => r.operatingSystem == "${influxDbPointTags.operatingSystem}") + |> filter(fn: (r) => r.hardware == "${influxDbPointTags.hardware}") + |> toFloat() + |> group() + |> pivot( + rowKey:["_measurement"], + columnKey: ["_field"], + valueColumn: "_value" + ) + |> sort(desc: true) + |> limit(n:1) + `; + try { + let previousResult: BenchmarkResult | undefined = undefined; + influxDbQueryClient.queryRows(fluxQuery, { + next(row, tableMeta) { + const tableObject = tableMeta.toObject(row); + if ( + !previousResult && + tableObject._measurement && + tableObject.mean && + tableObject.variance && + tableObject.size + ) { + const measurement = tableObject._measurement; + previousResult = { + label: measurement + .substring(0, measurement.lastIndexOf('-')) + .trim(), + mean: parseFloat(tableObject.mean), + variance: parseFloat(tableObject.variance), + size: parseInt(tableObject.size, 10), + }; + } + }, + error(e) { + console.error('Error querying InfluxDB: ', e); + resolve(undefined); + }, + complete() { + resolve(previousResult); + }, + }); + } catch (e) { + console.error('Error querying InfluxDB: ', e); + resolve(undefined); + } + }); +} diff --git a/benchmarks/ecdsa.ts b/benchmarks/ecdsa.ts deleted file mode 100644 index 5bc29a9a87..0000000000 --- a/benchmarks/ecdsa.ts +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Benchmark runner example - * - * Run with - * ``` - * ./run benchmarks/ecdsa.ts --bundle - * ``` - */ -import { Provable } from 'o1js'; -import { - keccakAndEcdsa, - Secp256k1, - Ecdsa, - Bytes32, -} from '../src/examples/crypto/ecdsa/ecdsa.js'; -import { BenchmarkResult, benchmark, printResult } from './benchmark.js'; - -let privateKey = Secp256k1.Scalar.random(); -let publicKey = Secp256k1.generator.scale(privateKey); -let message = Bytes32.fromString("what's up"); -let signature = Ecdsa.sign(message.toBytes(), privateKey.toBigInt()); - -const EcdsaBenchmark = benchmark( - 'ecdsa', - async (tic, toc) => { - tic('build constraint system'); - await keccakAndEcdsa.analyzeMethods(); - toc(); - - tic('witness generation'); - await Provable.runAndCheck(() => { - let message_ = Provable.witness(Bytes32.provable, () => message); - let signature_ = Provable.witness(Ecdsa.provable, () => signature); - let publicKey_ = Provable.witness(Secp256k1.provable, () => publicKey); - keccakAndEcdsa.rawMethods.verifyEcdsa(message_, signature_, publicKey_); - }); - toc(); - }, - // two warmups to ensure full caching - { numberOfWarmups: 2, numberOfRuns: 5 } -); - -// mock: load previous results - -let previousResults: BenchmarkResult[] = [ - { - label: 'ecdsa - build constraint system', - mean: 3103.639612600001, - variance: 72678.9751211293, - size: 5, - }, - { - label: 'ecdsa - witness generation', - mean: 2062.8708897999995, - variance: 13973.913943626918, - size: 5, - }, -]; - -// run benchmark - -let results = await EcdsaBenchmark.run(); - -// example for how to log results -// criterion-style comparison of result to previous one, check significant improvement - -for (let i = 0; i < results.length; i++) { - let result = results[i]; - let previous = previousResults[i]; - printResult(result, previous); -} diff --git a/package-lock.json b/package-lock.json index 585bd562cf..e94b7fdc43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "snarky-run": "src/build/run.js" }, "devDependencies": { + "@influxdata/influxdb-client": "^1.33.2", "@noble/hashes": "^1.3.2", "@playwright/test": "^1.25.2", "@types/isomorphic-fetch": "^0.0.36", @@ -815,6 +816,12 @@ "integrity": "sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==", "dev": true }, + "node_modules/@influxdata/influxdb-client": { + "version": "1.33.2", + "resolved": "https://registry.npmjs.org/@influxdata/influxdb-client/-/influxdb-client-1.33.2.tgz", + "integrity": "sha512-RT5SxH+grHAazo/YK3UTuWK/frPWRM0N7vkrCUyqVprDgQzlLP+bSK4ak2Jv3QVF/pazTnsxWjvtKZdwskV5Xw==", + "dev": true + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", diff --git a/package.json b/package.json index 18c56fa056..93fdc88923 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "build:update-bindings": "./src/bindings/scripts/update-o1js-bindings.sh", "build:wasm": "./src/bindings/scripts/update-wasm-and-types.sh", "build:web": "rimraf ./dist/web && node src/build/build-web.js", - "build:examples": "npm run build && rimraf ./dist/examples && npx tsc -p tsconfig.examples.json && npx tsc -p benchmarks/tsconfig.json", + "build:examples": "npm run build && rimraf ./dist/examples && npx tsc -p tsconfig.examples.json && npx tsc -p benchmark/tsconfig.json", "build:docs": "npx typedoc --tsconfig ./tsconfig.web.json", "prepublish:web": "NODE_ENV=production node src/build/build-web.js", "prepublish:node": "node src/build/copy-artifacts.js && rimraf ./dist/node && npx tsc -p tsconfig.node.json && node src/build/copy-to-dist.js && NODE_ENV=production node src/build/build-node.js", @@ -69,6 +69,7 @@ }, "author": "O(1) Labs", "devDependencies": { + "@influxdata/influxdb-client": "^1.33.2", "@noble/hashes": "^1.3.2", "@playwright/test": "^1.25.2", "@types/isomorphic-fetch": "^0.0.36", diff --git a/run-ci-benchmarks.sh b/run-ci-benchmarks.sh new file mode 100644 index 0000000000..927df47fab --- /dev/null +++ b/run-ci-benchmarks.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e + +echo "" +echo "Running o1js benchmarks." +echo "" + +./run benchmark/runners/with-cloud-history.ts --bundle >>benchmarks.log 2>&1