Confidence interval around the data.

o1-labs · Mar 14, 2024 · 6465c4e · 6465c4e
1 parent 546e112
commit 6465c4e
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/benchmark/base-benchmark.ts b/benchmark/base-benchmark.ts
@@ -3,7 +3,14 @@
  */
 
 import jStat from 'jstat';
-export { Benchmark, BenchmarkResult, benchmark, logResult, pValue };
+export {
+  Benchmark,
+  BenchmarkResult,
+  benchmark,
+  calculateBounds,
+  logResult,
+  pValue,
+};
 
 type BenchmarkResult = {
   label: string;
@@ -158,3 +165,10 @@ function pValue(sample1: BenchmarkResult, sample2: BenchmarkResult): number {
   const pValue = 2 * (1 - jStat.studentt.cdf(Math.abs(tStatistic), df));
   return pValue;
 }
+
+function calculateBounds(result: BenchmarkResult) {
+  const percentage = (Math.sqrt(result.variance) / result.mean) * 100;
+  const upperBound = result.mean + (result.mean * percentage) / 100;
+  const lowerBound = result.mean - (result.mean * percentage) / 100;
+  return { upperBound, lowerBound };
+}
diff --git a/benchmark/utils/influxdb-utils.ts b/benchmark/utils/influxdb-utils.ts
@@ -4,7 +4,7 @@
 
 import { InfluxDB, Point } from '@influxdata/influxdb-client';
 import os from 'node:os';
-import { BenchmarkResult } from '../base-benchmark.js';
+import { BenchmarkResult, calculateBounds } from '../base-benchmark.js';
 
 const INFLUXDB_CLIENT_OPTIONS = {
   url: process.env.INFLUXDB_URL,
@@ -36,11 +36,15 @@ export function writeResultToInfluxDb(result: BenchmarkResult): void {
     console.log('Writing result to InfluxDB.');
     const influxDbWriteClient = influxDbClient.getWriteApi(org, bucket, 'ms');
     try {
+      const sampleName = result.label.split('-')[1].trim();
+      const { upperBound, lowerBound } = calculateBounds(result);
       const point = new Point(`${result.label} - ${result.size} samples`)
         .tag('benchmarkName', result.label.trim())
         .tag('sampledTimes', result.size.toString())
         .floatField('mean', result.mean)
         .floatField('variance', result.variance)
+        .floatField(`${sampleName} - upperBound`, upperBound)
+        .floatField(`${sampleName} - lowerBound`, lowerBound)
         .intField('size', result.size);
       for (const [key, value] of Object.entries(INFLUXDB_COMMON_POINT_TAGS)) {
         point.tag(key, value.trim());