Improve benchmark reporting

.github/workflows/benchmarks.yml

@@ -0,0 +1,102 @@
+name: benchmarks
+
+on: [pull_request]
+
+jobs:
+  size:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 50
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '20.x'
+    - name: 'Setup temporary files'
+      run: |
+        echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV
+        echo "PATCH_JSON=$(mktemp)" >> $GITHUB_ENV
+    - name: 'Benchmark base'
+      run: |
+        git checkout -f ${{ github.event.pull_request.base.sha }}
+        npm install --loglevel error
+        if npm run benchmark:size -- -o ${{ env.BASE_JSON }}; then
+          echo "Ran successfully on base branch"
+        else
+          echo "{}" > ${{ env.BASE_JSON }}  # Empty JSON as default
+          echo "Benchmark script not found on base branch, using default values"
+        fi
+    - name: 'Benchmark patch'
+      run: |
+        git checkout -f ${{ github.event.pull_request.head.sha }}
+        npm install --loglevel error
+        npm run benchmark:size -- -o ${{ env.PATCH_JSON }}
+        echo "Ran successfully on patch branch"
+    - name: 'Collect results'
+      id: collect
+      run: |
+        echo "table<<EOF" >> $GITHUB_OUTPUT
+        npm run benchmark:compare -- ${{ env.BASE_JSON }} ${{ env.PATCH_JSON }} >> markdown
+        cat markdown >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+    - name: 'Post comment'
+      uses: edumserrano/find-create-or-update-comment@v3
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        body-includes: '<!-- workflow-benchmarks-size-data -->'
+        comment-author: 'github-actions[bot]'
+        body: |
+          <!-- workflow-benchmarks-size-data -->
+          ### workflow: benchmarks/size
+          Comparison of minified (terser) and compressed (brotli) size results, measured in bytes. Smaller is better.
+          ${{ steps.collect.outputs.table }}
+        edit-mode: replace
+
+  perf:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 50
+    - uses: actions/setup-node@v4
+      with:
+        node-version: '20.x'
+    - name: 'Setup temporary files'
+      run: |
+        echo "BASE_JSON=$(mktemp)" >> $GITHUB_ENV
+        echo "PATCH_JSON=$(mktemp)" >> $GITHUB_ENV
+    - name: 'Benchmark base'
+      run: |
+        git checkout -f ${{ github.event.pull_request.base.sha }}
+        npm install --loglevel error
+        if npm run benchmark:perf -- -o ${{ env.BASE_JSON }}; then
+          echo "Ran successfully on base branch"
+        else
+          echo "{}" > ${{ env.BASE_JSON }}  # Empty JSON as default
+          echo "Benchmark script not found on base branch, using default values"
+        fi
+    - name: 'Benchmark patch'
+      run: |
+        git checkout -f ${{ github.event.pull_request.head.sha }}
+        npm install --loglevel error
+        npm run benchmark:perf -- -o ${{ env.PATCH_JSON }}
+        echo "Ran successfully on patch branch"
+    - name: 'Collect results'
+      id: collect
+      run: |
+        echo "table<<EOF" >> $GITHUB_OUTPUT
+        npm run benchmark:compare -- ${{ env.BASE_JSON }} ${{ env.PATCH_JSON }} >> markdown
+        cat markdown >> $GITHUB_OUTPUT
+        echo "EOF" >> $GITHUB_OUTPUT
+    - name: 'Post comment'
+      uses: edumserrano/find-create-or-update-comment@v3
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        body-includes: '<!-- workflow-benchmarks-perf-data -->'
+        comment-author: 'github-actions[bot]'
+        body: |
+          <!-- workflow-benchmarks-perf-data -->
+          ### workflow: benchmarks/perf
+          Comparison of performance test results, measured in operations per second. Larger is better.
+          ${{ steps.collect.outputs.table }}
+        edit-mode: replace

.gitignore

@@ -1,4 +1,4 @@
-*.log*
 coverage
 dist
+logs
 node_modules

benchmark/compare.js

@@ -0,0 +1,103 @@
+/**
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+'use strict';
+
+const fs = require('fs');
+
+function readJsonFile(filePath) {
+  try {
+    const fileContents = fs.readFileSync(filePath, 'utf8');
+    const data = JSON.parse(fileContents);
+    return data;
+  } catch (error) {
+    console.error(`Error reading file ${filePath}:`, error);
+    return null;
+  }
+}
+
+function mergeData(base, patch) {
+  const merged = {};
+  function addToMerged(data, fileIndex) {
+    Object.keys(data).forEach((key) => {
+      if (merged[key] == null) {
+        merged[key] = {};
+      }
+      Object.keys(data[key]).forEach((subKey) => {
+        if (merged[key][subKey] == null) {
+          merged[key][subKey] = {};
+        }
+        merged[key][subKey][fileIndex] = data[key][subKey];
+      });
+    });
+  }
+  if (base != null) {
+    addToMerged(base, 1);
+  }
+  if (patch != null) {
+    addToMerged(patch, 2);
+  }
+  return merged;
+}
+
+function generateComparisonData(results) {
+  const baseResult = parseInt(results[1], 10);
+  const patchResult = parseInt(results[2], 10);
+  const isValidBase = !isNaN(baseResult);
+  const isValidPatch = !isNaN(patchResult);
+  let icon = '',
+    ratioFixed = '';
+
+  if (isValidBase && isValidPatch) {
+    const ratio = patchResult / baseResult;
+    ratioFixed = ratio.toFixed(2);
+    if (ratio < 0.95 || ratio > 1.05) {
+      icon = '**!!**';
+    } else if (ratio < 1) {
+      icon = '-';
+    } else if (ratio > 1) {
+      icon = '+';
+    }
+  }
+
+  return {
+    baseResult: isValidBase ? baseResult.toLocaleString() : '',
+    patchResult: isValidPatch ? patchResult.toLocaleString() : '',
+    ratio: ratioFixed,
+    icon,
+  };
+}
+
+function generateMarkdownTable(mergedData) {
+  const rows = [];
+  rows.push('| **Results** | **Base** | **Patch** | **Ratio** |  |');
+  rows.push('| :--- | ---: | ---: | ---: | ---: |');
+  Object.keys(mergedData).forEach((suiteName) => {
+    rows.push('|  |  |  |  |');
+    rows.push(`| **${suiteName}** |  |  |  |  |`);
+    Object.keys(mergedData[suiteName]).forEach((test) => {
+      const results = mergedData[suiteName][test];
+      const { baseResult, patchResult, ratio, icon } =
+        generateComparisonData(results);
+      rows.push(
+        `| &middot; ${test} | ${baseResult} | ${patchResult} | ${ratio} | ${icon} |`
+      );
+    });
+  });
+  return rows.join('\n');
+}
+
+/**
+ * Compare up to 2 different benchmark runs
+ */
+const args = process.argv.slice(2);
+const baseResults = args[0] ? readJsonFile(args[0]) : null;
+const patchResults = args[1] ? readJsonFile(args[1]) : null;
+const mergedData = mergeData(baseResults, patchResults);
+const markdownTable = generateMarkdownTable(mergedData);
+
+console.log(markdownTable);