Get actions running with published results

.github/workflows/ci-reports.yaml

@@ -0,0 +1,113 @@
+name: "PR Reports"
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+
+permissions:
+  actions: read
+  checks: write
+  contents: read
+  pull-requests: write
+
+jobs:
+  report:
+    name: "Report"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Find workflow run
+        id: find-run
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        uses: octokit/[email protected]
+        with:
+          route: GET /repos/${{ github.event.workflow_run.repository.full_name }}/actions/runs/${{ github.event.workflow_run.id }}
+
+      - name: Extract PR number
+        id: get-pr
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          head_branch=$(echo '${{ steps.find-run.outputs.data }}' | jq -r '.head_branch')
+          head_sha=$(echo '${{ steps.find-run.outputs.data }}' | jq -r '.head_sha')
+          head_repo=$(echo '${{ steps.find-run.outputs.data }}' | jq -r '.head_repository.full_name')
+
+          if [[ "${head_branch}" != "" && "${head_repo}" != "" ]]; then
+            pr_number="$(gh pr view -R "${head_repo}" "${head_branch}" --json number -q '.number' || true)"
+          else
+            pr_number=""
+          fi
+
+          if [[ "${pr_number}" != "" ]]; then
+            echo "This workflow run was for PR #${pr_number}."
+          else
+            echo "This workflow run was not for a PR."
+          fi
+
+          echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT"
+          echo "head_branch=${head_branch}" >> "$GITHUB_OUTPUT"
+          echo "head_sha=${head_sha}" >> "$GITHUB_OUTPUT"
+          echo "head_repo=${head_repo}" >> "$GITHUB_OUTPUT"
+
+      - name: Download code coverage reports
+        if: steps.get-pr.outputs.pr_number != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: codecov-reports
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: reports/
+
+      - name: Download performance reports
+        if: steps.get-pr.outputs.pr_number != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: perf-reports
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: reports/
+
+      - name: Download test results
+        if: steps.get-pr.outputs.pr_number != ''
+        uses: actions/download-artifact@v4
+        with:
+          name: test-reports
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: reports/
+
+      - name: Show published reports
+        if: steps.get-pr.outputs.pr_number != ''
+        run: |
+          ls -lR reports/
+
+      - name: "Synthesize event file"
+        run: |
+          echo \
+            '{
+               "pull_request": {
+                 "head": {
+                   "sha": "${{ steps.get-pr.outputs.head_sha }}",
+                   "repo": {
+                     "full_name": "${{ steps.get-pr.outputs.head_repo }}"
+                   }
+                 }
+               }
+             }' >event-file.json
+
+      - name: "Publish test results"
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: event-file.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: reports/test-results-*.xml
+
+      - name: "Publish available .md reports to PR"
+        uses: marocchino/sticky-pull-request-comment@v2
+        if: steps.get-pr.outputs.pr_number != ''
+        with:
+          path: reports/*.md
+          number: ${{ steps.get-pr.outputs.pr_number }}

.github/workflows/ci.yaml

@@ -19,6 +19,10 @@ env:
   CLICOLOR: 1
   CLICOLOR_FORCE: 1
 
+permissions:
+  actions: read
+  contents: read
+
 jobs:
   build:
     name: "Build"
@@ -93,12 +97,11 @@ jobs:
 
           exit ${result}
 
-      - name: "Publish test results"
-        uses: EnricoMi/publish-unit-test-result-action@v2
-        if: always()
+      - name: "Upload test results"
+        uses: actions/upload-artifact@v4
         with:
-          files: |
-            test-results-*.xml
+          name: test-reports
+          path: test-results-*.xml
 
       - name: "Generate code coverage report"
         uses: clearlyip/code-coverage-report-action@v4
@@ -108,12 +111,18 @@ jobs:
           artifact_download_workflow_names: "CI"
           filename: "codecov.xml"
 
-      - name: "Publish code coverage data to PR"
-        uses: marocchino/sticky-pull-request-comment@v2
-        if: ${{ github.actor != 'dependabot[bot]' && steps.code_coverage_report.outputs.file != '' && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') }}
+      - name: "Upload code coverage report"
+        uses: actions/upload-artifact@v4
         with:
+          name: codecov-reports
           path: code-coverage-results.md
 
+      - name: "Upload event file"
+        uses: actions/upload-artifact@v4
+        with:
+          name: event-file
+          path: ${{ github.event_path }}
+
   check:
     name: "Source code checks"
     runs-on: ubuntu-latest
@@ -190,16 +199,14 @@ jobs:
         working-directory: main
 
       - name: Compare benchmark results
-        uses: openpgpjs/github-action-pull-request-benchmark@v1
-        if: ${{ github.actor != 'dependabot[bot]' }}
+        run: |
+          ./pr/scripts/compare-benchmark-results.py -b main/benchmarks.txt -t pr/benchmarks.txt >benchmark-results.md
+
+      - name: Upload performance results
+        uses: actions/upload-artifact@v4
         with:
-          name: "Time benchmark"
-          tool: "cargo"
-          pr-benchmark-file-path: pr/benchmarks.txt
-          base-benchmark-file-path: main/benchmarks.txt
-          comment-always: true
-          alert-threshold: "130%"
-          fail-on-alert: false
-          fail-threshold: "150%"
-          # A token is needed to leave commit comments
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          name: perf-reports
+          path: |
+            pr/benchmarks.txt
+            main/benchmarks.txt
+            benchmark-results.md

.github/workflows/devcontainer.yaml

@@ -5,12 +5,40 @@ on:
       - main
   pull_request:
     paths:
-      - '.devcontainer/**'
+      - ".devcontainer/**"
 
 jobs:
   build:
-    name: "Build and publish"
+    name: "Build devcontainer"
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pre-build dev container image
+        uses: devcontainers/[email protected]
+        with:
+          imageName: ghcr.io/reubeno/brush/devcontainer
+          imageTag: latest
+          cacheFrom: ghcr.io/reubeno/brush/devcontainer
+          push: never
+
+  build_and_publish:
+    name: "Build and publish devcontainer"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4

scripts/compare-benchmark-results.py

@@ -0,0 +1,80 @@
+#!/usr/bin/python3
+import argparse
+import datetime
+import re
+from dataclasses import dataclass
+from typing import Dict
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-b", "--base-results", dest="base_results_file_path", type=str, help="Path to base results output file", required=True)
+parser.add_argument("-t", "--test-results", dest="test_results_file_path", type=str, help="Path to test results output file", required=True)
+
+args = parser.parse_args()
+
+@dataclass
+class Benchmark:
+    test_name: str
+    duration_in_ns: int
+    deviation_in_ns: int
+
+def parse_benchmarks_results(file_path: str) -> Dict[str, Benchmark]:
+    benchmarks = {}
+
+    with open(file_path, "r") as file:
+        for line in file.readlines():
+            match = re.match(r"test (.*) \.\.\. bench: +(\d+) ns/iter \(\+/- (\d+)\)", line.strip())
+            if match:
+                benchmark = Benchmark(
+                    test_name=match.group(1),
+                    duration_in_ns=int(match.group(2)),
+                    deviation_in_ns=int(match.group(3))
+                )
+
+                benchmarks[benchmark.test_name] = benchmark
+
+    return benchmarks
+
+base_results = parse_benchmarks_results(args.base_results_file_path)
+test_results = parse_benchmarks_results(args.test_results_file_path)
+
+base_test_names = set(base_results.keys())
+test_test_names = set(test_results.keys())
+
+removed_from_base = base_test_names - test_test_names
+added_by_test = test_test_names - base_test_names
+common = base_test_names & test_test_names
+
+print("# Performance Benchmark Report")
+
+if common:
+    print(f"| {'Benchmark name':36} | {'Baseline (ns)':>13} | {'Test/PR (ns)':>13} | {'Delta (ns)':>13} | {'Delta %'} |")
+    print(f"| {'-' * 36} | {'-' * 13} | {'-' * 13} | {'-' * 13} | {'-' * 7}")
+    for name in sorted(common):
+        base_duration = base_results[name].duration_in_ns
+        test_duration = test_results[name].duration_in_ns
+
+        delta_duration = test_duration - base_duration
+        delta_str = str(delta_duration)
+        if delta_duration > 0:
+            delta_str = "+" + delta_str
+
+        delta_percentage = (100.0 * delta_duration) / base_duration
+        delta_percentage_str = f"{delta_percentage:.2f}%"
+        if delta_percentage < 0:
+            delta_percentage_str = "🟢 " + delta_percentage_str
+        elif delta_percentage > 0:
+            delta_percentage_str = "🟠 +" + delta_percentage_str
+        else:
+            delta_percentage_str = "⚪  " + delta_percentage_str
+
+        print(f"| `{name:36}` | `{base_duration:10} ns` | `{test_duration:10} ns` | `{delta_str:>10} ns` | `{delta_percentage_str:>7}` |")
+
+if removed_from_base:
+    print("Benchmarks removed:")
+    for name in removed_from_base:
+        print(f"  - {name}")
+
+if added_by_test:
+    print("Benchmarks added:")
+    for name in added_by_test:
+        print(f"  - {name}")