Release 2.14.0

.github/scripts/publish-benchmark.cjs

@@ -0,0 +1,169 @@
+const { readFile } = require('node:fs/promises')
+
+const marker = '<!-- blits-benchmark-results -->'
+const branch = 'benchmark-reports'
+
+module.exports = async ({ github, context, core }) => {
+  const { comparePairs, formatComparison } = await import('../../benchmarks/comparison.js')
+  const repo = context.repo
+  const run = context.payload.workflow_run
+  let pulls = run.pull_requests
+  if (!pulls?.length) {
+    pulls = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
+      ...repo,
+      commit_sha: run.head_sha,
+    })
+  }
+  const report = JSON.parse(await readFile('benchmark-data/comparison.json', 'utf8'))
+  const prs = []
+  for (const pull of pulls) {
+    const { data: pr } = await github.rest.pulls.get({ ...repo, pull_number: pull.number })
+    if (
+      pr.state === 'open' &&
+      pr.head.sha === run.head_sha &&
+      pr.head.sha === report.candidate?.commit &&
+      pr.base.sha === report.base?.commit
+    ) {
+      prs.push(pr)
+    }
+  }
+  if (!prs.length) {
+    core.info('No open PR matches these head and base commits; skipping stale results.')
+    return
+  }
+
+  // Artifacts are untrusted data. Validate numeric inputs and escape display text.
+  const positiveInteger = (value) => Number.isSafeInteger(value) && value > 0 && value <= 100_000
+  if (
+    !positiveInteger(report.size) ||
+    !positiveInteger(report.samples) ||
+    !Number.isSafeInteger(report.warmup) ||
+    report.warmup < 0 ||
+    report.warmup > 100_000 ||
+    !Array.isArray(report.pairs) ||
+    !report.pairs.length ||
+    report.pairs.length > 100
+  ) {
+    throw new Error('Invalid benchmark configuration')
+  }
+  for (const pair of report.pairs) {
+    if (
+      !Array.isArray(pair.order) ||
+      pair.order.length !== 2 ||
+      [...pair.order].sort().join(',') !== 'base,candidate'
+    ) {
+      throw new Error('Invalid benchmark execution order')
+    }
+    for (const name of ['base', 'candidate']) {
+      for (const metric of ['medianGenerationSyncMs', 'medianTotalToIdleMs']) {
+        if (!Number.isFinite(pair[name]?.[metric]) || pair[name][metric] < 0) {
+          throw new Error('Invalid benchmark timing')
+        }
+      }
+    }
+  }
+  const text = (value) =>
+    String(value)
+      .slice(0, 120)
+      .replace(/[\\`*_[\]<>|#\r\n]/g, ' ')
+  report.title = text(report.title)
+  report.browserVersion = text(report.browserVersion)
+  report.cpuSlowdown = 20
+  report.candidate.dirty = false
+  report.comparison = comparePairs(report.pairs)
+  if (
+    report.comparison.some(
+      (row) =>
+        !Number.isFinite(row.differenceMs) ||
+        (row.changePercent !== null && !Number.isFinite(row.changePercent))
+    )
+  ) {
+    throw new Error('Invalid benchmark comparison')
+  }
+
+  const path = `runs/${run.id}-${run.run_attempt || 1}`
+  const markdown = `${formatComparison(report)}\n[JSON data](comparison.json) · [Workflow run](${run.html_url})\n`
+  let parent
+  let baseTree
+  try {
+    const { data: ref } = await github.rest.git.getRef({ ...repo, ref: `heads/${branch}` })
+    parent = ref.object.sha
+    const { data: commit } = await github.rest.git.getCommit({ ...repo, commit_sha: parent })
+    baseTree = commit.tree.sha
+  } catch (error) {
+    if (error.status !== 404) throw error
+  }
+  const { data: tree } = await github.rest.git.createTree({
+    ...repo,
+    ...(baseTree ? { base_tree: baseTree } : {}),
+    tree: [
+      { path: `${path}/report.md`, mode: '100644', type: 'blob', content: markdown },
+      {
+        path: `${path}/comparison.json`,
+        mode: '100644',
+        type: 'blob',
+        content: JSON.stringify(report, null, 2),
+      },
+    ],
+  })
+  const { data: commit } = await github.rest.git.createCommit({
+    ...repo,
+    message: `Added benchmark report for run ${run.id}.`,
+    tree: tree.sha,
+    parents: parent ? [parent] : [],
+  })
+  if (parent) {
+    await github.rest.git.updateRef({
+      ...repo,
+      ref: `heads/${branch}`,
+      sha: commit.sha,
+      force: false,
+    })
+  } else {
+    await github.rest.git.createRef({ ...repo, ref: `refs/heads/${branch}`, sha: commit.sha })
+  }
+
+  const url = `${context.serverUrl}/${repo.owner}/${repo.repo}/blob/${commit.sha}/${path}/report.md`
+  const change = (metric) => {
+    const value = report.comparison.find((row) => row.metric === metric).changePercent
+    return value === null ? 'n/a' : `${value > 0 ? '+' : ''}${value.toFixed(2)}%`
+  }
+  const body = [
+    marker,
+    '### Benchmark comparison',
+    '',
+    '| Timing | Change |',
+    '| --- | ---: |',
+    `| Synchronous generation | ${change('medianGenerationSyncMs')} |`,
+    `| Total time to idle | ${change('medianTotalToIdleMs')} |`,
+    '',
+    'Negative means faster. Informational results; small changes can be measurement noise.',
+    '',
+    `[Full report](${url}) · [Workflow run](${run.html_url})`,
+    '',
+    `Head: \`${report.candidate.commit.slice(0, 12)}\` · Base: \`${report.base.commit.slice(0, 12)}\``,
+  ].join('\n')
+  for (const pr of prs) {
+    // Check again in case the PR changed while publishing.
+    const { data: current } = await github.rest.pulls.get({ ...repo, pull_number: pr.number })
+    if (
+      current.state !== 'open' ||
+      current.head.sha !== report.candidate.commit ||
+      current.base.sha !== report.base.commit
+    )
+      continue
+    const comments = await github.paginate(github.rest.issues.listComments, {
+      ...repo,
+      issue_number: pr.number,
+    })
+    const existing = comments.find(
+      (comment) => comment.user.login === 'github-actions[bot]' && comment.body?.includes(marker)
+    )
+    if (existing) {
+      await github.rest.issues.updateComment({ ...repo, comment_id: existing.id, body })
+    } else {
+      await github.rest.issues.createComment({ ...repo, issue_number: pr.number, body })
+    }
+  }
+  core.info(`Published benchmark report: ${url}`)
+}

.github/scripts/publish-benchmark.test.cjs

@@ -0,0 +1,139 @@
+const assert = require('node:assert/strict')
+const { before, after, test } = require('node:test')
+const { mkdtemp, mkdir, writeFile, rm } = require('node:fs/promises')
+const { tmpdir } = require('node:os')
+const { join } = require('node:path')
+const publish = require('./publish-benchmark.cjs')
+
+const head = 'a'.repeat(40)
+const base = 'b'.repeat(40)
+const originalDirectory = process.cwd()
+let directory
+before(async () => {
+  directory = await mkdtemp(join(tmpdir(), 'benchmark-publisher-test-'))
+  process.chdir(directory)
+  await mkdir('benchmark-data')
+})
+after(async () => {
+  process.chdir(originalDirectory)
+  await rm(directory, { recursive: true, force: true })
+})
+
+const fixture = async ({ existing = false, stale = false, invalid = false } = {}) => {
+  const calls = []
+  const report = {
+    title: 'Element generation benchmark',
+    base: { commit: base },
+    candidate: { commit: head },
+    size: 1000,
+    warmup: 5,
+    samples: 20,
+    browserVersion: '156.0',
+    pairs: [
+      {
+        order: ['base', 'candidate'],
+        base: { medianGenerationSyncMs: 100, medianTotalToIdleMs: 200 },
+        candidate: { medianGenerationSyncMs: invalid ? 'bad' : 80, medianTotalToIdleMs: 220 },
+      },
+    ],
+    comparison: [{ changePercent: 999 }], // Recomputed rather than trusted.
+  }
+  await writeFile('benchmark-data/comparison.json', JSON.stringify(report))
+  const method = (name, data) => async (args) => {
+    calls.push({ name, args })
+    return { data }
+  }
+  const github = {
+    rest: {
+      pulls: {
+        get: method('pulls.get', {
+          number: 12,
+          state: 'open',
+          head: { sha: stale ? 'c'.repeat(40) : head },
+          base: { sha: base },
+        }),
+      },
+      repos: { listPullRequestsAssociatedWithCommit: () => {} },
+      issues: {
+        listComments: () => {},
+        createComment: method('createComment', {}),
+        updateComment: method('updateComment', {}),
+      },
+      git: {
+        getRef: existing
+          ? method('getRef', { object: { sha: 'parent' } })
+          : async () => {
+              throw Object.assign(new Error('missing'), { status: 404 })
+            },
+        getCommit: method('getCommit', { tree: { sha: 'previous-tree' } }),
+        createTree: method('createTree', { sha: 'tree' }),
+        createCommit: method('createCommit', { sha: 'report-commit' }),
+        createRef: method('createRef', {}),
+        updateRef: method('updateRef', {}),
+      },
+    },
+  }
+  github.paginate = async () =>
+    existing
+      ? [
+          {
+            id: 42,
+            user: { login: 'github-actions[bot]' },
+            body: '<!-- blits-benchmark-results -->',
+          },
+        ]
+      : []
+  const context = {
+    repo: { owner: 'example', repo: 'blits' },
+    serverUrl: 'https://github.com',
+    payload: {
+      workflow_run: {
+        id: 123,
+        run_attempt: 1,
+        head_sha: head,
+        pull_requests: [{ number: 12 }],
+        html_url: 'https://github.com/example/blits/actions/runs/123',
+      },
+    },
+  }
+  return { github, context, core: { info: () => {} }, calls }
+}
+
+test('creates an isolated report branch and links its immutable commit with computed percentages', async () => {
+  const mock = await fixture()
+  await publish(mock)
+  const tree = mock.calls.find((call) => call.name === 'createTree').args
+  assert.equal(tree.base_tree, undefined)
+  assert.match(tree.tree[0].content, /Synchronous generation/)
+  assert.deepEqual(mock.calls.find((call) => call.name === 'createCommit').args.parents, [])
+  assert.equal(
+    mock.calls.find((call) => call.name === 'createRef').args.ref,
+    'refs/heads/benchmark-reports'
+  )
+  const body = mock.calls.find((call) => call.name === 'createComment').args.body
+  assert.match(body, /-20\.00%/)
+  assert.match(body, /\+10\.00%/)
+  assert.match(body, /blob\/report-commit\/runs\/123-1\/report.md/)
+})
+
+test('preserves previous reports and updates the existing bot comment', async () => {
+  const mock = await fixture({ existing: true })
+  await publish(mock)
+  assert.equal(
+    mock.calls.find((call) => call.name === 'createTree').args.base_tree,
+    'previous-tree'
+  )
+  assert.deepEqual(mock.calls.find((call) => call.name === 'createCommit').args.parents, ['parent'])
+  assert.equal(mock.calls.find((call) => call.name === 'updateRef').args.force, false)
+  assert.equal(mock.calls.find((call) => call.name === 'updateComment').args.comment_id, 42)
+  assert.ok(!mock.calls.some((call) => call.name === 'createComment'))
+})
+
+test('skips stale results and rejects invalid timings before publication', async () => {
+  const stale = await fixture({ stale: true })
+  await publish(stale)
+  assert.ok(!stale.calls.some((call) => call.name === 'createTree'))
+  const invalid = await fixture({ invalid: true })
+  await assert.rejects(publish(invalid), /Invalid benchmark timing/)
+  assert.ok(!invalid.calls.some((call) => call.name === 'createTree'))
+})

.github/workflows/publish-benchmark-results.yml

@@ -0,0 +1,43 @@
+name: Publish Benchmark Results
+
+on:
+  workflow_run:
+    workflows: [Run Benchmarks]
+    types: [completed]
+
+permissions:
+  contents: write
+  actions: read
+  pull-requests: write
+
+concurrency:
+  group: publish-benchmark-reports
+  cancel-in-progress: false
+
+jobs:
+  publish:
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      # Only execute trusted default-branch code, never the PR checkout/artifact.
+      - name: Checkout report publisher
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+
+      - name: Download benchmark data
+        uses: actions/download-artifact@v8
+        with:
+          name: benchmark-results
+          path: benchmark-data
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish report and update PR comment
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const publish = require('./.github/scripts/publish-benchmark.cjs');
+            await publish({ github, context, core });