perf-to-benchmark-json.mjs

// @ts-check
/**
 * Convert the raw perf-benchmark result JSON files (written by the
 * ``perf`` Playwright project under ``tests/benchmark/results/``) into the
 * flat ``customSmallerIsBetter`` format consumed by
 * ``benchmark-action/github-action-benchmark``.
 *
 * Two output files are produced so the on-demand ``perf`` workflow can
 * track them with different alerting policies:
 *
 *   - ``bench-determinist.json`` — memory / payload-size metrics. These
 *     are deterministic with respect to the code (they do not depend on
 *     wall-clock timing) and therefore make trustworthy regression
 *     gates: a real increase means a genuine memory/serialisation
 *     regression. The workflow alerts (and fails on PRs) on these.
 *
 *   - ``bench-timings.json`` — trend-only metrics. Millisecond timings
 *     vary run-to-run on shared CI runners, and a few memory metrics
 *     (e.g. RAM-mode peak heap growth) are allocator-dependent rather
 *     than code-deterministic; both are tracked here with a wide
 *     threshold and never fail the workflow.
 *
 * For each benchmark the most recent ``<prefix>_<timestamp>.json`` file
 * is used (the timestamp embedded in the filename sorts lexicographically).
 *
 * Usage:
 *   node scripts/perf-to-benchmark-json.mjs [resultsDir] [outDir]
 *
 * Both arguments default to ``tests/benchmark/results``.
 */
import { readdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const here = dirname(fileURLToPath(import.meta.url));
const repoRoot = join(here, "..");
const resultsDir =
  process.argv[2] || join(repoRoot, "tests", "benchmark", "results");
const outDir = process.argv[3] || resultsDir;

/** @typedef {{ name: string, unit: string, value: number }} Metric */

/**
 * Mean of a numeric array (0 when empty).
 * @param {number[]} xs
 * @returns {number}
 */
function mean(xs) {
  if (!xs || xs.length === 0) return 0;
  return xs.reduce((a, b) => a + b, 0) / xs.length;
}

const MiB = 1024 * 1024;

/**
 * Latest ``<prefix>*.json`` file in ``resultsDir`` (by filename sort), or
 * ``null`` when none is present.
 * @param {string} prefix
 * @returns {{ file: string, data: any } | null}
 */
function latest(prefix) {
  if (!existsSync(resultsDir)) return null;
  const matches = readdirSync(resultsDir)
    .filter((f) => f.startsWith(prefix) && f.endsWith(".json"))
    .sort();
  if (matches.length === 0) return null;
  const file = join(resultsDir, matches[matches.length - 1]);
  return { file, data: JSON.parse(readFileSync(file, "utf8")) };
}

const determinist = /** @type {Metric[]} */ ([]);
const timings = /** @type {Metric[]} */ ([]);
const used = /** @type {string[]} */ ([]);

/**
 * Human-readable config label for one benchmark result row.
 * @param {{ side: number, n: number, dtype: string }} r
 * @returns {string}
 */
function cfgLabel(r) {
  return `${r.side}² ×${r.n} ${r.dtype}`;
}

// --- image_perf -----------------------------------------------------
{
  const got = latest("image_perf_");
  if (got) {
    used.push(got.file);
    const r = got.data.result;
    // Deterministic: actual typed-array bytes in the batched fetch.
    determinist.push({
      name: `image_perf · payload (${r.imageCount} imgs)`,
      unit: "MB",
      value: +(r.payloadBytes / 1e6).toFixed(3),
    });
    // Timings (trend only).
    timings.push({
      name: "image_perf · multi-select → grid",
      unit: "ms",
      value: +r.multiSelectToGridMs.toFixed(1),
    });
    timings.push({
      name: "image_perf · getImagesData (×4)",
      unit: "ms",
      value: +r.getImagesData.totalMs.toFixed(1),
    });
    timings.push({
      name: "image_perf · single Plotly render",
      unit: "ms",
      value: +r.singlePlotlyRenderMs.toFixed(1),
    });
    timings.push({
      name: "image_perf · grid React commit",
      unit: "ms",
      value: +r.multiSelectToCommitMs.toFixed(1),
    });
    timings.push({
      name: "image_perf · grid canvas paint",
      unit: "ms",
      value: +r.commitToCanvasPaintMs.toFixed(1),
    });
  }
}

// --- signal_perf ----------------------------------------------------
{
  const got = latest("signal_perf_");
  if (got) {
    used.push(got.file);
    for (const r of got.data.results) {
      const label = `${r.size} pts`;
      determinist.push({
        name: `signal_perf · payload [${label}]`,
        unit: "MB",
        value: +(r.fetch.payloadBytes / 1e6).toFixed(3),
      });
      determinist.push({
        name: `signal_perf · Plotly points [${label}]`,
        unit: "points",
        value: r.plottedPoints,
      });
      timings.push({
        name: `signal_perf · select → visible [${label}]`,
        unit: "ms",
        value: +r.selectToVisibleMs.toFixed(1),
      });
      // Small transfers complete below the useful timing floor of a shared
      // runner (the 10k samples span 1–17 ms in one run). Keep them in the
      // raw artifact, but only publish stable large-transfer trends. This is
      // the median of five post-warmup transfers; the distinct name also
      // prevents comparison with the historical single cold sample.
      if (r.size >= 500_000) {
        timings.push({
          name: `signal_perf · binary fetch median [${label}]`,
          unit: "ms",
          value: +r.fetch.totalMs.toFixed(1),
        });
      }
      timings.push({
        name: `signal_perf · longest task [${label}]`,
        unit: "ms",
        value: +r.longestTaskMs.toFixed(1),
      });
      for (const [interaction, value] of [
        ["hover", r.hoverMs],
        ["pan", r.panMs],
        ["zoom", r.zoomMs],
      ]) {
        if (typeof value === "number") {
          timings.push({
            name: `signal_perf · ${interaction} [${label}]`,
            unit: "ms",
            value: +value.toFixed(1),
          });
        }
      }
    }
  }
}

// --- opfs_storage (RAM vs disk) -------------------------------------
{
  const got = latest("opfs_storage_");
  if (got) {
    used.push(got.file);
    for (const r of got.data.results) {
      const label = cfgLabel(r);
      const diskGrowth = (r.disk.createPeak - r.disk.base) / MiB;
      const ramGrowth = (r.ram.createPeak - r.ram.base) / MiB;
      // Deterministic headline: on-disk peak heap growth must stay near
      // zero. A regression (disk mode no longer keeping the heap flat)
      // shows up as this number rising — exactly what smaller-is-better
      // alerting catches.
      determinist.push({
        name: `opfs_storage · disk Δheap [${label}]`,
        unit: "MiB",
        value: +diskGrowth.toFixed(1),
      });
      // RAM-mode peak growth holds the full working set on purpose, so it is
      // expected to be large. It is also allocator-dependent: the Pyodide WASM
      // linear memory grows in coarse chunks and never shrinks, so unrelated
      // changes (e.g. an extra package loaded at init shifting the baseline
      // arena) move it by 2× on a single sample without any real leak. It is
      // therefore tracked as a trend-only metric, never as a regression gate —
      // the deterministic guard is the `disk Δheap` above.
      timings.push({
        name: `opfs_storage · ram Δheap [${label}]`,
        unit: "MiB",
        value: +ramGrowth.toFixed(1),
      });
      timings.push({
        name: `opfs_storage · disk add [${label}]`,
        unit: "ms",
        value: +mean(r.disk.createMs).toFixed(1),
      });
      timings.push({
        name: `opfs_storage · disk read [${label}]`,
        unit: "ms",
        value: +mean(r.disk.readMs).toFixed(1),
      });
    }
  }
}

// --- opfs_worker (async vs sync) ------------------------------------
{
  const got = latest("opfs_worker_");
  if (got) {
    used.push(got.file);
    for (const r of got.data.results) {
      const label = cfgLabel(r);
      determinist.push({
        name: `opfs_worker · async Δheap [${label}]`,
        unit: "MiB",
        value: +((r.async.createPeak - r.async.base) / MiB).toFixed(1),
      });
      determinist.push({
        name: `opfs_worker · sync Δheap [${label}]`,
        unit: "MiB",
        value: +((r.sync.createPeak - r.sync.base) / MiB).toFixed(1),
      });
      timings.push({
        name: `opfs_worker · async add [${label}]`,
        unit: "ms",
        value: +mean(r.async.createMs).toFixed(1),
      });
      timings.push({
        name: `opfs_worker · sync add [${label}]`,
        unit: "ms",
        value: +mean(r.sync.createMs).toFixed(1),
      });
    }
  }
}

// --- provenance_capsule_perf (report version 1) ----------------------
{
  const got = latest("provenance_capsule_perf_");
  if (got) {
    used.push(got.file);
    /** @type {Record<string, any>} */
    const sizeMetrics = {
      "workspace.save": ["hdf5_bytes", "workspace HDF5", 1e6, "MB"],
      "capsule.export": ["archive_bytes", "capsule archive", 1e6, "MB"],
      "ledger.export": ["archive_bytes", "ledger capsule archive", 1e6, "MB"],
    };
    const timed = new Set([
      "process.gaussian_filter",
      "workspace.save",
      "workspace.reopen",
      "consult.describe",
      "consult.check-data",
      "consult.verify",
      "capsule.export",
      "capsule.inspect",
      "capsule.open-empty",
      "capsule.open-over-current",
      "ledger.describe-last",
      "ledger.replay-list",
    ]);
    for (const c of got.data.cases) {
      if (c.status !== "ok") continue;
      const mode = `${c.mode.execution.replace("browser-", "")}/${c.mode.storage}`;
      const label = `[${got.data.profile} ${mode}]`;
      // Deterministic: structural sizes of what is saved and exported. The
      // manifest also records dates and versions, so it is not tracked here.
      const size = sizeMetrics[c.id];
      if (size && c.sizes && typeof c.sizes[size[0]] === "number") {
        determinist.push({
          name: `provenance · ${size[1]} ${label}`,
          unit: size[3],
          value: +(c.sizes[size[0]] / size[2]).toFixed(3),
        });
      }
      // Trend only: durations and WebAssembly heap growth depend on the
      // runner and on the allocator.
      if (timed.has(c.id) && typeof c.median_ms === "number") {
        timings.push({
          name: `provenance · ${c.id} ${label}`,
          unit: "ms",
          value: +c.median_ms.toFixed(1),
        });
      }
      const growth = c.memory?.wasm_heap?.growth_bytes;
      if (c.id === "capsule.open-empty" && typeof growth === "number") {
        timings.push({
          name: `provenance · capsule.open-empty Δheap ${label}`,
          unit: "MiB",
          value: +(growth / MiB).toFixed(1),
        });
      }
    }
  }
}

if (used.length === 0) {
  console.error(
    `[perf-to-benchmark] no result files found in ${resultsDir} — did the perf project run?`,
  );
  process.exit(1);
}

const determPath = join(outDir, "bench-determinist.json");
const timingsPath = join(outDir, "bench-timings.json");
writeFileSync(determPath, JSON.stringify(determinist, null, 2));
writeFileSync(timingsPath, JSON.stringify(timings, null, 2));

console.log(`[perf-to-benchmark] sources:\n  ${used.join("\n  ")}`);
console.log(
  `[perf-to-benchmark] wrote ${determinist.length} deterministic metric(s) → ${determPath}`,
);
console.log(
  `[perf-to-benchmark] wrote ${timings.length} timing metric(s) → ${timingsPath}`,
);