diff --git a/index.js b/index.js
index 9f5d168..fe231aa 100644
--- a/index.js
+++ b/index.js
@@ -26,6 +26,7 @@ import shell from 'shelljs';
import { runBenchmark, getBrowserVersion } from "./testrunner/runBenchmark.js";
import { runStartupBenchmark } from "./testrunner/runStartup.js";
import { processResults, processStartupResults } from "./testrunner/processResults.js";
+import { compareBenchmarks, formatComparisonMarkdown } from "./testrunner/compareResults.js";
import { writeResults } from "./testrunner/writeResults.js";
import { getJavaScriptBundleSize } from "./testrunner/totalBundlefileSize.js";
@@ -47,6 +48,48 @@ if (args.includes('--results') || args.includes('-r')) {
*/
const startupOnlyFlag = args.includes('--startup-only');
+/**
+ * Parses a `--name value` or `--name=value` CLI argument.
+ * @param {string} name - Argument name without leading dashes.
+ * @returns {string|undefined} The argument value, if present.
+ */
+const getArgValue = (name) => {
+ const flag = `--${name}`;
+ for (let i = 0; i < args.length; i++) {
+ if (args[i] === flag && i + 1 < args.length && !args[i + 1].startsWith('--')) {
+ return args[i + 1];
+ }
+ if (args[i].startsWith(`${flag}=`)) {
+ return args[i].slice(flag.length + 1);
+ }
+ }
+ return undefined;
+};
+
+/**
+ * Compare mode: `node index.js --compare --base
--current
+ * [--threshold ] [--strict] [--startup-only]`.
+ * Compares two already-built `dist/` bundles on the same machine/browser.
+ */
+const compareFlag = args.includes('--compare');
+const compareBase = getArgValue('base');
+const compareCurrent = getArgValue('current');
+const compareThresholdRaw = getArgValue('threshold');
+const compareThreshold = compareThresholdRaw === undefined ? 10 : parseFloat(compareThresholdRaw);
+const compareStrict = args.includes('--strict');
+
+// Fail fast before launching a browser when compare args are invalid.
+if (compareFlag) {
+ if (!compareBase || !compareCurrent) {
+ console.error('Compare mode requires --base and --current , e.g. node index.js --compare --base renderer-base --current renderer-pr [--threshold 10] [--strict] [--startup-only]');
+ process.exit(1);
+ }
+ if (!Number.isFinite(compareThreshold) || compareThreshold < 0) {
+ console.error(`Invalid --threshold '${compareThresholdRaw}', expected a non-negative number (percent)`);
+ process.exit(1);
+ }
+}
+
/**
* Checks if the './dist' directory exists.
* If it doesn't exist, it logs an error message and exits the process.
@@ -348,5 +391,127 @@ const writeOfficialResults = (filename) => {
fs.writeFileSync(resultsJsonPath, JSON.stringify(resultsJson, null, 2));
}
+/**
+ * Runs the full benchmark pipeline for a single already-built `dist/` bundle.
+ * Shared by normal and `--compare` runs so both variants are measured identically.
+ * @param {string} dir - The dist subdirectory to benchmark.
+ * @returns {Promise<{result: Object, memoryResult: Object, startupResult: Object, fileSize: number}>}
+ */
+const runOne = async (dir) => {
+ let result = null;
+ let memoryResult = null;
+
+ if (!startupOnlyFlag) {
+ console.log(`Running benchmark for ${dir}`);
+ result = await runBenchmark(`http://localhost:8080/${dir}/`);
+
+ console.log(`Running memory benchmark for ${dir}`);
+ memoryResult = await runBenchmark(`http://localhost:8080/${dir}/#memory`);
+ } else {
+ console.log(`Skipping throughput and memory benchmarks for ${dir} (--startup-only)`);
+ }
+
+ console.log(`Running startup benchmark for ${dir}`);
+ const startupResult = await runStartupBenchmark(`http://localhost:8080/${dir}/`);
+
+ console.log(`Getting total size of the javascript bundle for ${dir}`);
+ const fileSize = getJavaScriptBundleSize(`./dist/${dir}`);
+
+ return { result, memoryResult, startupResult, fileSize };
+};
+
+/**
+ * Compare mode entrypoint: benchmark `--base` and `--current` back-to-back on
+ * the same browser, then fail the process when the candidate regresses beyond
+ * `--threshold` percent. Intended for release/version-bump PRs in renderer/FTL:
+ * build the base bundle (e.g. `main`) and the PR bundle into two `dist/` dirs,
+ * then `node index.js --compare --base renderer-base --current renderer-pr`.
+ */
+const runCompare = async (base, current) => {
+ if (!Number.isFinite(compareThreshold) || compareThreshold < 0) {
+ console.error(`Invalid --threshold '${compareThresholdRaw}', expected a non-negative number (percent)`);
+ process.exit(1);
+ }
+
+ for (const dir of [base, current]) {
+ if (!fs.existsSync(`./dist/${dir}`)) {
+ console.error(`Compare target './dist/${dir}' does not exist. Build both bundles first (e.g. copy dist/renderer to dist/${dir}).`);
+ process.exit(1);
+ }
+ }
+
+ const compareVersions = {
+ [base]: lookupFrameworkVersion(base),
+ [current]: lookupFrameworkVersion(current),
+ };
+
+ const compareResults = {};
+ const compareMemory = {};
+ const compareFileSize = {};
+ const compareStartup = {};
+
+ // Sequential on the same machine/browser setup keeps the comparison fair.
+ for (const dir of [base, current]) {
+ const { result, memoryResult, startupResult, fileSize } = await runOne(dir);
+ if (result !== null) compareResults[dir] = result;
+ if (memoryResult !== null) compareMemory[dir] = memoryResult;
+ compareStartup[dir] = startupResult;
+ compareFileSize[dir] = fileSize;
+ }
+
+ let processed;
+ if (startupOnlyFlag) {
+ processed = processStartupResults(compareStartup, compareFileSize);
+ } else {
+ processed = processResults(compareResults, compareMemory, compareFileSize, compareStartup);
+ }
+ console.log('Processed compare results:');
+ console.log(processed);
+
+ const comparison = compareBenchmarks({
+ results: compareResults,
+ memoryResults: compareMemory,
+ startupResults: compareStartup,
+ fileSizeResults: compareFileSize,
+ processedResults: processed,
+ base,
+ current,
+ thresholdPct: compareThreshold,
+ strict: compareStrict,
+ startupOnly: startupOnlyFlag,
+ });
+
+ console.log(`Compare ${base} vs ${current} (threshold +${compareThreshold}%${compareStrict ? ', strict' : ''}):`);
+ console.log(comparison);
+
+ const markdown = formatComparisonMarkdown(comparison, {
+ browserVersion,
+ frameworkVersions: compareVersions,
+ });
+ console.log(markdown);
+
+ if (!fs.existsSync('./results')) {
+ fs.mkdirSync('./results');
+ }
+ const stem = `compare_${base}_vs_${current}`;
+ fs.writeFileSync(`./results/${stem}.json`, JSON.stringify({ ...comparison, browserVersion, frameworkVersions: compareVersions }, null, 2));
+ fs.writeFileSync(`./results/${stem}.md`, markdown);
+ console.log(`Comparison written to ./results/${stem}.json and ./results/${stem}.md`);
+
+ // Also emit the standard side-by-side HTML report for visual inspection.
+ const htmlFile = await writeResults(processed, compareVersions, browserVersion);
+ console.log(`Side-by-side HTML report: ./results/${htmlFile}`);
+
+ if (comparison.regressed) {
+ console.error(`❌ REGRESSION: ${current} regressed vs ${base} beyond +${compareThreshold}%`);
+ process.exitCode = 1;
+ } else {
+ console.log(`✅ NO REGRESSION: ${current} is within +${compareThreshold}% of ${base}`);
+ }
+};
-run(dirs[0]);
\ No newline at end of file
+if (compareFlag) {
+ await runCompare(compareBase, compareCurrent);
+} else {
+ run(dirs[0]);
+}
\ No newline at end of file
diff --git a/package.json b/package.json
index 325f9a4..cfb9c0e 100644
--- a/package.json
+++ b/package.json
@@ -10,7 +10,8 @@
"setup": "npm install && npm run build",
"setup:playwright": "npx playwright install",
"start": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js\"",
- "start:official": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js --results\""
+ "start:official": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js --results\"",
+ "compare": "node index.js --compare"
},
"author": "",
"license": "MIT",
diff --git a/readme.md b/readme.md
index 424c3f1..df2e4e3 100644
--- a/readme.md
+++ b/readme.md
@@ -26,6 +26,29 @@ To run only the startup benchmark (skips the throughput and memory benchmarks, u
Tune the number of cold loads per framework with `STARTUP_RUNS` (default 7) and force headless Chromium with `HEADLESS=true`.
+## Compare mode (regression check for renderer / FTL release PRs)
+
+Compares two already-built `dist/` bundles back-to-back on the same machine/browser and fails when the candidate regresses beyond a threshold. Intended to run infrequently (e.g. version bumps / release PRs), since the full throughput suite is slow.
+
+`npx http-server -c-1 ./dist/` (terminal 1)
+
+`node index.js --compare --base renderer-base --current renderer-pr --threshold 10` (terminal 2)
+
+- `--base ` / `--current `: required, must exist under `dist/`. Build them first, e.g. build `main` into `dist/renderer-base` and the PR into `dist/renderer-pr` (for FTL: `ftl-base` / `ftl-pr`).
+- `--threshold `: allowed % slowdown, default `10`. Gates on overall `avgMean`, startup average, and bundle size.
+- `--strict`: also fail when any single op (or memory heap/time) exceeds the threshold. Default reports per-op regressions without failing.
+- `--startup-only`: only compare cold startup + bundle size (much faster, more stable — best for frequent CI).
+- Writes `results/compare__vs_.json` (machine-readable artifact), `results/compare__vs_.md` (paste into a PR comment or CI step summary), plus the standard side-by-side HTML report. Exit code is `1` on regression, `0` otherwise.
+
+Example for a renderer release PR (run once, same job/machine):
+
+```
+# base from main, current from the PR
+cp -r dist/renderer dist/renderer-base
+# ... rebuild dist/renderer-pr from the PR source ...
+node index.js --compare --base renderer-base --current renderer-pr --threshold 10
+```
+
## Why not use js-framework-benchmark
Two reasons:
diff --git a/testrunner/compareResults.js b/testrunner/compareResults.js
new file mode 100644
index 0000000..d25a018
--- /dev/null
+++ b/testrunner/compareResults.js
@@ -0,0 +1,287 @@
+/*
+ * Copyright 2024 Comcast Cable Communications Management, LLC
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * Pure comparison logic for `--compare` runs.
+ * No browser or filesystem access here so the math stays unit-testable.
+ * All I/O (running benchmarks, writing JSON/markdown) lives in `index.js`.
+ * @module compareResults
+ */
+
+/**
+ * Throughput ops measured by `runBenchmark` (in-page `Done!` payload).
+ */
+export const COMPARE_OPS = [
+ 'create',
+ 'update',
+ 'skipNth',
+ 'select',
+ 'swap',
+ 'remove',
+ 'createLots',
+ 'append',
+ 'clear',
+];
+
+/**
+ * Parses values like `40.00ms ±5.00`, `40.00`, `58.67` or `40` into ms.
+ * Returns NaN when unparseable (e.g. missing/failed runs reported as -1 stay -1).
+ * @param {number|string} value
+ * @returns {number}
+ */
+export const parseMs = (value) => {
+ if (typeof value === 'number') return value;
+ if (typeof value !== 'string') return NaN;
+ return parseFloat(value);
+};
+
+/**
+ * Percent change of current vs base.
+ * @param {number} base
+ * @param {number} current
+ * @returns {{delta: number, pct: number}}
+ */
+export const deltaPct = (base, current) => {
+ if (!Number.isFinite(base) || !Number.isFinite(current) || base <= 0) {
+ return { delta: NaN, pct: NaN };
+ }
+ const delta = current - base;
+ return { delta, pct: (delta / base) * 100 };
+};
+
+/**
+ * Compares one metric against the regression threshold.
+ * Positive pct above threshold = regression. Improvements never fail.
+ * @param {number} base
+ * @param {number} current
+ * @param {number} thresholdPct
+ * @returns {{base: number, current: number, delta: number, pct: number, regressed: boolean}}
+ */
+const compareMetric = (base, current, thresholdPct) => {
+ const { delta, pct } = deltaPct(base, current);
+ return {
+ base,
+ current,
+ delta,
+ pct,
+ regressed: Number.isFinite(pct) && pct > thresholdPct,
+ };
+};
+
+/**
+ * Compares base vs current benchmark runs.
+ *
+ * Gating (overall `regressed`):
+ * - full mode: `avgMean` OR `startup` OR `fileSize` exceeding threshold fails.
+ * - startup-only mode: `startup` OR `fileSize` exceeding threshold fails.
+ * - per-op / memory regressions are reported always, but only fail with `strict: true`
+ * (memory heap/time stays advisory unless strict, it is too noisy for shared CI runners).
+ *
+ * @param {Object} input
+ * @param {import('./types/results.js').Results} input.results - Raw throughput results per dir.
+ * @param {import('./types/memoryResults.js').MemoryResults} input.memoryResults - Raw memory results per dir.
+ * @param {import('./types/startupResults.js').StartupResults} input.startupResults - Raw startup results per dir.
+ * @param {import('./types/fileSizeResults.js').FileSizeResults} input.fileSizeResults - Raw bundle bytes per dir.
+ * @param {Object} input.processedResults - Output of `processResults`/`processStartupResults` for [base, current].
+ * @param {string} input.base - Dist dir of the baseline (e.g. `renderer-base`).
+ * @param {string} input.current - Dist dir of the candidate (e.g. `renderer-pr`).
+ * @param {number} [input.thresholdPct=10] - Allowed % slowdown before failing.
+ * @param {boolean} [input.strict=false] - Fail on any single op/memory regression too.
+ * @param {boolean} [input.startupOnly=false] - Only startup + fileSize were measured.
+ * @returns {Object} Comparison with per-metric details and overall `regressed` flag.
+ */
+export const compareBenchmarks = ({
+ results = {},
+ memoryResults = {},
+ startupResults = {},
+ fileSizeResults = {},
+ processedResults = {},
+ base,
+ current,
+ thresholdPct = 10,
+ strict = false,
+ startupOnly = false,
+}) => {
+ if (!base) throw new Error('base is required');
+ if (!current) throw new Error('current is required');
+
+ const ops = {};
+ const regressedOps = [];
+
+ if (!startupOnly) {
+ for (const op of COMPARE_OPS) {
+ const baseMs = parseMs(results[base]?.[op]);
+ const currentMs = parseMs(results[current]?.[op]);
+ // Skip ops that failed on either side (missing or -1); they cannot be compared.
+ if (!Number.isFinite(baseMs) || !Number.isFinite(currentMs) || baseMs < 0 || currentMs < 0) {
+ ops[op] = { base: baseMs, current: currentMs, delta: NaN, pct: NaN, regressed: false, skipped: true };
+ continue;
+ }
+ const compared = compareMetric(baseMs, currentMs, thresholdPct);
+ ops[op] = { ...compared, skipped: false };
+ if (compared.regressed) regressedOps.push(op);
+ }
+ }
+
+ // avgMean is dimensionless (mean of weighted means, ~1.0); compare directly.
+ let avgMean = null;
+ if (!startupOnly) {
+ const baseAvg = parseFloat(processedResults[base]?.avgMean);
+ const currentAvg = parseFloat(processedResults[current]?.avgMean);
+ if (Number.isFinite(baseAvg) && Number.isFinite(currentAvg)) {
+ avgMean = compareMetric(baseAvg, currentAvg, thresholdPct);
+ } else {
+ avgMean = { base: baseAvg, current: currentAvg, delta: NaN, pct: NaN, regressed: false, skipped: true };
+ }
+ }
+
+ // Startup: prefer the raw numeric average over parsing the display string.
+ const baseStartupRaw = startupResults[base];
+ const currentStartupRaw = startupResults[current];
+ const baseStartupMs = baseStartupRaw ? (Number.isFinite(baseStartupRaw.average) ? baseStartupRaw.average : parseMs(baseStartupRaw.startup)) : NaN;
+ const currentStartupMs = currentStartupRaw ? (Number.isFinite(currentStartupRaw.average) ? currentStartupRaw.average : parseMs(currentStartupRaw.startup)) : NaN;
+ let startup = null;
+ if (Number.isFinite(baseStartupMs) && Number.isFinite(currentStartupMs) && baseStartupMs >= 0 && currentStartupMs >= 0) {
+ startup = compareMetric(baseStartupMs, currentStartupMs, thresholdPct);
+ } else {
+ startup = { base: baseStartupMs, current: currentStartupMs, delta: NaN, pct: NaN, regressed: false, skipped: true };
+ }
+
+ // Bundle size in bytes (raw); displayed as KB elsewhere.
+ const baseBytes = fileSizeResults[base];
+ const currentBytes = fileSizeResults[current];
+ let fileSize = null;
+ if (Number.isFinite(baseBytes) && Number.isFinite(currentBytes)) {
+ fileSize = compareMetric(baseBytes, currentBytes, thresholdPct);
+ } else {
+ fileSize = { base: baseBytes, current: currentBytes, delta: NaN, pct: NaN, regressed: false, skipped: true };
+ }
+
+ // Memory is advisory-only unless strict (shared CI runners make heap noisy).
+ let memory = null;
+ const baseMem = memoryResults[base];
+ const currentMem = memoryResults[current];
+ if (!startupOnly && baseMem && currentMem) {
+ const baseHeap = parseFloat(baseMem.heap);
+ const currentHeap = parseFloat(currentMem.heap);
+ const baseTime = parseMs(baseMem.create);
+ const currentTime = parseMs(currentMem.create);
+ memory = {
+ heap: Number.isFinite(baseHeap) && Number.isFinite(currentHeap)
+ ? compareMetric(baseHeap, currentHeap, thresholdPct)
+ : { base: baseHeap, current: currentHeap, delta: NaN, pct: NaN, regressed: false, skipped: true },
+ time: Number.isFinite(baseTime) && Number.isFinite(currentTime) && baseTime >= 0 && currentTime >= 0
+ ? compareMetric(baseTime, currentTime, thresholdPct)
+ : { base: baseTime, current: currentTime, delta: NaN, pct: NaN, regressed: false, skipped: true },
+ };
+ }
+
+ const memoryRegressed = Boolean(
+ strict && memory && (!memory.heap.skipped && !memory.time.skipped) &&
+ (memory.heap.regressed || memory.time.regressed)
+ );
+
+ const regressed = Boolean(
+ (avgMean && !avgMean.skipped && avgMean.regressed) ||
+ (startup && !startup.skipped && startup.regressed) ||
+ (fileSize && !fileSize.skipped && fileSize.regressed) ||
+ (strict && regressedOps.length > 0) ||
+ memoryRegressed
+ );
+
+ return {
+ base,
+ current,
+ thresholdPct,
+ strict,
+ startupOnly,
+ ops,
+ regressedOps,
+ avgMean,
+ startup,
+ fileSize,
+ memory,
+ regressed,
+ };
+};
+
+/**
+ * Formats a ms delta/pct cell for markdown.
+ * @param {{delta: number, pct: number, skipped?: boolean}} metric
+ * @param {string} unit
+ * @returns {string}
+ */
+const formatChange = (metric, unit = 'ms') => {
+ if (!metric || metric.skipped || !Number.isFinite(metric.pct)) return 'n/a';
+ const sign = metric.delta > 0 ? '+' : '';
+ return `${sign}${metric.delta.toFixed(2)}${unit} (${sign}${metric.pct.toFixed(2)}%)`;
+};
+
+/**
+ * Renders a comparison as Markdown for PR comments / CI step summaries.
+ * @param {Object} comparison - Output of `compareBenchmarks`.
+ * @param {Object} [meta]
+ * @param {string} [meta.browserVersion]
+ * @param {Object} [meta.frameworkVersions]
+ * @returns {string}
+ */
+export const formatComparisonMarkdown = (comparison, meta = {}) => {
+ const { base, current, thresholdPct, strict, startupOnly, ops, avgMean, startup, fileSize, memory, regressed } = comparison;
+ const lines = [];
+ lines.push(`# Benchmark comparison: \`${base}\` vs \`${current}\``);
+ lines.push('');
+ lines.push(`Threshold: +${thresholdPct}% ${strict ? '(strict: any op fails)' : '(gates on overall/startup/bundle)'}` + (startupOnly ? ' · startup-only' : ''));
+ if (meta.browserVersion) lines.push(`Browser: ${meta.browserVersion}`);
+ if (meta.frameworkVersions) {
+ lines.push(`Versions: \`${base}\` ${meta.frameworkVersions[base] ?? ''} vs \`${current}\` ${meta.frameworkVersions[current] ?? ''}`);
+ }
+ lines.push('');
+ lines.push(`## Verdict: ${regressed ? '❌ REGRESSION' : '✅ NO REGRESSION'}`);
+ lines.push('');
+
+ if (!startupOnly && avgMean && !avgMean.skipped) {
+ lines.push(`- Overall (avgMean): ${avgMean.base.toFixed(2)} → ${avgMean.current.toFixed(2)} ${formatChange(avgMean, '')} ${avgMean.regressed ? '❌' : '✅'}`);
+ }
+ if (startup && !startup.skipped) {
+ lines.push(`- Startup (avg): ${startup.base.toFixed(2)}ms → ${startup.current.toFixed(2)}ms (${formatChange(startup)}) ${startup.regressed ? '❌' : '✅'}`);
+ } else {
+ lines.push(`- Startup: n/a (a side failed to report)`);
+ }
+ if (fileSize && !fileSize.skipped) {
+ lines.push(`- Bundle: ${(fileSize.base / 1024).toFixed(2)}KB → ${(fileSize.current / 1024).toFixed(2)}KB (${formatChange({ ...fileSize, delta: fileSize.delta / 1024, }, 'KB')}) ${fileSize.regressed ? '❌' : '✅'}`);
+ }
+ if (memory && (!memory.heap.skipped || !memory.time.skipped)) {
+ const heap = memory.heap.skipped ? 'n/a' : `${memory.heap.base}MB → ${memory.heap.current}MB (${formatChange(memory.heap, 'MB')})`;
+ lines.push(`- Memory heap: ${heap}${strict && memory.heap.regressed ? ' ❌' : ''}`);
+ }
+ lines.push('');
+
+ if (!startupOnly) {
+ lines.push('| Op | Base | Current | Change | Status |');
+ lines.push('| --- | --- | --- | --- | --- |');
+ for (const op of COMPARE_OPS) {
+ const m = ops[op];
+ if (!m || m.skipped) {
+ lines.push(`| ${op} | n/a | n/a | n/a | ⚠️ skipped |`);
+ continue;
+ }
+ lines.push(`| ${op} | ${m.base.toFixed(2)}ms | ${m.current.toFixed(2)}ms | ${formatChange(m)} | ${m.regressed ? '❌' : '✅'} |`);
+ }
+ }
+
+ return lines.join('\n') + '\n';
+};