diff --git a/index.js b/index.js index 9f5d168..fe231aa 100644 --- a/index.js +++ b/index.js @@ -26,6 +26,7 @@ import shell from 'shelljs'; import { runBenchmark, getBrowserVersion } from "./testrunner/runBenchmark.js"; import { runStartupBenchmark } from "./testrunner/runStartup.js"; import { processResults, processStartupResults } from "./testrunner/processResults.js"; +import { compareBenchmarks, formatComparisonMarkdown } from "./testrunner/compareResults.js"; import { writeResults } from "./testrunner/writeResults.js"; import { getJavaScriptBundleSize } from "./testrunner/totalBundlefileSize.js"; @@ -47,6 +48,48 @@ if (args.includes('--results') || args.includes('-r')) { */ const startupOnlyFlag = args.includes('--startup-only'); +/** + * Parses a `--name value` or `--name=value` CLI argument. + * @param {string} name - Argument name without leading dashes. + * @returns {string|undefined} The argument value, if present. + */ +const getArgValue = (name) => { + const flag = `--${name}`; + for (let i = 0; i < args.length; i++) { + if (args[i] === flag && i + 1 < args.length && !args[i + 1].startsWith('--')) { + return args[i + 1]; + } + if (args[i].startsWith(`${flag}=`)) { + return args[i].slice(flag.length + 1); + } + } + return undefined; +}; + +/** + * Compare mode: `node index.js --compare --base --current + * [--threshold ] [--strict] [--startup-only]`. + * Compares two already-built `dist/` bundles on the same machine/browser. + */ +const compareFlag = args.includes('--compare'); +const compareBase = getArgValue('base'); +const compareCurrent = getArgValue('current'); +const compareThresholdRaw = getArgValue('threshold'); +const compareThreshold = compareThresholdRaw === undefined ? 10 : parseFloat(compareThresholdRaw); +const compareStrict = args.includes('--strict'); + +// Fail fast before launching a browser when compare args are invalid. +if (compareFlag) { + if (!compareBase || !compareCurrent) { + console.error('Compare mode requires --base and --current , e.g. node index.js --compare --base renderer-base --current renderer-pr [--threshold 10] [--strict] [--startup-only]'); + process.exit(1); + } + if (!Number.isFinite(compareThreshold) || compareThreshold < 0) { + console.error(`Invalid --threshold '${compareThresholdRaw}', expected a non-negative number (percent)`); + process.exit(1); + } +} + /** * Checks if the './dist' directory exists. * If it doesn't exist, it logs an error message and exits the process. @@ -348,5 +391,127 @@ const writeOfficialResults = (filename) => { fs.writeFileSync(resultsJsonPath, JSON.stringify(resultsJson, null, 2)); } +/** + * Runs the full benchmark pipeline for a single already-built `dist/` bundle. + * Shared by normal and `--compare` runs so both variants are measured identically. + * @param {string} dir - The dist subdirectory to benchmark. + * @returns {Promise<{result: Object, memoryResult: Object, startupResult: Object, fileSize: number}>} + */ +const runOne = async (dir) => { + let result = null; + let memoryResult = null; + + if (!startupOnlyFlag) { + console.log(`Running benchmark for ${dir}`); + result = await runBenchmark(`http://localhost:8080/${dir}/`); + + console.log(`Running memory benchmark for ${dir}`); + memoryResult = await runBenchmark(`http://localhost:8080/${dir}/#memory`); + } else { + console.log(`Skipping throughput and memory benchmarks for ${dir} (--startup-only)`); + } + + console.log(`Running startup benchmark for ${dir}`); + const startupResult = await runStartupBenchmark(`http://localhost:8080/${dir}/`); + + console.log(`Getting total size of the javascript bundle for ${dir}`); + const fileSize = getJavaScriptBundleSize(`./dist/${dir}`); + + return { result, memoryResult, startupResult, fileSize }; +}; + +/** + * Compare mode entrypoint: benchmark `--base` and `--current` back-to-back on + * the same browser, then fail the process when the candidate regresses beyond + * `--threshold` percent. Intended for release/version-bump PRs in renderer/FTL: + * build the base bundle (e.g. `main`) and the PR bundle into two `dist/` dirs, + * then `node index.js --compare --base renderer-base --current renderer-pr`. + */ +const runCompare = async (base, current) => { + if (!Number.isFinite(compareThreshold) || compareThreshold < 0) { + console.error(`Invalid --threshold '${compareThresholdRaw}', expected a non-negative number (percent)`); + process.exit(1); + } + + for (const dir of [base, current]) { + if (!fs.existsSync(`./dist/${dir}`)) { + console.error(`Compare target './dist/${dir}' does not exist. Build both bundles first (e.g. copy dist/renderer to dist/${dir}).`); + process.exit(1); + } + } + + const compareVersions = { + [base]: lookupFrameworkVersion(base), + [current]: lookupFrameworkVersion(current), + }; + + const compareResults = {}; + const compareMemory = {}; + const compareFileSize = {}; + const compareStartup = {}; + + // Sequential on the same machine/browser setup keeps the comparison fair. + for (const dir of [base, current]) { + const { result, memoryResult, startupResult, fileSize } = await runOne(dir); + if (result !== null) compareResults[dir] = result; + if (memoryResult !== null) compareMemory[dir] = memoryResult; + compareStartup[dir] = startupResult; + compareFileSize[dir] = fileSize; + } + + let processed; + if (startupOnlyFlag) { + processed = processStartupResults(compareStartup, compareFileSize); + } else { + processed = processResults(compareResults, compareMemory, compareFileSize, compareStartup); + } + console.log('Processed compare results:'); + console.log(processed); + + const comparison = compareBenchmarks({ + results: compareResults, + memoryResults: compareMemory, + startupResults: compareStartup, + fileSizeResults: compareFileSize, + processedResults: processed, + base, + current, + thresholdPct: compareThreshold, + strict: compareStrict, + startupOnly: startupOnlyFlag, + }); + + console.log(`Compare ${base} vs ${current} (threshold +${compareThreshold}%${compareStrict ? ', strict' : ''}):`); + console.log(comparison); + + const markdown = formatComparisonMarkdown(comparison, { + browserVersion, + frameworkVersions: compareVersions, + }); + console.log(markdown); + + if (!fs.existsSync('./results')) { + fs.mkdirSync('./results'); + } + const stem = `compare_${base}_vs_${current}`; + fs.writeFileSync(`./results/${stem}.json`, JSON.stringify({ ...comparison, browserVersion, frameworkVersions: compareVersions }, null, 2)); + fs.writeFileSync(`./results/${stem}.md`, markdown); + console.log(`Comparison written to ./results/${stem}.json and ./results/${stem}.md`); + + // Also emit the standard side-by-side HTML report for visual inspection. + const htmlFile = await writeResults(processed, compareVersions, browserVersion); + console.log(`Side-by-side HTML report: ./results/${htmlFile}`); + + if (comparison.regressed) { + console.error(`❌ REGRESSION: ${current} regressed vs ${base} beyond +${compareThreshold}%`); + process.exitCode = 1; + } else { + console.log(`✅ NO REGRESSION: ${current} is within +${compareThreshold}% of ${base}`); + } +}; -run(dirs[0]); \ No newline at end of file +if (compareFlag) { + await runCompare(compareBase, compareCurrent); +} else { + run(dirs[0]); +} \ No newline at end of file diff --git a/package.json b/package.json index 325f9a4..cfb9c0e 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "setup": "npm install && npm run build", "setup:playwright": "npx playwright install", "start": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js\"", - "start:official": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js --results\"" + "start:official": "concurrently --kill-others --success last \"npx http-server -c-1 ./dist/\" \"node index.js --results\"", + "compare": "node index.js --compare" }, "author": "", "license": "MIT", diff --git a/readme.md b/readme.md index 424c3f1..df2e4e3 100644 --- a/readme.md +++ b/readme.md @@ -26,6 +26,29 @@ To run only the startup benchmark (skips the throughput and memory benchmarks, u Tune the number of cold loads per framework with `STARTUP_RUNS` (default 7) and force headless Chromium with `HEADLESS=true`. +## Compare mode (regression check for renderer / FTL release PRs) + +Compares two already-built `dist/` bundles back-to-back on the same machine/browser and fails when the candidate regresses beyond a threshold. Intended to run infrequently (e.g. version bumps / release PRs), since the full throughput suite is slow. + +`npx http-server -c-1 ./dist/` (terminal 1) + +`node index.js --compare --base renderer-base --current renderer-pr --threshold 10` (terminal 2) + +- `--base ` / `--current `: required, must exist under `dist/`. Build them first, e.g. build `main` into `dist/renderer-base` and the PR into `dist/renderer-pr` (for FTL: `ftl-base` / `ftl-pr`). +- `--threshold `: allowed % slowdown, default `10`. Gates on overall `avgMean`, startup average, and bundle size. +- `--strict`: also fail when any single op (or memory heap/time) exceeds the threshold. Default reports per-op regressions without failing. +- `--startup-only`: only compare cold startup + bundle size (much faster, more stable — best for frequent CI). +- Writes `results/compare__vs_.json` (machine-readable artifact), `results/compare__vs_.md` (paste into a PR comment or CI step summary), plus the standard side-by-side HTML report. Exit code is `1` on regression, `0` otherwise. + +Example for a renderer release PR (run once, same job/machine): + +``` +# base from main, current from the PR +cp -r dist/renderer dist/renderer-base +# ... rebuild dist/renderer-pr from the PR source ... +node index.js --compare --base renderer-base --current renderer-pr --threshold 10 +``` + ## Why not use js-framework-benchmark Two reasons: diff --git a/testrunner/compareResults.js b/testrunner/compareResults.js new file mode 100644 index 0000000..d25a018 --- /dev/null +++ b/testrunner/compareResults.js @@ -0,0 +1,287 @@ +/* + * Copyright 2024 Comcast Cable Communications Management, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Pure comparison logic for `--compare` runs. + * No browser or filesystem access here so the math stays unit-testable. + * All I/O (running benchmarks, writing JSON/markdown) lives in `index.js`. + * @module compareResults + */ + +/** + * Throughput ops measured by `runBenchmark` (in-page `Done!` payload). + */ +export const COMPARE_OPS = [ + 'create', + 'update', + 'skipNth', + 'select', + 'swap', + 'remove', + 'createLots', + 'append', + 'clear', +]; + +/** + * Parses values like `40.00ms ±5.00`, `40.00`, `58.67` or `40` into ms. + * Returns NaN when unparseable (e.g. missing/failed runs reported as -1 stay -1). + * @param {number|string} value + * @returns {number} + */ +export const parseMs = (value) => { + if (typeof value === 'number') return value; + if (typeof value !== 'string') return NaN; + return parseFloat(value); +}; + +/** + * Percent change of current vs base. + * @param {number} base + * @param {number} current + * @returns {{delta: number, pct: number}} + */ +export const deltaPct = (base, current) => { + if (!Number.isFinite(base) || !Number.isFinite(current) || base <= 0) { + return { delta: NaN, pct: NaN }; + } + const delta = current - base; + return { delta, pct: (delta / base) * 100 }; +}; + +/** + * Compares one metric against the regression threshold. + * Positive pct above threshold = regression. Improvements never fail. + * @param {number} base + * @param {number} current + * @param {number} thresholdPct + * @returns {{base: number, current: number, delta: number, pct: number, regressed: boolean}} + */ +const compareMetric = (base, current, thresholdPct) => { + const { delta, pct } = deltaPct(base, current); + return { + base, + current, + delta, + pct, + regressed: Number.isFinite(pct) && pct > thresholdPct, + }; +}; + +/** + * Compares base vs current benchmark runs. + * + * Gating (overall `regressed`): + * - full mode: `avgMean` OR `startup` OR `fileSize` exceeding threshold fails. + * - startup-only mode: `startup` OR `fileSize` exceeding threshold fails. + * - per-op / memory regressions are reported always, but only fail with `strict: true` + * (memory heap/time stays advisory unless strict, it is too noisy for shared CI runners). + * + * @param {Object} input + * @param {import('./types/results.js').Results} input.results - Raw throughput results per dir. + * @param {import('./types/memoryResults.js').MemoryResults} input.memoryResults - Raw memory results per dir. + * @param {import('./types/startupResults.js').StartupResults} input.startupResults - Raw startup results per dir. + * @param {import('./types/fileSizeResults.js').FileSizeResults} input.fileSizeResults - Raw bundle bytes per dir. + * @param {Object} input.processedResults - Output of `processResults`/`processStartupResults` for [base, current]. + * @param {string} input.base - Dist dir of the baseline (e.g. `renderer-base`). + * @param {string} input.current - Dist dir of the candidate (e.g. `renderer-pr`). + * @param {number} [input.thresholdPct=10] - Allowed % slowdown before failing. + * @param {boolean} [input.strict=false] - Fail on any single op/memory regression too. + * @param {boolean} [input.startupOnly=false] - Only startup + fileSize were measured. + * @returns {Object} Comparison with per-metric details and overall `regressed` flag. + */ +export const compareBenchmarks = ({ + results = {}, + memoryResults = {}, + startupResults = {}, + fileSizeResults = {}, + processedResults = {}, + base, + current, + thresholdPct = 10, + strict = false, + startupOnly = false, +}) => { + if (!base) throw new Error('base is required'); + if (!current) throw new Error('current is required'); + + const ops = {}; + const regressedOps = []; + + if (!startupOnly) { + for (const op of COMPARE_OPS) { + const baseMs = parseMs(results[base]?.[op]); + const currentMs = parseMs(results[current]?.[op]); + // Skip ops that failed on either side (missing or -1); they cannot be compared. + if (!Number.isFinite(baseMs) || !Number.isFinite(currentMs) || baseMs < 0 || currentMs < 0) { + ops[op] = { base: baseMs, current: currentMs, delta: NaN, pct: NaN, regressed: false, skipped: true }; + continue; + } + const compared = compareMetric(baseMs, currentMs, thresholdPct); + ops[op] = { ...compared, skipped: false }; + if (compared.regressed) regressedOps.push(op); + } + } + + // avgMean is dimensionless (mean of weighted means, ~1.0); compare directly. + let avgMean = null; + if (!startupOnly) { + const baseAvg = parseFloat(processedResults[base]?.avgMean); + const currentAvg = parseFloat(processedResults[current]?.avgMean); + if (Number.isFinite(baseAvg) && Number.isFinite(currentAvg)) { + avgMean = compareMetric(baseAvg, currentAvg, thresholdPct); + } else { + avgMean = { base: baseAvg, current: currentAvg, delta: NaN, pct: NaN, regressed: false, skipped: true }; + } + } + + // Startup: prefer the raw numeric average over parsing the display string. + const baseStartupRaw = startupResults[base]; + const currentStartupRaw = startupResults[current]; + const baseStartupMs = baseStartupRaw ? (Number.isFinite(baseStartupRaw.average) ? baseStartupRaw.average : parseMs(baseStartupRaw.startup)) : NaN; + const currentStartupMs = currentStartupRaw ? (Number.isFinite(currentStartupRaw.average) ? currentStartupRaw.average : parseMs(currentStartupRaw.startup)) : NaN; + let startup = null; + if (Number.isFinite(baseStartupMs) && Number.isFinite(currentStartupMs) && baseStartupMs >= 0 && currentStartupMs >= 0) { + startup = compareMetric(baseStartupMs, currentStartupMs, thresholdPct); + } else { + startup = { base: baseStartupMs, current: currentStartupMs, delta: NaN, pct: NaN, regressed: false, skipped: true }; + } + + // Bundle size in bytes (raw); displayed as KB elsewhere. + const baseBytes = fileSizeResults[base]; + const currentBytes = fileSizeResults[current]; + let fileSize = null; + if (Number.isFinite(baseBytes) && Number.isFinite(currentBytes)) { + fileSize = compareMetric(baseBytes, currentBytes, thresholdPct); + } else { + fileSize = { base: baseBytes, current: currentBytes, delta: NaN, pct: NaN, regressed: false, skipped: true }; + } + + // Memory is advisory-only unless strict (shared CI runners make heap noisy). + let memory = null; + const baseMem = memoryResults[base]; + const currentMem = memoryResults[current]; + if (!startupOnly && baseMem && currentMem) { + const baseHeap = parseFloat(baseMem.heap); + const currentHeap = parseFloat(currentMem.heap); + const baseTime = parseMs(baseMem.create); + const currentTime = parseMs(currentMem.create); + memory = { + heap: Number.isFinite(baseHeap) && Number.isFinite(currentHeap) + ? compareMetric(baseHeap, currentHeap, thresholdPct) + : { base: baseHeap, current: currentHeap, delta: NaN, pct: NaN, regressed: false, skipped: true }, + time: Number.isFinite(baseTime) && Number.isFinite(currentTime) && baseTime >= 0 && currentTime >= 0 + ? compareMetric(baseTime, currentTime, thresholdPct) + : { base: baseTime, current: currentTime, delta: NaN, pct: NaN, regressed: false, skipped: true }, + }; + } + + const memoryRegressed = Boolean( + strict && memory && (!memory.heap.skipped && !memory.time.skipped) && + (memory.heap.regressed || memory.time.regressed) + ); + + const regressed = Boolean( + (avgMean && !avgMean.skipped && avgMean.regressed) || + (startup && !startup.skipped && startup.regressed) || + (fileSize && !fileSize.skipped && fileSize.regressed) || + (strict && regressedOps.length > 0) || + memoryRegressed + ); + + return { + base, + current, + thresholdPct, + strict, + startupOnly, + ops, + regressedOps, + avgMean, + startup, + fileSize, + memory, + regressed, + }; +}; + +/** + * Formats a ms delta/pct cell for markdown. + * @param {{delta: number, pct: number, skipped?: boolean}} metric + * @param {string} unit + * @returns {string} + */ +const formatChange = (metric, unit = 'ms') => { + if (!metric || metric.skipped || !Number.isFinite(metric.pct)) return 'n/a'; + const sign = metric.delta > 0 ? '+' : ''; + return `${sign}${metric.delta.toFixed(2)}${unit} (${sign}${metric.pct.toFixed(2)}%)`; +}; + +/** + * Renders a comparison as Markdown for PR comments / CI step summaries. + * @param {Object} comparison - Output of `compareBenchmarks`. + * @param {Object} [meta] + * @param {string} [meta.browserVersion] + * @param {Object} [meta.frameworkVersions] + * @returns {string} + */ +export const formatComparisonMarkdown = (comparison, meta = {}) => { + const { base, current, thresholdPct, strict, startupOnly, ops, avgMean, startup, fileSize, memory, regressed } = comparison; + const lines = []; + lines.push(`# Benchmark comparison: \`${base}\` vs \`${current}\``); + lines.push(''); + lines.push(`Threshold: +${thresholdPct}% ${strict ? '(strict: any op fails)' : '(gates on overall/startup/bundle)'}` + (startupOnly ? ' · startup-only' : '')); + if (meta.browserVersion) lines.push(`Browser: ${meta.browserVersion}`); + if (meta.frameworkVersions) { + lines.push(`Versions: \`${base}\` ${meta.frameworkVersions[base] ?? ''} vs \`${current}\` ${meta.frameworkVersions[current] ?? ''}`); + } + lines.push(''); + lines.push(`## Verdict: ${regressed ? '❌ REGRESSION' : '✅ NO REGRESSION'}`); + lines.push(''); + + if (!startupOnly && avgMean && !avgMean.skipped) { + lines.push(`- Overall (avgMean): ${avgMean.base.toFixed(2)} → ${avgMean.current.toFixed(2)} ${formatChange(avgMean, '')} ${avgMean.regressed ? '❌' : '✅'}`); + } + if (startup && !startup.skipped) { + lines.push(`- Startup (avg): ${startup.base.toFixed(2)}ms → ${startup.current.toFixed(2)}ms (${formatChange(startup)}) ${startup.regressed ? '❌' : '✅'}`); + } else { + lines.push(`- Startup: n/a (a side failed to report)`); + } + if (fileSize && !fileSize.skipped) { + lines.push(`- Bundle: ${(fileSize.base / 1024).toFixed(2)}KB → ${(fileSize.current / 1024).toFixed(2)}KB (${formatChange({ ...fileSize, delta: fileSize.delta / 1024, }, 'KB')}) ${fileSize.regressed ? '❌' : '✅'}`); + } + if (memory && (!memory.heap.skipped || !memory.time.skipped)) { + const heap = memory.heap.skipped ? 'n/a' : `${memory.heap.base}MB → ${memory.heap.current}MB (${formatChange(memory.heap, 'MB')})`; + lines.push(`- Memory heap: ${heap}${strict && memory.heap.regressed ? ' ❌' : ''}`); + } + lines.push(''); + + if (!startupOnly) { + lines.push('| Op | Base | Current | Change | Status |'); + lines.push('| --- | --- | --- | --- | --- |'); + for (const op of COMPARE_OPS) { + const m = ops[op]; + if (!m || m.skipped) { + lines.push(`| ${op} | n/a | n/a | n/a | ⚠️ skipped |`); + continue; + } + lines.push(`| ${op} | ${m.base.toFixed(2)}ms | ${m.current.toFixed(2)}ms | ${formatChange(m)} | ${m.regressed ? '❌' : '✅'} |`); + } + } + + return lines.join('\n') + '\n'; +};