diff --git a/frameworks/blits/src/App.js b/frameworks/blits/src/App.js index 304e181..77ec37a 100644 --- a/frameworks/blits/src/App.js +++ b/frameworks/blits/src/App.js @@ -31,6 +31,10 @@ export default Blits.Application({ routes: [ { path: '/', component: Benchmark }, { path: '/memory', component: Memory }, + // Hash-routed alias so `#startup` mounts Benchmark (same as `#memory` + // mounts Memory). Benchmark.ready branches on the hash and runs the + // cold-startup measurement instead of the throughput sequence. + { path: '/startup', component: Benchmark }, ], hooks: { async ready() { diff --git a/frameworks/blits/src/pages/Benchmark.js b/frameworks/blits/src/pages/Benchmark.js index a1248fb..cfe14af 100644 --- a/frameworks/blits/src/pages/Benchmark.js +++ b/frameworks/blits/src/pages/Benchmark.js @@ -17,11 +17,13 @@ import Blits from '@lightningjs/blits' import { warmup } from '../../../../shared/utils/warmup.js' +import { reportStartup } from '../../../../shared/utils/startup.js' import { clearTest, sequence, printResults, createMany, + createStartup, updateMany, updateRandom, swapRows, @@ -56,6 +58,15 @@ export default Blits.Component('Benchmark', { }, hooks: { async ready() { + // `#startup` shares the `/` route (hash only), so branch here instead + // of adding a route: single cold load of 1,000 rows, timed from + // navigation start to first idle (see createStartup in perf.js). + const hash = window.location.hash.substring(1); + if (hash === 'startup') { + const { time } = await createStartup.call(this, 1000) + reportStartup(time) + return + } sequence([ () => this.testCreateMany(), () => this.testUpdateMany(), diff --git a/frameworks/blits/src/perf.js b/frameworks/blits/src/perf.js index e2de14a..3ae8e43 100644 --- a/frameworks/blits/src/perf.js +++ b/frameworks/blits/src/perf.js @@ -125,6 +125,32 @@ const createMany = function (amount) { }) } +const createStartup = function (amount = 1000) { + return new Promise((resolve) => { + // Cold startup: navigation start -> first idle after rendering `amount` + // rows. A startTime of 0 makes waitUntilIdle resolve with + // `performance.now()`, i.e. time since navigation start, so JS + // download/parse/execute, framework init and font load are all included. + // No clear() first: items are empty on fresh load. + waitUntilIdle(renderer, 0).then((time) => { + resolve({ time }) + }) + + const items = [] + + let x = 0 + let y = 0 + for (let i = 1; i < amount + 1; i++) { + items.push(createItem(x, y)) + + x = (i % 27) * 40 + y = ~~(i / 27) * 40 + } + + this.items = items + }) +} + const createManyWithoutText = function (amount = 20000) { return new Promise((resolve) => { clear.call(this).then(() => { @@ -242,6 +268,7 @@ export { printResults, createMany, createManyWithoutText, + createStartup, updateMany, updateRandom, swapRows, diff --git a/frameworks/ftl-canvas-bitmap/index.js b/frameworks/ftl-canvas-bitmap/index.js index d8a97ef..384ed49 100644 --- a/frameworks/ftl-canvas-bitmap/index.js +++ b/frameworks/ftl-canvas-bitmap/index.js @@ -5,6 +5,7 @@ import bitmapTextEngine from 'ftl/text/bitmap'; import { adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; @@ -297,6 +298,18 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); }; +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // waitUntilIdle resolves `performance.now() - startTime`, so a startTime of 0 + // yields time since navigation start: JS download/parse/execute, framework + // init, bitmap font load and scene build are all included. Runs after + // fontLoadedPromise resolves, same as the regular benchmark. + const startupPromise = waitUntilIdle(0); + for (let i = 0; i < 1000; i++) createRow(rootNode, i); + const time = await startupPromise; + reportStartup(time); +}; + const runBenchmark = async () => { const results = {}; @@ -354,6 +367,8 @@ const hash = window.location.hash.substring(1); fontLoadedPromise.then(() => { if (hash === 'memory') { createMemoryBenchmark(); + } else if (hash === 'startup') { + runStartupBenchmark(); } else { runBenchmark(); } diff --git a/frameworks/ftl-canvas/index.js b/frameworks/ftl-canvas/index.js index 057f5ee..d96a979 100644 --- a/frameworks/ftl-canvas/index.js +++ b/frameworks/ftl-canvas/index.js @@ -5,6 +5,7 @@ import canvasTextEngine from 'ftl/text/canvas'; import { adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; const canvas = document.querySelector('canvas') @@ -287,6 +288,17 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); }; +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // waitUntilIdle resolves `performance.now() - startTime`, so a startTime of 0 + // yields time since navigation start: JS download/parse/execute, framework + // init and scene build are all included. + const startupPromise = waitUntilIdle(0); + for (let i = 0; i < 1000; i++) createRow(rootNode, i); + const time = await startupPromise; + reportStartup(time); +}; + const runBenchmark = async () => { const results = {}; @@ -342,6 +354,8 @@ const runAndLog = async (fn, args, repeat) => { const hash = window.location.hash.substring(1); if (hash === 'memory') { createMemoryBenchmark(); +} else if (hash === 'startup') { + runStartupBenchmark(); } else { runBenchmark(); } diff --git a/frameworks/ftl-webgl-bitmap/index.js b/frameworks/ftl-webgl-bitmap/index.js index f32ba34..946e6ca 100644 --- a/frameworks/ftl-webgl-bitmap/index.js +++ b/frameworks/ftl-webgl-bitmap/index.js @@ -7,6 +7,7 @@ import { createShader } from 'ftl/shaders/create'; import { adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; const shaders = { @@ -301,6 +302,18 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); }; +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // waitUntilIdle resolves `performance.now() - startTime`, so a startTime of 0 + // yields time since navigation start: JS download/parse/execute, framework + // init, bitmap font load and scene build are all included. Runs after + // fontLoadedPromise resolves, same as the regular benchmark. + const startupPromise = waitUntilIdle(0); + for (let i = 0; i < 1000; i++) createRow(rootNode, i); + const time = await startupPromise; + reportStartup(time); +}; + const runBenchmark = async () => { const results = {}; @@ -358,6 +371,8 @@ const hash = window.location.hash.substring(1); fontLoadedPromise.then(() => { if (hash === 'memory') { createMemoryBenchmark(); + } else if (hash === 'startup') { + runStartupBenchmark(); } else { runBenchmark(); } diff --git a/frameworks/ftl/index.js b/frameworks/ftl/index.js index 420c966..64985d8 100644 --- a/frameworks/ftl/index.js +++ b/frameworks/ftl/index.js @@ -7,6 +7,7 @@ import { createShader } from 'ftl/shaders/create'; import { adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; const shaders = { @@ -304,6 +305,23 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); }; +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // waitUntilIdle resolves `performance.now() - startTime`, so a startTime of 0 + // yields time since navigation start: JS download/parse/execute, framework + // init, font atlas load and scene build are all included. + await loadFont('msdf', { + family: 'Ubuntu', + atlas: './assets/Ubuntu-Regular.msdf.png', + fontData: './assets/UbuntuRegularMsdf.json' + }) + + const startupPromise = waitUntilIdle(0); + for (let i = 0; i < 1000; i++) createRow(rootNode, i); + const time = await startupPromise; + reportStartup(time); +}; + const runBenchmark = async () => { await loadFont('msdf', { @@ -367,6 +385,8 @@ const runAndLog = async (fn, args, repeat) => { const hash = window.location.hash.substring(1); if (hash === 'memory') { createMemoryBenchmark(); +} else if (hash === 'startup') { + runStartupBenchmark(); } else { runBenchmark(); } diff --git a/frameworks/lightning2-core/index.js b/frameworks/lightning2-core/index.js index 93704cb..dfd7450 100644 --- a/frameworks/lightning2-core/index.js +++ b/frameworks/lightning2-core/index.js @@ -7,6 +7,7 @@ import Lightning from '@lightningjs/core'; import { adjectives, nouns } from '../../shared/data'; import { warmup } from '../../shared/utils/warmup'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; // apply CSS styling @@ -81,6 +82,9 @@ export class App extends Lightning.Application { if (hash === 'memory') { console.log('Running memory benchmark'); this.createMemoryBenchmark(); + } else if (hash === 'startup') { + console.log('Running startup benchmark'); + this.runStartupBenchmark(); } else { this.runBenchmark(); } @@ -376,6 +380,23 @@ export class App extends Lightning.Application { console.log('Done!', results); } + async runStartupBenchmark() { + // Cold startup: navigation start -> first stage idle after rendering + // 1,000 rows. Items is empty on fresh load; a startTime of 0 makes + // waitUntilIdle resolve with `performance.now()`, i.e. time since + // navigation start, so JS download/parse/execute and framework init + // are included. + const startupPromise = this.waitUntilIdle(0); + + const items = this.tag('Items'); + for (let i = 0; i < 1000; i++) { + this._createRow(items, i); + } + + const time = await startupPromise; + reportStartup(time); + } + async createMemoryBenchmark() { const results = {}; diff --git a/frameworks/lightning2/index.js b/frameworks/lightning2/index.js index 7d5bb7b..d2aa5cb 100644 --- a/frameworks/lightning2/index.js +++ b/frameworks/lightning2/index.js @@ -6,6 +6,7 @@ import Lightning from '@lightningjs/core'; import { adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; // apply CSS styling @@ -129,6 +130,9 @@ export class App extends Lightning.Application { if (hash === 'memory') { console.log('Running memory benchmark'); this.createMemoryBenchmark(); + } else if (hash === 'startup') { + console.log('Running startup benchmark'); + this.runStartupBenchmark(); } else { this.runBenchmark(); } @@ -383,6 +387,23 @@ export class App extends Lightning.Application { console.log('Done!', results); } + async runStartupBenchmark() { + // Cold startup: navigation start -> first stage idle after rendering + // 1,000 rows. Items is empty on fresh load; a startTime of 0 makes + // waitUntilIdle resolve with `performance.now()`, i.e. time since + // navigation start, so JS download/parse/execute and framework init + // are included. + const startupPromise = this.waitUntilIdle(0); + + const items = this.tag('Items'); + for (let i = 0; i < 1000; i++) { + this._createRow(items, i); + } + + const time = await startupPromise; + reportStartup(time); + } + async createMemoryBenchmark() { const results = {}; diff --git a/frameworks/renderer-canvasfonts/index.js b/frameworks/renderer-canvasfonts/index.js index ca7da0f..0db0075 100644 --- a/frameworks/renderer-canvasfonts/index.js +++ b/frameworks/renderer-canvasfonts/index.js @@ -22,6 +22,7 @@ import { CanvasCoreRenderer, CanvasTextRenderer } from '@lightningjs/renderer/ca import { colours, adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; import { waitUntilIdle } from '../../shared/utils/waitUntilIdle.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; const appHeight = 1080; @@ -278,6 +279,26 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); } +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // A startTime of 0 makes waitUntilIdle resolve with `performance.now()`, + // i.e. time since navigation start, so JS download/parse/execute, + // framework init and font atlas load are all included. + const startupPromise = waitUntilIdle(renderer, 0); + + for (let i = 0; i < 1000; i++) { + createRow(rootNode, { + index: i, + color: pick(colours), + textColor: pick(colours), + text: `${pick(adjectives)} ${pick(nouns)}` + }); + } + + const time = await startupPromise; + reportStartup(time); +} + const runBenchmark = async () => { const results = {}; @@ -357,6 +378,9 @@ const hash = window.location.hash.substring(1); if (hash === 'memory') { console.log('Running memory benchmark'); createMemoryBenchmark(); +} else if (hash === 'startup') { + console.log('Running startup benchmark'); + runStartupBenchmark(); } else { runBenchmark(); } \ No newline at end of file diff --git a/frameworks/renderer/index.js b/frameworks/renderer/index.js index be50a1b..5772c8c 100644 --- a/frameworks/renderer/index.js +++ b/frameworks/renderer/index.js @@ -22,6 +22,7 @@ import { WebGlCoreRenderer, SdfTextRenderer } from '@lightningjs/renderer/webgl' import { colours, adjectives, nouns } from '../../shared/data.js'; import { warmup } from '../../shared/utils/warmup.js'; import { waitUntilIdle } from '../../shared/utils/waitUntilIdle.js'; +import { reportStartup } from '../../shared/utils/startup.js'; import { run } from '../../shared/utils/run.js'; const appHeight = 1080; @@ -294,6 +295,26 @@ const createMemoryBenchmark = async () => { console.log('Memory!', results); } +const runStartupBenchmark = async () => { + // Cold startup: navigation start -> first idle after rendering 1,000 rows. + // A startTime of 0 makes waitUntilIdle resolve with `performance.now()`, + // i.e. time since navigation start, so JS download/parse/execute, + // framework init and font atlas load are all included. + const startupPromise = waitUntilIdle(renderer, 0); + + for (let i = 0; i < 1000; i++) { + createRow(rootNode, { + index: i, + color: pick(colours), + textColor: pick(colours), + text: `${pick(adjectives)} ${pick(nouns)}` + }); + } + + const time = await startupPromise; + reportStartup(time); +} + const runBenchmark = async () => { const results = {}; @@ -373,6 +394,9 @@ const hash = window.location.hash.substring(1); if (hash === 'memory') { console.log('Running memory benchmark'); createMemoryBenchmark(); +} else if (hash === 'startup') { + console.log('Running startup benchmark'); + runStartupBenchmark(); } else { runBenchmark(); } \ No newline at end of file diff --git a/index.js b/index.js index eeddf16..9f5d168 100644 --- a/index.js +++ b/index.js @@ -24,7 +24,8 @@ import fs from "fs"; import shell from 'shelljs'; import { runBenchmark, getBrowserVersion } from "./testrunner/runBenchmark.js"; -import { processResults } from "./testrunner/processResults.js"; +import { runStartupBenchmark } from "./testrunner/runStartup.js"; +import { processResults, processStartupResults } from "./testrunner/processResults.js"; import { writeResults } from "./testrunner/writeResults.js"; import { getJavaScriptBundleSize } from "./testrunner/totalBundlefileSize.js"; @@ -39,6 +40,13 @@ if (args.includes('--results') || args.includes('-r')) { writeOfficialResultsFlag = true; } +/** + * Check for --startup-only argument: run only the cold-startup benchmark + * (plus bundle sizes) and skip the throughput and memory benchmarks. + * Useful for validating the startup functionality quickly. + */ +const startupOnlyFlag = args.includes('--startup-only'); + /** * Checks if the './dist' directory exists. * If it doesn't exist, it logs an error message and exits the process. @@ -131,6 +139,7 @@ dirs.forEach( (f) => { * @typedef {import('./testrunner/types/results.js').Results} Results * @typedef {import('./testrunner/types/memoryResults.js').MemoryResults} MemoryResults * @typedef {import('./testrunner/types/fileSizeResults.js').FileSizeResults} FileSizeResults + * @typedef {import('./testrunner/types/startupResults.js').StartupResults} StartupResults * * @typedef {?Results} results */ @@ -147,6 +156,12 @@ let memoryResults = null; * @typedef {?FileSizeResults} fileSizeResults */ let fileSizeResults = null; + +/** + * The startup results object. + * @typedef {?StartupResults} startupResults +*/ +let startupResults = null; let idx = 0; /** @@ -182,17 +197,27 @@ const run = async (dir) => { if (results === null) results = {}; if (memoryResults === null) memoryResults = {}; if (fileSizeResults === null) fileSizeResults = {}; + if (startupResults === null) startupResults = {}; results[dir] = createNewResult(); - console.log(`Running benchmark for ${dir}`); - const result = await runBenchmark(`http://localhost:8080/${dir}/`); - results[dir] = result; + if (!startupOnlyFlag) { + console.log(`Running benchmark for ${dir}`); + const result = await runBenchmark(`http://localhost:8080/${dir}/`); + results[dir] = result; - // run memory benchmark - console.log(`Running memory benchmark for ${dir}`); - const memoryResult = await runBenchmark(`http://localhost:8080/${dir}/#memory`); - memoryResults[dir] = memoryResult; + // run memory benchmark + console.log(`Running memory benchmark for ${dir}`); + const memoryResult = await runBenchmark(`http://localhost:8080/${dir}/#memory`); + memoryResults[dir] = memoryResult; + } else { + console.log(`Skipping throughput and memory benchmarks for ${dir} (--startup-only)`); + } + + // run startup benchmark (cold load to first frame, fresh context per run) + console.log(`Running startup benchmark for ${dir}`); + const startupResult = await runStartupBenchmark(`http://localhost:8080/${dir}/`); + startupResults[dir] = startupResult; // get the total size of the javascript bundle console.log(`Getting total size of the javascript bundle for ${dir}`); @@ -211,10 +236,36 @@ const run = async (dir) => { console.log('Memory results:'); console.log(memoryResults); + console.log('Startup results:'); + console.log(startupResults); + console.log('File size results:'); console.log(fileSizeResults); - const processedResults = processResults(results, memoryResults, fileSizeResults); + if (startupOnlyFlag) { + const processedStartup = processStartupResults(startupResults, fileSizeResults); + console.log('Processed startup results:'); + console.log(processedStartup); + console.table(Object.fromEntries( + Object.entries(processedStartup).map(([fw, r]) => [fw, { + startup: r.startup?.time ?? 'Failed', + mean: r.startup?.mean ?? '-', + min: r.startup && r.startup.min >= 0 ? `${r.startup.min.toFixed(2)}ms` : '-', + runs: r.startup?.runs?.length ?? 0, + bundleKB: r.fileSize ?? 'n/a', + }]) + )); + + console.log('Writing results...'); + const filename = await writeResults(processedStartup, frameworkVersions, browserVersion); + + if (writeOfficialResultsFlag) { + writeOfficialResults(filename); + } + return; + } + + const processedResults = processResults(results, memoryResults, fileSizeResults, startupResults); console.log('Processed results:'); console.log(processedResults); diff --git a/readme.md b/readme.md index 7037cf7..424c3f1 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,15 @@ The following operations will be executed: - Remove row, remove 1 row (5 warm up runs) - Create many rows, create 10,000 rows (5 warmup runs) - Append rows to large table, appending 1,000 rows to a table of 1,000 rows (5 warmup runs) +- Cold startup, load the app and render 1,000 rows to the first frame (7 cold loads, fresh browser context per run, 6x CPU slowdown applied before navigation so JS fetch/parse is included; reported separately from the throughput average) + +To run only the startup benchmark (skips the throughput and memory benchmarks, useful for validating startup functionality), serve `dist/` and pass the flag directly to the runner: + +`npx http-server -c-1 ./dist/` (terminal 1) + +`node index.js --startup-only` (terminal 2, combine with `--results` to also copy to `official_results/`) + +Tune the number of cold loads per framework with `STARTUP_RUNS` (default 7) and force headless Chromium with `HEADLESS=true`. ## Why not use js-framework-benchmark diff --git a/shared/utils/startup.js b/shared/utils/startup.js new file mode 100644 index 0000000..a93bb69 --- /dev/null +++ b/shared/utils/startup.js @@ -0,0 +1,37 @@ +/* + * Copyright 2024 Comcast Cable Communications Management, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Contract string the startup benchmark logs to the console. + * The test runner (`testrunner/runStartup.js`) listens for exactly this prefix. + */ +export const startupLogPrefix = 'Startup!'; + +/** + * Report a cold-startup measurement to the console in the shape the runner expects. + * + * @param {number} time - Milliseconds since navigation start (i.e. `performance.now()` + * captured when the renderer goes idle after the first frame, which equals time + * since `performance.timeOrigin` and therefore includes JS download/parse/execute, + * framework init, font atlas load and scene build). + * @returns {{startup: string}} The logged results object. + */ +export const reportStartup = (time) => { + const results = { startup: `${time.toFixed(2)}ms` }; + console.log(startupLogPrefix, results); + return results; +} diff --git a/testrunner/processResults.js b/testrunner/processResults.js index 956114d..9b423af 100644 --- a/testrunner/processResults.js +++ b/testrunner/processResults.js @@ -4,13 +4,15 @@ * @typedef {import('./types/results.js').Results} Results * @typedef {import('./types/memoryResults.js').MemoryResults} MemoryResults * @typedef {import('./types/fileSizeResults.js').FileSizeResults} FileSizeResultsObject + * @typedef {import('./types/startupResults.js').StartupResults} StartupResultsObject * * @param {Results} results - The benchmark results object. * @param {MemoryResults} memoryResults - The memory results object. * @param {FileSizeResultsObject} fileSizeResults - The file size results object. + * @param {StartupResultsObject} [startupResults] - The startup results object. * @returns {Object} - The processed and sorted results object. */ -export const processResults = (results, memoryResults, fileSizeResults) => { +export const processResults = (results, memoryResults, fileSizeResults, startupResults = {}) => { const processedResults = {}; const fastestTimes = { create: null, @@ -93,6 +95,16 @@ export const processResults = (results, memoryResults, fileSizeResults) => { processedResults[f].fileSize = (fileSizeResults[f] / 1024).toFixed(2); }); + // add startup results (cold load to first frame, reported separately and + // deliberately excluded from avgMean). Uses a plain relative mean against + // the fastest framework rather than the Krause op weights. + const { cells: startupCells } = buildStartupCells(frameworks, startupResults); + frameworks.forEach(f => { + if (startupCells[f]) { + processedResults[f].startup = startupCells[f]; + } + }); + // sort the frameworks by avgMean const sortedFrameworks = frameworks.sort((a, b) => { return processedResults[a].avgMean - processedResults[b].avgMean; @@ -107,6 +119,107 @@ export const processResults = (results, memoryResults, fileSizeResults) => { return sortedResults; } +/** + * Processes only startup results (used by `--startup-only` runs, where no + * throughput/memory results exist). Returns per-framework startup cells plus + * bundle sizes, sorted by startup mean (fastest first). Frameworks whose + * startup failed get no `startup` cell, same as in the full report. + * + * @typedef {import('./types/fileSizeResults.js').FileSizeResults} FileSizeResultsObject + * @typedef {import('./types/startupResults.js').StartupResults} StartupResultsObject + * + * @param {StartupResultsObject} startupResults - The startup results object. + * @param {FileSizeResultsObject} [fileSizeResults] - The file size results object. + * @returns {Object} - The processed startup-only results object, in the same + * shape as `processResults` output (per-framework `startup`/`fileSize` cells). + */ +export const processStartupResults = (startupResults, fileSizeResults = {}) => { + const frameworks = Object.keys(startupResults); + const { cells } = buildStartupCells(frameworks, startupResults); + + const processedResults = {}; + frameworks.forEach(f => { + processedResults[f] = {}; + if (cells[f]) { + processedResults[f].startup = cells[f]; + } + if (fileSizeResults[f] !== undefined) { + processedResults[f].fileSize = (fileSizeResults[f] / 1024).toFixed(2); + } + }); + + // sort fastest startup first; frameworks without a startup cell go last + const sortedFrameworks = frameworks.sort((a, b) => { + const aMean = processedResults[a].startup ? parseFloat(processedResults[a].startup.mean) : Infinity; + const bMean = processedResults[b].startup ? parseFloat(processedResults[b].startup.mean) : Infinity; + return aMean - bMean; + }); + + const sortedResults = {}; + sortedFrameworks.forEach(f => { + sortedResults[f] = processedResults[f]; + }); + + return sortedResults; +} + +/** + * Builds the per-framework startup report cells shared by the full and the + * startup-only reports. Uses a plain relative mean against the fastest + * framework (no Krause op weights). + * + * @param {string[]} frameworks - Framework names. + * @param {Object} startupResults - Raw startup results per framework. + * @returns {{cells: Object, fastestStartup: (number|null)}} + */ +const buildStartupCells = (frameworks, startupResults) => { + let fastestStartup = null; + (frameworks || []).forEach(f => { + if (!startupResults[f]) { + return; + } + + const current = parseFloat(startupResults[f].startup); + if (!Number.isFinite(current) || current < 0) { + return; + } + + if (fastestStartup === null || current < fastestStartup) { + fastestStartup = current; + } + }); + + const cells = {}; + (frameworks || []).forEach(f => { + if (!startupResults[f] || fastestStartup === null) { + return; + } + + const value = parseFloat(startupResults[f].startup); + if (!Number.isFinite(value) || value < 0) { + cells[f] = { + mean: '-1', + color: computeColor(1), + time: 'Failed', + min: -1, + runs: [], + }; + return; + } + + const mean = calculateMean(value, fastestStartup); + cells[f] = { + mean: mean.toFixed(2), + color: computeColor(mean), + time: startupResults[f].startup, + min: startupResults[f].min ?? -1, + runs: startupResults[f].runs ?? [], + }; + }); + + return { cells, fastestStartup }; +} + /** * Calculates the mean value relative to a baseline value. * @param {number} value - The value to calculate the mean for. diff --git a/testrunner/runStartup.js b/testrunner/runStartup.js new file mode 100644 index 0000000..c8d6e10 --- /dev/null +++ b/testrunner/runStartup.js @@ -0,0 +1,177 @@ +/* + * Copyright 2024 Comcast Cable Communications Management, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + + +/** + * This file contains the cold-startup benchmark runner. + * @module runStartup + */ + +import { chromium } from 'playwright'; + +/** + * Number of cold loads per framework. Tunable via `STARTUP_RUNS`. + * Cold starts are noisier than warm ops, hence more runs than the usual 5. + */ +const defaultIterations = () => { + const parsed = parseInt(process.env.STARTUP_RUNS || '7', 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 7; +}; + +/** + * Runs a single cold startup measurement: fresh context, CPU throttling applied + * *before* navigation (so JS parse/compile is throttled too), then goto `#startup` + * and wait for the in-page `Startup!` console signal. + * + * @param {import('playwright').Browser} browser - Reused browser instance. + * @param {string} url - Base URL of the framework (without hash). + * @param {Object} options + * @param {number} options.cpuThrottleRate - CDP CPU throttling rate. + * @param {boolean} options.headless - Unused here, kept for signature symmetry. + * @param {number} options.timeoutPerRun - Per-iteration timeout in ms. + * @returns {Promise<{startupMs: number, wallMs: number}>} + */ +const runSingleStartup = async (browser, url, { cpuThrottleRate, timeoutPerRun }) => { + const context = await browser.newContext({ + viewport: { width: 1920, height: 1080 }, + deviceScaleFactor: 1, + isMobile: false, + hasTouch: false, + }); + + const page = await context.newPage(); + + // Apply CPU throttling BEFORE navigation so it covers JS parse/compile, + // unlike the throughput benchmark which throttles after page load. + const client = await page.context().newCDPSession(page); + await client.send('Emulation.setCPUThrottlingRate', { rate: cpuThrottleRate }); + try { + await client.send('Network.enable'); + await client.send('Network.clearBrowserCache'); + } catch { + // Non-fatal: a fresh context already starts with a cold cache. + } + + const wallStart = Date.now(); + + const resultPromise = new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Startup run timed out after ${timeoutPerRun}ms for ${url}`)); + }, timeoutPerRun); + + page.on('console', async (msg) => { + const text = msg.text(); + if (msg.type() === 'error') { + console.log('[browser error]', text); + } + + if (text.includes('Startup!')) { + clearTimeout(timer); + try { + const results = msg.args().length > 1 ? await msg.args()[1].jsonValue() : {}; + const startupMs = parseFloat(results.startup); + if (!Number.isFinite(startupMs)) { + reject(new Error(`Invalid Startup! payload: ${JSON.stringify(results)}`)); + return; + } + resolve({ startupMs, wallMs: Date.now() - wallStart }); + } catch (err) { + reject(err); + } + } + }); + }); + + try { + await page.goto(`${url}#startup`, { waitUntil: 'load' }); + const title = await page.title(); + console.log('Loaded (startup): ', title); + return await resultPromise; + } finally { + await context.close(); + } +}; + +/** + * Runs the cold-startup benchmark for one framework: N sequential cold loads. + * + * Each iteration uses a fresh browser context (cold disk/memory cache) and + * reports the in-page time (navigation start -> first idle after 1,000 rows, + * which includes JS download/parse/execute) plus a Playwright wall-clock + * cross-check. + * + * @typedef {import('./types/startupResults.js').StartupResult} StartupResult + * + * @param {string} url - The base URL of the framework (without hash). + * @param {Object} [options] + * @param {number} [options.iterations] - Cold loads to perform (default 7 / `STARTUP_RUNS`). + * @param {number} [options.cpuThrottleRate] - CDP CPU throttling rate (default 6). + * @param {boolean} [options.headless] - Run headless (default false, matches throughput benchmark; set `HEADLESS=true` to override). + * @param {number} [options.timeoutPerRun] - Per-iteration timeout in ms (default 2 min). + * @returns {Promise} Startup result; `startup` is `-1` when every iteration failed. + */ +export const runStartupBenchmark = async (url, options = {}) => { + if (!url) throw new Error('url is required'); + + const iterations = options.iterations ?? defaultIterations(); + const cpuThrottleRate = options.cpuThrottleRate ?? 6; + const headless = options.headless ?? process.env.HEADLESS === 'true'; + const timeoutPerRun = options.timeoutPerRun ?? 1000 * 60 * 2; + + const browser = await chromium.launch({ headless }); + + const runs = []; + const wallRuns = []; + + try { + for (let i = 0; i < iterations; i++) { + try { + console.log(`Startup run ${i + 1}/${iterations} for ${url}`); + const { startupMs, wallMs } = await runSingleStartup(browser, url, { + cpuThrottleRate, + timeoutPerRun, + }); + console.log(`Startup run ${i + 1}: ${startupMs.toFixed(2)}ms (wall ${wallMs}ms)`); + runs.push(startupMs); + wallRuns.push(wallMs); + } catch (err) { + console.error(`Startup run ${i + 1} failed:`, err.message); + } + } + } finally { + await browser.close(); + } + + if (runs.length === 0) { + console.error(`All startup runs failed for ${url}`); + return { startup: -1, average: -1, spread: -1, min: -1, runs: [], wallAverage: -1 }; + } + + const average = runs.reduce((acc, time) => acc + time, 0) / runs.length; + const spread = Math.max(...runs) - Math.min(...runs); + const min = Math.min(...runs); + const wallAverage = wallRuns.reduce((acc, time) => acc + time, 0) / wallRuns.length; + + return { + startup: `${average.toFixed(2)}ms ±${spread.toFixed(2)}`, + average, + spread, + min, + runs, + wallAverage, + }; +} diff --git a/testrunner/test.js b/testrunner/test.js index d5641ff..a70fd95 100644 --- a/testrunner/test.js +++ b/testrunner/test.js @@ -59,7 +59,12 @@ const test = () => { solid: 512 } - const processedResults = processResults(results, memoryResults, fileSize); + const startupResults = { + renderer: { startup: '1200.00ms ±100.00', average: 1200, spread: 100, min: 1150, runs: [1150, 1200, 1250], wallAverage: 1300 }, + solid: { startup: '1500.00ms ±120.00', average: 1500, spread: 120, min: 1440, runs: [1440, 1500, 1560], wallAverage: 1600 } + } + + const processedResults = processResults(results, memoryResults, fileSize, startupResults); console.log(JSON.stringify(processedResults)); console.log(processedResults); diff --git a/testrunner/types/startupResults.js b/testrunner/types/startupResults.js new file mode 100644 index 0000000..dc41095 --- /dev/null +++ b/testrunner/types/startupResults.js @@ -0,0 +1,19 @@ +/** + * A single cold-startup measurement for one framework: N sequential cold loads + * (fresh browser context each) from navigation start to first idle after + * rendering 1,000 rows, including JS download/parse/execute. + * + * @typedef {Object} StartupResult + * @property {string|number} startup - Mean startup time formatted as `"ms ±"`, or `-1` when every run failed. + * @property {number} average - Mean of the successful runs in milliseconds. + * @property {number} spread - Max minus min of the successful runs in milliseconds. + * @property {number} min - Fastest successful run in milliseconds. + * @property {number[]} runs - All successful per-run measurements in milliseconds. + * @property {number} wallAverage - Mean Playwright wall-clock time per run in milliseconds (cross-check). + * + * @typedef {Object} StartupResults + * @property {string} framework - The framework name. + * @property {StartupResult} results - The startup benchmark results. + */ + +export {} diff --git a/testrunner/writeResults.js b/testrunner/writeResults.js index e59f275..b106ab5 100644 --- a/testrunner/writeResults.js +++ b/testrunner/writeResults.js @@ -70,13 +70,19 @@ const header = `
const resultsHeader = `

Results

+ {{resultsDescription}} +
`; + +const opsDescription = `

Tests are run by executing an operation and waiting for the WebGL rennder to be Idle

Results are the time in milliseconds to execute the operation

-

All operations are executed on 6x CPU slowdown

-
`; +

All operations are executed on 6x CPU slowdown

`; +const startupDescription = ` +

Startup is measured from navigation start to the first idle after rendering 1,000 nodes (fresh browser context per run, 6x CPU slowdown applied before navigation, so JS fetch/parse is included)

`; -const table = ` + +const opsSection = `
@@ -159,7 +165,9 @@ const table = ` {{overall}} - + `; + +const memorySection = `
@@ -178,7 +186,30 @@ const table = ` {{rowsMemory}} - + `; + +const startupSection = ` + + + + {{frameworks}} + + + + + + {{rowsStartup}} + + `; + +const fileSizeSection = ` `).join(''); - const createRows = frameworks.map(f => ``).join(''); - const updateRows = frameworks.map(f => ``).join(''); - const partialUpdateRows = frameworks.map(f => ``).join(''); - const selectRow = frameworks.map(f => ``).join(''); - const swapRows = frameworks.map(f => ``).join(''); - const removeRow = frameworks.map(f => ``).join(''); - const createManyRows = frameworks.map(f => ``).join(''); - const appendRows = frameworks.map(f => ``).join(''); - const clearRows = frameworks.map(f => ``).join(''); - const overall = frameworks.map(f => ``).join(''); - const rowsSize = frameworks.map(f => ``).join(''); + const opCell = (f, key) => { + const entry = results[f][key]; + if (entry && entry.time !== undefined) { + return ``; + } + return ``; + }; + const createRows = frameworks.map(f => opCell(f, 'create')).join(''); + const updateRows = frameworks.map(f => opCell(f, 'update')).join(''); + const partialUpdateRows = frameworks.map(f => opCell(f, 'skipNth')).join(''); + const selectRow = frameworks.map(f => opCell(f, 'select')).join(''); + const swapRows = frameworks.map(f => opCell(f, 'swap')).join(''); + const removeRow = frameworks.map(f => opCell(f, 'remove')).join(''); + const createManyRows = frameworks.map(f => opCell(f, 'createLots')).join(''); + const appendRows = frameworks.map(f => opCell(f, 'append')).join(''); + const clearRows = frameworks.map(f => opCell(f, 'clear')).join(''); + const overall = frameworks.map(f => ``).join(''); + const rowsSize = frameworks.map(f => ``).join(''); const rowsMemory = frameworks.map(f => { if (results[f].memory && (results[f].memory.memory !== 'NaN' || results[f].memory.time !== -1)) { return `` @@ -266,24 +319,59 @@ export const writeResults = (results, frameworkVersions, browserVersion) => { return ``; } }).join(''); + const rowsStartup = frameworks.map(f => { + const startup = results[f].startup; + if (startup && startup.time && startup.time !== 'Failed') { + const detail = startup.min >= 0 && startup.runs + ? ` (min ${startup.min.toFixed(2)}ms, n=${startup.runs.length})` + : ''; + return ``; + } else { + return ``; + } + }).join(''); - fileStream.write(table - .replace('{{frameworks}}', frameworksHeader) - .replace('{{createRows}}', createRows) - .replace('{{updateRows}}', updateRows) - .replace('{{partialUpdateRows}}', partialUpdateRows) - .replace('{{selectRow}}', selectRow) - .replace('{{swapRows}}', swapRows) - .replace('{{removeRow}}', removeRow) - .replace('{{createManyRows}}', createManyRows) - .replace('{{appendRows}}', appendRows) - .replace('{{clearRows}}', clearRows) - .replace('{{overall}}', overall) - .replace('{{frameworks}}', frameworksHeader) - .replace('{{rowsMemory}}', rowsMemory) - .replace('{{frameworks}}', frameworksHeader) - .replace('{{rowsSize}}', rowsSize) - ); + // Sections are assembled into a single table so framework columns + // share widths and stay aligned across ops / memory / startup rows. + const sections = []; + if (hasOps) { + sections.push(opsSection + .replace('{{frameworks}}', frameworksHeader) + .replace('{{createRows}}', createRows) + .replace('{{updateRows}}', updateRows) + .replace('{{partialUpdateRows}}', partialUpdateRows) + .replace('{{selectRow}}', selectRow) + .replace('{{swapRows}}', swapRows) + .replace('{{removeRow}}', removeRow) + .replace('{{createManyRows}}', createManyRows) + .replace('{{appendRows}}', appendRows) + .replace('{{clearRows}}', clearRows) + .replace('{{overall}}', overall) + ); + } + + if (hasMemory) { + sections.push(memorySection + .replace('{{frameworks}}', frameworksHeader) + .replace('{{rowsMemory}}', rowsMemory) + ); + } + + if (hasStartup) { + sections.push(startupSection + .replace('{{frameworks}}', frameworksHeader) + .replace('{{rowsStartup}}', rowsStartup) + ); + } + + if (hasFileSize) { + sections.push(fileSizeSection + .replace('{{frameworks}}', frameworksHeader) + .replace('{{rowsSize}}', rowsSize) + ); + } + + fileStream.write(table.replace('{{sections}}', sections.join('\n'))); // write the footer fileStream.write(indexFooter); @@ -296,4 +384,5 @@ export const writeResults = (results, frameworkVersions, browserVersion) => { resolve(filename); }); }); -} \ No newline at end of file +} +
+ Name +
+ Startup for... +
+ cold startup +
cold load to first frame of 1,000 nodes. Includes JS fetch/parse, font atlas and first idle. Fresh context per run.
+
@@ -197,7 +228,13 @@ const table = ` {{rowsSize}} - + `; + +// Single table: all sections share the same column widths, so framework +// columns stay aligned across ops / memory / startup / file-size rows. +// Only sections with data are included. +const table = `
+{{sections}}
`; const createFile = (filename) => { @@ -210,7 +247,9 @@ const createFile = (filename) => { } /** - * Writes benchmark results to an HTML file. + * Writes benchmark results to a single HTML file. + * Only sections with data are rendered, so both full runs and + * `--startup-only` runs produce the same `results_*.html` file. * @param {Object} results - The benchmark results. * @param {Object} frameworkVersions - The versions of the frameworks used in the benchmark. * @param {string} browserVersion - The version of the browser used for the benchmark. @@ -240,25 +279,39 @@ export const writeResults = (results, frameworkVersions, browserVersion) => { .replace('{{browserVersion}}', browserVersion) ); - // write the results header - fileStream.write(resultsHeader); - const frameworks = Object.keys(results); - const frameworkNamesAndVersions = frameworks.map(f => `${f} ${frameworkVersions[f]}`); + const frameworkNamesAndVersions = frameworks.map(f => `${f} ${frameworkVersions[f] ?? ''}`); + + const hasOps = frameworks.some(f => results[f].create); + const hasMemory = frameworks.some(f => results[f].memory); + const hasStartup = frameworks.some(f => results[f].startup); + const hasFileSize = frameworks.some(f => results[f].fileSize !== undefined); + + // write the results header with only the relevant descriptions + fileStream.write(resultsHeader.replace('{{resultsDescription}}', + `${hasOps ? opsDescription : ''}${hasStartup ? startupDescription : ''}` + )); // write some html const frameworksHeader = frameworkNamesAndVersions.map(f => `		${f}${results[f].create.time} (${results[f].create.mean})${results[f].update.time} (${results[f].update.mean})${results[f].skipNth.time} (${results[f].skipNth.mean})${results[f].select.time} (${results[f].select.mean})${results[f].swap.time} (${results[f].swap.mean})${results[f].remove.time} (${results[f].remove.mean})${results[f].createLots.time} (${results[f].createLots.mean})${results[f].append.time} (${results[f].append.mean})${results[f].clear.time} (${results[f].clear.mean})${results[f].avgMean}${results[f].fileSize} KB${entry.time} (${entry.mean})n/a${results[f].avgMean ?? 'n/a'}${results[f].fileSize ?? 'n/a'} KB${results[f].memory.memory}MB (created in ${results[f].memory.time}ms)Failed${startup.time} (${startup.mean})${detail}Failed