From 451af59a0e4fefd59240089e9a9c2acbcd1d0b39 Mon Sep 17 00:00:00 2001 From: Ethan Date: Thu, 22 Oct 2026 23:49:43 -0400 Subject: [PATCH 01/16] feat(capture): --exclude leaves parts of the host out: never requested, sitemaps listed skipped, links kept and listed apart, meta.json and the baseline carry the prefixes --- skills/capture-site/SKILL.md | 15 +- skills/capture-site/scripts/assets.mjs | 27 ++- skills/capture-site/scripts/capture.mjs | 72 +++++-- skills/capture-site/scripts/lib/exclude.mjs | 35 ++++ skills/capture-site/scripts/lib/wayback.mjs | 13 +- skills/capture-site/scripts/media.mjs | 26 ++- skills/convert-site/references/intake.md | 9 +- skills/convert-site/references/phases.md | 4 +- .../parity-gates/scripts/build-baseline.mjs | 4 +- test/capture/exclude.test.mjs | 198 ++++++++++++++++++ test/gates/build-baseline.test.mjs | 5 +- 11 files changed, 373 insertions(+), 35 deletions(-) create mode 100644 skills/capture-site/scripts/lib/exclude.mjs create mode 100644 test/capture/exclude.test.mjs diff --git a/skills/capture-site/SKILL.md b/skills/capture-site/SKILL.md index 806af4d..3bd8fff 100644 --- a/skills/capture-site/SKILL.md +++ b/skills/capture-site/SKILL.md @@ -82,6 +82,15 @@ source. Run each step as its own command, in this order: as one host) is listed there as `{ url, host, skipped: "other host" }` and never fetched, so none of its URLs enter the crawl. Two sitemap URLs with the same file name never overwrite each other: the second is saved with a suffix from its directory path (`page-sitemap-es.xml`). + `--exclude [,...]` leaves parts of the host out (a WordPress multisite's + sub-sites such as `/president/`, or a section the owner keeps): a URL whose path starts with + one is never requested, whether a sitemap, the REST lists, a link or a redirect leads there. + A sitemap under one is listed in `sitemaps/index.json` as `{ url, skipped: "excluded" }`; a + redirect into one is recorded with that target, unrequested; links to one stay in the saved + HTML and are listed in `links.json` `excluded`, not `internal`; `meta.json` `excluded` names + the prefixes and the baseline carries them. `--resume` keeps them (more given there are + added, and what was queued under them is dropped); `media.mjs` and `assets.mjs` skip files + under them too. node /scripts/capture.mjs https://www.example.com --out capture @@ -220,7 +229,8 @@ redirects to a captcha path. Every page then looks the same, so a capture of it inventory.json one record per page: status, head tags, JSON-LD, headings, forms, links pages/ text/ raw HTML and visible text per page redirects.json every redirect, chains collapsed, hops kept; shortlinks marked verified or not - links.json internal and external links, with the pages that carry them + links.json internal and external links, with the pages that carry them; with + --exclude, links under an excluded prefix in "excluded" scripts.json third-party vendors with their ids and pages media.json every media file with sources, pages, status, type and bytes; a size only the REST list names: downloaded: false, "unreferenced size" @@ -244,7 +254,8 @@ redirects to a captcha path. Every page then looks the same, so a capture of it pages/404.html the site's own 404 page (meta.json notFound: path, status, snapshot, text) sitemaps/ every sitemap robots.txt names, the default index, and their children; index.json: every sitemap URL asked for { url, status, file }, and - { url, host, skipped: "other host" } for one on another host + { url, host, skipped: "other host" } for one on another host, + { url, skipped: "excluded" } for one under an --exclude prefix feeds/ robots.txt wp-rest/ wayback-unverified.txt meta.json state.json README.md the generated summary and its ## Notes diff --git a/skills/capture-site/scripts/assets.mjs b/skills/capture-site/scripts/assets.mjs index 75d9b0c..1ac47d2 100644 --- a/skills/capture-site/scripts/assets.mjs +++ b/skills/capture-site/scripts/assets.mjs @@ -13,9 +13,10 @@ import * as csstree from 'css-tree'; import { assertNodeVersion } from './lib/node-version.mjs'; import { serializeOuter } from 'parse5'; import { attr, children, findAll, isElement, parseHtml } from '../../parity-gates/scripts/lib/html.mjs'; +import { excludedBy, parseExclude } from './lib/exclude.mjs'; import { BLOCK_LIMIT, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_UA, fetchPaced, http, MAX_FILE_BYTES, pacing, POLITE_DELAY_MS } from './lib/http.mjs'; -const USAGE = `Usage: node assets.mjs --capture [--delay ] [--concurrency ] [--polite] [--ua ""] [--max-file-bytes ] +const USAGE = `Usage: node assets.mjs --capture [--exclude [,...]] [--delay ] [--concurrency ] [--polite] [--ua ""] [--max-file-bytes ] Save the site's logo, favicon, apple-touch-icon and web fonts into /assets/ (logo.*, favicon.*, apple-touch-icon.*, fonts/*). The logo is found on the home page inside @@ -26,6 +27,10 @@ alt, class or id says "logo", else an inline that does (saved as logo.svg) capture.mjs; without it, fonts are reported as not captured). --capture the capture folder written by capture.mjs + --exclude [,...] also leave out assets on the site whose path starts with + one of these (repeatable); the capture's own --exclude prefixes (meta.json + excluded) always apply. One under a prefix is listed with the note + "excluded (under )" and never requested --delay time between request starts on a host (default ${DEFAULT_DELAY_MS}) --concurrency requests in flight on a host at once (default ${DEFAULT_CONCURRENCY}) --polite one request at a time, ${POLITE_DELAY_MS} ms apart; --delay and @@ -194,13 +199,20 @@ function extensionFor(url, contentType) { } function parseArgs(argv) { - const args = { capture: null, delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; + const args = { capture: null, exclude: [], delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; const value = (i) => (i < argv.length && !argv[i].startsWith('--') ? argv[i] : null); for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === '--help' || arg === '-h') args.help = true; else if (arg === '--capture' && value(i + 1)) args.capture = argv[(i += 1)]; else if (arg === '--ua' && value(i + 1)) args.ua = argv[(i += 1)]; + else if (arg === '--exclude' && value(i + 1)) { + try { + args.exclude = parseExclude([...args.exclude, argv[(i += 1)]]); + } catch (err) { + return { error: err.message }; + } + } else if (arg === '--delay' && value(i + 1)) { args.delayMs = Number(argv[(i += 1)]); if (!Number.isInteger(args.delayMs) || args.delayMs < 0) return { error: '--delay needs a whole number' }; @@ -247,7 +259,18 @@ async function main() { const fontNames = new Map(); // file name -> url, so two fonts with one name do not overwrite // Saves url as /, or fonts/ (kept when already saved). + const prefixes = [...(meta.excluded ?? []), ...args.exclude]; + const site = origin ? new URL(origin).host.replace(/^www\./i, '').toLowerCase() : null; + const excludedPrefix = (url) => { + const target = new URL(url); + return target.host.replace(/^www\./i, '').toLowerCase() === site ? excludedBy(prefixes, target.pathname) : null; + }; const save = async (kind, url, stem, { keepName = false } = {}) => { + const prefix = excludedPrefix(url); + if (prefix) { + entries.push({ kind, url, file: null, status: null, note: `excluded (under ${prefix})` }); + return; + } let name = null; if (keepName) { const original = basename(new URL(url).pathname) || 'font'; diff --git a/skills/capture-site/scripts/capture.mjs b/skills/capture-site/scripts/capture.mjs index 141ef7e..195842a 100644 --- a/skills/capture-site/scripts/capture.mjs +++ b/skills/capture-site/scripts/capture.mjs @@ -33,6 +33,7 @@ import { CDX_ENDPOINT, WAYBACK_FILE, waybackPaths } from './lib/wayback.mjs'; import { extensionOf, NEVER, NOT_PAGES, PAGE_EXTENSIONS } from './lib/page-rules.mjs'; import { cssImports } from './lib/css-urls.mjs'; import { CHALLENGE, SUSPECT, detectChallenge } from './lib/challenge.mjs'; +import { excludedBy, parseExclude } from './lib/exclude.mjs'; import { parse } from 'parse5'; const USAGE = `Usage: node capture.mjs --out [options] @@ -73,6 +74,12 @@ to list and download the media. --seed, --seed-file and --include-query also apply on --resume --no-rest do not use the platform's REST API for discovery --no-wayback do not ask the Wayback Machine for archived URLs + --exclude [,...] never request a URL whose path starts with one of these + (repeatable), e.g. a multisite's sub-sites: --exclude /president/,/news-room/. + Sitemaps under one are listed in sitemaps/index.json as skipped: "excluded", + links to one stay in the HTML and are listed in links.json "excluded", + and meta.json "excluded" names the prefixes. --resume keeps them; more + given on --resume are added --help show this help Never requests wp-admin, wp-login, xmlrpc.php or ?s= searches, and /wp-json/ only for the @@ -106,6 +113,8 @@ const MAX_HOPS = 10; const SITEMAP_PATH = /sitemap|\.xml(\.gz)?$/i; // Why a sitemap on another host is listed in sitemaps/index.json but never fetched. const OTHER_HOST = 'other host'; +// Why a sitemap under an --exclude prefix is listed in sitemaps/index.json but never fetched. +const EXCLUDED = 'excluded'; // The root icons a launch needs, asked for once so a missing one is evidence live lacked it. const ROOT_ICONS = ['/favicon.ico', '/apple-touch-icon.png']; // HTML responses in a row matching only the tiny-page heuristic that stop the run. @@ -147,7 +156,7 @@ function readSeedFile(file) { function parseArgs(argv) { const args = { origin: null, out: null, delayMs: null, concurrency: null, polite: false, max: null, resume: false, ua: DEFAULT_UA, - seeds: [], verifyShortlinks: false, includeQuery: false, noRest: false, noWayback: false, help: false, + seeds: [], exclude: [], verifyShortlinks: false, includeQuery: false, noRest: false, noWayback: false, help: false, }; const value = (i, flag) => { if (i >= argv.length || argv[i].startsWith('--')) throw new UsageError(`${flag} needs a value`); @@ -166,6 +175,13 @@ function parseArgs(argv) { } else if (arg === '--polite') args.polite = true; else if (arg === '--ua') args.ua = value((i += 1), arg); else if (arg === '--seed') args.seeds.push(value((i += 1), arg)); else if (arg === '--seed-file') args.seeds.push(...readSeedFile(value((i += 1), arg))); + else if (arg === '--exclude') { + try { + args.exclude = parseExclude([...args.exclude, value((i += 1), arg)]); + } catch (err) { + throw new UsageError(err.message); + } + } else if (arg === '--resume') args.resume = true; else if (arg === '--verify-shortlinks') args.verifyShortlinks = true; else if (arg === '--include-query') args.includeQuery = true; @@ -302,6 +318,14 @@ class Crawler { return this.scopeFn; } + // True for a URL on the site whose path is under an --exclude prefix: never requested. + excluded(href) { + const prefixes = this.state.options.exclude ?? []; + if (prefixes.length === 0) return false; + const url = new URL(href); + return bareHost(url) === bareHost(new URL(this.state.origin)) && excludedBy(prefixes, url.pathname) !== null; + } + retryWait(attempts) { return Math.max(RETRY_MIN_MS, this.delayMs) * 2 ** Math.max(0, attempts - 1); } @@ -319,7 +343,7 @@ class Crawler { enqueue(href, source, base, kind = 'page') { const key = this.scope(href, base); - if (!key) return null; + if (!key || this.excluded(key)) return null; if (isShortlink(new URL(key))) return null; // recorded as an unverified redirect instead const sources = (this.state.foundBy[key] ??= []); if (!sources.includes(source)) sources.push(source); @@ -344,6 +368,11 @@ class Crawler { this.noteSitemap(url, null, null, { host: target.host, skipped: OTHER_HOST }); return { status: null, skipped: OTHER_HOST, headers: {}, body: '' }; } + // Under an --exclude prefix (a multisite's sub-site sitemap index): never asked for. + if (this.excluded(url)) { + if (SITEMAP_PATH.test(target.pathname)) this.noteSitemap(url, null, null, { skipped: EXCLUDED }); + return { status: null, skipped: EXCLUDED, headers: {}, body: '' }; + } for (let attempt = 1; ; attempt += 1) { const res = await this.fetch(url, options); const verdict = this.judge(res, url); @@ -375,7 +404,7 @@ class Crawler { } // Every sitemap the capture asked for, with its answer: sitemaps/index.json. - // A skipped one (on another host) is { url, host, skipped } and never fetched. + // A skipped one is never fetched: { url, host, skipped: "other host" } or { url, skipped: "excluded" }. noteSitemap(url, status, file, skipped = null) { const log = (this.state.sitemapLog ??= {}); log[url] = skipped ?? { status, file }; @@ -573,8 +602,9 @@ class Crawler { next.hash = ''; const offSite = bareHost(next) !== bareHost(new URL(this.state.canonicalOrigin)); const looped = chain.some((c) => c.url === next.href); - if (offSite || looped || forbidden(next) || notAPage(next) || hop === MAX_HOPS) { - return { end: { res: null, finalUrl: next.href, chain, offSite, looped } }; + const excluded = !offSite && this.excluded(next.href); + if (offSite || looped || excluded || forbidden(next) || notAPage(next) || hop === MAX_HOPS) { + return { end: { res: null, finalUrl: next.href, chain, offSite, looped, excluded } }; } return { next: next.href }; } @@ -689,7 +719,7 @@ class Crawler { } catch { continue; } - if (!/^https?:$/.test(url.protocol) || forbidden(url)) continue; + if (!/^https?:$/.test(url.protocol) || forbidden(url) || this.excluded(url.href)) continue; let entry = css.entries.find((e) => e.href === style.href); if (!entry) { const n = css.entries.filter((e) => e.href).length + 1; @@ -720,7 +750,7 @@ class Crawler { followImports(importer, text, base) { const css = this.state.css; for (const href of cssImports(text, base)) { - if (forbidden(new URL(href))) continue; + if (forbidden(new URL(href)) || this.excluded(href)) continue; let entry = css.entries.find((e) => e.href === href); if (!entry) { const depth = (importer.depth ?? 0) + 1; @@ -773,7 +803,7 @@ class Crawler { } catch { continue; } - if (!/^https?:$/.test(url.protocol) || bareHost(url) !== site || forbidden(url)) continue; + if (!/^https?:$/.test(url.protocol) || bareHost(url) !== site || forbidden(url) || this.excluded(url.href)) continue; let entry = js.entries.find((e) => e.src === script.src); if (!entry) { const kind = script.kind ?? 'script'; @@ -855,7 +885,9 @@ class Crawler { const host = new URL(this.state.canonicalOrigin).host.replace(/^www\./i, ''); for (let attempt = 1; ; attempt += 1) { try { - const paths = await waybackPaths(host, { fetch: this.fetch, known, out: this.out, endpoint }); + const prefixes = this.state.options.exclude ?? []; + const skip = (path) => excludedBy(prefixes, path) !== null; + const paths = await waybackPaths(host, { fetch: this.fetch, known, out: this.out, endpoint, skip }); return { status: 'ok', paths: paths.length }; } catch (err) { if (attempt < WAYBACK_TRIES) { @@ -887,7 +919,7 @@ class Crawler { return {}; } - const finalKey = result.offSite ? null : this.scope(result.finalUrl); + const finalKey = result.offSite || result.excluded ? null : this.scope(result.finalUrl); const landed = res && finalKey && upperEscapes(result.finalUrl) === finalKey; // A redirect that only changes the escapes' case lands on this same page: no redirect row. if (landed && finalKey === item.url) { @@ -1112,24 +1144,32 @@ class Crawler { challenge: this.state.challenge ?? null, notFound: this.state.notFound ?? null, icons: this.state.icons ?? {}, + excluded: this.state.options.exclude ?? [], }); json('sitemaps/index.json', Object.entries(this.state.sitemapLog ?? {}) - .map(([url, e]) => (e.skipped ? { url, host: e.host, skipped: e.skipped } : { url, status: e.status, file: e.file })) + .map(([url, e]) => (e.skipped ? { url, ...(e.host ? { host: e.host } : {}), skipped: e.skipped } : { url, status: e.status, file: e.file })) .sort(byKey('url'))); json('css/manifest.json', this.cssManifest()); json('js/manifest.json', this.jsManifest()); + // Links to a path under an --exclude prefix are listed apart (excluded), never as internal: + // the build serves no such page, and the gates treat them like links off the site. const internal = {}; const external = {}; + const excluded = {}; + const prefixes = this.state.options.exclude ?? []; const add = (map, target, from) => { (map[target] ??= new Set()).add(from); }; for (const r of inventory) { - for (const link of r.internalLinks) if (link !== r.path) add(internal, link, r.path); + for (const link of r.internalLinks) { + if (link === r.path) continue; + add(excludedBy(prefixes, link.split('?')[0]) ? excluded : internal, link, r.path); + } for (const link of r.externalLinks) add(external, link, r.path); } const plain = (map) => Object.fromEntries(Object.keys(map).sort().map((k) => [k, [...map[k]].sort()])); - json('links.json', { internal: plain(internal), external: plain(external) }); + json('links.json', { internal: plain(internal), external: plain(external), ...(prefixes.length ? { excluded: plain(excluded) } : {}) }); const vendors = {}; for (const r of inventory) { @@ -1183,11 +1223,14 @@ async function main() { const known = new Set(state.options.seeds ?? []); newSeeds = [...new Set(args.seeds.filter((seed) => !known.has(seed)))]; state.options.seeds = [...known, ...newSeeds]; + // Prefixes given on --resume join the saved ones; what is queued under them is dropped. + state.options.exclude = parseExclude([...(state.options.exclude ?? []), ...args.exclude]); } else { state = newState(args.origin, { includeQuery: args.includeQuery, noRest: args.noRest, seeds: args.seeds, + exclude: args.exclude, verifyShortlinks: args.verifyShortlinks, noWayback: args.noWayback, }); @@ -1197,6 +1240,9 @@ async function main() { http.resetBlocked(); const fetch = (url, options) => fetchPaced(url, { ...options, delayMs: args.delayMs, concurrency: args.concurrency, ua: args.ua, trusted: [args.origin, process.env.CAPTURE_WAYBACK_CDX] }); const crawler = new Crawler(state, { out, fetch, delayMs: args.delayMs, concurrency: args.concurrency }); + if (args.resume && args.exclude.length > 0) { + state.queue = state.queue.filter((item) => !crawler.excluded(item.url)); + } // Seeds given on --resume: once the home page is known they join the queue here (before that, // discovery queues every seed in the options), and a finished capture crawls again for them. if (state.fingerprint && newSeeds.length > 0) { diff --git a/skills/capture-site/scripts/lib/exclude.mjs b/skills/capture-site/scripts/lib/exclude.mjs new file mode 100644 index 0000000..d1b3b19 --- /dev/null +++ b/skills/capture-site/scripts/lib/exclude.mjs @@ -0,0 +1,35 @@ +// Excluded path prefixes (--exclude): parts of the host the capture never requests, such as a +// WordPress multisite's sub-sites (/president/, /media-relations/) or a section the owner leaves +// out. A prefix is matched against a URL's path as written; one ending in / also covers that +// folder without its slash (/services/ excludes /services and /services/roofing/). +// parseExclude(values) -> the prefixes of each comma-separated value, each starting with /; +// throws on an empty one, "/" (the whole site) or one that is not a path or URL. +// excludedBy(prefixes, pathname) -> the first prefix covering pathname, or null. + +export function parseExclude(values) { + const prefixes = []; + for (const part of values.flatMap((v) => String(v).split(','))) { + const raw = part.trim(); + if (!raw) continue; + let prefix = raw; + if (/^https?:\/\//i.test(raw)) { + try { + prefix = new URL(raw).pathname; + } catch { + throw new Error(`--exclude ${raw} is not a path or URL`); + } + } + if (!prefix.startsWith('/')) throw new Error(`--exclude ${raw} must be a path starting with /, e.g. /president/`); + if (prefix === '/') throw new Error('--exclude / would leave out the whole site'); + if (!prefixes.includes(prefix)) prefixes.push(prefix); + } + return prefixes; +} + +export function excludedBy(prefixes, pathname) { + for (const prefix of prefixes ?? []) { + if (pathname.startsWith(prefix)) return prefix; + if (prefix.endsWith('/') && pathname === prefix.slice(0, -1)) return prefix; + } + return null; +} diff --git a/skills/capture-site/scripts/lib/wayback.mjs b/skills/capture-site/scripts/lib/wayback.mjs index abaf80e..d2954f4 100644 --- a/skills/capture-site/scripts/lib/wayback.mjs +++ b/skills/capture-site/scripts/lib/wayback.mjs @@ -1,8 +1,9 @@ // Legacy URLs from the Wayback Machine's CDX API. -// waybackPaths(host, { fetch, known, out, endpoint }) asks for every URL archived with a 200 on the +// waybackPaths(host, { fetch, known, out, endpoint, skip }) asks for every URL archived with a 200 on the // host (one request), keeps page-like paths on that host (www or not; query strings dropped; -// never wp-admin, wp-login, xmlrpc.php or /wp-json/), drops the `known` paths, and returns the -// rest sorted. With `out`, writes them one per line to /wayback-unverified.txt. +// never wp-admin, wp-login, xmlrpc.php or /wp-json/), drops the `known` paths and any path +// `skip(path)` is true for (an --exclude prefix), and returns the rest sorted. +// With `out`, writes them one per line to /wayback-unverified.txt. // Throws when the API cannot be reached or does not answer with JSON. import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -23,7 +24,7 @@ function isPage(pathname) { return PAGE_EXTENSIONS.has(extensionOf(pathname)) && !NEVER.test(pathname) && !NOT_PAGES.test(pathname); } -export async function waybackPaths(host, { fetch, known = [], out = null, endpoint = CDX_ENDPOINT } = {}) { +export async function waybackPaths(host, { fetch, known = [], out = null, endpoint = CDX_ENDPOINT, skip = () => false } = {}) { const res = await fetch(cdxUrl(host, endpoint), { ...CDX_OPTIONS }); if (res.status !== 200) { throw new Error(`the Wayback CDX API answered ${res.status || 'nothing'}${res.error ? ` (${res.error})` : ''}`); @@ -39,7 +40,7 @@ export async function waybackPaths(host, { fetch, known = [], out = null, endpoi if (!Array.isArray(rows)) throw new Error('the Wayback CDX API did not answer with a JSON list'); } const site = bareHost(host); - const skip = new Set(known); + const seen = new Set(known); const paths = new Set(); for (const row of rows.slice(1)) { let url; @@ -49,7 +50,7 @@ export async function waybackPaths(host, { fetch, known = [], out = null, endpoi continue; } if (!/^https?:$/.test(url.protocol) || bareHost(url.host) !== site) continue; - if (!isPage(url.pathname) || skip.has(url.pathname)) continue; + if (!isPage(url.pathname) || seen.has(url.pathname) || skip(url.pathname)) continue; paths.add(url.pathname); } const sorted = [...paths].sort(); diff --git a/skills/capture-site/scripts/media.mjs b/skills/capture-site/scripts/media.mjs index 747b2e1..91c9ff6 100644 --- a/skills/capture-site/scripts/media.mjs +++ b/skills/capture-site/scripts/media.mjs @@ -12,6 +12,7 @@ import { BLOCK_LIMIT, DEFAULT_DELAY_MS, DEFAULT_UA, fetchPaced, MAX_FILE_BYTES, import { cssUrls, FONT_EXTENSIONS, IMAGE_EXTENSIONS } from './lib/css-urls.mjs'; import { extensionOf } from './lib/page-rules.mjs'; import { mediaUrlsOf } from './lib/wp-rest.mjs'; +import { excludedBy, parseExclude } from './lib/exclude.mjs'; import { srcsetUrls } from '../../parity-gates/scripts/lib/srcset.mjs'; export { srcsetUrls }; @@ -19,7 +20,7 @@ export { srcsetUrls }; // Media files are many and small: more in flight than the crawl's default. const MEDIA_CONCURRENCY = 6; -const USAGE = `Usage: node media.mjs --capture [--download ] [--sizes referenced|all] [--delay ] [--concurrency ] [--polite] [--max-file-bytes ] +const USAGE = `Usage: node media.mjs --capture [--download ] [--sizes referenced|all] [--exclude [,...]] [--delay ] [--concurrency ] [--polite] [--max-file-bytes ] List every media file of a captured site in /media.json: [{ url, path, source[], referencedBy[], status, contentType, bytes }] @@ -49,6 +50,9 @@ requested and is listed "blocked-address". --sizes referenced|all which REST-only image sizes to fetch (default referenced: none; all: every registered size, for when old emails or other sites may link to them) + --exclude [,...] also leave out files whose path starts with one of these + (repeatable); the capture's own --exclude prefixes (meta.json excluded) + always apply: a file under one is never listed or requested --max-file-bytes largest file to download or measure (default ${MAX_FILE_BYTES}, 2 GB) --delay time between request starts on the site (default ${DEFAULT_DELAY_MS}) --concurrency requests in flight on the site at once (default ${MEDIA_CONCURRENCY}) @@ -94,7 +98,7 @@ const sleep = (ms) => new Promise((done) => setTimeout(done, ms)); const bareHost = (url) => url.host.toLowerCase().replace(/^www\./, ''); function parseArgs(argv) { - const args = { capture: null, download: null, sizes: 'referenced', delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; + const args = { capture: null, download: null, sizes: 'referenced', exclude: [], delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; const value = (i, flag) => { if (i >= argv.length || argv[i].startsWith('--')) throw new UsageError(`${flag} needs a value`); return argv[i]; @@ -109,7 +113,13 @@ function parseArgs(argv) { if (!SIZES.has(args.sizes)) throw new UsageError(`--sizes is referenced or all, got "${args.sizes}"`); } else if (arg === '--ua') args.ua = value((i += 1), arg); - else if (arg === '--delay') { + else if (arg === '--exclude') { + try { + args.exclude = parseExclude([...args.exclude, value((i += 1), arg)]); + } catch (err) { + throw new UsageError(err.message); + } + } else if (arg === '--delay') { const n = Number(value((i += 1), arg)); if (!Number.isInteger(n) || n < 0) throw new UsageError('--delay needs a whole number'); args.delayMs = n; @@ -188,9 +198,11 @@ function sitemapImages(dir) { return urls; } -// Every media file the capture points at, keyed by path. -export function collectMedia(captureDir) { +// Every media file the capture points at, keyed by path; none under an excluded prefix (the +// capture's meta.json excluded, plus `exclude`). +export function collectMedia(captureDir, { exclude = [] } = {}) { const meta = existsSync(join(captureDir, 'meta.json')) ? readJson(join(captureDir, 'meta.json')) : {}; + const prefixes = [...(meta.excluded ?? []), ...exclude]; const inventory = readJson(join(captureDir, 'inventory.json')); const origin = meta.canonicalOrigin ?? meta.origin ?? (inventory[0] ? new URL(inventory[0].url).origin : null); if (!origin) throw new UsageError(`${captureDir} has no origin in meta.json and no pages`); @@ -208,7 +220,7 @@ export function collectMedia(captureDir) { url.protocol = site.protocol; url.host = site.host; url.hash = ''; - if (url.pathname.endsWith('/')) return; + if (url.pathname.endsWith('/') || excludedBy(prefixes, url.pathname)) return; const ext = extensionOf(url.pathname); if (needsExtension && !MEDIA_EXTENSIONS.has(ext)) return; const entry = found.get(url.pathname) ?? { url: url.href, path: url.pathname, source: new Set(), referencedBy: new Set() }; @@ -353,7 +365,7 @@ async function main() { let media; try { - media = collectMedia(captureDir); + media = collectMedia(captureDir, { exclude: args.exclude }); } catch (err) { console.error(`Cannot read the capture in ${captureDir}: ${err.message}`); process.exit(2); diff --git a/skills/convert-site/references/intake.md b/skills/convert-site/references/intake.md index b39c5d9..1bae690 100644 --- a/skills/convert-site/references/intake.md +++ b/skills/convert-site/references/intake.md @@ -58,6 +58,8 @@ opening explaining what a repository, a staging deploy and a pull request are. 8. Redirects. Default: every redirect the crawl finds. If you have an old .htaccess, a redirect plugin export or a list of old URLs, send the file. 9. Pages that must not change, or must be left out. Default: none; every page converts. + Name a section to leave out by its path (`/president/`): nothing under it is read or + built. A multisite's sub-sites on the same host are left out this way. 10. Components. Default: off. Say "components" if you want repeated blocks (cards, calls to action) turned into Astro components after everything matches. @@ -93,7 +95,7 @@ lives in `$SITE/.conversion/` from the start; `init-site.mjs` scaffolds around i node $KIT/skills/convert-site/scripts/state.mjs --dir $SITE set site '""' node $KIT/skills/convert-site/scripts/state.mjs --dir $SITE set hosting '"vercel"' node $KIT/skills/convert-site/scripts/state.mjs --dir $SITE set repo '"/"' - node $KIT/skills/convert-site/scripts/state.mjs --dir $SITE set intake '{"forms":{"service":"formspree"},"testAddress":"you@example.com","domain":"","vercelScope":"","publicStaging":true,"consent":[],"keepPages":[],"components":false}' + node $KIT/skills/convert-site/scripts/state.mjs --dir $SITE set intake '{"forms":{"service":"formspree"},"testAddress":"you@example.com","domain":"","vercelScope":"","publicStaging":true,"consent":[],"keepPages":[],"excludePaths":[],"components":false}' `intake.vercelScope` is preflight's `vercel.scope` (the default team's slug) unless the person named another team slug; never the username, which the CLI refuses ("You cannot set your @@ -103,6 +105,11 @@ answer: `true` (public and noindexed, the default) unless the person asked for p `false`. Record it as a decision too (topic "staging protection"), so `docs/plan.md` and the PR show it. +`intake.keepPages` holds the pages that must not change; `intake.excludePaths` the path +prefixes to leave out (each starting and, for a folder, ending with `/`, e.g. `["/president/"]`), +which the capture reads as `--exclude`. Record a non-empty `excludePaths` as a decision too (topic +"excluded paths"). + Every default you applied is also a decision (`state.mjs decision`), so the PR shows it. Add each missing item to `needsYou` as `{ "item": "...", "command": "...", "before": "" }`: read the array with `state.mjs get needsYou`, append, and write it back with `set needsYou`. diff --git a/skills/convert-site/references/phases.md b/skills/convert-site/references/phases.md index 83b6dd9..ea587b0 100644 --- a/skills/convert-site/references/phases.md +++ b/skills/convert-site/references/phases.md @@ -29,7 +29,9 @@ Then the one message (`intake.md`) and the answers into state. Follow the `capture-site` skill exactly, from `SITE`, with `KIT` paths: S phase capture - node $KIT/skills/capture-site/scripts/capture.mjs $LIVE --out capture + node $KIT/skills/capture-site/scripts/capture.mjs $LIVE --out capture --exclude + +`--exclude` takes `intake.excludePaths` joined with commas; leave it off when that is empty. When `capture/wayback-unverified.txt` is not empty, run the Wayback second pass, then carry on (`readme.mjs` runs last below): diff --git a/skills/parity-gates/scripts/build-baseline.mjs b/skills/parity-gates/scripts/build-baseline.mjs index fa82181..334fc39 100644 --- a/skills/parity-gates/scripts/build-baseline.mjs +++ b/skills/parity-gates/scripts/build-baseline.mjs @@ -34,7 +34,8 @@ Freeze a capture (capture.mjs, media.mjs and templates.mjs output) into a baseli which the routes gate lets the build serve at the root text/ the visible text of each page, copied from the capture counts.json routes, byStatus, byTemplate, redirects (by source), media, forms, scripts - meta.json { origin, canonicalOrigin, capturedAt, builtAt, platform } of the capture + meta.json { origin, canonicalOrigin, capturedAt, builtAt, platform, excluded } of the + capture; excluded: the capture's --exclude path prefixes ([] when none) and redirects.csv (from,to,status,source,note) for the host. A redirect with no target (an unverified shortlink the site did not resolve) is left out and named on stderr. @@ -298,6 +299,7 @@ export function buildMeta(meta, state, builtAt = new Date().toISOString()) { capturedAt: meta.finishedAt ?? meta.startedAt ?? null, builtAt, platform: state.fingerprint?.platform ?? null, + excluded: meta.excluded ?? [], }; } diff --git a/test/capture/exclude.test.mjs b/test/capture/exclude.test.mjs new file mode 100644 index 0000000..46e586a --- /dev/null +++ b/test/capture/exclude.test.mjs @@ -0,0 +1,198 @@ +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { startFixture } from '../fixture/serve.mjs'; +import { startStub } from './helpers.mjs'; +import { excludedBy, parseExclude } from '../../skills/capture-site/scripts/lib/exclude.mjs'; +import { collectMedia } from '../../skills/capture-site/scripts/media.mjs'; + +const exec = promisify(execFile); +const script = (path) => fileURLToPath(new URL(`../../skills/${path}`, import.meta.url)); +const CAPTURE = script('capture-site/scripts/capture.mjs'); +const readJson = (dir, file) => JSON.parse(readFileSync(join(dir, file), 'utf8')); + +async function run(path, args, env = process.env) { + try { + const { stdout, stderr } = await exec(process.execPath, [path, ...args], { env }); + return { code: 0, stdout, stderr }; + } catch (err) { + if (typeof err.code !== 'number') throw err; + return { code: err.code, stdout: err.stdout, stderr: err.stderr }; + } +} + +const dirs = []; +const tempDir = () => { + const dir = mkdtempSync(join(tmpdir(), 'exclude-test-')); + dirs.push(dir); + return dir; +}; + +let fixture; +let cap; +let base; +let captured; +let requests; +before(async () => { + fixture = await startFixture(); + const root = tempDir(); + cap = join(root, 'capture'); + base = join(root, 'baseline'); + captured = await run(CAPTURE, [fixture.origin, '--out', cap, '--delay', '0', '--no-wayback', '--seed', '/old-services', '--exclude', '/services/']); + requests = [...fixture.requests]; + for (const [name, args] of [ + ['capture-site/scripts/media.mjs', ['--capture', cap, '--delay', '0']], + ['capture-site/scripts/templates.mjs', ['--capture', cap]], + ['parity-gates/scripts/build-baseline.mjs', ['--capture', cap, '--out', base]], + ]) { + const result = await run(script(name), args); + assert.equal(result.code, 0, `${name}: ${result.stderr}`); + } +}); +after(async () => { + await fixture.close(); + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); +}); + +test('parseExclude splits commas, keeps URL paths and refuses / and bare words', () => { + assert.deepEqual(parseExclude(['/president/,/media-relations/', 'https://www.example.com/team/', '/president/']), ['/president/', '/media-relations/', '/team/']); + assert.throws(() => parseExclude(['/']), /whole site/); + assert.throws(() => parseExclude(['president']), /starting with \//); + assert.equal(excludedBy(['/services/'], '/services/roofing/'), '/services/'); + assert.equal(excludedBy(['/services/'], '/services'), '/services/'); + assert.equal(excludedBy(['/services/'], '/services-old/'), null); +}); + +test('--exclude /services/: no page under it is requested or inventoried, and meta.json lists it', () => { + assert.equal(captured.code, 0, captured.stderr); + const asked = requests.map((u) => new URL(u, fixture.origin).pathname); + assert.deepEqual(asked.filter((p) => p.startsWith('/services')), [], 'nothing under /services/ is requested'); + const inventory = readJson(cap, 'inventory.json'); + assert.deepEqual(inventory.filter((r) => r.path.startsWith('/services')), []); + // 13 pages at 200 less the three services pages, plus the sitemap's 404. + assert.equal(inventory.length, 11); + assert.equal(inventory.filter((r) => r.status === 200).length, 10); + assert.deepEqual(readJson(cap, 'meta.json').excluded, ['/services/']); + assert.deepEqual(readJson(cap, 'state.json').options.exclude, ['/services/']); +}); + +test('a redirect into an excluded path is recorded and its target never requested', () => { + const old = readJson(cap, 'redirects.json').find((r) => r.from === '/old-services'); + assert.deepEqual(old, { from: '/old-services', to: '/services/', status: 301, source: 'seed', hops: ['/old-services/'] }); +}); + +test('links to excluded paths stay in the HTML and move to links.json excluded', () => { + assert.match(readFileSync(join(cap, 'pages/home.html'), 'utf8'), /href="[^"]*\/services\/roofing\/"/); + const links = readJson(cap, 'links.json'); + assert.deepEqual(Object.keys(links.internal).filter((p) => p.startsWith('/services')), []); + assert.ok(links.excluded['/services/'].includes('/'), 'the home page links to /services/'); + assert.ok(links.excluded['/services/roofing/'].length > 0); +}); + +test('media lists no page under the excluded prefix, and media under a prefix is not listed', () => { + const media = readJson(cap, 'media.json'); + assert.ok(media.length > 0); + assert.deepEqual(media.flatMap((m) => m.referencedBy).filter((p) => p.startsWith('/services')), []); + const none = collectMedia(cap, { exclude: ['/wp-content/uploads/'] }); + assert.deepEqual(none.filter((m) => m.path.startsWith('/wp-content/uploads/')), []); + assert.ok(none.some((m) => m.path.startsWith('/wp-content/themes/')), 'other media is still listed'); +}); + +test('the baseline carries the excluded prefixes in meta.json and has no route under them', () => { + assert.deepEqual(readJson(base, 'meta.json').excluded, ['/services/']); + assert.deepEqual(readJson(base, 'routes.json').filter((r) => r.path.startsWith('/services')), []); +}); + +test('assets.mjs --exclude leaves out an asset under the prefix without asking for it', async () => { + const before = fixture.requests.length; + const result = await run(script('capture-site/scripts/assets.mjs'), ['--capture', cap, '--delay', '0', '--exclude', '/wp-content/themes/fixture/fonts/']); + assert.equal(result.code, 0, result.stderr); + const asked = fixture.requests.slice(before); + assert.deepEqual(asked.filter((u) => u.includes('/fonts/')), []); + const fonts = readJson(cap, 'assets/index.json').filter((e) => e.kind === 'font'); + assert.equal(fonts.length, 2); + for (const font of fonts) assert.deepEqual([font.file, font.status, font.note], [null, null, 'excluded (under /wp-content/themes/fixture/fonts/)']); +}); + +test('a sitemap index under an excluded prefix is listed skipped, and nothing under it is asked for', async () => { + const home = (origin) => ({ body: `HomePresidentNews` }); + const page = { body: 'Newsnews' }; + const stub = await startStub({ + '/': home, + '/robots.txt': (origin) => ({ headers: { 'Content-Type': 'text/plain' }, body: `Sitemap: ${origin}/president/sitemap_index.xml\nSitemap: ${origin}/sitemap_index.xml\n` }), + '/sitemap_index.xml': (origin) => ({ headers: { 'Content-Type': 'application/xml' }, body: `${origin}/page-sitemap.xml${origin}/president/page-sitemap.xml` }), + '/page-sitemap.xml': (origin) => ({ headers: { 'Content-Type': 'application/xml' }, body: `${origin}/news/${origin}/president/bio/` }), + '/news/': page, + }); + try { + const out = tempDir(); + const result = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--no-wayback', '--exclude', '/president/']); + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(stub.requests.filter((u) => u.startsWith('/president')), []); + const index = readJson(out, 'sitemaps/index.json'); + assert.deepEqual(index.filter((e) => e.skipped), [ + { url: `${stub.origin}/president/page-sitemap.xml`, skipped: 'excluded' }, + { url: `${stub.origin}/president/sitemap_index.xml`, skipped: 'excluded' }, + ]); + const meta = readJson(out, 'meta.json'); + assert.deepEqual(meta.discoveryErrors, []); + assert.deepEqual(readJson(out, 'inventory.json').map((r) => r.path), ['/', '/news/']); + assert.deepEqual(readJson(out, 'links.json').excluded, { '/president/': ['/'] }); + } finally { + await stub.close(); + } +}); + +test('--resume keeps the excluded prefixes, and one added on --resume drops what is queued under it', async () => { + const fresh = await startFixture(); + try { + const out = tempDir(); + const first = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--no-rest', '--max', '2', '--exclude', '/services/']); + assert.equal(first.code, 0, first.stderr); + const queued = readJson(out, 'state.json').queue.map((q) => new URL(q.url).pathname); + assert.ok(queued.some((p) => p.startsWith('/blog/')), `blog is still queued: ${queued}`); + const again = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--resume', '--exclude', '/blog/']); + assert.equal(again.code, 0, again.stderr); + const asked = fresh.requests.map((u) => new URL(u, fresh.origin).pathname); + assert.deepEqual(asked.filter((p) => p.startsWith('/services') || p.startsWith('/blog')), []); + assert.deepEqual(readJson(out, 'meta.json').excluded, ['/services/', '/blog/']); + } finally { + await fresh.close(); + } +}); + +test('Wayback paths under an excluded prefix are not listed for a second pass', async () => { + const server = createServer((req, res) => { + const host = new URL(req.url, 'http://cdx').searchParams.get('url').replace(/\/\*$/, ''); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify([['original'], [`http://${host}/old-team/`], [`http://${host}/services/old-gutters/`]])); + }); + await new Promise((done) => server.listen(0, '127.0.0.1', done)); + const stub = await startStub({ '/': { body: 'Homehome' } }); + try { + const out = tempDir(); + const env = { ...process.env, CAPTURE_WAYBACK_CDX: `http://127.0.0.1:${server.address().port}/cdx/search/cdx`, CAPTURE_WAYBACK_RETRY_MS: '0' }; + const result = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--exclude', '/services/'], env); + assert.equal(result.code, 0, result.stderr); + assert.equal(readFileSync(join(out, 'wayback-unverified.txt'), 'utf8'), '/old-team/\n'); + } finally { + await stub.close(); + server.closeAllConnections(); + await new Promise((done) => server.close(done)); + } +}); + +test('--exclude needs a value and refuses /', async () => { + const out = tempDir(); + const bare = await run(CAPTURE, ['http://127.0.0.1:9', '--out', out, '--exclude']); + assert.equal(bare.code, 2); + const all = await run(CAPTURE, ['http://127.0.0.1:9', '--out', out, '--exclude', '/']); + assert.equal(all.code, 2); + assert.match(all.stderr, /whole site/); +}); diff --git a/test/gates/build-baseline.test.mjs b/test/gates/build-baseline.test.mjs index 41c6bb5..737e29b 100644 --- a/test/gates/build-baseline.test.mjs +++ b/test/gates/build-baseline.test.mjs @@ -282,10 +282,11 @@ test('--check passes on the built baseline and exits 1 after a route is deleted' assert.match(drifted.stdout, /routes: expected 16, got 15/); }); -test('meta.json records the capture origin, times and platform', () => { +test('meta.json records the capture origin, times, platform and excluded prefixes', () => { const meta = readJson(baseline, 'meta.json'); const captured = readJson(capture, 'meta.json'); - assert.deepEqual(Object.keys(meta), ['origin', 'canonicalOrigin', 'capturedAt', 'builtAt', 'platform']); + assert.deepEqual(Object.keys(meta), ['origin', 'canonicalOrigin', 'capturedAt', 'builtAt', 'platform', 'excluded']); + assert.deepEqual(meta.excluded, [], 'a capture without --exclude excludes nothing'); assert.equal(meta.origin, captured.origin); assert.equal(meta.canonicalOrigin, captured.canonicalOrigin); assert.equal(meta.capturedAt, captured.finishedAt); From a26fc90e95c2493205dd5103c476fefb7332b356 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 23 Oct 2026 00:00:07 -0400 Subject: [PATCH 02/16] feat(capture): save the root /favicon.ico and /apple-touch-icon.png bytes (assets/root-*), record { status, file, contentType, bytes } in meta.json icons, and list them in assets/index.json as root-icon --- skills/capture-site/SKILL.md | 11 ++++-- skills/capture-site/scripts/assets.mjs | 16 +++++++- skills/capture-site/scripts/capture.mjs | 30 +++++++++++---- test/capture/capture.test.mjs | 47 ++++++++++++++++++++++-- test/capture/readme.test.mjs | 10 +++++ test/fixture/serve.mjs | 1 + test/fixture/site/favicon.ico | Bin 0 -> 70 bytes 7 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 test/fixture/site/favicon.ico diff --git a/skills/capture-site/SKILL.md b/skills/capture-site/SKILL.md index 3bd8fff..f89e1de 100644 --- a/skills/capture-site/SKILL.md +++ b/skills/capture-site/SKILL.md @@ -74,8 +74,12 @@ source. Run each step as its own command, in this order: pass `--verify-shortlinks`. After the crawl it requests one path that cannot exist (`/<32 hex>/`) once, so the site's own 404 page is saved as `pages/404.html` (`meta.json` `notFound`; a site that answers 200 or redirects there is recorded with no page), and asks - for `/favicon.ico` and `/apple-touch-icon.png` once each (`meta.json` `icons`: path to - status). URLs are keyed with upper-case percent escapes (`%c3%a9` and `%C3%A9` are one URL), + for `/favicon.ico` and `/apple-touch-icon.png` once each, redirects followed (`meta.json` + `icons`: path to `{ status, file, contentType, bytes }`); one that answers 200 is saved byte + for byte as `assets/root-favicon.ico` or `assets/root-apple-touch-icon.png` (an HTML answer + is not saved and has a `note`; one over 5 MB is status `"too-large"`), and `assets.mjs` lists + it in `assets/index.json` as kind `"root-icon"` with its `path`, so the build ships it there. + URLs are keyed with upper-case percent escapes (`%c3%a9` and `%C3%A9` are one URL), so a page linked with both spellings is captured once. A sitemap robots.txt names that answers 404 (or any non-200) is listed in `sitemaps/index.json` with its status and in `meta.json` `discoveryErrors`. A sitemap on another host (robots.txt naming a sister site's, or an index listing one; www and apex count @@ -248,7 +252,8 @@ redirects to a captcha path. Every page then looks the same, so a capture of it not saved (an HTTP status, "off-site" for a redirect off the host, "too-large" over 5 MB); [] when the site has none templates.json template clusters with a representative page each - assets/ logo, icons, fonts, index.json + assets/ logo, icons, fonts, index.json; root-favicon.ico and + root-apple-touch-icon.png when live served them at the root screens/live/ live screenshots and manifest.json (gitignored) lighthouse-live/ Lighthouse reports, when run (gitignored) pages/404.html the site's own 404 page (meta.json notFound: path, status, snapshot, text) diff --git a/skills/capture-site/scripts/assets.mjs b/skills/capture-site/scripts/assets.mjs index 1ac47d2..6ff891e 100644 --- a/skills/capture-site/scripts/assets.mjs +++ b/skills/capture-site/scripts/assets.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // Brand assets: the logo, the favicon, the apple-touch-icon and every web font, saved byte for -// byte into /assets/ so the build can use the originals. +// byte into /assets/ so the build can use the originals; the root icons capture.mjs +// saved (assets/root-favicon.ico, assets/root-apple-touch-icon.png) are listed with them. // Sources: and the logo (findLogo: in
first, then in // other logo elements, an image or inline named logo, the home link's image, else the // first ) on the home page snapshot, and every @font-face src in the stylesheets under /css/. @@ -40,7 +41,10 @@ capture.mjs; without it, fonts are reported as not captured). listed with status "too-large" --help show this help -Writes /assets/index.json listing each asset looked for. An asset that does not answer +Writes /assets/index.json listing each asset looked for, and the root icons capture.mjs +asked for (kind "root-icon", with path: /favicon.ico or /apple-touch-icon.png, and file: +assets/root-favicon.ico or assets/root-apple-touch-icon.png when live served it), which the +build copies to public/ at that path. An asset that does not answer 200 is a warning on stderr and is listed there as missing. Files stream to disk; a host that resolves to a loopback, private or link-local address (other than the site's own) is not requested and is listed with status "blocked-address". @@ -339,6 +343,14 @@ async function main() { } } + // The root icons capture.mjs saved (meta.json icons), listed so the build ships each at its + // live path: { kind: "root-icon", url, path, file, status, note }. Never requested here. + for (const [path, icon] of Object.entries(meta.icons ?? {})) { + if (!icon || typeof icon !== 'object' || !origin) continue; // a capture older than the saved bytes + const note = icon.file ? null : icon.note ?? `live answered ${icon.status}`; + entries.push({ kind: 'root-icon', url: new URL(path, origin).href, path, file: icon.file, status: icon.status, note }); + } + const fonts = origin ? fontUrls(dir, origin) : null; if (fonts === null) absent('fonts', 'not captured (no css/manifest.json in the capture; run capture.mjs first)'); else if (fonts.length === 0) absent('fonts', 'none (no @font-face src in the captured CSS)'); diff --git a/skills/capture-site/scripts/capture.mjs b/skills/capture-site/scripts/capture.mjs index 195842a..b10721e 100644 --- a/skills/capture-site/scripts/capture.mjs +++ b/skills/capture-site/scripts/capture.mjs @@ -42,8 +42,9 @@ Crawl a live site, read-only and paced, into a capture folder: one record per pa inventory.json, the raw HTML in pages/.html, the visible text in text/.txt, redirects.json (chains collapsed, hops kept), links.json, scripts.json (third-party vendors), meta.json (notFound: the site's 404 page, from one request for a path that cannot exist, saved -as pages/404.html and text/404.txt; icons: the status /favicon.ico and /apple-touch-icon.png -answered, one request each), sitemaps/ (every sitemap robots.txt names, and their children; +as pages/404.html and text/404.txt; icons: { status, file, contentType, bytes } for +/favicon.ico and /apple-touch-icon.png, one request each, redirects followed, a 200 saved as +assets/root-favicon.ico and assets/root-apple-touch-icon.png), sitemaps/ (every sitemap robots.txt names, and their children; index.json lists every sitemap URL asked for with its status and file, and one robots.txt names that did not answer 200 is also in meta.json discoveryErrors; one on another host is listed with skipped: "other host" and never fetched), robots.txt, feeds/, css/ (every stylesheet and the sheets it @imports, @@ -115,8 +116,11 @@ const SITEMAP_PATH = /sitemap|\.xml(\.gz)?$/i; const OTHER_HOST = 'other host'; // Why a sitemap under an --exclude prefix is listed in sitemaps/index.json but never fetched. const EXCLUDED = 'excluded'; -// The root icons a launch needs, asked for once so a missing one is evidence live lacked it. +// The root icons a launch needs, asked for once so a missing one is evidence live lacked it; +// one that answers 200 is saved as assets/root- for the build to ship at the same path. const ROOT_ICONS = ['/favicon.ico', '/apple-touch-icon.png']; +// A root icon larger than this is recorded "too-large", not saved. +const ICON_MAX_BYTES = 5 * 1024 * 1024; // HTML responses in a row matching only the tiny-page heuristic that stop the run. const SUSPECT_LIMIT = 3; const MAX_ATTEMPTS = 3; @@ -573,21 +577,33 @@ class Crawler { } // /favicon.ico and /apple-touch-icon.png, one request each per capture (redirects followed), - // their status in meta.json `icons`: proof of what live served at the root, whatever the pages - // link. A blocked answer is left unset, so the next run asks again. + // each in meta.json `icons` as { status, file, contentType, bytes }: proof of what live served + // at the root, whatever the pages link. A 200 is saved byte for byte as assets/root- + // (file); a 200 that is an HTML page (a soft 404) is not saved and says so in `note`; one + // over ICON_MAX_BYTES is status "too-large". A blocked answer is left unset, so the next run + // asks again. async probeIcons() { this.state.icons ??= {}; for (const path of ROOT_ICONS) { if (this.state.icons[path] !== undefined) continue; const url = new URL(path, this.state.canonicalOrigin).href; - const res = await this.fetch(url, { redirect: 'follow' }); + const res = await this.fetch(url, { redirect: 'follow', raw: true, maxBytes: ICON_MAX_BYTES }); const verdict = this.judge(res, url); if (verdict?.stop) { this.noteChallenge(url, verdict.stop, { row: false }); return 'challenge'; } if (BLOCK_STATUSES.has(res.status)) continue; - this.state.icons[path] = res.status; + const contentType = res.headers?.['content-type'] ?? null; + const entry = { status: res.status, file: null, contentType, bytes: null }; + if (res.status === 200 && HTML_TYPE.test(contentType ?? '')) entry.note = 'answered HTML, not an icon'; + else if (res.status === 200) { + const body = Buffer.isBuffer(res.body) ? res.body : Buffer.from(String(res.body ?? '')); + entry.file = `assets/root-${basename(path)}`; + entry.bytes = body.length; + writeText(join(this.out, entry.file), body); + } + this.state.icons[path] = entry; } return 'done'; } diff --git a/test/capture/capture.test.mjs b/test/capture/capture.test.mjs index 19157df..12b8dc8 100644 --- a/test/capture/capture.test.mjs +++ b/test/capture/capture.test.mjs @@ -271,7 +271,7 @@ test('a sitemap robots.txt names that answers 404 is recorded in sitemaps/index. } }); -test('the capture asks for /favicon.ico and /apple-touch-icon.png once and records their status in meta.json icons', async () => { +test('the capture asks for /favicon.ico and /apple-touch-icon.png once, saves what answered 200 and records each in meta.json icons', async () => { const stub = await startStub({ '/': { body: 'Pagehome' }, '/favicon.ico': { status: 200, headers: { 'Content-Type': 'image/x-icon' }, body: 'ico' }, @@ -280,7 +280,12 @@ test('the capture asks for /favicon.ico and /apple-touch-icon.png once and recor const out = tempDir(); const run = await capture([stub.origin, '--out', out, '--delay', '0']); assert.equal(run.code, 0, run.stderr); - assert.deepEqual(readJson(out, 'meta.json').icons, { '/favicon.ico': 200, '/apple-touch-icon.png': 404 }); + assert.deepEqual(readJson(out, 'meta.json').icons, { + '/favicon.ico': { status: 200, file: 'assets/root-favicon.ico', contentType: 'image/x-icon', bytes: 3 }, + '/apple-touch-icon.png': { status: 404, file: null, contentType: 'text/html; charset=UTF-8', bytes: null }, + }); + assert.equal(readFileSync(join(out, 'assets/root-favicon.ico'), 'utf8'), 'ico'); + assert.ok(!existsSync(join(out, 'assets/root-apple-touch-icon.png'))); const asked = stub.requests.filter((u) => u === '/favicon.ico' || u === '/apple-touch-icon.png'); assert.deepEqual(asked.sort(), ['/apple-touch-icon.png', '/favicon.ico']); const again = await capture([stub.origin, '--out', out, '--delay', '0', '--resume']); @@ -291,6 +296,39 @@ test('the capture asks for /favicon.ico and /apple-touch-icon.png once and recor } }); +test('a root icon is saved where its redirects end; an HTML answer or one over 5 MB is recorded, not saved', async () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + const stub = await startStub({ + '/': { body: 'Pagehome' }, + '/apple-touch-icon.png': { status: 301, headers: { Location: '/wp-content/uploads/site-icon-180.png' } }, + '/wp-content/uploads/site-icon-180.png': { headers: { 'Content-Type': 'image/png' }, body: png }, + '/favicon.ico': { body: 'Homea page, not an icon' }, + }); + const big = await startStub({ + '/': { body: 'Pagehome' }, + '/favicon.ico': { headers: { 'Content-Type': 'image/x-icon' }, body: Buffer.alloc(5 * 1024 * 1024 + 1) }, + }); + try { + const out = tempDir(); + const run = await capture([stub.origin, '--out', out, '--delay', '0']); + assert.equal(run.code, 0, run.stderr); + const icons = readJson(out, 'meta.json').icons; + assert.deepEqual(icons['/apple-touch-icon.png'], { status: 200, file: 'assets/root-apple-touch-icon.png', contentType: 'image/png', bytes: png.length }); + assert.ok(readFileSync(join(out, 'assets/root-apple-touch-icon.png')).equals(png)); + assert.deepEqual(icons['/favicon.ico'], { status: 200, file: null, contentType: 'text/html; charset=UTF-8', bytes: null, note: 'answered HTML, not an icon' }); + assert.ok(!existsSync(join(out, 'assets/root-favicon.ico'))); + + const large = tempDir(); + const second = await capture([big.origin, '--out', large, '--delay', '0']); + assert.equal(second.code, 0, second.stderr); + assert.deepEqual(readJson(large, 'meta.json').icons['/favicon.ico'], { status: 'too-large', file: null, contentType: 'image/x-icon', bytes: null }); + assert.ok(!existsSync(join(large, 'assets/root-favicon.ico'))); + } finally { + await stub.close(); + await big.close(); + } +}); + test('robots.txt and sitemaps that always answer 403 are noted, not fatal', async () => { const stub = await startStub({ '/': { body: 'Pagehome' }, @@ -409,7 +447,10 @@ test('snapshots, text, robots, sitemaps, feeds, links and scripts are written', } const index = readJson(full, 'sitemaps/index.json'); assert.deepEqual(index.find((e) => e.url.endsWith('/page-sitemap.xml')), { url: `${fixture.origin}/page-sitemap.xml`, status: 200, file: 'sitemaps/page-sitemap.xml' }); - assert.deepEqual(readJson(full, 'meta.json').icons, { '/favicon.ico': 404, '/apple-touch-icon.png': 404 }); + const icons = readJson(full, 'meta.json').icons; + assert.deepEqual(icons['/favicon.ico'], { status: 200, file: 'assets/root-favicon.ico', contentType: 'image/x-icon', bytes: 70 }); + assert.ok(readFileSync(join(full, 'assets/root-favicon.ico')).equals(readFileSync(join(SITE, 'favicon.ico'))), 'saved byte for byte'); + assert.deepEqual(icons['/apple-touch-icon.png'], { status: 404, file: null, contentType: 'text/html; charset=UTF-8', bytes: null }); assert.match(readFileSync(join(full, 'feeds/feed.xml'), 'utf8'), / f.path), ['/feed/']); diff --git a/test/capture/readme.test.mjs b/test/capture/readme.test.mjs index 7250794..7c659fa 100644 --- a/test/capture/readme.test.mjs +++ b/test/capture/readme.test.mjs @@ -101,6 +101,16 @@ test('assets.mjs saves the logo, the favicon and both fonts byte for byte', () = assert.match(assets.stdout, /apple-touch-icon: none/); }); +test('assets.mjs lists the root icons the capture saved, for the build to ship at the same paths', () => { + const index = JSON.parse(readFileSync(join(dir, 'assets', 'index.json'), 'utf8')); + const roots = index.filter((e) => e.kind === 'root-icon'); + assert.deepEqual(roots, [ + { kind: 'root-icon', url: `${fixture.origin}/favicon.ico`, path: '/favicon.ico', file: 'assets/root-favicon.ico', status: 200, note: null }, + { kind: 'root-icon', url: `${fixture.origin}/apple-touch-icon.png`, path: '/apple-touch-icon.png', file: null, status: 404, note: 'live answered 404' }, + ]); + sameBytes(join(dir, 'assets', 'root-favicon.ico'), join(SITE, 'favicon.ico')); +}); + test('assets.mjs without captured CSS still saves the logo and says fonts were not captured', () => { assert.equal(bareAssets.code, 0, bareAssets.stderr); assert.ok(existsSync(join(bare, 'assets', 'logo.svg'))); diff --git a/test/fixture/serve.mjs b/test/fixture/serve.mjs index 8a45d56..422e03f 100644 --- a/test/fixture/serve.mjs +++ b/test/fixture/serve.mjs @@ -33,6 +33,7 @@ const CONTENT_TYPES = { '.jpg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif', + '.ico': 'image/x-icon', '.pdf': 'application/pdf', '.woff2': 'font/woff2', }; diff --git a/test/fixture/site/favicon.ico b/test/fixture/site/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..eefafbb8e7d402f28273afc7b9f72a56d34cd7d5 GIT binary patch literal 70 ucmZQzU<5%%1|X@xV8Fn@AO^%5KnxUOU;@($K$3xh15Bb4#%X8%gG2x`Bm*P> literal 0 HcmV?d00001 From f5bc698ad099b35363df23ab7faaf03acf5c843b Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 23 Oct 2026 00:06:28 -0400 Subject: [PATCH 03/16] fix(gates): links and the robots.txt sitemap check skip paths under meta.excluded; reconcile reads the meta.json icons objects; build-rules ships root-icon assets --- skills/capture-site/SKILL.md | 5 +-- skills/convert-site/references/build-rules.md | 5 +++ .../scripts/prelaunch-reconcile.mjs | 3 +- skills/parity-gates/SKILL.md | 5 +++ skills/parity-gates/scripts/verify/links.mjs | 14 ++++++++ .../parity-gates/scripts/verify/sitemaps.mjs | 19 +++++++---- test/convert/prelaunch-reconcile.test.mjs | 17 ++++++++++ test/gates/fixtures/baseline.mjs | 10 +++--- test/gates/links.test.mjs | 24 +++++++++++++- test/gates/sitemaps.test.mjs | 32 +++++++++++++++++++ 10 files changed, 119 insertions(+), 15 deletions(-) diff --git a/skills/capture-site/SKILL.md b/skills/capture-site/SKILL.md index f89e1de..f85afa6 100644 --- a/skills/capture-site/SKILL.md +++ b/skills/capture-site/SKILL.md @@ -233,8 +233,9 @@ redirects to a captcha path. Every page then looks the same, so a capture of it inventory.json one record per page: status, head tags, JSON-LD, headings, forms, links pages/ text/ raw HTML and visible text per page redirects.json every redirect, chains collapsed, hops kept; shortlinks marked verified or not - links.json internal and external links, with the pages that carry them; with - --exclude, links under an excluded prefix in "excluded" + links.json { internal, external } maps of link -> [pages that carry it]; with + --exclude also "excluded": { "": [pages] }, the links under an + excluded prefix (kept in the HTML, never in internal, never requested) scripts.json third-party vendors with their ids and pages media.json every media file with sources, pages, status, type and bytes; a size only the REST list names: downloaded: false, "unreferenced size" diff --git a/skills/convert-site/references/build-rules.md b/skills/convert-site/references/build-rules.md index c4fc4ae..37f0df4 100644 --- a/skills/convert-site/references/build-rules.md +++ b/skills/convert-site/references/build-rules.md @@ -124,6 +124,11 @@ parity and this file is wrong: say so and fix this file. with the same families, weights and styles. Replacing a font service link with local `@font-face` rules is a decision row; the visual review proves it. - Logo, favicon, apple-touch-icon and the web manifest from `capture/assets/`, same paths. +- Every `root-icon` entry in `capture/assets/index.json` that has a `file` is copied byte for + byte to `public/` (`assets/root-favicon.ico` → `public/favicon.ico`, + `assets/root-apple-touch-icon.png` → `public/apple-touch-icon.png`). One with `file: null` is + an icon live did not serve at the root (its status and note say why); copy-assets copies each + `root-icon` entry with a `file` to `public/` plus its `path`. ## Scripts diff --git a/skills/convert-site/scripts/prelaunch-reconcile.mjs b/skills/convert-site/scripts/prelaunch-reconcile.mjs index e611eaf..955a8f6 100644 --- a/skills/convert-site/scripts/prelaunch-reconcile.mjs +++ b/skills/convert-site/scripts/prelaunch-reconcile.mjs @@ -190,7 +190,8 @@ export function loadBaseline(dir, captureDir = null) { if (route) return route.status; if (requestedMedia.has(path)) return 200; if (mediaPaths.has(path)) return null; - return assetAt(path)?.status ?? icons[path] ?? null; + // meta.json icons: { status, file, contentType, bytes } per path (a bare status in older captures). + return assetAt(path)?.status ?? icons[path]?.status ?? icons[path] ?? null; }, // The size of live's PNG at path, from the capture's saved asset. livePngSize(path) { diff --git a/skills/parity-gates/SKILL.md b/skills/parity-gates/SKILL.md index 2c1356e..22dc242 100644 --- a/skills/parity-gates/SKILL.md +++ b/skills/parity-gates/SKILL.md @@ -93,6 +93,11 @@ rebuild, rerun. One gate alone: `node /scripts/verify/.mjs`. | `sitemaps` | Every top sitemap in `dist/` (Astro's `sitemap-index.xml`, or a live name in `baseline/sitemaps.json` such as `wp-sitemap.xml` or `sitemap_index.xml`) lists exactly the indexable 200 pages on the origin (percent escapes compare case-blind: `%c3` is `%C3`); every robots.txt `Sitemap:` line is on the origin and names a file in `dist/`, or is a line the live robots.txt (`capture/robots.txt`) carried verbatim; every feed is built with as many items | | | `text` | Each page's visible text equals `baseline/text/` (word diff, 5 words of context) | | +A capture made with `--exclude` records its path prefixes in `baseline/meta.json` `excluded`: the +`links` gate does not check a link under one (like a link to another host, noted "N links under +excluded paths ... not checked"), and the `sitemaps` gate accepts a robots.txt `Sitemap:` line under +one when the live robots.txt carried it verbatim (a multisite sub-site's sitemap on the same host). + ## 4. Differences and decisions When the build cannot or should not match live, read `references/exceptions.md`. In short: an diff --git a/skills/parity-gates/scripts/verify/links.mjs b/skills/parity-gates/scripts/verify/links.mjs index 9805c74..9d7f86c 100644 --- a/skills/parity-gates/scripts/verify/links.mjs +++ b/skills/parity-gates/scripts/verify/links.mjs @@ -31,6 +31,9 @@ dist/_redirects; the slash rules and every redirects.csv row) take it to a built one hop, as live redirected it; the summary notes "N links resolved through redirects". Two hops, or a hop to a page that is not built, still fail. Without a host config the note says so and no link is resolved that way. +A link whose path is under a prefix the capture left out (--exclude, baseline/meta.json +excluded: a multisite's sub-site, a section the owner kept) is not checked, like a link to +another host: the build serves no such page. `; export const BROKEN = 'a built page, file, redirect source or media path'; const SKIP_SCHEME = /^(?:mailto|tel|sms|javascript|data|blob|about):/i; @@ -83,6 +86,11 @@ export function resolveLink(url, { present, format, redirectSources, mediaPaths return BROKEN; } +// True when path is under one of the capture's --exclude prefixes (baseline/meta.json excluded); +// a prefix ending in / also covers its path without the slash. +const underExcluded = (path, prefixes = []) => + prefixes.some((p) => path.startsWith(p) || (p.endsWith('/') && path === p.slice(0, -1))); + // A link that lands on a page or file in dist in one hop through the host's rules. const oneHop = (resolve, url) => { const { hops, final } = resolve(`${url.pathname}${url.search}`); @@ -102,6 +110,7 @@ export function checkLinks({ dist, redirects, media, meta, exceptions, origin = const relevant = exceptions.filter((e) => e.field === 'link'); const pages = htmlPages(files).filter((p) => paths(p.path)); let viaRedirect = 0; + let excluded = 0; const known = new Set(pages.map((p) => p.path)); const failures = relevant.filter((e) => paths(e.path) && !known.has(e.path)).map((e) => exceptionFailure(e, 'path not checked')); for (const page of pages) { @@ -117,6 +126,10 @@ export function checkLinks({ dist, redirects, media, meta, exceptions, origin = continue; } if (!internal.has(url.origin)) continue; + if (underExcluded(decode(url.pathname), meta.excluded)) { + excluded += 1; + continue; + } const expected = resolveLink(url, context); if (expected && resolve && oneHop(resolve, url)) viaRedirect += 1; else if (expected) broken.push({ href, expected }); @@ -131,6 +144,7 @@ export function checkLinks({ dist, redirects, media, meta, exceptions, origin = } } const notes = [resolve ? `${viaRedirect} links resolved through redirects` : 'no host config (vercel.json or dist/_redirects): no link resolved through redirects']; + if (meta.excluded?.length) notes.push(`${excluded} links under excluded paths (${meta.excluded.join(', ')}) not checked`); return { checked: pages.length, failures, notes }; } diff --git a/skills/parity-gates/scripts/verify/sitemaps.mjs b/skills/parity-gates/scripts/verify/sitemaps.mjs index f355567..7a1f220 100644 --- a/skills/parity-gates/scripts/verify/sitemaps.mjs +++ b/skills/parity-gates/scripts/verify/sitemaps.mjs @@ -27,7 +27,9 @@ exactly the expected pages; with more than one, a failure names it ("sitemap wp- Every "Sitemap:" line of dist/robots.txt must be a URL on the origin (--origin, else canonicalOrigin) naming a file in dist, or a line the live robots.txt carried verbatim: a URL on another host (a sister site's sitemap) passes when /robots.txt, the live file the -capture saved, has a Sitemap: line with exactly that URL. No live request is made; without +capture saved, has a Sitemap: line with exactly that URL. So does a line the live robots.txt +carried verbatim whose path is under a prefix the capture left out (--exclude, baseline/meta.json +excluded: a multisite's sub-site sitemap on the same host). No live request is made; without /robots.txt every other-host line fails. Feed items are (RSS) or (Atom). `; @@ -102,8 +104,10 @@ const hostOf = (url) => { // Every "Sitemap:" line of dist/robots.txt is a URL on compareOrigin naming a file in dist, or a // URL on another host (never the site's own, `ownOrigins`, www or not) that the live robots.txt -// carried verbatim (`live`, its Sitemap: URLs). -export function robotsSitemapFailures(dist, compareOrigin, live = [], ownOrigins = []) { +// carried verbatim (`live`, its Sitemap: URLs), or a carried line under an `excluded` prefix. +const underExcluded = (path, prefixes) => prefixes.some((p) => path.startsWith(p) || (p.endsWith('/') && path === p.slice(0, -1))); + +export function robotsSitemapFailures(dist, compareOrigin, live = [], ownOrigins = [], excluded = []) { const file = join(dist, 'robots.txt'); if (!existsSync(file)) return { checked: 0, failures: [] }; const lines = sitemapLines(readFileSync(file, 'utf8')); @@ -115,6 +119,7 @@ export function robotsSitemapFailures(dist, compareOrigin, live = [], ownOrigins const path = pathOf(url); const rel = path.replace(/^\/+/, ''); if (origin !== compareOrigin && carried.has(url) && !ownHosts.has(hostOf(url))) continue; + if (carried.has(url) && underExcluded(path, excluded)) continue; if (origin !== compareOrigin) failures.push({ path: '/robots.txt', field: 'sitemap line', expected: `${compareOrigin}${origin ? new URL(url).pathname : ''}`, actual: url }); else if (!rel || !existsSync(join(dist, rel)) || !statSync(join(dist, rel)).isFile()) { failures.push({ path: '/robots.txt', field: 'sitemap line', expected: 'a file in dist', actual: url }); @@ -126,7 +131,7 @@ export function robotsSitemapFailures(dist, compareOrigin, live = [], ownOrigins // %e2 and %E2 are the same octet (RFC 3986 6.2.2.1), so URLs compare with upper-case escapes. const upperEscapes = (url) => url?.replace(/%[0-9a-f]{2}/gi, (m) => m.toUpperCase()) ?? null; -export function checkSitemaps({ dist, capture, routes, compareOrigin, ownOrigins = [], sitemaps = [], paths = () => true }) { +export function checkSitemaps({ dist, capture, routes, compareOrigin, ownOrigins = [], excluded = [], sitemaps = [], paths = () => true }) { const pages = routes.filter((r) => r.status === 200 && PAGE_KINDS.has(r.kind) && !r.noindex && paths(r.path)); const feeds = routes.filter((r) => r.status === 200 && r.kind === 'feed' && paths(r.path)); const { roots, failures } = readSitemaps(dist, sitemaps); @@ -138,13 +143,13 @@ export function checkSitemaps({ dist, capture, routes, compareOrigin, ownOrigins for (const path of expected) { const want = `${compareOrigin}${new URL(path, compareOrigin).pathname}`; const got = listed.get(path) ?? null; - if (upperEscapes(got) !== want) failures.push({ path, field, expected: want, actual: got }); + if (upperEscapes(got) !== upperEscapes(want)) failures.push({ path, field, expected: want, actual: got }); } for (const [path, url] of listed) { if (!expected.has(path) && paths(path)) failures.push({ path, field, expected: null, actual: url }); } } - const robots = paths('/robots.txt') ? robotsSitemapFailures(dist, compareOrigin, liveSitemapLines(capture), ownOrigins) : { checked: 0, failures: [] }; + const robots = paths('/robots.txt') ? robotsSitemapFailures(dist, compareOrigin, liveSitemapLines(capture), ownOrigins, excluded) : { checked: 0, failures: [] }; failures.push(...robots.failures); for (const feed of feeds) { const candidates = feedFiles(feed.path); @@ -188,7 +193,7 @@ function main() { const sitemapsFile = join(args.baseline, 'sitemaps.json'); const sitemaps = existsSync(sitemapsFile) ? readJson(sitemapsFile) : []; const ownOrigins = [meta.canonicalOrigin, meta.origin].map((o) => originOf(o ?? '')).filter(Boolean); - const results = checkSitemaps({ dist: args.dist, capture: args.capture, routes, compareOrigin, ownOrigins, sitemaps, paths: args.paths }); + const results = checkSitemaps({ dist: args.dist, capture: args.capture, routes, compareOrigin, ownOrigins, excluded: meta.excluded ?? [], sitemaps, paths: args.paths }); report(NAME, results, { json: args.json }); } diff --git a/test/convert/prelaunch-reconcile.test.mjs b/test/convert/prelaunch-reconcile.test.mjs index b4c4386..52c0ad4 100644 --- a/test/convert/prelaunch-reconcile.test.mjs +++ b/test/convert/prelaunch-reconcile.test.mjs @@ -336,6 +336,23 @@ test('favicon rows read meta.json icons: a root icon live answered 404 for is re assert.match(parsed(served).blocking[0].rows[0].reason, /live served \/apple-touch-icon\.png/); }); +test('favicon rows read the meta.json icons objects a capture writes ({ status, file, ... })', async () => { + // The fixture captured with --exclude: its meta.json icons say /favicon.ico was served (and saved) + // and /apple-touch-icon.png answered 404. + const excluded = join(root, 'excluded'); + await buildFixtureBaseline(excluded, { exclude: '/services/' }); + const { icons } = JSON.parse(readFileSync(join(excluded, 'capture', 'meta.json'), 'utf8')); + assert.equal(icons['/favicon.ico'].status, 200); + assert.equal(icons['/apple-touch-icon.png'].status, 404); + const captureDir = notFoundCapture('capture-icons-objects', { icons }); + const planFile = planWith('plan-icons-objects', ['d2', 'output.favicon-set: icons as live had them']); + const apple = await reconcile([failing('output.favicon-set', 'apple-touch-icon.png is missing', ['apple-touch-icon.png is missing'], 'file')], { planFile, captureDir }); + assert.equal(apple.code, 0, apple.stdout); + const favicon = await reconcile([failing('output.favicon-set', 'favicon.ico is missing', ['favicon.ico is missing'], 'file')], { planFile, captureDir }); + assert.equal(favicon.code, 1, favicon.stdout); + assert.match(parsed(favicon).blocking[0].rows[0].reason, /live served \/favicon\.ico/); +}); + // A baseline whose live site published no sitemap, and a site whose dist builds the given routes. function noSitemapSite(name, { drop = [] } = {}) { const base = join(root, `${name}-baseline`); diff --git a/test/gates/fixtures/baseline.mjs b/test/gates/fixtures/baseline.mjs index 21626fc..ceb2b07 100644 --- a/test/gates/fixtures/baseline.mjs +++ b/test/gates/fixtures/baseline.mjs @@ -2,7 +2,8 @@ // run once per directory and cached for the test file. buildFixtureBaseline(dir) resolves to the // baseline folder (/baseline); the capture is in /capture and redirects.csv in . // The capture seeds /old-services and /about so the baseline has a redirect chain and a slash -// redirect besides the shortlinks. +// redirect besides the shortlinks. buildFixtureBaseline(dir, { exclude }) captures with +// --exclude (a comma-separated prefix list) instead. import { execFile } from 'node:child_process'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -21,7 +22,7 @@ async function run(path, args) { } } -async function build(dir) { +async function build(dir, { exclude = null } = {}) { const capture = join(dir, 'capture'); const baseline = join(dir, 'baseline'); // Async child processes: the fixture server runs in this process and must keep answering. @@ -29,6 +30,7 @@ async function build(dir) { try { await run('capture-site/scripts/capture.mjs', [ fixture.origin, '--out', capture, '--delay', '0', '--no-wayback', '--seed', '/old-services', '--seed', '/about', + ...(exclude ? ['--exclude', exclude] : []), ]); await run('capture-site/scripts/media.mjs', ['--capture', capture, '--delay', '0']); await run('capture-site/scripts/templates.mjs', ['--capture', capture]); @@ -39,7 +41,7 @@ async function build(dir) { return baseline; } -export function buildFixtureBaseline(dir) { - if (!cache.has(dir)) cache.set(dir, build(dir)); +export function buildFixtureBaseline(dir, options = {}) { + if (!cache.has(dir)) cache.set(dir, build(dir, options)); return cache.get(dir); } diff --git a/test/gates/links.test.mjs b/test/gates/links.test.mjs index 66b12e7..f5eb81b 100644 --- a/test/gates/links.test.mjs +++ b/test/gates/links.test.mjs @@ -1,6 +1,6 @@ import { test, before, after } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { buildFixtureBaseline } from './fixtures/baseline.mjs'; @@ -147,6 +147,28 @@ test('cloudflare: Pages slash handling resolves an unslashed link in one hop', a assert.match(result.stdout, /^ {2}1 links resolved through redirects$/m); }); +test('a link under a path the capture excluded (--exclude) is not checked; without the exclusion it fails', async () => { + // The fixture captured with --exclude /services/: the build serves no services page, but the + // other pages keep their links to them, as live had them. + const excludedBaseline = await buildFixtureBaseline(join(root, 'excluded'), { exclude: '/services/' }); + assert.deepEqual(JSON.parse(readFileSync(join(excludedBaseline, 'meta.json'), 'utf8')).excluded, ['/services/']); + const dist = makeDist(join(root, 'dist-excluded')); + rmSync(join(dist, 'services'), { recursive: true }); + const passed = await runGate('links', ['--dist', dist, '--baseline', excludedBaseline, '--origin', DIST_ORIGIN], { cwd: root }); + assert.equal(passed.code, 0, passed.stdout + passed.stderr); + assert.match(passed.stdout, /links under excluded paths \(\/services\/\) not checked/); + assert.match(passed.stdout, /^links: 11 checked, 0 failed$/m); + + const failed = await links(dist); + assert.equal(failed.code, 1, 'the full baseline has no exclusion'); + assert.match(failed.stdout, /^ {2}\/ link: expected "a built page, file, redirect source or media path", got "\/services\/roofing\/"$/m); + + edit(dist, 'about/index.html', '', ''); + const broken = await runGate('links', ['--dist', dist, '--baseline', excludedBaseline, '--origin', DIST_ORIGIN], { cwd: root }); + assert.equal(broken.code, 1, 'a link outside the excluded prefix is still checked'); + assert.match(broken.stdout, /\/about\/ link: .*got "\/contact-us\/"/); +}); + test('usage and missing inputs exit 2; --help exits 0', async () => { assert.equal((await links(good, ['--bogus'])).code, 2); assert.equal((await links(good, ['--origin', 'not a url'])).code, 2); diff --git a/test/gates/sitemaps.test.mjs b/test/gates/sitemaps.test.mjs index 06f00c0..0b461e2 100644 --- a/test/gates/sitemaps.test.mjs +++ b/test/gates/sitemaps.test.mjs @@ -144,6 +144,38 @@ test('a carried Sitemap: line on the site\'s own host is never skipped: with --o assert.match(result.stdout, /\/robots\.txt sitemap line: expected "https:\/\/staging\.example\.com\//); }); +test('a carried Sitemap: line under a path the capture excluded passes (a multisite sub-site); not without the exclusion', async () => { + const dir = join(root, 'excluded'); + const excludedBaseline = await buildFixtureBaseline(dir, { exclude: '/services/' }); + const excludedCapture = join(dir, 'capture'); + const liveOrigin = JSON.parse(readFileSync(join(excludedCapture, 'meta.json'), 'utf8')).origin; + // The live robots.txt also names the excluded sub-site's sitemap index, on the same host. + const subSite = `${liveOrigin}/services/sitemap_index.xml`; + writeFileSync(join(excludedCapture, 'robots.txt'), `${readFileSync(join(excludedCapture, 'robots.txt'), 'utf8')}Sitemap: ${subSite}\n`); + const dist = makeDist(join(root, 'dist-robots-excluded')); + rmSync(join(dist, 'services'), { recursive: true }); + for (const path of ['/services/', '/services/roofing/', '/services/siding/']) edit(dist, 'sitemap-0.xml', `${DIST_ORIGIN}${path}`, ''); + const own = `User-agent: *\nSitemap: ${DIST_ORIGIN}/sitemap-index.xml\n`; + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${subSite}\n`); + const gate = (base, capt) => runGate('sitemaps', ['--dist', dist, '--baseline', base, '--capture', capt, '--origin', DIST_ORIGIN], { cwd: root }); + + const carried = await gate(excludedBaseline, excludedCapture); + assert.equal(carried.code, 0, carried.stdout + carried.stderr); + assert.match(carried.stdout, /^sitemaps: 12 checked, 0 failed$/m); + + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${liveOrigin}/services/other-sitemap.xml\n`); + const notLive = await gate(excludedBaseline, excludedCapture); + assert.equal(notLive.code, 1, 'a line live did not carry still fails'); + + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${subSite}\n`); + const copied = join(root, 'capture-with-subsite'); + cpSync(capture, copied, { recursive: true }); + writeFileSync(join(copied, 'robots.txt'), `${readFileSync(join(copied, 'robots.txt'), 'utf8')}Sitemap: ${subSite}\n`); + const unexcluded = await gate(baseline, copied); + assert.equal(unexcluded.code, 1, 'without the exclusion the same line fails'); + assert.match(unexcluded.stdout, /\/robots\.txt sitemap line: .*services\/sitemap_index\.xml/); +}); + test('a robots.txt Sitemap: line must point at a file in dist', async () => { const dist = makeDist(join(root, 'dist-robots')); writeFileSync(join(dist, 'robots.txt'), `User-agent: *\nSitemap: ${DIST_ORIGIN}/sitemap-index.xml\nSitemap: ${DIST_ORIGIN}/sitemap_index.xml\n`); From c2b3a96015d8a0fa56ebd0320d525a9a0178fae1 Mon Sep 17 00:00:00 2001 From: Ethan Date: Fri, 23 Oct 2026 00:16:17 -0400 Subject: [PATCH 04/16] fix(capture): --exclude compares paths in one canonical form; a prefix added on --resume removes what was captured under it; stylesheets and root icons never follow a redirect under a prefix --- skills/capture-site/SKILL.md | 10 +- skills/capture-site/scripts/capture.mjs | 156 +++++++++++++++--- skills/capture-site/scripts/lib/exclude.mjs | 21 +-- skills/parity-gates/scripts/lib/excluded.mjs | 31 ++++ skills/parity-gates/scripts/verify/links.mjs | 8 +- .../parity-gates/scripts/verify/sitemaps.mjs | 5 +- test/capture/exclude.test.mjs | 89 +++++++++- 7 files changed, 275 insertions(+), 45 deletions(-) create mode 100644 skills/parity-gates/scripts/lib/excluded.mjs diff --git a/skills/capture-site/SKILL.md b/skills/capture-site/SKILL.md index f85afa6..9dac815 100644 --- a/skills/capture-site/SKILL.md +++ b/skills/capture-site/SKILL.md @@ -92,9 +92,13 @@ source. Run each step as its own command, in this order: A sitemap under one is listed in `sitemaps/index.json` as `{ url, skipped: "excluded" }`; a redirect into one is recorded with that target, unrequested; links to one stay in the saved HTML and are listed in `links.json` `excluded`, not `internal`; `meta.json` `excluded` names - the prefixes and the baseline carries them. `--resume` keeps them (more given there are - added, and what was queued under them is dropped); `media.mjs` and `assets.mjs` skip files - under them too. + the prefixes and the baseline carries them. Prefixes and paths compare in one form, so + `/présidence/`, `/pr%c3%a9sidence/` and `/pr%C3%A9sidence/` are the same prefix. A stylesheet + or root icon that redirects under one is not followed there (the sheet is listed with status + `"excluded"`). `--resume` keeps them; a prefix added on `--resume` removes what was already + captured under it (its pages with their `pages/` and `text/` files, redirects from it, feeds + and sitemaps under it, and stylesheets and scripts only those pages used) and prints what it + removed. `media.mjs` and `assets.mjs` skip files under them too. node /scripts/capture.mjs https://www.example.com --out capture diff --git a/skills/capture-site/scripts/capture.mjs b/skills/capture-site/scripts/capture.mjs index b10721e..924ec80 100644 --- a/skills/capture-site/scripts/capture.mjs +++ b/skills/capture-site/scripts/capture.mjs @@ -18,7 +18,7 @@ // After the crawl one path that cannot exist (/<32 hex>/) is requested once, so the site's own // 404 page lands in pages/404.html and meta.json `notFound` (it is not a route). import { createHash, randomBytes } from 'node:crypto'; -import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { basename, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { assertNodeVersion } from './lib/node-version.mjs'; @@ -79,8 +79,12 @@ to list and download the media. (repeatable), e.g. a multisite's sub-sites: --exclude /president/,/news-room/. Sitemaps under one are listed in sitemaps/index.json as skipped: "excluded", links to one stay in the HTML and are listed in links.json "excluded", - and meta.json "excluded" names the prefixes. --resume keeps them; more - given on --resume are added + and meta.json "excluded" names the prefixes. Paths compare in one form + (/présidence/ = /pr%c3%a9sidence/). A stylesheet or root icon that + redirects under one is not followed there. --resume keeps them; one + added on --resume removes what was already captured under it (pages and + their files, redirects from it, stylesheets and scripts only those pages + used) and says what it removed --help show this help Never requests wp-admin, wp-login, xmlrpc.php or ?s= searches, and /wp-json/ only for the @@ -285,6 +289,10 @@ function sitemapFileName(href, taken = new Map()) { for (let n = 2; ; n += 1) if (free(withSuffix(base, String(n)))) return withSuffix(base, String(n)); } +// The next free numbered file name (.css, .js) among entries: one past the highest, so a +// name is never reused after entries are dropped (--resume with a new --exclude prefix). +const nextFile = (entries, ext) => `${Math.max(0, ...entries.map((e) => Number.parseInt(e.file, 10)).filter(Number.isFinite)) + 1}.${ext}`; + const byKey = (key) => (a, b) => (a[key] < b[key] ? -1 : a[key] > b[key] ? 1 : 0); // The text of every