diff --git a/skills/capture-site/SKILL.md b/skills/capture-site/SKILL.md index 806af4d..9dac815 100644 --- a/skills/capture-site/SKILL.md +++ b/skills/capture-site/SKILL.md @@ -74,14 +74,31 @@ source. Run each step as its own command, in this order: pass `--verify-shortlinks`. After the crawl it requests one path that cannot exist (`/<32 hex>/`) once, so the site's own 404 page is saved as `pages/404.html` (`meta.json` `notFound`; a site that answers 200 or redirects there is recorded with no page), and asks - for `/favicon.ico` and `/apple-touch-icon.png` once each (`meta.json` `icons`: path to - status). URLs are keyed with upper-case percent escapes (`%c3%a9` and `%C3%A9` are one URL), + for `/favicon.ico` and `/apple-touch-icon.png` once each, redirects followed (`meta.json` + `icons`: path to `{ status, file, contentType, bytes }`); one that answers 200 is saved byte + for byte as `assets/root-favicon.ico` or `assets/root-apple-touch-icon.png` (an HTML answer + is not saved and has a `note`; one over 5 MB is status `"too-large"`), and `assets.mjs` lists + it in `assets/index.json` as kind `"root-icon"` with its `path`, so the build ships it there. + URLs are keyed with upper-case percent escapes (`%c3%a9` and `%C3%A9` are one URL), so a page linked with both spellings is captured once. A sitemap robots.txt names that answers 404 (or any non-200) is listed in `sitemaps/index.json` with its status and in `meta.json` `discoveryErrors`. A sitemap on another host (robots.txt naming a sister site's, or an index listing one; www and apex count as one host) is listed there as `{ url, host, skipped: "other host" }` and never fetched, so none of its URLs enter the crawl. Two sitemap URLs with the same file name never overwrite each other: the second is saved with a suffix from its directory path (`page-sitemap-es.xml`). + `--exclude [,...]` leaves parts of the host out (a WordPress multisite's + sub-sites such as `/president/`, or a section the owner keeps): a URL whose path starts with + one is never requested, whether a sitemap, the REST lists, a link or a redirect leads there. + A sitemap under one is listed in `sitemaps/index.json` as `{ url, skipped: "excluded" }`; a + redirect into one is recorded with that target, unrequested; links to one stay in the saved + HTML and are listed in `links.json` `excluded`, not `internal`; `meta.json` `excluded` names + the prefixes and the baseline carries them. Prefixes and paths compare in one form, so + `/présidence/`, `/pr%c3%a9sidence/` and `/pr%C3%A9sidence/` are the same prefix. A stylesheet + or root icon that redirects under one is not followed there (the sheet is listed with status + `"excluded"`). `--resume` keeps them; a prefix added on `--resume` removes what was already + captured under it (its pages with their `pages/` and `text/` files, redirects from it, feeds + and sitemaps under it, and stylesheets and scripts only those pages used) and prints what it + removed. `media.mjs` and `assets.mjs` skip files under them too. node /scripts/capture.mjs https://www.example.com --out capture @@ -220,7 +237,9 @@ redirects to a captcha path. Every page then looks the same, so a capture of it inventory.json one record per page: status, head tags, JSON-LD, headings, forms, links pages/ text/ raw HTML and visible text per page redirects.json every redirect, chains collapsed, hops kept; shortlinks marked verified or not - links.json internal and external links, with the pages that carry them + links.json { internal, external } maps of link -> [pages that carry it]; with + --exclude also "excluded": { "": [pages] }, the links under an + excluded prefix (kept in the HTML, never in internal, never requested) scripts.json third-party vendors with their ids and pages media.json every media file with sources, pages, status, type and bytes; a size only the REST list names: downloaded: false, "unreferenced size" @@ -238,13 +257,15 @@ redirects to a captcha path. Every page then looks the same, so a capture of it not saved (an HTTP status, "off-site" for a redirect off the host, "too-large" over 5 MB); [] when the site has none templates.json template clusters with a representative page each - assets/ logo, icons, fonts, index.json + assets/ logo, icons, fonts, index.json; root-favicon.ico and + root-apple-touch-icon.png when live served them at the root screens/live/ live screenshots and manifest.json (gitignored) lighthouse-live/ Lighthouse reports, when run (gitignored) pages/404.html the site's own 404 page (meta.json notFound: path, status, snapshot, text) sitemaps/ every sitemap robots.txt names, the default index, and their children; index.json: every sitemap URL asked for { url, status, file }, and - { url, host, skipped: "other host" } for one on another host + { url, host, skipped: "other host" } for one on another host, + { url, skipped: "excluded" } for one under an --exclude prefix feeds/ robots.txt wp-rest/ wayback-unverified.txt meta.json state.json README.md the generated summary and its ## Notes diff --git a/skills/capture-site/scripts/assets.mjs b/skills/capture-site/scripts/assets.mjs index 75d9b0c..d2127da 100644 --- a/skills/capture-site/scripts/assets.mjs +++ b/skills/capture-site/scripts/assets.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // Brand assets: the logo, the favicon, the apple-touch-icon and every web font, saved byte for -// byte into /assets/ so the build can use the originals. +// byte into /assets/ so the build can use the originals; the root icons capture.mjs +// saved (assets/root-favicon.ico, assets/root-apple-touch-icon.png) are listed with them. // Sources: and the logo (findLogo: in
first, then in // other logo elements, an image or inline named logo, the home link's image, else the // first ) on the home page snapshot, and every @font-face src in the stylesheets under /css/. @@ -13,9 +14,10 @@ import * as csstree from 'css-tree'; import { assertNodeVersion } from './lib/node-version.mjs'; import { serializeOuter } from 'parse5'; import { attr, children, findAll, isElement, parseHtml } from '../../parity-gates/scripts/lib/html.mjs'; +import { excludedBy, parseExclude } from './lib/exclude.mjs'; import { BLOCK_LIMIT, DEFAULT_CONCURRENCY, DEFAULT_DELAY_MS, DEFAULT_UA, fetchPaced, http, MAX_FILE_BYTES, pacing, POLITE_DELAY_MS } from './lib/http.mjs'; -const USAGE = `Usage: node assets.mjs --capture [--delay ] [--concurrency ] [--polite] [--ua ""] [--max-file-bytes ] +const USAGE = `Usage: node assets.mjs --capture [--exclude [,...]] [--delay ] [--concurrency ] [--polite] [--ua ""] [--max-file-bytes ] Save the site's logo, favicon, apple-touch-icon and web fonts into /assets/ (logo.*, favicon.*, apple-touch-icon.*, fonts/*). The logo is found on the home page inside @@ -26,6 +28,10 @@ alt, class or id says "logo", else an inline that does (saved as logo.svg) capture.mjs; without it, fonts are reported as not captured). --capture the capture folder written by capture.mjs + --exclude [,...] also leave out assets on the site whose path starts with + one of these (repeatable); the capture's own --exclude prefixes (meta.json + excluded) always apply. One under a prefix is listed with the note + "excluded (under )" and never requested --delay time between request starts on a host (default ${DEFAULT_DELAY_MS}) --concurrency requests in flight on a host at once (default ${DEFAULT_CONCURRENCY}) --polite one request at a time, ${POLITE_DELAY_MS} ms apart; --delay and @@ -35,7 +41,10 @@ capture.mjs; without it, fonts are reported as not captured). listed with status "too-large" --help show this help -Writes /assets/index.json listing each asset looked for. An asset that does not answer +Writes /assets/index.json listing each asset looked for, and the root icons capture.mjs +asked for (kind "root-icon", with path: /favicon.ico or /apple-touch-icon.png, and file: +assets/root-favicon.ico or assets/root-apple-touch-icon.png when live served it), which the +build copies to public/ at that path. An asset that does not answer 200 is a warning on stderr and is listed there as missing. Files stream to disk; a host that resolves to a loopback, private or link-local address (other than the site's own) is not requested and is listed with status "blocked-address". @@ -194,13 +203,20 @@ function extensionFor(url, contentType) { } function parseArgs(argv) { - const args = { capture: null, delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; + const args = { capture: null, exclude: [], delayMs: null, concurrency: null, polite: false, ua: DEFAULT_UA, maxFileBytes: MAX_FILE_BYTES, help: false }; const value = (i) => (i < argv.length && !argv[i].startsWith('--') ? argv[i] : null); for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === '--help' || arg === '-h') args.help = true; else if (arg === '--capture' && value(i + 1)) args.capture = argv[(i += 1)]; else if (arg === '--ua' && value(i + 1)) args.ua = argv[(i += 1)]; + else if (arg === '--exclude' && value(i + 1)) { + try { + args.exclude = parseExclude([...args.exclude, argv[(i += 1)]]); + } catch (err) { + return { error: err.message }; + } + } else if (arg === '--delay' && value(i + 1)) { args.delayMs = Number(argv[(i += 1)]); if (!Number.isInteger(args.delayMs) || args.delayMs < 0) return { error: '--delay needs a whole number' }; @@ -247,7 +263,18 @@ async function main() { const fontNames = new Map(); // file name -> url, so two fonts with one name do not overwrite // Saves url as /, or fonts/ (kept when already saved). + const prefixes = [...(meta.excluded ?? []), ...args.exclude]; + const site = origin ? new URL(origin).host.replace(/^www\./i, '').toLowerCase() : null; + const excludedPrefix = (url) => { + const target = new URL(url); + return target.host.replace(/^www\./i, '').toLowerCase() === site ? excludedBy(prefixes, target.pathname) : null; + }; const save = async (kind, url, stem, { keepName = false } = {}) => { + const prefix = excludedPrefix(url); + if (prefix) { + entries.push({ kind, url, file: null, status: null, note: `excluded (under ${prefix})` }); + return; + } let name = null; if (keepName) { const original = basename(new URL(url).pathname) || 'font'; @@ -309,11 +336,14 @@ async function main() { if (url) await save(kind, url, kind); else absent(kind, missing); } - // The root icons browsers and launch checks ask for whatever the head links; saved when the - // capture's probe (meta.json icons) saw live answer 200. - for (const [path, stem] of [['/favicon.ico', 'root-favicon'], ['/apple-touch-icon.png', 'root-apple-touch-icon']]) { - if (origin && meta.icons?.[path] === 200) await save(stem, new URL(path, origin).href, stem); - } + } + + // The root icons capture.mjs saved (meta.json icons), listed so the build ships each at its + // live path: { kind: "root-icon", url, path, file, status, note }. Never requested here. + for (const [path, icon] of Object.entries(meta.icons ?? {})) { + if (!icon || typeof icon !== 'object' || !origin) continue; // a capture older than the saved bytes + const note = icon.file ? null : icon.note ?? `live answered ${icon.status}`; + entries.push({ kind: 'root-icon', url: new URL(path, origin).href, path, file: icon.file, status: icon.status, note }); } const fonts = origin ? fontUrls(dir, origin) : null; diff --git a/skills/capture-site/scripts/capture.mjs b/skills/capture-site/scripts/capture.mjs index 141ef7e..924ec80 100644 --- a/skills/capture-site/scripts/capture.mjs +++ b/skills/capture-site/scripts/capture.mjs @@ -18,7 +18,7 @@ // After the crawl one path that cannot exist (/<32 hex>/) is requested once, so the site's own // 404 page lands in pages/404.html and meta.json `notFound` (it is not a route). import { createHash, randomBytes } from 'node:crypto'; -import { mkdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { basename, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { assertNodeVersion } from './lib/node-version.mjs'; @@ -33,6 +33,7 @@ import { CDX_ENDPOINT, WAYBACK_FILE, waybackPaths } from './lib/wayback.mjs'; import { extensionOf, NEVER, NOT_PAGES, PAGE_EXTENSIONS } from './lib/page-rules.mjs'; import { cssImports } from './lib/css-urls.mjs'; import { CHALLENGE, SUSPECT, detectChallenge } from './lib/challenge.mjs'; +import { excludedBy, parseExclude } from './lib/exclude.mjs'; import { parse } from 'parse5'; const USAGE = `Usage: node capture.mjs --out [options] @@ -41,8 +42,9 @@ Crawl a live site, read-only and paced, into a capture folder: one record per pa inventory.json, the raw HTML in pages/.html, the visible text in text/.txt, redirects.json (chains collapsed, hops kept), links.json, scripts.json (third-party vendors), meta.json (notFound: the site's 404 page, from one request for a path that cannot exist, saved -as pages/404.html and text/404.txt; icons: the status /favicon.ico and /apple-touch-icon.png -answered, one request each), sitemaps/ (every sitemap robots.txt names, and their children; +as pages/404.html and text/404.txt; icons: { status, file, contentType, bytes } for +/favicon.ico and /apple-touch-icon.png, one request each, redirects followed, a 200 saved as +assets/root-favicon.ico and assets/root-apple-touch-icon.png), sitemaps/ (every sitemap robots.txt names, and their children; index.json lists every sitemap URL asked for with its status and file, and one robots.txt names that did not answer 200 is also in meta.json discoveryErrors; one on another host is listed with skipped: "other host" and never fetched), robots.txt, feeds/, css/ (every stylesheet and the sheets it @imports, @@ -73,6 +75,16 @@ to list and download the media. --seed, --seed-file and --include-query also apply on --resume --no-rest do not use the platform's REST API for discovery --no-wayback do not ask the Wayback Machine for archived URLs + --exclude [,...] never request a URL whose path starts with one of these + (repeatable), e.g. a multisite's sub-sites: --exclude /president/,/news-room/. + Sitemaps under one are listed in sitemaps/index.json as skipped: "excluded", + links to one stay in the HTML and are listed in links.json "excluded", + and meta.json "excluded" names the prefixes. Paths compare in one form + (/présidence/ = /pr%c3%a9sidence/). A stylesheet or root icon that + redirects under one is not followed there. --resume keeps them; one + added on --resume removes what was already captured under it (pages and + their files, redirects from it, stylesheets and scripts only those pages + used) and says what it removed --help show this help Never requests wp-admin, wp-login, xmlrpc.php or ?s= searches, and /wp-json/ only for the @@ -106,8 +118,13 @@ const MAX_HOPS = 10; const SITEMAP_PATH = /sitemap|\.xml(\.gz)?$/i; // Why a sitemap on another host is listed in sitemaps/index.json but never fetched. const OTHER_HOST = 'other host'; -// The root icons a launch needs, asked for once so a missing one is evidence live lacked it. +// Why a sitemap under an --exclude prefix is listed in sitemaps/index.json but never fetched. +const EXCLUDED = 'excluded'; +// The root icons a launch needs, asked for once so a missing one is evidence live lacked it; +// one that answers 200 is saved as assets/root- for the build to ship at the same path. const ROOT_ICONS = ['/favicon.ico', '/apple-touch-icon.png']; +// A root icon larger than this is recorded "too-large", not saved. +const ICON_MAX_BYTES = 5 * 1024 * 1024; // HTML responses in a row matching only the tiny-page heuristic that stop the run. const SUSPECT_LIMIT = 3; const MAX_ATTEMPTS = 3; @@ -147,7 +164,7 @@ function readSeedFile(file) { function parseArgs(argv) { const args = { origin: null, out: null, delayMs: null, concurrency: null, polite: false, max: null, resume: false, ua: DEFAULT_UA, - seeds: [], verifyShortlinks: false, includeQuery: false, noRest: false, noWayback: false, help: false, + seeds: [], exclude: [], verifyShortlinks: false, includeQuery: false, noRest: false, noWayback: false, help: false, }; const value = (i, flag) => { if (i >= argv.length || argv[i].startsWith('--')) throw new UsageError(`${flag} needs a value`); @@ -166,6 +183,13 @@ function parseArgs(argv) { } else if (arg === '--polite') args.polite = true; else if (arg === '--ua') args.ua = value((i += 1), arg); else if (arg === '--seed') args.seeds.push(value((i += 1), arg)); else if (arg === '--seed-file') args.seeds.push(...readSeedFile(value((i += 1), arg))); + else if (arg === '--exclude') { + try { + args.exclude = parseExclude([...args.exclude, value((i += 1), arg)]); + } catch (err) { + throw new UsageError(err.message); + } + } else if (arg === '--resume') args.resume = true; else if (arg === '--verify-shortlinks') args.verifyShortlinks = true; else if (arg === '--include-query') args.includeQuery = true; @@ -265,6 +289,10 @@ function sitemapFileName(href, taken = new Map()) { for (let n = 2; ; n += 1) if (free(withSuffix(base, String(n)))) return withSuffix(base, String(n)); } +// The next free numbered file name (.css, .js) among entries: one past the highest, so a +// name is never reused after entries are dropped (--resume with a new --exclude prefix). +const nextFile = (entries, ext) => `${Math.max(0, ...entries.map((e) => Number.parseInt(e.file, 10)).filter(Number.isFinite)) + 1}.${ext}`; + const byKey = (key) => (a, b) => (a[key] < b[key] ? -1 : a[key] > b[key] ? 1 : 0); // The text of every ', ''); const meta = JSON.parse(readFileSync(join(dir, 'meta.json'), 'utf8')); - writeFileSync(join(dir, 'meta.json'), JSON.stringify({ ...meta, icons: { '/favicon.ico': 200, '/apple-touch-icon.png': 404 } })); + mkdirSync(join(dir, 'assets'), { recursive: true }); + writeFileSync(join(dir, 'assets/root-favicon.ico'), 'ico'); + writeFileSync(join(dir, 'meta.json'), JSON.stringify({ + ...meta, + icons: { + '/favicon.ico': { status: 200, file: 'assets/root-favicon.ico', contentType: 'image/x-icon', bytes: 3 }, + '/apple-touch-icon.png': { status: 404, file: null, contentType: null, bytes: null }, + }, + })); try { const result = await run(['--capture', dir, '--delay', '0']); assert.equal(result.code, 0, result.stderr); @@ -206,10 +214,9 @@ test('assets.mjs saves the web manifest, and the root icons meta.json icons saw const manifest = index.find((e) => e.kind === 'manifest'); assert.equal(manifest.url, `${site.origin}/site.webmanifest`); assert.equal(readFileSync(join(dir, manifest.file), 'utf8'), '{"name":"x"}'); - const root = index.find((e) => e.kind === 'root-favicon'); - assert.equal(readFileSync(join(dir, root.file), 'utf8'), 'ico'); - assert.ok(!index.some((e) => e.kind === 'root-apple-touch-icon')); - assert.ok(!site.requests.includes('/apple-touch-icon.png'), 'a root icon live answered 404 for is not asked'); + const roots = index.filter((e) => e.kind === 'root-icon'); + assert.deepEqual(roots.map((e) => [e.path, e.file, e.status]), [['/favicon.ico', 'assets/root-favicon.ico', 200], ['/apple-touch-icon.png', null, 404]]); + assert.ok(!site.requests.includes('/favicon.ico') && !site.requests.includes('/apple-touch-icon.png'), 'root icons come from the capture, never requested here'); } finally { rmSync(dir, { recursive: true, force: true }); await site.close(); diff --git a/test/capture/capture.test.mjs b/test/capture/capture.test.mjs index 19157df..12b8dc8 100644 --- a/test/capture/capture.test.mjs +++ b/test/capture/capture.test.mjs @@ -271,7 +271,7 @@ test('a sitemap robots.txt names that answers 404 is recorded in sitemaps/index. } }); -test('the capture asks for /favicon.ico and /apple-touch-icon.png once and records their status in meta.json icons', async () => { +test('the capture asks for /favicon.ico and /apple-touch-icon.png once, saves what answered 200 and records each in meta.json icons', async () => { const stub = await startStub({ '/': { body: 'Pagehome' }, '/favicon.ico': { status: 200, headers: { 'Content-Type': 'image/x-icon' }, body: 'ico' }, @@ -280,7 +280,12 @@ test('the capture asks for /favicon.ico and /apple-touch-icon.png once and recor const out = tempDir(); const run = await capture([stub.origin, '--out', out, '--delay', '0']); assert.equal(run.code, 0, run.stderr); - assert.deepEqual(readJson(out, 'meta.json').icons, { '/favicon.ico': 200, '/apple-touch-icon.png': 404 }); + assert.deepEqual(readJson(out, 'meta.json').icons, { + '/favicon.ico': { status: 200, file: 'assets/root-favicon.ico', contentType: 'image/x-icon', bytes: 3 }, + '/apple-touch-icon.png': { status: 404, file: null, contentType: 'text/html; charset=UTF-8', bytes: null }, + }); + assert.equal(readFileSync(join(out, 'assets/root-favicon.ico'), 'utf8'), 'ico'); + assert.ok(!existsSync(join(out, 'assets/root-apple-touch-icon.png'))); const asked = stub.requests.filter((u) => u === '/favicon.ico' || u === '/apple-touch-icon.png'); assert.deepEqual(asked.sort(), ['/apple-touch-icon.png', '/favicon.ico']); const again = await capture([stub.origin, '--out', out, '--delay', '0', '--resume']); @@ -291,6 +296,39 @@ test('the capture asks for /favicon.ico and /apple-touch-icon.png once and recor } }); +test('a root icon is saved where its redirects end; an HTML answer or one over 5 MB is recorded, not saved', async () => { + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3]); + const stub = await startStub({ + '/': { body: 'Pagehome' }, + '/apple-touch-icon.png': { status: 301, headers: { Location: '/wp-content/uploads/site-icon-180.png' } }, + '/wp-content/uploads/site-icon-180.png': { headers: { 'Content-Type': 'image/png' }, body: png }, + '/favicon.ico': { body: 'Homea page, not an icon' }, + }); + const big = await startStub({ + '/': { body: 'Pagehome' }, + '/favicon.ico': { headers: { 'Content-Type': 'image/x-icon' }, body: Buffer.alloc(5 * 1024 * 1024 + 1) }, + }); + try { + const out = tempDir(); + const run = await capture([stub.origin, '--out', out, '--delay', '0']); + assert.equal(run.code, 0, run.stderr); + const icons = readJson(out, 'meta.json').icons; + assert.deepEqual(icons['/apple-touch-icon.png'], { status: 200, file: 'assets/root-apple-touch-icon.png', contentType: 'image/png', bytes: png.length }); + assert.ok(readFileSync(join(out, 'assets/root-apple-touch-icon.png')).equals(png)); + assert.deepEqual(icons['/favicon.ico'], { status: 200, file: null, contentType: 'text/html; charset=UTF-8', bytes: null, note: 'answered HTML, not an icon' }); + assert.ok(!existsSync(join(out, 'assets/root-favicon.ico'))); + + const large = tempDir(); + const second = await capture([big.origin, '--out', large, '--delay', '0']); + assert.equal(second.code, 0, second.stderr); + assert.deepEqual(readJson(large, 'meta.json').icons['/favicon.ico'], { status: 'too-large', file: null, contentType: 'image/x-icon', bytes: null }); + assert.ok(!existsSync(join(large, 'assets/root-favicon.ico'))); + } finally { + await stub.close(); + await big.close(); + } +}); + test('robots.txt and sitemaps that always answer 403 are noted, not fatal', async () => { const stub = await startStub({ '/': { body: 'Pagehome' }, @@ -409,7 +447,10 @@ test('snapshots, text, robots, sitemaps, feeds, links and scripts are written', } const index = readJson(full, 'sitemaps/index.json'); assert.deepEqual(index.find((e) => e.url.endsWith('/page-sitemap.xml')), { url: `${fixture.origin}/page-sitemap.xml`, status: 200, file: 'sitemaps/page-sitemap.xml' }); - assert.deepEqual(readJson(full, 'meta.json').icons, { '/favicon.ico': 404, '/apple-touch-icon.png': 404 }); + const icons = readJson(full, 'meta.json').icons; + assert.deepEqual(icons['/favicon.ico'], { status: 200, file: 'assets/root-favicon.ico', contentType: 'image/x-icon', bytes: 70 }); + assert.ok(readFileSync(join(full, 'assets/root-favicon.ico')).equals(readFileSync(join(SITE, 'favicon.ico'))), 'saved byte for byte'); + assert.deepEqual(icons['/apple-touch-icon.png'], { status: 404, file: null, contentType: 'text/html; charset=UTF-8', bytes: null }); assert.match(readFileSync(join(full, 'feeds/feed.xml'), 'utf8'), / f.path), ['/feed/']); diff --git a/test/capture/exclude.test.mjs b/test/capture/exclude.test.mjs new file mode 100644 index 0000000..d31bfd5 --- /dev/null +++ b/test/capture/exclude.test.mjs @@ -0,0 +1,285 @@ +import { test, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { startFixture } from '../fixture/serve.mjs'; +import { startStub } from './helpers.mjs'; +import { excludedBy, parseExclude } from '../../skills/capture-site/scripts/lib/exclude.mjs'; +import { collectMedia } from '../../skills/capture-site/scripts/media.mjs'; + +const exec = promisify(execFile); +const script = (path) => fileURLToPath(new URL(`../../skills/${path}`, import.meta.url)); +const CAPTURE = script('capture-site/scripts/capture.mjs'); +const readJson = (dir, file) => JSON.parse(readFileSync(join(dir, file), 'utf8')); + +async function run(path, args, env = process.env) { + try { + const { stdout, stderr } = await exec(process.execPath, [path, ...args], { env }); + return { code: 0, stdout, stderr }; + } catch (err) { + if (typeof err.code !== 'number') throw err; + return { code: err.code, stdout: err.stdout, stderr: err.stderr }; + } +} + +const dirs = []; +const tempDir = () => { + const dir = mkdtempSync(join(tmpdir(), 'exclude-test-')); + dirs.push(dir); + return dir; +}; + +let fixture; +let cap; +let base; +let captured; +let requests; +before(async () => { + fixture = await startFixture(); + const root = tempDir(); + cap = join(root, 'capture'); + base = join(root, 'baseline'); + captured = await run(CAPTURE, [fixture.origin, '--out', cap, '--delay', '0', '--no-wayback', '--seed', '/old-services', '--exclude', '/services/']); + requests = [...fixture.requests]; + for (const [name, args] of [ + ['capture-site/scripts/media.mjs', ['--capture', cap, '--delay', '0']], + ['capture-site/scripts/templates.mjs', ['--capture', cap]], + ['parity-gates/scripts/build-baseline.mjs', ['--capture', cap, '--out', base]], + ]) { + const result = await run(script(name), args); + assert.equal(result.code, 0, `${name}: ${result.stderr}`); + } +}); +after(async () => { + await fixture.close(); + for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); +}); + +test('parseExclude splits commas, keeps URL paths and refuses / and bare words', () => { + assert.deepEqual(parseExclude(['/president/,/media-relations/', 'https://www.example.com/team/', '/president/']), ['/president/', '/media-relations/', '/team/']); + assert.throws(() => parseExclude(['/']), /whole site/); + assert.throws(() => parseExclude(['president']), /starting with \//); + assert.equal(excludedBy(['/services/'], '/services/roofing/'), '/services/'); + assert.equal(excludedBy(['/services/'], '/services'), '/services/'); + assert.equal(excludedBy(['/services/'], '/services-old/'), null); +}); + +test('--exclude /services/: no page under it is requested or inventoried, and meta.json lists it', () => { + assert.equal(captured.code, 0, captured.stderr); + const asked = requests.map((u) => new URL(u, fixture.origin).pathname); + assert.deepEqual(asked.filter((p) => p.startsWith('/services')), [], 'nothing under /services/ is requested'); + const inventory = readJson(cap, 'inventory.json'); + assert.deepEqual(inventory.filter((r) => r.path.startsWith('/services')), []); + // 13 pages at 200 less the three services pages, plus the sitemap's 404. + assert.equal(inventory.length, 11); + assert.equal(inventory.filter((r) => r.status === 200).length, 10); + assert.deepEqual(readJson(cap, 'meta.json').excluded, ['/services/']); + assert.deepEqual(readJson(cap, 'state.json').options.exclude, ['/services/']); +}); + +test('a redirect into an excluded path is recorded and its target never requested', () => { + const old = readJson(cap, 'redirects.json').find((r) => r.from === '/old-services'); + assert.deepEqual(old, { from: '/old-services', to: '/services/', status: 301, source: 'seed', hops: ['/old-services/'] }); +}); + +test('links to excluded paths stay in the HTML and move to links.json excluded', () => { + assert.match(readFileSync(join(cap, 'pages/home.html'), 'utf8'), /href="[^"]*\/services\/roofing\/"/); + const links = readJson(cap, 'links.json'); + assert.deepEqual(Object.keys(links.internal).filter((p) => p.startsWith('/services')), []); + assert.ok(links.excluded['/services/'].includes('/'), 'the home page links to /services/'); + assert.ok(links.excluded['/services/roofing/'].length > 0); +}); + +test('media lists no page under the excluded prefix, and media under a prefix is not listed', () => { + const media = readJson(cap, 'media.json'); + assert.ok(media.length > 0); + assert.deepEqual(media.flatMap((m) => m.referencedBy).filter((p) => p.startsWith('/services')), []); + const none = collectMedia(cap, { exclude: ['/wp-content/uploads/'] }); + assert.deepEqual(none.filter((m) => m.path.startsWith('/wp-content/uploads/')), []); + assert.ok(none.some((m) => m.path.startsWith('/wp-content/themes/')), 'other media is still listed'); +}); + +test('the baseline carries the excluded prefixes in meta.json and has no route under them', () => { + assert.deepEqual(readJson(base, 'meta.json').excluded, ['/services/']); + assert.deepEqual(readJson(base, 'routes.json').filter((r) => r.path.startsWith('/services')), []); +}); + +test('assets.mjs --exclude leaves out an asset under the prefix without asking for it', async () => { + const before = fixture.requests.length; + const result = await run(script('capture-site/scripts/assets.mjs'), ['--capture', cap, '--delay', '0', '--exclude', '/wp-content/themes/fixture/fonts/']); + assert.equal(result.code, 0, result.stderr); + const asked = fixture.requests.slice(before); + assert.deepEqual(asked.filter((u) => u.includes('/fonts/')), []); + const fonts = readJson(cap, 'assets/index.json').filter((e) => e.kind === 'font'); + assert.equal(fonts.length, 2); + for (const font of fonts) assert.deepEqual([font.file, font.status, font.note], [null, null, 'excluded (under /wp-content/themes/fixture/fonts/)']); +}); + +test('a sitemap index under an excluded prefix is listed skipped, and nothing under it is asked for', async () => { + const home = (origin) => ({ body: `HomePresidentNews` }); + const page = { body: 'Newsnews' }; + const stub = await startStub({ + '/': home, + '/robots.txt': (origin) => ({ headers: { 'Content-Type': 'text/plain' }, body: `Sitemap: ${origin}/president/sitemap_index.xml\nSitemap: ${origin}/sitemap_index.xml\n` }), + '/sitemap_index.xml': (origin) => ({ headers: { 'Content-Type': 'application/xml' }, body: `${origin}/page-sitemap.xml${origin}/president/page-sitemap.xml` }), + '/page-sitemap.xml': (origin) => ({ headers: { 'Content-Type': 'application/xml' }, body: `${origin}/news/${origin}/president/bio/` }), + '/news/': page, + }); + try { + const out = tempDir(); + const result = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--no-wayback', '--exclude', '/president/']); + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(stub.requests.filter((u) => u.startsWith('/president')), []); + const index = readJson(out, 'sitemaps/index.json'); + assert.deepEqual(index.filter((e) => e.skipped), [ + { url: `${stub.origin}/president/page-sitemap.xml`, skipped: 'excluded' }, + { url: `${stub.origin}/president/sitemap_index.xml`, skipped: 'excluded' }, + ]); + const meta = readJson(out, 'meta.json'); + assert.deepEqual(meta.discoveryErrors, []); + assert.deepEqual(readJson(out, 'inventory.json').map((r) => r.path), ['/', '/news/']); + assert.deepEqual(readJson(out, 'links.json').excluded, { '/president/': ['/'] }); + } finally { + await stub.close(); + } +}); + +test('--resume keeps the excluded prefixes, and one added on --resume drops what is queued under it', async () => { + const fresh = await startFixture(); + try { + const out = tempDir(); + const first = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--no-rest', '--max', '2', '--exclude', '/services/']); + assert.equal(first.code, 0, first.stderr); + const queued = readJson(out, 'state.json').queue.map((q) => new URL(q.url).pathname); + assert.ok(queued.some((p) => p.startsWith('/blog/')), `blog is still queued: ${queued}`); + const again = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--resume', '--exclude', '/blog/']); + assert.equal(again.code, 0, again.stderr); + const asked = fresh.requests.map((u) => new URL(u, fresh.origin).pathname); + assert.deepEqual(asked.filter((p) => p.startsWith('/services') || p.startsWith('/blog')), []); + assert.deepEqual(readJson(out, 'meta.json').excluded, ['/services/', '/blog/']); + } finally { + await fresh.close(); + } +}); + +test('Wayback paths under an excluded prefix are not listed for a second pass', async () => { + const server = createServer((req, res) => { + const host = new URL(req.url, 'http://cdx').searchParams.get('url').replace(/\/\*$/, ''); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify([['original'], [`http://${host}/old-team/`], [`http://${host}/services/old-gutters/`]])); + }); + await new Promise((done) => server.listen(0, '127.0.0.1', done)); + const stub = await startStub({ '/': { body: 'Homehome' } }); + try { + const out = tempDir(); + const env = { ...process.env, CAPTURE_WAYBACK_CDX: `http://127.0.0.1:${server.address().port}/cdx/search/cdx`, CAPTURE_WAYBACK_RETRY_MS: '0' }; + const result = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--exclude', '/services/'], env); + assert.equal(result.code, 0, result.stderr); + assert.equal(readFileSync(join(out, 'wayback-unverified.txt'), 'utf8'), '/old-team/\n'); + } finally { + await stub.close(); + server.closeAllConnections(); + await new Promise((done) => server.close(done)); + } +}); + +test('--exclude needs a value and refuses /', async () => { + const out = tempDir(); + const bare = await run(CAPTURE, ['http://127.0.0.1:9', '--out', out, '--exclude']); + assert.equal(bare.code, 2); + const all = await run(CAPTURE, ['http://127.0.0.1:9', '--out', out, '--exclude', '/']); + assert.equal(all.code, 2); + assert.match(all.stderr, /whole site/); +}); + +test('prefixes and paths compare in one form: non-ASCII, percent-encoded and either-case escapes', () => { + const accented = parseExclude(['/présidence/']); + assert.deepEqual(accented, ['/pr%C3%A9sidence/']); + assert.equal(excludedBy(accented, new URL('http://x/présidence/bio/').pathname), '/pr%C3%A9sidence/'); + assert.equal(excludedBy(accented, '/pr%c3%a9sidence/bio/'), '/pr%C3%A9sidence/', 'lowercase escapes match'); + assert.equal(excludedBy(parseExclude(['/pr%c3%a9sidence/']), '/pr%C3%A9sidence/'), '/pr%C3%A9sidence/'); + assert.equal(excludedBy(['/president/'], '/%70resident/'), '/president/', 'an escaped unreserved letter is that letter'); + assert.equal(excludedBy(['/President/'], '/president/'), null, 'paths stay case-sensitive'); +}); + +test('a prefix added on --resume removes what was already captured under it, everywhere', async () => { + const fresh = await startFixture(); + try { + const out = tempDir(); + const first = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--seed', '/old-services']); + assert.equal(first.code, 0, first.stderr); + const blog = readJson(out, 'inventory.json').find((r) => r.path === '/blog/'); + assert.ok(blog?.snapshot && existsSync(join(out, blog.snapshot)), 'the first run captured /blog/'); + const text = `text/${blog.snapshot.replace(/^pages\//, '').replace(/\.html$/, '')}.txt`; + assert.ok(existsSync(join(out, text))); + + const again = await run(CAPTURE, [fresh.origin, '--out', out, '--delay', '0', '--no-wayback', '--resume', '--exclude', '/blog/,/old-services']); + assert.equal(again.code, 0, again.stderr); + assert.match(again.stdout, /Removed 1 page and 1 redirect now under --exclude \/blog\/, \/old-services/); + assert.deepEqual(readJson(out, 'inventory.json').filter((r) => r.path.startsWith('/blog')), []); + assert.ok(!existsSync(join(out, blog.snapshot)), 'its snapshot is deleted'); + assert.ok(!existsSync(join(out, text)), 'its text is deleted'); + const state = readJson(out, 'state.json'); + assert.deepEqual(Object.keys(state.foundBy).filter((k) => new URL(k).pathname.startsWith('/blog')), []); + assert.deepEqual(Object.values(state.slugs).filter((k) => new URL(k).pathname.startsWith('/blog')), []); + assert.deepEqual(readJson(out, 'redirects.json').filter((r) => r.from.startsWith('/old-services')), []); + assert.ok(readJson(out, 'links.json').excluded['/blog/'].length > 0, 'links to it are listed excluded'); + assert.deepEqual(readJson(out, 'meta.json').excluded, ['/blog/', '/old-services']); + } finally { + await fresh.close(); + } +}); + +const sheetPage = (title, sheet) => ({ body: `${title}only` }); + +test('a stylesheet only excluded pages used is dropped on --resume, its file deleted', async () => { + const stub = await startStub({ + '/': sheetPage('Home', '/a.css'), + '/only/': sheetPage('Only', '/b.css'), + '/a.css': { headers: { 'Content-Type': 'text/css' }, body: 'a{}' }, + '/b.css': { headers: { 'Content-Type': 'text/css' }, body: 'b{}' }, + }); + try { + const out = tempDir(); + const first = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--no-wayback']); + assert.equal(first.code, 0, first.stderr); + const b = readJson(out, 'css/manifest.json').find((e) => e.href.endsWith('/b.css')); + assert.ok(b.file && existsSync(join(out, 'css', b.file))); + const again = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--no-wayback', '--resume', '--exclude', '/only/']); + assert.equal(again.code, 0, again.stderr); + assert.match(again.stdout, /Removed 1 page and 1 stylesheet now under --exclude \/only\//); + assert.deepEqual(readJson(out, 'css/manifest.json').map((e) => new URL(e.href).pathname), ['/a.css']); + assert.ok(!existsSync(join(out, 'css', b.file))); + } finally { + await stub.close(); + } +}); + +test('a stylesheet or root icon that redirects under an excluded prefix is not followed there', async () => { + const stub = await startStub({ + '/': sheetPage('Home', '/style.css'), + '/style.css': { status: 301, headers: { Location: '/president/style.css' } }, + '/favicon.ico': { status: 302, headers: { Location: '/president/favicon.ico' } }, + '/president/style.css': { headers: { 'Content-Type': 'text/css' }, body: 'p{}' }, + '/president/favicon.ico': { headers: { 'Content-Type': 'image/x-icon' }, body: 'ico' }, + }); + try { + const out = tempDir(); + const result = await run(CAPTURE, [stub.origin, '--out', out, '--delay', '0', '--no-wayback', '--exclude', '/president/']); + assert.equal(result.code, 0, result.stderr); + assert.deepEqual(stub.requests.filter((u) => u.startsWith('/president')), []); + const sheet = readJson(out, 'css/manifest.json').find((e) => e.href.endsWith('/style.css')); + assert.deepEqual([sheet.file, sheet.status], [null, 'excluded']); + assert.deepEqual(readJson(out, 'meta.json').icons['/favicon.ico'], { + status: 302, file: null, contentType: 'text/html; charset=UTF-8', bytes: null, note: 'redirects to /president/favicon.ico, under an excluded prefix', + }); + assert.ok(!existsSync(join(out, 'assets/root-favicon.ico'))); + } finally { + await stub.close(); + } +}); diff --git a/test/capture/readme.test.mjs b/test/capture/readme.test.mjs index 7250794..7c659fa 100644 --- a/test/capture/readme.test.mjs +++ b/test/capture/readme.test.mjs @@ -101,6 +101,16 @@ test('assets.mjs saves the logo, the favicon and both fonts byte for byte', () = assert.match(assets.stdout, /apple-touch-icon: none/); }); +test('assets.mjs lists the root icons the capture saved, for the build to ship at the same paths', () => { + const index = JSON.parse(readFileSync(join(dir, 'assets', 'index.json'), 'utf8')); + const roots = index.filter((e) => e.kind === 'root-icon'); + assert.deepEqual(roots, [ + { kind: 'root-icon', url: `${fixture.origin}/favicon.ico`, path: '/favicon.ico', file: 'assets/root-favicon.ico', status: 200, note: null }, + { kind: 'root-icon', url: `${fixture.origin}/apple-touch-icon.png`, path: '/apple-touch-icon.png', file: null, status: 404, note: 'live answered 404' }, + ]); + sameBytes(join(dir, 'assets', 'root-favicon.ico'), join(SITE, 'favicon.ico')); +}); + test('assets.mjs without captured CSS still saves the logo and says fonts were not captured', () => { assert.equal(bareAssets.code, 0, bareAssets.stderr); assert.ok(existsSync(join(bare, 'assets', 'logo.svg'))); diff --git a/test/convert/prelaunch-reconcile.test.mjs b/test/convert/prelaunch-reconcile.test.mjs index b4c4386..52c0ad4 100644 --- a/test/convert/prelaunch-reconcile.test.mjs +++ b/test/convert/prelaunch-reconcile.test.mjs @@ -336,6 +336,23 @@ test('favicon rows read meta.json icons: a root icon live answered 404 for is re assert.match(parsed(served).blocking[0].rows[0].reason, /live served \/apple-touch-icon\.png/); }); +test('favicon rows read the meta.json icons objects a capture writes ({ status, file, ... })', async () => { + // The fixture captured with --exclude: its meta.json icons say /favicon.ico was served (and saved) + // and /apple-touch-icon.png answered 404. + const excluded = join(root, 'excluded'); + await buildFixtureBaseline(excluded, { exclude: '/services/' }); + const { icons } = JSON.parse(readFileSync(join(excluded, 'capture', 'meta.json'), 'utf8')); + assert.equal(icons['/favicon.ico'].status, 200); + assert.equal(icons['/apple-touch-icon.png'].status, 404); + const captureDir = notFoundCapture('capture-icons-objects', { icons }); + const planFile = planWith('plan-icons-objects', ['d2', 'output.favicon-set: icons as live had them']); + const apple = await reconcile([failing('output.favicon-set', 'apple-touch-icon.png is missing', ['apple-touch-icon.png is missing'], 'file')], { planFile, captureDir }); + assert.equal(apple.code, 0, apple.stdout); + const favicon = await reconcile([failing('output.favicon-set', 'favicon.ico is missing', ['favicon.ico is missing'], 'file')], { planFile, captureDir }); + assert.equal(favicon.code, 1, favicon.stdout); + assert.match(parsed(favicon).blocking[0].rows[0].reason, /live served \/favicon\.ico/); +}); + // A baseline whose live site published no sitemap, and a site whose dist builds the given routes. function noSitemapSite(name, { drop = [] } = {}) { const base = join(root, `${name}-baseline`); diff --git a/test/fixture/serve.mjs b/test/fixture/serve.mjs index 8a45d56..422e03f 100644 --- a/test/fixture/serve.mjs +++ b/test/fixture/serve.mjs @@ -33,6 +33,7 @@ const CONTENT_TYPES = { '.jpg': 'image/jpeg', '.webp': 'image/webp', '.gif': 'image/gif', + '.ico': 'image/x-icon', '.pdf': 'application/pdf', '.woff2': 'font/woff2', }; diff --git a/test/fixture/site/favicon.ico b/test/fixture/site/favicon.ico new file mode 100644 index 0000000..eefafbb Binary files /dev/null and b/test/fixture/site/favicon.ico differ diff --git a/test/gates/build-baseline.test.mjs b/test/gates/build-baseline.test.mjs index 41c6bb5..737e29b 100644 --- a/test/gates/build-baseline.test.mjs +++ b/test/gates/build-baseline.test.mjs @@ -282,10 +282,11 @@ test('--check passes on the built baseline and exits 1 after a route is deleted' assert.match(drifted.stdout, /routes: expected 16, got 15/); }); -test('meta.json records the capture origin, times and platform', () => { +test('meta.json records the capture origin, times, platform and excluded prefixes', () => { const meta = readJson(baseline, 'meta.json'); const captured = readJson(capture, 'meta.json'); - assert.deepEqual(Object.keys(meta), ['origin', 'canonicalOrigin', 'capturedAt', 'builtAt', 'platform']); + assert.deepEqual(Object.keys(meta), ['origin', 'canonicalOrigin', 'capturedAt', 'builtAt', 'platform', 'excluded']); + assert.deepEqual(meta.excluded, [], 'a capture without --exclude excludes nothing'); assert.equal(meta.origin, captured.origin); assert.equal(meta.canonicalOrigin, captured.canonicalOrigin); assert.equal(meta.capturedAt, captured.finishedAt); diff --git a/test/gates/build-redirects.test.mjs b/test/gates/build-redirects.test.mjs index 1e14729..84a2882 100644 --- a/test/gates/build-redirects.test.mjs +++ b/test/gates/build-redirects.test.mjs @@ -248,3 +248,99 @@ test('usage errors exit 2; --help exits 0 and notes static-file precedence', asy assert.match(help.stdout, /dist\/ when it exists, else public\//); assert.match(help.stdout, /static file/i); }); + +// A live href like "trackshttps://www.example.com/x/" resolved against a page becomes this source. +const COLON_SOURCE = '/trackshttps://www.example.com/exhibitor-contact/'; + +test('vercel: a source with a bare ":" (not a parameter) is escaped as "\\:", and the resolver answers it in one hop', async () => { + const out = join(root, 'colon-vercel.json'); + const rows = [`${COLON_SOURCE},/contact/,301,crawl,`, '/o/:slug,/about/,301,decision,']; + const result = await compile(['--csv', csv('colon.csv', rows), '--host', 'vercel', '--out', out, '--trailing-slash', 'always']); + assert.equal(result.code, 0, result.stderr); + const config = JSON.parse(readFileSync(out, 'utf8')); + const sources = config.redirects.map((r) => r.source); + assert.ok(sources.includes('/trackshttps\\://www.example.com/exhibitor-contact/'), sources.join('\n')); + assert.ok(sources.includes('/o/:slug'), 'a real parameter is kept'); + const dist = join(root, 'colon-dist'); + mkdirSync(join(dist, 'contact'), { recursive: true }); + writeFileSync(join(dist, 'contact', 'index.html'), '

Contact

'); + assert.deepEqual(resolveVercel(COLON_SOURCE, { config, dist }).hops, [{ from: COLON_SOURCE, to: '/contact/', status: 301 }]); + assert.doesNotMatch(result.stderr, /skipped/); + assert.ok(!existsSync(join(root, 'redirects-skipped.json'))); +}); + +test('vercel: a captured row escapes every ":" and "*" (live sources hold no parameters); a decision row keeps its :name parameters but never a ":" before a digit', async () => { + const out = join(root, 'colons-vercel.json'); + const rows = [ + '/foo:bar/,/contact/,301,crawl,', + '/x/http://host:8080/y,/contact/,301,sitemap,', + '/star*/,/contact/,301,crawl,', + '/o/:slug,/about/,301,decision,', + '/d/a:8080/,/about/,301,decision,', + ]; + const result = await compile(['--csv', csv('colons.csv', rows), '--host', 'vercel', '--out', out, '--trailing-slash', 'always']); + assert.equal(result.code, 0, result.stderr); + const config = JSON.parse(readFileSync(out, 'utf8')); + const sources = config.redirects.map((r) => r.source); + for (const want of ['/foo\\:bar/', '/x/http\\://host\\:8080/y', '/star\\*/', '/o/:slug', '/d/a\\:8080/']) { + assert.ok(sources.includes(want), `${want} in:\n${sources.join('\n')}`); + } + const dist = join(root, 'colons-dist'); + for (const page of ['contact', 'about']) { + mkdirSync(join(dist, page), { recursive: true }); + writeFileSync(join(dist, page, 'index.html'), `

${page}

`); + } + const hops = (url) => resolveVercel(url, { config, dist }).hops.map((h) => h.to); + assert.deepEqual(hops('/foo:bar/'), ['/contact/']); + assert.deepEqual(hops('/foo:baz/'), [], 'a literal colon matches only itself'); + assert.deepEqual(hops('/x/http://host:8080/y'), ['/contact/']); + assert.deepEqual(hops('/star*/'), ['/contact/']); + assert.deepEqual(hops('/starry/'), [], 'a literal "*" is no wildcard'); + assert.deepEqual(hops('/o/anything'), ['/about/']); +}); + +test('cloudflare: a captured source with ":" or "*" is dropped (_redirects reads them as a placeholder or splat); a decision row keeps them', async () => { + const dir = join(root, 'colons-cloudflare'); + mkdirSync(dir, { recursive: true }); + const file = join(dir, 'redirects.csv'); + writeFileSync(file, `${[HEADER, '/foo:bar/,/contact/,301,crawl,', '/star*/,/contact/,301,crawl,', '/o/:slug,/about/,301,decision,'].join('\n')}\n`); + const out = join(dir, '_redirects'); + const result = await compile(['--csv', file, '--host', 'cloudflare', '--out', out]); + assert.equal(result.code, 0, result.stderr); + const skipped = JSON.parse(readFileSync(join(dir, 'redirects-skipped.json'), 'utf8')); + assert.deepEqual(skipped.map((s) => s.source), ['/foo:bar/', '/star*/']); + assert.match(skipped[0].reason, /placeholder/); + const text = readFileSync(out, 'utf8'); + assert.ok(!text.includes('/foo:bar/') && !text.includes('/star*/')); + assert.match(text, /^\/o\/:slug \/about\/ 301$/m); +}); + +// Sources no host rule can hold: no leading "/", whitespace, a control character. +const UNESCAPABLE = ['trackshttps-no-slash/', '/two words/', '/tab\there/', '/ctl\u0001char/']; + +for (const host of ['vercel', 'cloudflare']) { + test(`${host}: a source with no valid form is dropped, named on stderr and listed in redirects-skipped.json beside the csv`, async () => { + const dir = join(root, `unescapable-${host}`); + mkdirSync(dir, { recursive: true }); + const file = join(dir, 'redirects.csv'); + writeFileSync(file, `${[HEADER, '/old-services,/services/,301,seed,', ...UNESCAPABLE.map((s) => `${s},/contact/,301,crawl,`)].join('\n')}\n`); + const out = join(dir, host === 'vercel' ? 'vercel.json' : '_redirects'); + const result = await compile(['--csv', file, '--host', host, '--out', out]); + assert.equal(result.code, 0, result.stderr); + for (const source of UNESCAPABLE) assert.ok(result.stderr.includes(JSON.stringify(source)), `${source} named on stderr:\n${result.stderr}`); + assert.match(result.stderr, /4 row\(s\) skipped/); + const skipped = JSON.parse(readFileSync(join(dir, 'redirects-skipped.json'), 'utf8')); + assert.deepEqual(skipped.map((s) => s.source), UNESCAPABLE); + for (const s of skipped) assert.ok(typeof s.reason === 'string' && s.reason.length > 0); + assert.match(skipped[0].reason, /does not start with \//); + assert.match(skipped[1].reason, /whitespace/); + assert.match(skipped[3].reason, /control character/); + const text = readFileSync(out, 'utf8'); + assert.ok(text.includes('/old-services'), 'the valid row is compiled'); + for (const source of UNESCAPABLE) assert.ok(!text.includes(JSON.stringify(source).slice(1, -1)) && !text.includes(source), `${source} is not compiled`); + // A clean recompile removes the stale listing. + writeFileSync(file, `${HEADER}\n/old-services,/services/,301,seed,\n`); + assert.equal((await compile(['--csv', file, '--host', host, '--out', out])).code, 0); + assert.ok(!existsSync(join(dir, 'redirects-skipped.json'))); + }); +} diff --git a/test/gates/exceptions.test.mjs b/test/gates/exceptions.test.mjs index c8c3492..3006557 100644 --- a/test/gates/exceptions.test.mjs +++ b/test/gates/exceptions.test.mjs @@ -89,6 +89,27 @@ test('loadExceptions validates every record', (t) => { assert.throws(() => loadExceptions(badField, { env: {} }), /field/); }); +test('loadExceptions accepts a "no redirect" record { kind: "redirect", path, reason, action: "none", approvedBy, date } and validates it', (t) => { + const dir = tempDir(t); + const none = { kind: 'redirect', path: '/two words/', reason: 'a broken live href', action: 'none', approvedBy: 'client', date: '2026-10-23' }; + const good = join(dir, 'none.json'); + writeFileSync(good, JSON.stringify([none, H1])); + assert.deepEqual(loadExceptions(good, { env: {} }), [none, H1]); + const bad = [ + [{ ...none, reason: '' }, /reason/], + [{ ...none, path: undefined }, /path/], + [{ ...none, approvedBy: undefined }, /approvedBy/], + [{ ...none, date: '' }, /date/], + [{ ...none, action: 'redirect' }, /action/], + [{ ...none, kind: 'link' }, /kind/], + ]; + bad.forEach(([record, message], i) => { + const file = join(dir, `bad-${i}.json`); + writeFileSync(file, JSON.stringify([record])); + assert.throws(() => loadExceptions(file, { env: {} }), message); + }); +}); + test('loadExceptions accepts an empty from or to', (t) => { const dir = tempDir(t); const file = join(dir, 'exceptions.json'); diff --git a/test/gates/fixtures/baseline.mjs b/test/gates/fixtures/baseline.mjs index 21626fc..ceb2b07 100644 --- a/test/gates/fixtures/baseline.mjs +++ b/test/gates/fixtures/baseline.mjs @@ -2,7 +2,8 @@ // run once per directory and cached for the test file. buildFixtureBaseline(dir) resolves to the // baseline folder (/baseline); the capture is in /capture and redirects.csv in . // The capture seeds /old-services and /about so the baseline has a redirect chain and a slash -// redirect besides the shortlinks. +// redirect besides the shortlinks. buildFixtureBaseline(dir, { exclude }) captures with +// --exclude (a comma-separated prefix list) instead. import { execFile } from 'node:child_process'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -21,7 +22,7 @@ async function run(path, args) { } } -async function build(dir) { +async function build(dir, { exclude = null } = {}) { const capture = join(dir, 'capture'); const baseline = join(dir, 'baseline'); // Async child processes: the fixture server runs in this process and must keep answering. @@ -29,6 +30,7 @@ async function build(dir) { try { await run('capture-site/scripts/capture.mjs', [ fixture.origin, '--out', capture, '--delay', '0', '--no-wayback', '--seed', '/old-services', '--seed', '/about', + ...(exclude ? ['--exclude', exclude] : []), ]); await run('capture-site/scripts/media.mjs', ['--capture', capture, '--delay', '0']); await run('capture-site/scripts/templates.mjs', ['--capture', capture]); @@ -39,7 +41,7 @@ async function build(dir) { return baseline; } -export function buildFixtureBaseline(dir) { - if (!cache.has(dir)) cache.set(dir, build(dir)); +export function buildFixtureBaseline(dir, options = {}) { + if (!cache.has(dir)) cache.set(dir, build(dir, options)); return cache.get(dir); } diff --git a/test/gates/forms.test.mjs b/test/gates/forms.test.mjs index 1520b7a..e0f2f53 100644 --- a/test/gates/forms.test.mjs +++ b/test/gates/forms.test.mjs @@ -6,6 +6,7 @@ import { join } from 'node:path'; import { buildFixtureBaseline } from './fixtures/baseline.mjs'; import { edit, makeDist } from './fixtures/make-dist.mjs'; import { runGate } from './fixtures/run-gate.mjs'; +import { externalHost, registrableDomain } from '../../skills/parity-gates/scripts/verify/forms.mjs'; // The fixture build keeps the live form's action, so that is the endpoint it is configured with. const ENDPOINT = '/contact/'; @@ -139,6 +140,91 @@ test('a GET search form is compared with live: action path, method and fields; n assert.match(result.stdout, /\/blog\/ method: expected "get", got "post"/); }); +// A newsletter signup that already posts to a third party (a Mailchimp-style list host). +const SIGNUP_ACTION = 'https://example.us1.list-manage.com/subscribe/post?u=abc&id=def'; +const SIGNUP_URL = 'https://example.us1.list-manage.com/subscribe/post?u=abc&id=def'; +const SIGNUP = ``; + +test('a POST form whose action is on another origin is kept as is: compared like live, no endpoint, honeypot or thank-you', async () => { + const base = join(root, 'baseline-signup'); + cpSync(baseline, base, { recursive: true }); + const captured = JSON.parse(readFileSync(join(base, 'forms.json'), 'utf8')); + const field = (name, type) => ({ name, type, required: false }); + captured['/about/'] = [{ action: SIGNUP_URL, method: 'post', fields: [field('EMAIL', 'email'), field('b_abc_def', 'text')] }]; + writeFileSync(join(base, 'forms.json'), JSON.stringify(captured)); + const dist = makeDist(join(root, 'dist-signup')); + edit(dist, 'about/index.html', '', `${SIGNUP}`); + + const kept = await forms(dist, ['--paths', '/about/'], base); + assert.equal(kept.code, 0, kept.stdout + kept.stderr); + assert.match(kept.stdout, /^ {2}1 form kept as is \(posts to example\.us1\.list-manage\.com\)$/m); + assert.match(kept.stdout, /^forms: 1 checked, 0 failed$/m); + const strict = await forms(dist, ['--paths', '/about/', '--endpoint', ENDPOINT, '--honeypot', '_gotcha', '--thank-you', '/thank-you/'], base); + assert.equal(strict.code, 0, strict.stdout + strict.stderr); + + // Rewired to the form service, it is no longer the live signup. + const rewired = makeDist(join(root, 'dist-signup-rewired')); + edit(rewired, 'about/index.html', '', `${SIGNUP.replace(SIGNUP_ACTION, ENDPOINT)}`); + const failed = await forms(rewired, ['--paths', '/about/', '--endpoint', ENDPOINT], base); + assert.equal(failed.code, 1, failed.stdout); + assert.match(failed.stdout, new RegExp(`^ {2}/about/ action: expected "${SIGNUP_URL.replace(/[?.]/g, '\\$&')}", got "/contact/"$`, 'm')); + + // The contact form still posts to the old site, so the whole baseline still needs --endpoint. + assert.equal((await forms(dist, [], base)).code, 2); + const all = await forms(dist, ['--endpoint', ENDPOINT], base); + assert.equal(all.code, 0, all.stdout); + assert.match(all.stdout, /^forms: 3 checked, 0 failed, 1 skipped \(thank-you not checked\)$/m); +}); + +// The site's own forms, however the action spells the host, and WordPress's form handlers on any host. +const OLD_SITE_ACTIONS = [ + 'https://www.example.com/wp-admin/admin-ajax.php', + 'http://example.com/wp-comments-post.php', + '//example.com/x', + 'https://staging.example.wpengine.com/?wpcf7=1', + 'https://forms.example.com/contact', + 'https://cdn.other.example.net/wp-json/contact-form-7/v1/contact-forms/5/feedback', +]; + +test('externalHost: only a POST action on another registrable domain posts elsewhere', () => { + const origins = ['https://example.com']; + for (const action of OLD_SITE_ACTIONS) assert.equal(externalHost({ action, method: 'post' }, '/about/', origins), null, action); + assert.equal(externalHost({ action: SIGNUP_URL, method: 'post' }, '/about/', origins), 'example.us1.list-manage.com'); + assert.equal(externalHost({ action: 'https://formspree.io/f/abc', method: 'post' }, '/', origins), 'formspree.io'); + assert.equal(externalHost({ action: 'https://forms.hsforms.com/submissions', method: 'post' }, '/', origins), 'forms.hsforms.com'); + assert.equal(externalHost({ action: SIGNUP_URL, method: 'get' }, '/', origins), null, 'a GET form is not a submission form'); + for (const action of ['https://example.wpengine.com/contact/', 'https://staging-site.kinsta.cloud/signup/', 'https://dev-x.pantheonsite.io/form/', 'https://Example.WPEngine.com./x']) { + assert.equal(externalHost({ action, method: 'post' }, '/', origins), null, `${action} is a managed-WordPress staging host`); + } + assert.equal(externalHost({ action: '', method: 'post' }, '/', origins), null); + // Two-level public suffixes: the registrable domain keeps three labels. + const uk = ['https://www.example.co.uk']; + assert.equal(externalHost({ action: 'https://forms.example.co.uk/x', method: 'post' }, '/', uk), null); + assert.equal(externalHost({ action: 'https://other.co.uk/x', method: 'post' }, '/', uk), 'other.co.uk'); + assert.equal(registrableDomain('a.b.example.com.au'), 'example.com.au'); + assert.equal(registrableDomain('127.0.0.1'), '127.0.0.1'); +}); + +test('a POST form posting to the old site under another spelling of its host still needs --endpoint', async () => { + const base = join(root, 'baseline-old-site'); + cpSync(baseline, base, { recursive: true }); + const meta = JSON.parse(readFileSync(join(base, 'meta.json'), 'utf8')); + writeFileSync(join(base, 'meta.json'), JSON.stringify({ ...meta, origin: 'https://example.com', canonicalOrigin: 'https://example.com' })); + const captured = JSON.parse(readFileSync(join(base, 'forms.json'), 'utf8')); + for (const action of OLD_SITE_ACTIONS) { + writeFileSync(join(base, 'forms.json'), JSON.stringify({ ...captured, '/about/': [{ action, method: 'post', fields: [{ name: 'email', type: 'email', required: true }] }] })); + const result = await forms(good, ['--paths', '/about/'], base); + assert.equal(result.code, 2, `${action}: ${result.stdout}${result.stderr}`); + assert.match(result.stderr, /--endpoint is required/); + } + writeFileSync(join(base, 'forms.json'), JSON.stringify({ ...captured, '/about/': [{ action: SIGNUP_URL, method: 'post', fields: [{ name: 'EMAIL', type: 'email', required: false }, { name: 'b_abc_def', type: 'text', required: false }] }] })); + const dist = makeDist(join(root, 'dist-signup-example')); + edit(dist, 'about/index.html', '', `${SIGNUP}`); + const kept = await forms(dist, ['--paths', '/about/'], base); + assert.equal(kept.code, 0, kept.stdout + kept.stderr); + assert.match(kept.stdout, /^ {2}1 form kept as is \(posts to example\.us1\.list-manage\.com\)$/m); +}); + test('--endpoint is required when the baseline has a form', async () => { const result = await forms(good); assert.equal(result.code, 2); diff --git a/test/gates/links.test.mjs b/test/gates/links.test.mjs index 66b12e7..f5eb81b 100644 --- a/test/gates/links.test.mjs +++ b/test/gates/links.test.mjs @@ -1,6 +1,6 @@ import { test, before, after } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { buildFixtureBaseline } from './fixtures/baseline.mjs'; @@ -147,6 +147,28 @@ test('cloudflare: Pages slash handling resolves an unslashed link in one hop', a assert.match(result.stdout, /^ {2}1 links resolved through redirects$/m); }); +test('a link under a path the capture excluded (--exclude) is not checked; without the exclusion it fails', async () => { + // The fixture captured with --exclude /services/: the build serves no services page, but the + // other pages keep their links to them, as live had them. + const excludedBaseline = await buildFixtureBaseline(join(root, 'excluded'), { exclude: '/services/' }); + assert.deepEqual(JSON.parse(readFileSync(join(excludedBaseline, 'meta.json'), 'utf8')).excluded, ['/services/']); + const dist = makeDist(join(root, 'dist-excluded')); + rmSync(join(dist, 'services'), { recursive: true }); + const passed = await runGate('links', ['--dist', dist, '--baseline', excludedBaseline, '--origin', DIST_ORIGIN], { cwd: root }); + assert.equal(passed.code, 0, passed.stdout + passed.stderr); + assert.match(passed.stdout, /links under excluded paths \(\/services\/\) not checked/); + assert.match(passed.stdout, /^links: 11 checked, 0 failed$/m); + + const failed = await links(dist); + assert.equal(failed.code, 1, 'the full baseline has no exclusion'); + assert.match(failed.stdout, /^ {2}\/ link: expected "a built page, file, redirect source or media path", got "\/services\/roofing\/"$/m); + + edit(dist, 'about/index.html', '', ''); + const broken = await runGate('links', ['--dist', dist, '--baseline', excludedBaseline, '--origin', DIST_ORIGIN], { cwd: root }); + assert.equal(broken.code, 1, 'a link outside the excluded prefix is still checked'); + assert.match(broken.stdout, /\/about\/ link: .*got "\/contact-us\/"/); +}); + test('usage and missing inputs exit 2; --help exits 0', async () => { assert.equal((await links(good, ['--bogus'])).code, 2); assert.equal((await links(good, ['--origin', 'not a url'])).code, 2); diff --git a/test/gates/media.test.mjs b/test/gates/media.test.mjs index ba1b5ef..5743546 100644 --- a/test/gates/media.test.mjs +++ b/test/gates/media.test.mjs @@ -151,14 +151,35 @@ test('live-temporary mode: the prefix is skipped (served from live), files outsi assert.match(failed.stdout, new RegExp(`^ {2}${outside[0].path} media: expected ${outside[0].bytes}, got null$`, 'm')); }); -test('live-temporary mode fails every file on a release build (PUBLIC_INDEXABLE=true)', async () => { - for (const [name, prefix] of [['site-live-release', UPLOADS], ['site-live-release-all', undefined]]) { - const site = siteWithPlan(name, 'live-temporary', prefix); - const result = await gate(good, [], site, 'true'); - assert.equal(result.code, 1, result.stdout); - assert.match(result.stdout, new RegExp(`^ {2}${media[0].path} media: expected "served by the build", got "release build may not serve media from the live site"$`, 'm')); - assert.match(result.stdout, new RegExp(`^media: ${media.length} checked, ${media.length} failed$`, 'm')); +test('live-temporary mode on a release build (PUBLIC_INDEXABLE=true) fails every file when the plan has no prefix', async () => { + const site = siteWithPlan('site-live-release-all', 'live-temporary'); + const result = await gate(good, [], site, 'true'); + assert.equal(result.code, 1, result.stdout); + assert.match(result.stdout, new RegExp(`^ {2}${media[0].path} media: expected "served by the build", got "release build may not serve media from the live site"$`, 'm')); + assert.match(result.stdout, new RegExp(`^media: ${media.length} checked, ${media.length} failed$`, 'm')); +}); + +test('live-temporary mode on a release build fails only the prefix; files outside it are checked in dist and pass', async () => { + const site = siteWithPlan('site-live-release', 'live-temporary', UPLOADS); + const inside = media.filter((m) => m.path.startsWith(UPLOADS)); + const outside = media.filter((m) => !m.path.startsWith(UPLOADS)); + assert.ok(inside.length > 0 && outside.length > 0, 'the fixture has media on both sides of the prefix'); + const result = await gate(good, [], site, 'true'); + assert.equal(result.code, 1, result.stdout); + assert.match(result.stdout, new RegExp(`^media: ${media.length} checked, ${inside.length} failed$`, 'm')); + for (const m of inside) { + assert.match(result.stdout, new RegExp(`^ {2}${m.path} media: expected "served by the build", got "release build may not serve media from the live site"$`, 'm')); } + for (const m of outside) assert.doesNotMatch(result.stdout, new RegExp(`^ {2}${m.path} `, 'm')); + const json = JSON.parse((await gate(good, ['--json'], site, 'true')).stdout); + assert.equal(json.checked, media.length); + assert.equal(json.failed, inside.length); + // An outside file missing from dist fails as in staging mode, with its sizes. + const noFont = makeDist(join(root, 'dist-live-release-no-font')); + rmSync(join(noFont, outside[0].path.slice(1))); + const failed = await gate(noFont, [], site, 'true'); + assert.match(failed.stdout, new RegExp(`^ {2}${outside[0].path} media: expected ${outside[0].bytes}, got null$`, 'm')); + assert.match(failed.stdout, new RegExp(`^media: ${media.length} checked, ${inside.length + 1} failed$`, 'm')); }); test('--paths limits the check', async () => { diff --git a/test/gates/redirects.test.mjs b/test/gates/redirects.test.mjs index 61eca46..43de5aa 100644 --- a/test/gates/redirects.test.mjs +++ b/test/gates/redirects.test.mjs @@ -1,7 +1,7 @@ import { test, before, after } from 'node:test'; import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; -import { appendFileSync, copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { appendFileSync, copyFileSync, cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -272,6 +272,102 @@ test('a built file whose extension the compiler never saw is caught as redirecte assert.match(result.stdout, /^ {2}\/wp-content\/uploads\/2026\/01\/data\.Qxz redirected: expected "no redirect", got "built file redirected by rule [^"]+: 308 \/wp-content\/uploads\/2026\/01\/data\.Qxz\/"$/m); }); +const COLON_SOURCE = '/trackshttps://www.example.com/exhibitor-contact/'; + +test('a source with a bare ":" compiles to its escaped form and the gate passes it; an unescaped one is rejected like Vercel does', async () => { + const dir = await site('site-colon', 'vercel', { rows: [`${COLON_SOURCE},/contact/,301,crawl,`] }); + assert.ok(vercelConfig(dir).redirects.some((r) => r.source === '/trackshttps\\://www.example.com/exhibitor-contact/')); + const passed = await gate(dir, good, ['--paths', COLON_SOURCE]); + assert.equal(passed.code, 0, passed.stdout + passed.stderr); + assert.match(passed.stdout, /^redirects: 1 checked, 0 failed$/m); + const raw = vercelConfig(dir); + raw.redirects.unshift({ source: COLON_SOURCE, destination: '/contact/', statusCode: 301 }); + writeFileSync(join(dir, 'vercel.json'), JSON.stringify(raw)); + const rejected = await gate(dir, good, ['--paths', COLON_SOURCE]); + assert.equal(rejected.code, 2, rejected.stdout); + assert.match(rejected.stderr, /vercel\.json: .*":" with no parameter name/); +}); + +test('a captured source with a mid-segment ":name" or a port is a literal URL: requested as it is, never as a sample', async () => { + // A sample (/p/sample/) would not match the literal rule /p/\:id/ and would fail as "no redirect". + const rows = ['/p/:id/,/contact/,301,crawl,', '/x/http://host:8080/y,/contact/,301,crawl,']; + const dir = await site('site-literal-colons', 'vercel', { rows }); + const result = await gate(dir, good, ['--paths', '/p/:id/,/x/http://host:8080/y']); + assert.equal(result.code, 0, result.stdout + result.stderr); + assert.match(result.stdout, /^redirects: 2 checked, 0 failed$/m); + assert.doesNotMatch(result.stdout, /sample/); +}); + +test('a target under a baseline meta.excluded prefix is checked like an external one: status and location of the first hop only', async () => { + const excluded = join(root, 'baseline-excluded'); + cpSync(baseline, excluded, { recursive: true }); + const meta = JSON.parse(readFileSync(join(excluded, 'meta.json'), 'utf8')); + writeFileSync(join(excluded, 'meta.json'), JSON.stringify({ ...meta, excluded: ['/private/'] })); + // /private/area is unslashed: on an "always" site the slash rule adds a second hop the gate ignores. + const rows = ['/go-private,/private/area/,301,decision,', '/go-private-bare,/private/area,301,decision,']; + const dir = await site('site-excluded', 'vercel', { rows }); + const paths = ['--paths', '/go-private,/go-private-bare']; + const run = (base) => runGate('redirects', ['--dist', good, '--baseline', base, ...paths], { cwd: dir }); + const plain = await run(baseline); + assert.equal(plain.code, 1, 'without the exclusion the target must be built'); + assert.match(plain.stdout, /\/go-private target: expected "a page or file in dist", got "404 at \/private\/area\/"/); + const passed = await run(excluded); + assert.equal(passed.code, 0, passed.stdout + passed.stderr); + assert.match(passed.stdout, /^redirects: 2 checked, 0 failed$/m); + // The first hop still has to land on the target with the row's status. + const config = vercelConfig(dir); + config.redirects = config.redirects.map((r) => (r.source === '/go-private' ? { ...r, destination: '/private/other/', statusCode: 302 } : r)); + writeFileSync(join(dir, 'vercel.json'), JSON.stringify(config)); + const wrong = await run(excluded); + assert.equal(wrong.code, 1); + assert.match(wrong.stdout, /\/go-private redirect: expected "\/private\/area\/", got "302 \/private\/other\/"/); +}); + +const SPACED = '/two words/'; +const NO_REDIRECT = { kind: 'redirect', path: SPACED, reason: 'a broken live href; nothing to redirect', action: 'none', approvedBy: 'client', date: '2026-10-23' }; + +for (const host of ['vercel', 'cloudflare']) { + test(`${host}: a source with no valid form passes only when redirects-skipped.json lists it or an exception says no redirect`, async () => { + const dir = await site(`site-unescapable-${host}`, host, { rows: [`${SPACED},/contact/,301,crawl,`] }); + const dist = host === 'cloudflare' ? cloudflareBuild(dir, `dist-unescapable-${host}`) : good; + const listing = join(dir, 'redirects-skipped.json'); + const listed = JSON.parse(readFileSync(listing, 'utf8')); + assert.deepEqual(listed.map((s) => s.source), [SPACED]); + const skipped = await gate(dir, dist, ['--paths', SPACED]); + assert.equal(skipped.code, 0, skipped.stdout + skipped.stderr); + assert.match(skipped.stdout, /^ {2}\/two words\/ skipped: not compiled \(redirects-skipped\.json\): .*whitespace/m); + assert.match(skipped.stdout, /^redirects: 0 checked, 0 failed, 1 skipped \(not compiled \(redirects-skipped\.json\)\)$/m); + + writeFileSync(listing, JSON.stringify([{ source: SPACED, reason: '' }])); + const noReason = await gate(dir, dist, ['--paths', SPACED]); + assert.equal(noReason.code, 1, 'a listing without a reason does not count'); + rmSync(listing); + const failed = await gate(dir, dist, ['--paths', SPACED]); + assert.equal(failed.code, 1, failed.stdout); + assert.match(failed.stdout, /\/two words\/ redirect: expected "\/contact\/", got "no redirect: 404/); + + writeFileSync(join(dir, 'exceptions.json'), JSON.stringify([NO_REDIRECT])); + const excepted = await gate(dir, dist, ['--paths', SPACED]); + assert.equal(excepted.code, 0, excepted.stdout + excepted.stderr); + assert.match(excepted.stdout, /^redirects: 1 checked, 0 failed$/m); + }); +} + +test('a "no redirect" exception fails when the build still redirects the path, and on a path the gate does not check', async () => { + writeFileSync(join(vercelDir, 'exceptions.json'), JSON.stringify([ + { ...NO_REDIRECT, path: '/old-services' }, + { ...NO_REDIRECT, path: '/nowhere' }, + ])); + try { + const result = await gate(vercelDir, good, ['--paths', '/old-services,/nowhere']); + assert.equal(result.code, 1, result.stdout); + assert.match(result.stdout, /^ {2}\/old-services redirect: expected "no redirect \(exception\)", got "301 \/services\/"$/m); + assert.match(result.stdout, /\/nowhere exception: .*"path not checked"/); + } finally { + rmSync(join(vercelDir, 'exceptions.json')); + } +}); + test('usage and missing inputs exit 2; --help exits 0', async () => { assert.equal((await gate(vercelDir, good, ['--bogus'])).code, 2); assert.equal((await gate(vercelDir, join(root, 'nope'))).code, 2); diff --git a/test/gates/sitemaps.test.mjs b/test/gates/sitemaps.test.mjs index 06f00c0..0b461e2 100644 --- a/test/gates/sitemaps.test.mjs +++ b/test/gates/sitemaps.test.mjs @@ -144,6 +144,38 @@ test('a carried Sitemap: line on the site\'s own host is never skipped: with --o assert.match(result.stdout, /\/robots\.txt sitemap line: expected "https:\/\/staging\.example\.com\//); }); +test('a carried Sitemap: line under a path the capture excluded passes (a multisite sub-site); not without the exclusion', async () => { + const dir = join(root, 'excluded'); + const excludedBaseline = await buildFixtureBaseline(dir, { exclude: '/services/' }); + const excludedCapture = join(dir, 'capture'); + const liveOrigin = JSON.parse(readFileSync(join(excludedCapture, 'meta.json'), 'utf8')).origin; + // The live robots.txt also names the excluded sub-site's sitemap index, on the same host. + const subSite = `${liveOrigin}/services/sitemap_index.xml`; + writeFileSync(join(excludedCapture, 'robots.txt'), `${readFileSync(join(excludedCapture, 'robots.txt'), 'utf8')}Sitemap: ${subSite}\n`); + const dist = makeDist(join(root, 'dist-robots-excluded')); + rmSync(join(dist, 'services'), { recursive: true }); + for (const path of ['/services/', '/services/roofing/', '/services/siding/']) edit(dist, 'sitemap-0.xml', `${DIST_ORIGIN}${path}`, ''); + const own = `User-agent: *\nSitemap: ${DIST_ORIGIN}/sitemap-index.xml\n`; + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${subSite}\n`); + const gate = (base, capt) => runGate('sitemaps', ['--dist', dist, '--baseline', base, '--capture', capt, '--origin', DIST_ORIGIN], { cwd: root }); + + const carried = await gate(excludedBaseline, excludedCapture); + assert.equal(carried.code, 0, carried.stdout + carried.stderr); + assert.match(carried.stdout, /^sitemaps: 12 checked, 0 failed$/m); + + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${liveOrigin}/services/other-sitemap.xml\n`); + const notLive = await gate(excludedBaseline, excludedCapture); + assert.equal(notLive.code, 1, 'a line live did not carry still fails'); + + writeFileSync(join(dist, 'robots.txt'), `${own}Sitemap: ${subSite}\n`); + const copied = join(root, 'capture-with-subsite'); + cpSync(capture, copied, { recursive: true }); + writeFileSync(join(copied, 'robots.txt'), `${readFileSync(join(copied, 'robots.txt'), 'utf8')}Sitemap: ${subSite}\n`); + const unexcluded = await gate(baseline, copied); + assert.equal(unexcluded.code, 1, 'without the exclusion the same line fails'); + assert.match(unexcluded.stdout, /\/robots\.txt sitemap line: .*services\/sitemap_index\.xml/); +}); + test('a robots.txt Sitemap: line must point at a file in dist', async () => { const dist = makeDist(join(root, 'dist-robots')); writeFileSync(join(dist, 'robots.txt'), `User-agent: *\nSitemap: ${DIST_ORIGIN}/sitemap-index.xml\nSitemap: ${DIST_ORIGIN}/sitemap_index.xml\n`); diff --git a/test/gates/verify-live.test.mjs b/test/gates/verify-live.test.mjs index 5ae21a2..74e4a6f 100644 --- a/test/gates/verify-live.test.mjs +++ b/test/gates/verify-live.test.mjs @@ -169,6 +169,39 @@ test('a redirect that takes two hops fails; a failing unverified shortlink fails assert.deepEqual(json.skipped, []); }); +test('a redirect build-redirects dropped (redirects-skipped.json) or a no-redirect exception covers is skipped with its reason', async () => { + const base = join(root, 'baseline-dropped'); + cpSync(baseline, base, { recursive: true }); + const extra = [ + { from: '/two words/', to: '/contact/', status: 301, source: 'crawl' }, + { from: '/no-more/', to: '/contact/', status: 301, source: 'crawl' }, + ]; + writeFileSync(join(base, 'redirects.json'), JSON.stringify([...JSON.parse(readFileSync(join(baseline, 'redirects.json'), 'utf8')), ...extra])); + const host = await serve(good); + const run = (args = []) => verifyLive(['--base', host.origin, '--baseline', base, '--delay', '0', ...args]); + const failed = await run(['--json']); + assert.equal(failed.code, 1, 'neither is redirected by the deploy'); + assert.deepEqual(JSON.parse(failed.stdout).failures.map((f) => f.path).sort(), ['/no-more/', '/two words/']); + + const skippedFile = join(root, 'dropped-redirects-skipped.json'); + writeFileSync(skippedFile, JSON.stringify([{ source: '/two words/', reason: 'contains whitespace: no vercel rule can match it' }])); + const exceptionsFile = join(root, 'dropped-exceptions.json'); + writeFileSync(exceptionsFile, JSON.stringify([{ kind: 'redirect', path: '/no-more/', reason: 'a broken live href', action: 'none', approvedBy: 'client', date: '2026-10-23' }])); + const args = ['--skipped', skippedFile, '--exceptions', exceptionsFile]; + const passed = await run(args); + assert.equal(passed.code, 0, passed.stdout + passed.stderr); + assert.match(passed.stdout, /^ {2}skipped redirect \/two words\/: not compiled \(redirects-skipped\.json\): contains whitespace/m); + assert.match(passed.stdout, /^ {2}skipped redirect \/no-more\/: no redirect \(exception\): a broken live href$/m); + assert.match(passed.stdout, /^verify-live: 2 redirects skipped with a reason$/m); + const json = JSON.parse((await run([...args, '--json'])).stdout); + assert.deepEqual(json.skipped.filter((s) => s.check === 'redirect').map((s) => s.path), ['/two words/', '/no-more/']); + assert.equal(json.failed, 0); + // A listing only counts for a source that really has no valid form. + writeFileSync(skippedFile, JSON.stringify([{ source: '/no-more/', reason: 'listed by hand' }])); + assert.equal((await run(['--skipped', skippedFile])).code, 1); + assert.equal((await run(['--skipped', join(root, 'missing.json')])).code, 2); +}); + test('a query redirect whose target is its own path must answer 200 in place', async () => { const host = await serve(good); const result = await live(host.origin);