feat(web): llms.txt, llms-full.txt, llms-small.txt and per-page Markdown for guides and API

.github/workflows/web-deploy.yml

@@ -62,6 +62,17 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: pnpm --filter @wheels-dev/site-${{ matrix.site }} build
 
+      # The guides and API sites build llms.txt, llms-full.txt, llms-small.txt, the per-version
+      # llms-txt/ sets and a .md for every page. Fail before deploying if a file is missing, a link
+      # in llms.txt doesn't resolve, a page lacks its .md, or a non-current version leaks into
+      # llms-full.txt / llms-small.txt (web/scripts/check-llms.mjs).
+      - name: Check llms files
+        if: matrix.site == 'guides' || matrix.site == 'api'
+        working-directory: web
+        env:
+          SITE: ${{ matrix.site }}
+        run: node scripts/check-llms.mjs "sites/${SITE}/dist" "${SITE}"
+
       - name: Deploy to Cloudflare Pages
         # Dependabot-triggered runs can't read repository secrets (they use
         # the separate Dependabot secrets store), so CLOUDFLARE_API_TOKEN is

web/packages/ui/package.json

@@ -18,6 +18,7 @@
 		"./components/starlight/EditLink.astro": "./src/components/starlight/EditLink.astro",
 		"./components/starlight/VersionSwitcher.astro": "./src/components/starlight/VersionSwitcher.astro",
 		"./data/versions": "./src/data/versions.ts",
+		"./data/llms": "./src/data/llms.ts",
 		"./styles/tokens.css": "./src/styles/tokens.css",
 		"./styles/fonts.css": "./src/styles/fonts.css",
 		"./styles/base.css": "./src/styles/base.css",

web/packages/ui/src/data/llms.ts

@@ -0,0 +1,213 @@
+/**
+ * Plain-text docs for language models on the guides and API sites (https://llmstxt.org):
+ *
+ * - /llms.txt: an index (what Wheels is, the documentation sets, and every current-version
+ *   page with a link to its Markdown);
+ * - /llms-full.txt: every current-version page as Markdown, in one file;
+ * - /llms-small.txt: an outline of the current version (each page's title, Markdown link,
+ *   description and section headings), sized for an assistant's context;
+ * - /llms-txt/<version>.txt: one file per docs version, current or not;
+ * - <page URL>.md: one page as Markdown.
+ *
+ * The current version is the `current` entry in versions.ts, so a version flip moves all of this
+ * with no edit here. Each site's src/pages/ holds thin endpoints that call these functions with
+ * its content collection.
+ */
+import type { VersionMeta } from './versions';
+
+export type LlmsSite = 'guides' | 'api';
+
+/** The fields of a docs collection entry these functions read. */
+export interface LlmsDoc {
+	id: string;
+	body?: string;
+	data: {
+		title: string;
+		description?: string;
+		draft?: boolean;
+		pagefind?: boolean;
+		sidebar?: { hidden?: boolean };
+	};
+}
+
+const SITE_TEXT: Record<LlmsSite, { name: string; what: string; origin: string }> = {
+	guides: {
+		name: 'Wheels Guides',
+		what: 'narrative guides: installation, the tutorial, models, controllers, views, routing, migrations, testing and deployment',
+		origin: 'https://guides.wheels.dev',
+	},
+	api: {
+		name: 'Wheels API Reference',
+		what: 'the function reference: every framework function with its signature, parameters, return type and examples',
+		origin: 'https://api.wheels.dev',
+	},
+};
+
+const DETAILS = [
+	'Wheels is an open-source MVC framework for CFML (Lucee, Adobe ColdFusion, BoxLang and RustCFML), inspired by Ruby on Rails.',
+	'Code examples are CFML, not Ruby: models extend "Model", controllers extend "Controller", and model associations, validations and callbacks are declared in a config() method.',
+	'Wheels functions take either all positional or all named arguments, never a mix.',
+	'Model finders return query objects by default, not arrays.',
+];
+
+const IMPORT_LINE = /^import\s.+\sfrom\s+['"][^'"]+['"];?\s*$/;
+
+export function currentVersion(versions: VersionMeta[]): VersionMeta {
+	const current = versions.find((v) => v.status === 'current');
+	if (!current) {
+		throw new Error(
+			'versions.ts has no `current` entry; llms.txt needs one to choose its version.'
+		);
+	}
+	return current;
+}
+
+/** The docs version a collection entry belongs to (its first path segment), or "" for the site root. */
+export function docVersion(id: string): string {
+	const slash = id.indexOf('/');
+	return slash === -1 ? (id === 'index' ? '' : id) : id.slice(0, slash);
+}
+
+/**
+ * Whether a page belongs in llms.txt and the bundles: not a draft, and not unlisted (hidden from the
+ * sidebar, or left out of search with `pagefind: false`).
+ */
+export function isListed(doc: LlmsDoc): boolean {
+	return !doc.data.draft && doc.data.pagefind !== false && doc.data.sidebar?.hidden !== true;
+}
+
+/** Listed entries of one version, in a stable order (the version's landing page first). */
+export function docsOfVersion<T extends LlmsDoc>(docs: T[], slug: string): T[] {
+	return docs
+		.filter((doc) => isListed(doc) && docVersion(doc.id) === slug)
+		.sort((a, b) => (a.id === slug ? -1 : b.id === slug ? 1 : a.id.localeCompare(b.id)));
+}
+
+function versionLabel(version: VersionMeta): string {
+	if (version.status === 'current') return `Wheels ${version.label} (current)`;
+	if (version.status === 'snapshot') return `Wheels ${version.label} (in development)`;
+	return `Wheels ${version.label}`;
+}
+
+/** The URL slug of a version's set: /llms-txt/<slug>.txt. */
+export function setSlug(version: VersionMeta): string {
+	return version.slug;
+}
+
+/** Page source without MDX import lines. */
+function pageSource(body: string | undefined): string {
+	return (body ?? '')
+		.split('\n')
+		.filter((line) => !IMPORT_LINE.test(line))
+		.join('\n')
+		.replace(/\n{3,}/g, '\n\n')
+		.trim();
+}
+
+/** A page's level-2 and level-3 headings as an indented list, skipping fenced code. */
+function pageOutline(body: string | undefined): string[] {
+	const out: string[] = [];
+	let fence = '';
+	for (const line of (body ?? '').split('\n')) {
+		const marker = /^\s*(`{3,}|~{3,})/.exec(line);
+		if (marker) {
+			if (!fence) fence = marker[1];
+			else if (marker[1].startsWith(fence)) fence = '';
+			continue;
+		}
+		if (fence) continue;
+		const heading = /^(#{2,3})\s+(.+?)\s*#*\s*$/.exec(line);
+		if (heading) out.push(`${heading[1].length === 3 ? '  ' : ''}- ${heading[2]}`);
+	}
+	return out;
+}
+
+/**
+ * The Markdown served at `<page URL>.md`: the page's title and description, then its source.
+ * MDX import lines are dropped; components such as <Aside> stay as written.
+ */
+export function markdownTwin(
+	title: string,
+	description: string | undefined,
+	body: string | undefined
+): string {
+	const head = [`# ${title}`];
+	if (description) head.push('', `> ${description}`);
+	return `${head.join('\n')}\n\n${pageSource(body)}\n`;
+}
+
+function pageUrl(site: LlmsSite, id: string): string {
+	return `${SITE_TEXT[site].origin}/${id}/`;
+}
+
+/** Many pages as one Markdown document (llms-full.txt and the version sets). */
+export function llmsBundle(site: LlmsSite, heading: string, docs: LlmsDoc[]): string {
+	const parts = [`# ${heading}`, '', `> ${SITE_TEXT[site].name}. ${DETAILS[0]}`];
+	for (const doc of docs) {
+		parts.push('', '---', '', `# ${doc.data.title}`, '', `Source: ${pageUrl(site, doc.id)}`);
+		if (doc.data.description) parts.push('', `> ${doc.data.description}`);
+		const source = pageSource(doc.body);
+		if (source) parts.push('', source);
+	}
+	return parts.join('\n') + '\n';
+}
+
+/** llms-small.txt: an outline of the pages, with a link to each page's Markdown. */
+export function llmsOutline(site: LlmsSite, heading: string, docs: LlmsDoc[]): string {
+	const origin = SITE_TEXT[site].origin;
+	const parts = [
+		`# ${heading}`,
+		'',
+		`> ${SITE_TEXT[site].name}, abridged: each page's title, description and section headings. Fetch a page's full text from its Markdown link.`,
+		'',
+		...DETAILS,
+	];
+	for (const doc of docs) {
+		parts.push('', `## [${doc.data.title}](${origin}/${doc.id}.md)`);
+		if (doc.data.description) parts.push('', doc.data.description);
+		const outline = pageOutline(doc.body);
+		if (outline.length) parts.push('', ...outline);
+	}
+	return parts.join('\n') + '\n';
+}
+
+/** /llms.txt: the index. */
+export function llmsIndex(site: LlmsSite, versions: VersionMeta[], docs: LlmsDoc[]): string {
+	const text = SITE_TEXT[site];
+	const current = currentVersion(versions);
+	const other: LlmsSite = site === 'guides' ? 'api' : 'guides';
+	const lines = [
+		`# ${text.name}`,
+		'',
+		`> ${text.name} for Wheels ${current.label}: ${text.what}.`,
+		'',
+		...DETAILS,
+		'Every page is also available as Markdown: append ".md" to its URL (without the trailing slash).',
+		'',
+		'## Documentation sets',
+		'',
+		`- [Complete documentation](${text.origin}/llms-full.txt): every Wheels ${current.label} page, as Markdown`,
+		`- [Abridged documentation](${text.origin}/llms-small.txt): an outline of every Wheels ${current.label} page, with section headings`,
+		...versions.map(
+			(v) =>
+				`- [${versionLabel(v)}](${text.origin}/llms-txt/${setSlug(v)}.txt): every page for that version`
+		),
+		'',
+		`## Wheels ${current.label} pages`,
+		'',
+		...docsOfVersion(docs, current.slug).map((doc) => {
+			const desc = doc.data.description ? `: ${doc.data.description}` : '';
+			return `- [${doc.data.title}](${text.origin}/${doc.id}.md)${desc}`;
+		}),
+		'',
+		'## Optional',
+		'',
+		`- [${SITE_TEXT[other].name}](${SITE_TEXT[other].origin}/llms.txt): ${SITE_TEXT[other].what}`,
+		'- [Wheels blog](https://blog.wheels.dev/): release notes and tutorials',
+		'- [Wheels packages](https://packages.wheels.dev/): first-party and community packages',
+	];
+	return lines.join('\n') + '\n';
+}
+
+export const TEXT_HEADERS = { 'Content-Type': 'text/plain; charset=utf-8' };
+export const MARKDOWN_HEADERS = { 'Content-Type': 'text/markdown; charset=utf-8' };

web/scripts/check-llms.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+/**
+ * Checks a built guides or API site's llms files (see packages/ui/src/data/llms.ts):
+ *
+ * - llms.txt, llms-full.txt and llms-small.txt exist and aren't empty;
+ * - every link in llms.txt to this site resolves to a built file;
+ * - llms-full.txt and llms-small.txt hold only the CURRENT version's pages (the `current` entry in
+ *   packages/ui/src/data/versions.ts): a page from any other version fails the check;
+ * - every built docs page has its .md, and every .md has its page.
+ *
+ *   node scripts/check-llms.mjs <dist dir> <guides|api>
+ *
+ * Exits 1 and lists every problem when a check fails.
+ */
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const ORIGINS = { guides: 'https://guides.wheels.dev', api: 'https://api.wheels.dev' };
+const LISTS = { guides: 'GUIDES_VERSIONS', api: 'API_VERSIONS' };
+
+/** The version slugs and the current one, read from versions.ts (same parse as build-docs.sh). */
+export function readVersions(versionsSource, site) {
+	const block = versionsSource.match(
+		new RegExp(`${LISTS[site]}[\\s\\S]*?=\\s*\\[([\\s\\S]*?)\\];`)
+	);
+	if (!block) throw new Error(`versions.ts has no ${LISTS[site]} list`);
+	const entries = [...block[1].matchAll(/\{[^}]*\}/g)].map((m) => m[0]);
+	const slugs = entries.map((e) => /slug:\s*'([^']+)'/.exec(e)?.[1]).filter(Boolean);
+	const current = entries.find((e) => /status:\s*'current'/.test(e));
+	const currentSlug = current && /slug:\s*'([^']+)'/.exec(current)?.[1];
+	if (!currentSlug) throw new Error(`${LISTS[site]} has no 'current' entry`);
+	return { slugs, currentSlug };
+}
+
+function walk(dir) {
+	const out = [];
+	for (const name of readdirSync(dir)) {
+		const path = join(dir, name);
+		if (statSync(path).isDirectory()) out.push(...walk(path));
+		else out.push(path);
+	}
+	return out;
+}
+
+/** The version slug a URL path under the site origin belongs to ("" when none). */
+function slugOfPath(path, slugs) {
+	const first = path.replace(/^\/+/, '').split('/')[0].replace(/\.md$/, '');
+	return slugs.includes(first) ? first : '';
+}
+
+export function checkLlms({ distDir, site, slugs, currentSlug }) {
+	const errors = [];
+	const origin = ORIGINS[site];
+	const read = (name) => {
+		const path = join(distDir, name);
+		if (!existsSync(path)) {
+			errors.push(`${name} is missing`);
+			return '';
+		}
+		const text = readFileSync(path, 'utf8');
+		if (!text.trim()) errors.push(`${name} is empty`);
+		return text;
+	};
+	const index = read('llms.txt');
+	const full = read('llms-full.txt');
+	const small = read('llms-small.txt');
+
+	// Every link in llms.txt to this site resolves to a built file.
+	for (const [, url] of index.matchAll(/\]\((https?:[^)\s]+)\)/g)) {
+		if (!url.startsWith(origin + '/')) continue;
+		const path = url.slice(origin.length);
+		const file = path.endsWith('/') ? join(distDir, path, 'index.html') : join(distDir, path);
+		if (!existsSync(file)) errors.push(`llms.txt links to ${url}, which isn't in the build`);
+	}
+
+	// llms-full.txt and llms-small.txt hold only the current version.
+	const pagesIn = (text, pattern) => [...text.matchAll(pattern)].map((m) => m[1]);
+	const fullPages = pagesIn(full, new RegExp(`^Source: ${origin}(/[^\\s]*)`, 'gm'));
+	const smallPages = pagesIn(small, new RegExp(`\\]\\(${origin}(/[^)\\s]+\\.md)\\)`, 'g'));
+	for (const [name, pages] of [
+		['llms-full.txt', fullPages],
+		['llms-small.txt', smallPages],
+	]) {
+		if (!pages.length && errors.every((e) => !e.startsWith(name)))
+			errors.push(`${name} lists no pages`);
+		for (const page of pages) {
+			const slug = slugOfPath(page, slugs);
+			if (slug !== currentSlug) {
+				errors.push(`${name} includes ${page}, which isn't the current version (${currentSlug})`);
+			}
+		}
+	}
+
+	// Every built docs page has its .md, and every .md its page. Redirect stubs (a meta refresh) have no page of their own.
+	for (const file of walk(distDir)) {
+		const rel = relative(distDir, file).split(sep).join('/');
+		if (!slugs.includes(rel.split('/')[0])) continue;
+		if (rel.endsWith('/index.html')) {
+			if (/http-equiv=["']?refresh/i.test(readFileSync(file, 'utf8'))) continue;
+			const twin = join(distDir, rel.slice(0, -'/index.html'.length) + '.md');
+			if (!existsSync(twin)) errors.push(`${rel} has no .md`);
+		} else if (rel.endsWith('.md')) {
+			const page = join(distDir, rel.slice(0, -'.md'.length), 'index.html');
+			if (!existsSync(page)) errors.push(`${rel} has no page`);
+		}
+	}
+	return errors;
+}
+
+const isMain = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+	const [distArg, site] = process.argv.slice(2);
+	if (!distArg || !ORIGINS[site]) {
+		console.error('usage: node scripts/check-llms.mjs <dist dir> <guides|api>');
+		process.exit(2);
+	}
+	const here = dirname(fileURLToPath(import.meta.url));
+	const versionsSource = readFileSync(resolve(here, '../packages/ui/src/data/versions.ts'), 'utf8');
+	const { slugs, currentSlug } = readVersions(versionsSource, site);
+	const errors = checkLlms({ distDir: resolve(distArg), site, slugs, currentSlug });
+	if (errors.length) {
+		console.error(`llms check failed for ${site} (${errors.length}):`);
+		for (const e of errors.slice(0, 50)) console.error(`  - ${e}`);
+		if (errors.length > 50) console.error(`  … and ${errors.length - 50} more`);
+		process.exit(1);
+	}
+	console.log(`llms check passed for ${site}: current version ${currentSlug}`);
+}