From 87890d7efd17b269ec13bc91ad8d2aaf396aaae0 Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Mon, 5 Oct 2026 09:12:25 -0700 Subject: [PATCH] feat(web): llms.txt, llms-full.txt, llms-small.txt and per-page Markdown for guides and API Serve plain-text docs for language models from the guides and API sites: an llms.txt index, llms-full.txt (every current-version page), llms-small.txt (an outline of the current version), one set per docs version under /llms-txt/, and every page as Markdown at .md. The current version comes from versions.ts. Built by plain Astro endpoints over the content collection, with no new dependencies. Add a wheels.dev/llms.txt index, and a CI check that fails the deploy when a file is missing, a link doesn't resolve, a page lacks its .md, or a non-current version leaks into llms-full / llms-small. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/web-deploy.yml | 11 + web/packages/ui/package.json | 1 + web/packages/ui/src/data/llms.ts | 213 ++++++++++++++++++ web/scripts/check-llms.mjs | 129 +++++++++++ web/scripts/check-llms.test.mjs | 105 +++++++++ web/sites/api/src/pages/[...slug].md.ts | 21 ++ web/sites/api/src/pages/llms-full.txt.ts | 14 ++ web/sites/api/src/pages/llms-small.txt.ts | 17 ++ .../api/src/pages/llms-txt/[slug].txt.ts | 20 ++ web/sites/api/src/pages/llms.txt.ts | 12 + web/sites/guides/src/pages/[...slug].md.ts | 21 ++ web/sites/guides/src/pages/llms-full.txt.ts | 14 ++ web/sites/guides/src/pages/llms-small.txt.ts | 17 ++ .../guides/src/pages/llms-txt/[slug].txt.ts | 20 ++ web/sites/guides/src/pages/llms.txt.ts | 12 + web/sites/landing/public/llms.txt | 23 ++ 16 files changed, 650 insertions(+) create mode 100644 web/packages/ui/src/data/llms.ts create mode 100644 web/scripts/check-llms.mjs create mode 100644 web/scripts/check-llms.test.mjs create mode 100644 web/sites/api/src/pages/[...slug].md.ts create mode 100644 web/sites/api/src/pages/llms-full.txt.ts create mode 100644 web/sites/api/src/pages/llms-small.txt.ts create mode 100644 web/sites/api/src/pages/llms-txt/[slug].txt.ts create mode 100644 web/sites/api/src/pages/llms.txt.ts create mode 100644 web/sites/guides/src/pages/[...slug].md.ts create mode 100644 web/sites/guides/src/pages/llms-full.txt.ts create mode 100644 web/sites/guides/src/pages/llms-small.txt.ts create mode 100644 web/sites/guides/src/pages/llms-txt/[slug].txt.ts create mode 100644 web/sites/guides/src/pages/llms.txt.ts create mode 100644 web/sites/landing/public/llms.txt diff --git a/.github/workflows/web-deploy.yml b/.github/workflows/web-deploy.yml index b68ce2438a..c9fe01d0f0 100644 --- a/.github/workflows/web-deploy.yml +++ b/.github/workflows/web-deploy.yml @@ -62,6 +62,17 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: pnpm --filter @wheels-dev/site-${{ matrix.site }} build + # The guides and API sites build llms.txt, llms-full.txt, llms-small.txt, the per-version + # llms-txt/ sets and a .md for every page. Fail before deploying if a file is missing, a link + # in llms.txt doesn't resolve, a page lacks its .md, or a non-current version leaks into + # llms-full.txt / llms-small.txt (web/scripts/check-llms.mjs). + - name: Check llms files + if: matrix.site == 'guides' || matrix.site == 'api' + working-directory: web + env: + SITE: ${{ matrix.site }} + run: node scripts/check-llms.mjs "sites/${SITE}/dist" "${SITE}" + - name: Deploy to Cloudflare Pages # Dependabot-triggered runs can't read repository secrets (they use # the separate Dependabot secrets store), so CLOUDFLARE_API_TOKEN is diff --git a/web/packages/ui/package.json b/web/packages/ui/package.json index c2887d3360..4d6209be03 100644 --- a/web/packages/ui/package.json +++ b/web/packages/ui/package.json @@ -18,6 +18,7 @@ "./components/starlight/EditLink.astro": "./src/components/starlight/EditLink.astro", "./components/starlight/VersionSwitcher.astro": "./src/components/starlight/VersionSwitcher.astro", "./data/versions": "./src/data/versions.ts", + "./data/llms": "./src/data/llms.ts", "./styles/tokens.css": "./src/styles/tokens.css", "./styles/fonts.css": "./src/styles/fonts.css", "./styles/base.css": "./src/styles/base.css", diff --git a/web/packages/ui/src/data/llms.ts b/web/packages/ui/src/data/llms.ts new file mode 100644 index 0000000000..ef6df437d0 --- /dev/null +++ b/web/packages/ui/src/data/llms.ts @@ -0,0 +1,213 @@ +/** + * Plain-text docs for language models on the guides and API sites (https://llmstxt.org): + * + * - /llms.txt: an index (what Wheels is, the documentation sets, and every current-version + * page with a link to its Markdown); + * - /llms-full.txt: every current-version page as Markdown, in one file; + * - /llms-small.txt: an outline of the current version (each page's title, Markdown link, + * description and section headings), sized for an assistant's context; + * - /llms-txt/.txt: one file per docs version, current or not; + * - .md: one page as Markdown. + * + * The current version is the `current` entry in versions.ts, so a version flip moves all of this + * with no edit here. Each site's src/pages/ holds thin endpoints that call these functions with + * its content collection. + */ +import type { VersionMeta } from './versions'; + +export type LlmsSite = 'guides' | 'api'; + +/** The fields of a docs collection entry these functions read. */ +export interface LlmsDoc { + id: string; + body?: string; + data: { + title: string; + description?: string; + draft?: boolean; + pagefind?: boolean; + sidebar?: { hidden?: boolean }; + }; +} + +const SITE_TEXT: Record = { + guides: { + name: 'Wheels Guides', + what: 'narrative guides: installation, the tutorial, models, controllers, views, routing, migrations, testing and deployment', + origin: 'https://guides.wheels.dev', + }, + api: { + name: 'Wheels API Reference', + what: 'the function reference: every framework function with its signature, parameters, return type and examples', + origin: 'https://api.wheels.dev', + }, +}; + +const DETAILS = [ + 'Wheels is an open-source MVC framework for CFML (Lucee, Adobe ColdFusion, BoxLang and RustCFML), inspired by Ruby on Rails.', + 'Code examples are CFML, not Ruby: models extend "Model", controllers extend "Controller", and model associations, validations and callbacks are declared in a config() method.', + 'Wheels functions take either all positional or all named arguments, never a mix.', + 'Model finders return query objects by default, not arrays.', +]; + +const IMPORT_LINE = /^import\s.+\sfrom\s+['"][^'"]+['"];?\s*$/; + +export function currentVersion(versions: VersionMeta[]): VersionMeta { + const current = versions.find((v) => v.status === 'current'); + if (!current) { + throw new Error( + 'versions.ts has no `current` entry; llms.txt needs one to choose its version.' + ); + } + return current; +} + +/** The docs version a collection entry belongs to (its first path segment), or "" for the site root. */ +export function docVersion(id: string): string { + const slash = id.indexOf('/'); + return slash === -1 ? (id === 'index' ? '' : id) : id.slice(0, slash); +} + +/** + * Whether a page belongs in llms.txt and the bundles: not a draft, and not unlisted (hidden from the + * sidebar, or left out of search with `pagefind: false`). + */ +export function isListed(doc: LlmsDoc): boolean { + return !doc.data.draft && doc.data.pagefind !== false && doc.data.sidebar?.hidden !== true; +} + +/** Listed entries of one version, in a stable order (the version's landing page first). */ +export function docsOfVersion(docs: T[], slug: string): T[] { + return docs + .filter((doc) => isListed(doc) && docVersion(doc.id) === slug) + .sort((a, b) => (a.id === slug ? -1 : b.id === slug ? 1 : a.id.localeCompare(b.id))); +} + +function versionLabel(version: VersionMeta): string { + if (version.status === 'current') return `Wheels ${version.label} (current)`; + if (version.status === 'snapshot') return `Wheels ${version.label} (in development)`; + return `Wheels ${version.label}`; +} + +/** The URL slug of a version's set: /llms-txt/.txt. */ +export function setSlug(version: VersionMeta): string { + return version.slug; +} + +/** Page source without MDX import lines. */ +function pageSource(body: string | undefined): string { + return (body ?? '') + .split('\n') + .filter((line) => !IMPORT_LINE.test(line)) + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +/** A page's level-2 and level-3 headings as an indented list, skipping fenced code. */ +function pageOutline(body: string | undefined): string[] { + const out: string[] = []; + let fence = ''; + for (const line of (body ?? '').split('\n')) { + const marker = /^\s*(`{3,}|~{3,})/.exec(line); + if (marker) { + if (!fence) fence = marker[1]; + else if (marker[1].startsWith(fence)) fence = ''; + continue; + } + if (fence) continue; + const heading = /^(#{2,3})\s+(.+?)\s*#*\s*$/.exec(line); + if (heading) out.push(`${heading[1].length === 3 ? ' ' : ''}- ${heading[2]}`); + } + return out; +} + +/** + * The Markdown served at `.md`: the page's title and description, then its source. + * MDX import lines are dropped; components such as