From 74eb0ee2c06f5afe54271354c82ff1cec803894e Mon Sep 17 00:00:00 2001
From: Josh Vaughen <josh@vaughen.net>
Date: Wed, 7 Oct 2026 12:39:20 -0700
Subject: [PATCH 1/2] feat(rsync): directory pages in the engine, behind INDEX

pages and index are ctan's, moved here; the closing paragraph is
PAGE_FOOT. run-pages/want was drawn by ctan's pages before the move.
---
 CLAUDE.md                                     |  10 ++
 README.md                                     |  40 +++++--
 engines/rsync.yml                             | 106 +++++++++++++++++-
 examples/rsync/Taskfile.yml                   |  14 +++
 examples/rsync/fixtures/run-pages/applied.txt |   4 +
 examples/rsync/fixtures/run-pages/indexed.txt |   3 +
 .../fixtures/run-pages/want/index-gone.txt    |   2 +
 .../fixtures/run-pages/want/index-make.txt    |   4 +
 .../rsync/fixtures/run-pages/want/slash/1/a   |   8 ++
 .../rsync/fixtures/run-pages/want/slash/2/a/b |   8 ++
 .../fixtures/run-pages/want/slash/2/a/sp ace  |   7 ++
 .../b/ctan.katoptra.org.directory.index.html  |   8 ++
 .../a/ctan.katoptra.org.directory.index.html  |   8 ++
 .../ctan.katoptra.org.directory.index.html    |   7 ++
 .../ctan.katoptra.org.directory.index.html    |   7 ++
 examples/rsync/render.txt                     |  52 ++++++++-
 toolbox.yml                                   |   4 +-
 17 files changed, 275 insertions(+), 17 deletions(-)
 create mode 100644 examples/rsync/fixtures/run-pages/applied.txt
 create mode 100644 examples/rsync/fixtures/run-pages/indexed.txt
 create mode 100644 examples/rsync/fixtures/run-pages/want/index-gone.txt
 create mode 100644 examples/rsync/fixtures/run-pages/want/index-make.txt
 create mode 100644 examples/rsync/fixtures/run-pages/want/slash/1/a
 create mode 100644 examples/rsync/fixtures/run-pages/want/slash/2/a/b
 create mode 100644 examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace
 create mode 100644 examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html
 create mode 100644 examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
 create mode 100644 examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html
 create mode 100644 examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html

diff --git a/CLAUDE.md b/CLAUDE.md
index d47632b..7cb7040 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -44,6 +44,16 @@ verified go-task or platform fact that cost a session to find.
   (`github.job_workflow_sha`) for the toolbox action and the lock, so a workflow pin is
   the one pin. The include and the image float at `v2` by design: moving that tag is
   the rollout.
+- GNU `xargs` runs its command once on empty input. A guard on the file feeding a pipe is
+  not a guard on what reaches `xargs`: `pages`'s `SLASH` awk drops the root, which has no
+  slashless key, so a run whose only dirty directory is the root sends it nothing. Every
+  `xargs` whose input can be filtered down to nothing takes `-r`.
+- A `>-` folded block keeps the newline when a continuation line is indented further than
+  the lines around it. `pages`'s awk programs depend on that; a shell line meant to
+  continue ends with a backslash. `task check` shows what actually renders.
+- A macOS disk merges directories that differ only in case. Pages drawn on a laptop from a
+  real listing come out short wherever upstream has two such directories (CTAN has
+  `obsolete/support/TeXshell/` and `texshell/`). The runner is ext4 and draws both.
 
 ## Verifying a change
 
diff --git a/README.md b/README.md
index 5b47b9f..01c8609 100644
--- a/README.md
+++ b/README.md
@@ -270,8 +270,8 @@ flowchart LR
   class prepare,verify,index,sm,re,rm hook
 ```
 
-Dashed verbs are hooks. `prepare` and `verify` do nothing until `TL_KEY` is set; `index`
-and `smoke-mirror` do nothing until a mirror fills them.
+Dashed verbs are hooks. `prepare` and `verify` do nothing until `TL_KEY` is set, `index`
+until `INDEX` is set, and `smoke-mirror` until a mirror fills it.
 
 | Verb | Does |
 |---|---|
@@ -289,7 +289,8 @@ and `smoke-mirror` do nothing until a mirror fills them.
 | `checkpoint` | `merge` what landed into the state and push it as one PutObject; empty staging |
 | `delete` | Remove the keys upstream dropped, 1,000 per call, once every batch has landed, and drop them from the state |
 | `reconcile` | When `due` left `.run/reconcile`: rebuild the state, delete what neither upstream, `OWN` nor the state's own directories own, then `reconciled` |
-| `index` | Hook. Nothing here; a mirror that draws directory pages or a landing page replaces it |
+| `index` | Hook. With `INDEX` set: `pages`, then both key sets uploaded, the keys of emptied directories removed, and `.state/indexed.txt.xz` moved forward. A mirror with a landing page of its own replaces it |
+| `pages` | A page for every directory the run touched, drawn from the state into staging, with a tree per depth for the slashless keys |
 | `smoke` | A sample of the run's keys read back through `HOST`, sizes against the listing; the tlpdb sha512 when `TL` is set; then `smoke-mirror` |
 | `smoke-mirror` | Hook. Nothing here; a mirror with more to read back defines it |
 | `report-engine` | Hook. The engine's rows of the run summary |
@@ -362,6 +363,24 @@ A batch that fails any check stays local; the previous good copy stays live. `sm
 reads `texlive.tlpdb.sha512` back through the domain afterwards and compares it with the
 verified copy. `tlmgr` repeats the signature check on the client.
 
+#### Directory pages
+
+R2 serves no listings. With `INDEX` set, `index` draws a page for every directory a run
+touched, from the state rather than from upstream, and writes it under two keys:
+`<dir>/<INDEX>`, which a zone Transform Rule serves for `/dir/`, and the bare `<dir>`,
+which serves `/dir` where a filesystem mirror would answer 301. A `<base href>` lets one
+document serve both. `.state/indexed.txt.xz` records the state the pages last showed, so a
+run that dies before moving it redraws the same pages next time. Deleting it redraws every
+page once, which is also how a change to the markup or to `PAGE_FOOT` reaches directories
+that have not changed. `reconcile` spares both keys.
+
+The zone rule, one per host:
+
+    when:         ends_with(http.request.uri.path, "/")
+    rewrite path: concat(http.request.uri.path, http.host, ".directory.index.html")
+
+It serves the root page for `/` too. ctan excludes `/`, which is CTAN's own `index.html`.
+
 #### The vars
 
 A mirror sets `SOURCE`, `BUCKET` and `HOST` in its root vars, always. Everything else
@@ -378,7 +397,8 @@ has an inline default, and a mirror sets only what differs:
 | `TL`, `TL_KEY` | empty | A signed TeX Live subtree and the fingerprint that signs it; empty, no signature checks |
 | `FILTER` | empty | rsync filter arguments that narrow the listing, for a mirror of a subtree |
 | `OWN` | empty | Bucket-root keys the mirror owns, space separated; `reconcile` never deletes them |
-| `INDEX` | empty | The key suffix of the directory pages a mirror's `index` draws; set, `reconcile` spares those pages and every bare directory of the state |
+| `INDEX` | empty | The key suffix of the directory pages; set, `index` draws them and `reconcile` spares them and every bare directory of the state |
+| `PAGE_FOOT` | empty | The HTML every directory page closes on; `%s` is the directory's encoded path, `%%` a literal percent |
 
 A var the mirror puts in its root `vars:` is fixed for every run: inside an included
 verb a root value shadows a `KEY=value` from the command line. One the mirror leaves to
@@ -557,13 +577,15 @@ Every mirror has one S3-compatible bucket. What it holds depends on the engine.
 |---|---|---|
 | rsync | every upstream path, at the root | The mirror. A public domain serves the bucket |
 | rsync | `.state/applied.txt.xz` | The state: what the bucket holds, at upstream's size and mtime |
-| rsync, a mirror's own | `.state/indexed.txt.xz`, `index.html` | ctan's record of what its directory pages show; tlnet's landing page, spared by `OWN` |
+| rsync, with `INDEX` | `.state/indexed.txt.xz`, `<dir>/<INDEX>`, `<dir>` | What the directory pages last showed, and the pages under both keys |
+| rsync, a mirror's own | `index.html` | tlnet's landing page, spared by `OWN` |
 | proton | `.state/session.tar.age` | The CLI session, encrypted. The only key |
 | a pipeline of its own | `.state/state.sqlite.xz.age`, `.state/history/<epoch>-<label>...` | dropbox's state and its dated copies; a lifecycle rule expires the history |
 | any that reconciles | `.state/reconciled` | The start epoch of the run that last completed a reconcile, through the engine's `push`: xz under rsync, with no `.xz` suffix, plain under dropbox |
 
-`.state/` is the one reserved prefix, chosen because no upstream in the org has a
-dot-prefixed root entry.
+`.state/` is the one reserved prefix, chosen because no upstream in the org has a `.state`
+root entry. gnu's root carries other dot-files, `.header.shtml` and `.message`, which are
+mirrored like any file.
 
 ### What a bucket needs
 
@@ -764,6 +786,10 @@ A mirror of one signed subtree, shaped like tlnet, adds `TL`, `TL_KEY`, `CEILING
 `OWN` and a `FILTER` to the root vars, excludes `index` on the engine include, and
 defines an `index` that uploads its landing page and a `report-mirror` with its row.
 
+A mirror with browsable directories, as ctan, gnu and nongnu are, adds `INDEX` and
+`PAGE_FOOT` to its root vars and the Transform Rule above to its zone. The engine does the
+rest.
+
 ### A proton mirror
 
 ```yaml
diff --git a/engines/rsync.yml b/engines/rsync.yml
index 3165110..0e2b7c7 100644
--- a/engines/rsync.yml
+++ b/engines/rsync.yml
@@ -12,13 +12,14 @@
 # and the fingerprint that signs it; empty: no signature checks), FILTER (rsync filter
 # arguments for the listing; empty: the whole tree), OWN (bucket-root keys the mirror owns
 # and reconcile never deletes, space separated; empty: none), INDEX (the directory-page
-# key suffix an index hook writes; empty: none).
+# key suffix; set, index draws a page for every directory), PAGE_FOOT (the HTML every page
+# closes on, %s the directory's encoded path; empty: none).
 #
 # Hooks: prepare, verify, index, smoke, smoke-mirror and report-engine are the verbs a
 # mirror may replace, by listing each under excludes: on this include and defining its
-# own. prepare and verify do nothing until TL_KEY is set; index and smoke-mirror do
-# nothing here. pipeline and plan-pipeline are the vocabulary in order; a mirror that
-# needs another order excludes and redefines them.
+# own. prepare and verify do nothing until TL_KEY is set, index until INDEX is set;
+# smoke-mirror does nothing here. pipeline and plan-pipeline are the vocabulary in order;
+# a mirror that needs another order excludes and redefines them.
 #
 # Rules this file keeps, as the toolbox does: no vars: default for anything a mirror
 # owns, because a value here shadows the mirror's root value, and every tunable is an
@@ -33,6 +34,7 @@ vars:
   S3: s3://{{.BUCKET}}
   URL: https://{{.HOST}}
   STATE: .state/applied.txt.xz                  # the listing of what the bucket holds; never in upstream
+  INDEXED: .state/indexed.txt.xz                # the state as the directory pages last showed it
   STAGING: '{{.ROOT_DIR}}/staging'
   BATCHES_MAX: '{{.MAX_BATCHES | default 4}}'   # batches per run; the rest chain the next run
   RETRY_S: '{{.RETRY_BASE | default 15}}'       # seconds; a retry sleeps RETRY_S * 2^i plus jitter
@@ -444,8 +446,100 @@ tasks:
 
   # ---- after the bytes moved ----
   index:
-    desc: A hook, empty here; a mirror that draws directory pages or a landing page replaces it
-    cmds: []
+    desc: 'Hook. With INDEX set: redraw the directory pages this run changed and publish them; INDEXED, the state the pages last showed, moves forward only after they landed. A mirror with a landing page of its own replaces it'
+    status: ['test -z "{{.INDEX}}"']
+    set: [pipefail]
+    vars:
+      SLASH: '{{.RUN}}/slash'
+    cmds:
+      # No INDEXED means no page has been drawn: every directory is drawn once (ctan's 27k take
+      # about ten minutes), and the file exists from then on.
+      - {task: pull, vars: {KEY: '{{.INDEXED}}', OUT: '{{.RUN}}/indexed.txt', MISSING: ': > {{.RUN}}/indexed.txt'}}
+      - {task: pages}
+      # cp --recursive guesses text/html from the suffix. The pages stay in staging for smoke;
+      # the next run's split clears them. Not appended to publish.txt: smoke sizes that
+      # sample against upstream, which has no pages.
+      - test ! -s {{.RUN}}/index-make.txt || aws s3 cp {{.AWS_FLAGS}} --recursive {{.STAGING}}/ {{.S3}}/
+      # One upload per depth, because no single tree holds both a/b and a/b/. A slashless key
+      # carries no suffix to guess from, so the type is given: without it the CLI sends
+      # binary/octet-stream and a browser downloads the page instead of drawing it.
+      - >-
+        for d in {{.SLASH}}/*/; do test -d "$d" || continue;
+        aws s3 cp {{.AWS_FLAGS}} --recursive --content-type text/html "$d" {{.S3}}/; done
+      - {task: remove, vars: {K: '{{.RUN}}/index-gone.txt'}}
+      # The pages now show the state, and the bucket already holds it: checkpoint put it there
+      # before this verb ran. One server-side copy moves INDEXED forward; no bytes leave the
+      # runner. A run that dies before this line redraws the same pages next run.
+      - aws s3 cp {{.AWS_FLAGS}} {{.S3}}/{{.STATE}} {{.S3}}/{{.INDEXED}}
+
+  pages:
+    desc: RUN/indexed.txt against RUN/applied.txt -> a page under both of its keys for every directory a changed line touches, RUN/index-gone.txt for the keys of directories that emptied
+    set: [pipefail]
+    vars:
+      # Every page is written under two keys: <dir>/INDEX, which the zone's rewrite serves for a
+      # directory URL, and <dir> without its trailing slash, which serves the same URL written
+      # without one, where every other mirror answers 301. No filesystem holds both a/b and
+      # a/b/, so the second copies stage one tree per directory depth and upload a tree at a
+      # time. A task var, because a global cannot follow a RUN the caller passes.
+      SLASH: '{{.RUN}}/slash'
+    cmds:
+      - mkdir -p {{.STAGING}} {{.SLASH}} && find {{.STAGING}} {{.SLASH}} -mindepth 1 -delete
+      # A page changes when a file in its directory or a directory below it appears, changes
+      # or goes, so every ancestor of every changed line is redrawn. ponytail: about five
+      # pages per touched directory where one would often do; ceiling is Class A, a few
+      # hundred a day against a million free. Upgrade: comm the directory sets of the two
+      # files and redraw a parent only for a new or gone child.
+      - >-
+        LC_ALL=C comm -3 {{.RUN}}/indexed.txt {{.RUN}}/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'
+        | {{.ANCESTORS}} | LC_ALL=C sort > {{.RUN}}/index-dirty.txt
+      - '{{.ANCESTORS}} {{.RUN}}/applied.txt | LC_ALL=C sort > {{.RUN}}/index-dirs.txt'
+      # A gone directory takes both of its keys with it. The root has no slashless key.
+      - >-
+        LC_ALL=C comm -23 {{.RUN}}/index-dirty.txt {{.RUN}}/index-dirs.txt
+        | awk -v i="{{.INDEX}}" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > {{.RUN}}/index-gone.txt
+      - LC_ALL=C comm -12 {{.RUN}}/index-dirty.txt {{.RUN}}/index-dirs.txt > {{.RUN}}/index-make.txt
+      - test ! -s {{.RUN}}/index-make.txt || awk -v s="{{.STAGING}}/" '{ print s $0 }' {{.RUN}}/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p
+      # SLASH/<depth>/ holds the slashless copies of the pages at that depth. Every file in
+      # one tree has the same number of components, so no file is another's parent directory
+      # and the tree can be uploaded whole. -r because the root is the one directory with no
+      # slashless key: a run whose only dirty directory is the root -- root files alone, or
+      # a deletion that leaves no dirty directory still standing -- gives this awk nothing to
+      # print, and xargs without it runs mkdir once with no operands.
+      - >-
+        test ! -s {{.RUN}}/index-make.txt ||
+        awk -v r="{{.SLASH}}/" '$0 != "" { n = split($0, c, "/"); d = ""
+          for (i = 1; i < n - 1; i++) d = d c[i] "/"; print r (n - 1) "/" d }' {{.RUN}}/index-make.txt \
+        | LC_ALL=C sort -u | tr '\n' '\0' | xargs -0r mkdir -p
+      # Entries, "dir TAB d|f TAB name TAB size TAB mtime", from the state: what the bucket
+      # holds, never upstream. Sorted, a directory's lines are contiguous with its
+      # subdirectories first, so each page is written once and closed. Names are
+      # percent-encoded in href and HTML-escaped in text. The page is built whole and written
+      # under both of its keys, so the two are the same bytes; the base href is what lets one
+      # document serve /dir/ and /dir alike, where a relative href would otherwise resolve
+      # against the parent for the second. PAGE_FOOT closes every page, %s standing for the
+      # directory's encoded path; empty, the page closes on its table.
+      - >-
+        test ! -s {{.RUN}}/index-make.txt ||
+        awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next }
+          { n = split($1, c, "/"); d = ""
+            for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" }
+            if (d in want) print d, "f", c[n], $2, $3 }' {{.RUN}}/index-make.txt {{.RUN}}/applied.txt \
+        | LC_ALL=C sort | awk -F'\t' -v S="{{.STAGING}}/" -v R="{{.SLASH}}/" -v I="{{.INDEX}}" -v F={{shellQuote (.PAGE_FOOT | default "")}} '{{.URLENC}}
+          function esc(s) { gsub(/&/, "\\&amp;", s); gsub(/</, "\\&lt;", s); gsub(/>/, "\\&gt;", s); gsub(/"/, "\\&quot;", s); return s }
+          function open(d) { cur = d; live = 1
+            p = "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Index of /" esc(d) "</title>\n"
+            p = p "<base href=\"/" enc(d) "\">\n"
+            p = p "<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>\n"
+            p = p "<h1>Index of /" esc(d) "</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>\n"
+            if (d != "") p = p "<tr><td><a href=\"../\">../</a></td><td></td><td></td></tr>\n" }
+          function shut(  f, b, k, e) { p = p "</table>" (F == "" ? "" : "<p>" sprintf(F, enc(cur)) "</p>") "</body></html>\n"
+            f = S cur I; printf("%s", p) > f; close(f)
+            if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/")
+              f = R k "/" b; printf("%s", p) > f; close(f) } }
+          !live || $1 != cur { if (live) shut(); open($1) }
+          $2 == "d" { p = p "<tr><td><a href=\"" enc($3) "/\">" esc($3) "/</a></td><td></td><td></td></tr>\n"; next }
+          { p = p "<tr><td><a href=\"" enc($3) "\">" esc($3) "</a></td><td>" $5 "</td><td>" $4 "</td></tr>\n" }
+          END { if (live) shut() }'
 
   smoke:
     desc: Read a sample of this run's keys back through the domain, sizes against the listing, and with TL set the tlpdb sha512 against the verified copy; then the mirror's own checks, smoke-mirror
diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml
index 04082c6..ab7a113 100644
--- a/examples/rsync/Taskfile.yml
+++ b/examples/rsync/Taskfile.yml
@@ -10,6 +10,8 @@ vars:
   BUCKET: example
   HOST: example.katoptra.org
   LIST_FLOOR: 1
+  INDEX: '{{.HOST}}.directory.index.html'
+  PAGE_FOOT: 'A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.'
 includes:
   toolbox:
     taskfile: ../../toolbox.yml
@@ -43,6 +45,10 @@ tasks:
       TREE: '{{.ROOT_DIR}}/fixtures/tree/'
       LOCAL: rsync --no-h   # a local tree; the engine's daemon-only flags are a usage error here
       FP: 3505F9E22FF030FE5D9C655A825C376AB4571163
+      # run-pages/want/ is ctan's page set for that state as ctan.katoptra.org serves it: its
+      # INDEX, and the closing line that links the directory on ctan.org.
+      CTAN_INDEX: ctan.katoptra.org.directory.index.html
+      CTAN_FOOT: '<a href="https://ctan.org/tex-archive/%s">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.'
     cmds:
       # list and normalise over a local tree: its two regular files, byte-sorted, and no directory.
       - {task: list, vars: {SOURCE: '{{.ROOT_DIR}}/fixtures/run-root/staging/', RSYNC: 'rsync --no-h'}}
@@ -107,6 +113,14 @@ tasks:
       - test "$(wc -l < {{.F}}/run-tl/tl/sums.txt)" -eq 3 && test ! -s {{.F}}/run-tl/tl/missing.txt
       # A container the tlpdb does not describe cannot pass, whatever its bytes.
       - '! task verify-refuses 2>/dev/null'
+      # run-pages: the state gained a file whose name carries & < > and a directory whose name
+      # carries a space, and lost gone/. pages draws the root, a/, a/b/ and a/sp ace/, each
+      # under both of its keys (the root under one), and names gone/'s two keys for removal.
+      # Every byte must be what want/ holds.
+      - {task: pages, vars: {RUN: '{{.F}}/run-pages', STAGING: '{{.F}}/run-pages/staging', INDEX: '{{.CTAN_INDEX}}', PAGE_FOOT: '{{.CTAN_FOOT}}'}}
+      - diff -r {{.F}}/run-pages/want/staging {{.F}}/run-pages/staging
+      - diff -r {{.F}}/run-pages/want/slash {{.F}}/run-pages/slash
+      - cmp {{.F}}/run-pages/want/index-make.txt {{.F}}/run-pages/index-make.txt && cmp {{.F}}/run-pages/want/index-gone.txt {{.F}}/run-pages/index-gone.txt
       - 'echo "offline: every check passed"'
   due-refuses:
     desc: due-rule over offline's due fixture with RECONCILE R and RECONCILE_HOURS H; offline expects it to refuse
diff --git a/examples/rsync/fixtures/run-pages/applied.txt b/examples/rsync/fixtures/run-pages/applied.txt
new file mode 100644
index 0000000..49a691d
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/applied.txt
@@ -0,0 +1,4 @@
+a/b/f.txt	10	2026/08/27 01:00:00
+a/b/new & <odd>.txt	7	2026/08/28 06:00:00
+a/sp ace/g.tar.gz	3	2026/08/28 06:00:00
+timestamp	20	2026/08/28 06:00:00
diff --git a/examples/rsync/fixtures/run-pages/indexed.txt b/examples/rsync/fixtures/run-pages/indexed.txt
new file mode 100644
index 0000000..847cd78
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/indexed.txt
@@ -0,0 +1,3 @@
+a/b/f.txt	10	2026/08/27 01:00:00
+gone/x.txt	5	2026/08/27 01:00:00
+timestamp	20	2026/08/27 01:00:00
diff --git a/examples/rsync/fixtures/run-pages/want/index-gone.txt b/examples/rsync/fixtures/run-pages/want/index-gone.txt
new file mode 100644
index 0000000..dcddfa7
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/index-gone.txt
@@ -0,0 +1,2 @@
+gone/ctan.katoptra.org.directory.index.html
+gone
diff --git a/examples/rsync/fixtures/run-pages/want/index-make.txt b/examples/rsync/fixtures/run-pages/want/index-make.txt
new file mode 100644
index 0000000..fd09e78
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/index-make.txt
@@ -0,0 +1,4 @@
+
+a/
+a/b/
+a/sp ace/
diff --git a/examples/rsync/fixtures/run-pages/want/slash/1/a b/examples/rsync/fixtures/run-pages/want/slash/1/a
new file mode 100644
index 0000000..1dac6ad
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/slash/1/a
@@ -0,0 +1,8 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/</title>
+<base href="/a/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="b/">b/</a></td><td></td><td></td></tr>
+<tr><td><a href="sp%20ace/">sp ace/</a></td><td></td><td></td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/b b/examples/rsync/fixtures/run-pages/want/slash/2/a/b
new file mode 100644
index 0000000..ce432b2
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/b
@@ -0,0 +1,8 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/b/</title>
+<base href="/a/b/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/b/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="f.txt">f.txt</a></td><td>2026/08/27 01:00:00</td><td>10</td></tr>
+<tr><td><a href="new%20%26%20%3Codd%3E.txt">new &amp; &lt;odd&gt;.txt</a></td><td>2026/08/28 06:00:00</td><td>7</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/b/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace b/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace
new file mode 100644
index 0000000..ae7c38b
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/slash/2/a/sp ace	
@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/sp ace/</title>
+<base href="/a/sp%20ace/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/sp ace/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="g.tar.gz">g.tar.gz</a></td><td>2026/08/28 06:00:00</td><td>3</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/sp%20ace/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html
new file mode 100644
index 0000000..ce432b2
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/staging/a/b/ctan.katoptra.org.directory.index.html
@@ -0,0 +1,8 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/b/</title>
+<base href="/a/b/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/b/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="f.txt">f.txt</a></td><td>2026/08/27 01:00:00</td><td>10</td></tr>
+<tr><td><a href="new%20%26%20%3Codd%3E.txt">new &amp; &lt;odd&gt;.txt</a></td><td>2026/08/28 06:00:00</td><td>7</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/b/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
new file mode 100644
index 0000000..1dac6ad
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/staging/a/ctan.katoptra.org.directory.index.html
@@ -0,0 +1,8 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/</title>
+<base href="/a/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="b/">b/</a></td><td></td><td></td></tr>
+<tr><td><a href="sp%20ace/">sp ace/</a></td><td></td><td></td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html
new file mode 100644
index 0000000..ae7c38b
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/staging/a/sp ace/ctan.katoptra.org.directory.index.html	
@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /a/sp ace/</title>
+<base href="/a/sp%20ace/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /a/sp ace/</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="../">../</a></td><td></td><td></td></tr>
+<tr><td><a href="g.tar.gz">g.tar.gz</a></td><td>2026/08/28 06:00:00</td><td>3</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/a/sp%20ace/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html
new file mode 100644
index 0000000..c4d322b
--- /dev/null
+++ b/examples/rsync/fixtures/run-pages/want/staging/ctan.katoptra.org.directory.index.html
@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /</title>
+<base href="/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="a/">a/</a></td><td></td><td></td></tr>
+<tr><td><a href="timestamp">timestamp</a></td><td>2026/08/28 06:00:00</td><td>20</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt
index 6565563..054bc94 100644
--- a/examples/rsync/render.txt
+++ b/examples/rsync/render.txt
@@ -133,8 +133,9 @@ task: [push] xz -T0 -c /work/examples/rsync/.run/applied.new > /work/examples/rs
 task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/applied.new.xz s3://example/.state/applied.txt.xz
 task: [rebuild] mv /work/examples/rsync/.run/applied.new /work/examples/rsync/.run/applied.txt
 task: [rebuild] rm -f /work/examples/rsync/.run/rebuild-now
-task: [reconcile] : > /work/examples/rsync/.run/index-keys.txt
-task: [reconcile] cut -f1 /work/examples/rsync/.run/bucket.txt | grep -vE '^\.state/'  | LC_ALL=C comm -23 - /work/examples/rsync/.run/index-keys.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/orphans.txt
+task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" }
+  { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/examples/rsync/.run/index-keys.txt
+task: [reconcile] cut -f1 /work/examples/rsync/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/example.katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/examples/rsync/.run/index-keys.txt | LC_ALL=C comm -23 - /work/examples/rsync/.run/paths.txt > /work/examples/rsync/.run/orphans.txt
 task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/orphans.txt /work/examples/rsync/.run/del-
 task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"")
      printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \
@@ -148,6 +149,53 @@ task: [checkpoint] find /work/examples/rsync/staging -mindepth 1 -delete
 task: [reconciled] cp /work/examples/rsync/.run/start.txt /work/examples/rsync/.run/reconciled
 task: [push] xz -T0 -c /work/examples/rsync/.run/reconciled > /work/examples/rsync/.run/reconciled.xz
 task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/examples/rsync/.run/reconciled.xz s3://example/.state/reconciled
+task: [pull] rc=0; aws s3 ls s3://example/.state/indexed.txt.xz > /dev/null || rc=$?
+case $rc in
+  0) aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://example/.state/indexed.txt.xz /work/examples/rsync/.run/indexed.txt.xz && xz -dc /work/examples/rsync/.run/indexed.txt.xz > /work/examples/rsync/.run/indexed.txt ;;
+  1) : > /work/examples/rsync/.run/indexed.txt ;;
+  *) exit $rc ;;
+esac
+
+task: [pages] mkdir -p /work/examples/rsync/staging /work/examples/rsync/.run/slash && find /work/examples/rsync/staging /work/examples/rsync/.run/slash -mindepth 1 -delete
+task: [pages] LC_ALL=C comm -3 /work/examples/rsync/.run/indexed.txt /work/examples/rsync/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" }
+  { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/examples/rsync/.run/index-dirty.txt
+task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" }
+  { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/examples/rsync/.run/applied.txt | LC_ALL=C sort > /work/examples/rsync/.run/index-dirs.txt
+task: [pages] LC_ALL=C comm -23 /work/examples/rsync/.run/index-dirty.txt /work/examples/rsync/.run/index-dirs.txt | awk -v i="example.katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/examples/rsync/.run/index-gone.txt
+task: [pages] LC_ALL=C comm -12 /work/examples/rsync/.run/index-dirty.txt /work/examples/rsync/.run/index-dirs.txt > /work/examples/rsync/.run/index-make.txt
+task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -v s="/work/examples/rsync/staging/" '{ print s $0 }' /work/examples/rsync/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p
+task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -v r="/work/examples/rsync/.run/slash/" '$0 != "" { n = split($0, c, "/"); d = ""
+  for (i = 1; i < n - 1; i++) d = d c[i] "/"; print r (n - 1) "/" d }' /work/examples/rsync/.run/index-make.txt \
+| LC_ALL=C sort -u | tr '\n' '\0' | xargs -0r mkdir -p
+task: [pages] test ! -s /work/examples/rsync/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next }
+  { n = split($1, c, "/"); d = ""
+    for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" }
+    if (d in want) print d, "f", c[n], $2, $3 }' /work/examples/rsync/.run/index-make.txt /work/examples/rsync/.run/applied.txt \
+| LC_ALL=C sort | awk -F'\t' -v S="/work/examples/rsync/staging/" -v R="/work/examples/rsync/.run/slash/" -v I="example.katoptra.org.directory.index.html" -v F='A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.' 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
+    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out }
+  function esc(s) { gsub(/&/, "\\&amp;", s); gsub(/</, "\\&lt;", s); gsub(/>/, "\\&gt;", s); gsub(/"/, "\\&quot;", s); return s }
+  function open(d) { cur = d; live = 1
+    p = "<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title>Index of /" esc(d) "</title>\n"
+    p = p "<base href=\"/" enc(d) "\">\n"
+    p = p "<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>\n"
+    p = p "<h1>Index of /" esc(d) "</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>\n"
+    if (d != "") p = p "<tr><td><a href=\"../\">../</a></td><td></td><td></td></tr>\n" }
+  function shut(  f, b, k, e) { p = p "</table>" (F == "" ? "" : "<p>" sprintf(F, enc(cur)) "</p>") "</body></html>\n"
+    f = S cur I; printf("%s", p) > f; close(f)
+    if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/")
+      f = R k "/" b; printf("%s", p) > f; close(f) } }
+  !live || $1 != cur { if (live) shut(); open($1) }
+  $2 == "d" { p = p "<tr><td><a href=\"" enc($3) "/\">" esc($3) "/</a></td><td></td><td></td></tr>\n"; next }
+  { p = p "<tr><td><a href=\"" enc($3) "\">" esc($3) "</a></td><td>" $5 "</td><td>" $4 "</td></tr>\n" }
+  END { if (live) shut() }'
+task: [index] test ! -s /work/examples/rsync/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive /work/examples/rsync/staging/ s3://example/
+task: [index] for d in /work/examples/rsync/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type text/html "$d" s3://example/; done
+task: [remove] rm -f /work/examples/rsync/.run/del-*; split -l 1000 -a 4 /work/examples/rsync/.run/index-gone.txt /work/examples/rsync/.run/del-
+task: [remove] for f in /work/examples/rsync/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"")
+     printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \
+> "$f.json" && aws s3api delete-objects --bucket example --delete "file://$f.json" | tee -a /work/examples/rsync/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done
+task: [index] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://example/.state/applied.txt.xz s3://example/.state/indexed.txt.xz
 task: [smoke] {  grep -h '^upload:' /work/examples/rsync/.run/publish.txt 2>/dev/null | sed 's|^upload: .* to s3://example/||' | head -3; } | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
 function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
     out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | while IFS="$(printf '\t')" read -r k u; do want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/examples/rsync/.run/upstream.txt); got=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }
diff --git a/toolbox.yml b/toolbox.yml
index 47ccf33..73cc1f5 100644
--- a/toolbox.yml
+++ b/toolbox.yml
@@ -10,8 +10,8 @@
 # one namespace, so the names below are reserved, and so is the engine vocabulary a
 # second include adds beside them. The rsync engine's: pipeline, plan-pipeline, list,
 # normalise, state, pull, push, rebuild, diff, split, batches, batch, fetch, publish,
-# merge, checkpoint, remove, delete, reconcile, retry, and the hooks prepare, verify,
-# index, smoke and smoke-mirror. The proton engine's: session, session-push, seal,
+# merge, checkpoint, remove, delete, reconcile, retry, pages, and the hooks prepare,
+# verify, index, smoke and smoke-mirror. The proton engine's: session, session-push, seal,
 # session-seal, age, pull, push, pd, destination, upload, confirm, list-folder, trash,
 # empty-trash, empty-trash-pipeline, and the hooks stage and prune. report-engine lives
 # here as a no-op and in every engine, so an engine consumer's toolbox include lists it

From b2b3b32c266286d2f0607518798280f958afbbe0 Mon Sep 17 00:00:00 2001
From: Josh Vaughen <josh@vaughen.net>
Date: Wed, 7 Oct 2026 12:40:23 -0700
Subject: [PATCH 2/2] feat(rsync): smoke reads pages and a canary back, refuses
 encodings

---
 CLAUDE.md                                     |  5 +-
 README.md                                     |  3 +-
 engines/rsync.yml                             | 53 ++++++++++++++++++-
 examples/rsync/Taskfile.yml                   | 15 ++++++
 examples/rsync/fixtures/run-smoke/applied.txt |  2 +
 .../rsync/fixtures/run-smoke/index-make.txt   |  1 +
 .../example.katoptra.org.directory.index.html |  7 +++
 .../fixtures/run-smoke/staging/timestamp      |  1 +
 .../rsync/fixtures/run-smoke/upstream.txt     |  2 +
 examples/rsync/render.txt                     | 10 ++++
 10 files changed, 94 insertions(+), 5 deletions(-)
 create mode 100644 examples/rsync/fixtures/run-smoke/applied.txt
 create mode 100644 examples/rsync/fixtures/run-smoke/index-make.txt
 create mode 100644 examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html
 create mode 100644 examples/rsync/fixtures/run-smoke/staging/timestamp
 create mode 100644 examples/rsync/fixtures/run-smoke/upstream.txt

diff --git a/CLAUDE.md b/CLAUDE.md
index 7cb7040..0e6b11e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -64,8 +64,9 @@ cd examples/proton && task image-build && task run -- task tools && task check &
 
 A verb change updates the `render.txt` files via `task render-update`; `offline` is
 each engine's own check. The rsync one runs over `examples/rsync/fixtures/`: the list
-diff over `run-root` and `run-empty`, `retry`'s exit codes, and `prepare` and `verify`
-over `tree/`, a signed subtree whose tlpdb is signed by a throwaway key pinned in the
+diff over `run-root` and `run-empty`, `retry`'s exit codes, `pages` over `run-pages`
+(whose `want/` is ctan's page set, matched byte for byte), `smoke`'s page read-back over
+`run-smoke`, and `prepare` and `verify` over `tree/`, a signed subtree whose tlpdb is signed by a throwaway key pinned in the
 example. Regenerate the tree with a new key only to change its shape; the private half
 was never kept. The proton one runs `confirm` over `examples/proton/fixtures/`, an
 accepting and a refusing upload summary, and the `age` verb round trip with a throwaway
diff --git a/README.md b/README.md
index 01c8609..77ee54c 100644
--- a/README.md
+++ b/README.md
@@ -291,7 +291,7 @@ until `INDEX` is set, and `smoke-mirror` until a mirror fills it.
 | `reconcile` | When `due` left `.run/reconcile`: rebuild the state, delete what neither upstream, `OWN` nor the state's own directories own, then `reconciled` |
 | `index` | Hook. With `INDEX` set: `pages`, then both key sets uploaded, the keys of emptied directories removed, and `.state/indexed.txt.xz` moved forward. A mirror with a landing page of its own replaces it |
 | `pages` | A page for every directory the run touched, drawn from the state into staging, with a tree per depth for the slashless keys |
-| `smoke` | A sample of the run's keys read back through `HOST`, sizes against the listing; the tlpdb sha512 when `TL` is set; then `smoke-mirror` |
+| `smoke` | A sample of the run's keys read back through `HOST`, sizes against the listing; the tlpdb sha512 when `TL` is set; with `INDEX`, one redrawn page under both keys; with `CANARY`, that file as `libwww-perl` against the bucket's copy; no `Content-Encoding` on the first non-empty `.tar.gz`; then `smoke-mirror` |
 | `smoke-mirror` | Hook. Nothing here; a mirror with more to read back defines it |
 | `report-engine` | Hook. The engine's rows of the run summary |
 | `retry` | Run a command, retrying rsync's transport exit codes with backoff; 23 (an unreadable path, skipped) and 24 (a file vanished mid-transfer) are successes |
@@ -399,6 +399,7 @@ has an inline default, and a mirror sets only what differs:
 | `OWN` | empty | Bucket-root keys the mirror owns, space separated; `reconcile` never deletes them |
 | `INDEX` | empty | The key suffix of the directory pages; set, `index` draws them and `reconcile` spares them and every bare directory of the state |
 | `PAGE_FOOT` | empty | The HTML every directory page closes on; `%s` is the directory's encoded path, `%%` a literal percent |
+| `CANARY` | empty | A path carrying plain `http://` links or `mailto:` addresses, no character of which a URL must encode; set, `smoke` fails if the domain serves it differently from the bucket or refuses a Perl client |
 
 A var the mirror puts in its root `vars:` is fixed for every run: inside an included
 verb a root value shadows a `KEY=value` from the command line. One the mirror leaves to
diff --git a/engines/rsync.yml b/engines/rsync.yml
index 0e2b7c7..6edc311 100644
--- a/engines/rsync.yml
+++ b/engines/rsync.yml
@@ -13,7 +13,9 @@
 # arguments for the listing; empty: the whole tree), OWN (bucket-root keys the mirror owns
 # and reconcile never deletes, space separated; empty: none), INDEX (the directory-page
 # key suffix; set, index draws a page for every directory), PAGE_FOOT (the HTML every page
-# closes on, %s the directory's encoded path; empty: none).
+# closes on, %s the directory's encoded path; empty: none), CANARY (a file whose bytes the
+# zone's HTML rewriters would alter; set, smoke reads it through the domain as a Perl
+# client).
 #
 # Hooks: prepare, verify, index, smoke, smoke-mirror and report-engine are the verbs a
 # mirror may replace, by listing each under excludes: on this include and defining its
@@ -542,7 +544,7 @@ tasks:
           END { if (live) shut() }'
 
   smoke:
-    desc: Read a sample of this run's keys back through the domain, sizes against the listing, and with TL set the tlpdb sha512 against the verified copy; then the mirror's own checks, smoke-mirror
+    desc: Read a sample of this run's keys back through the domain, sizes against the listing; with TL set the tlpdb sha512 against the verified copy; with INDEX set one redrawn page under both keys; with CANARY set the canary, as a Perl client, against the bucket's copy; no Content-Encoding on a .tar.gz; then the mirror's own checks, smoke-mirror
     vars:
       # The check below that names /timestamp asks for it only once the state says it landed.
       # It rides the decision batch, the last one, so a first fill capped at MAX_BATCHES has
@@ -567,6 +569,51 @@ tasks:
         {{if .TL}}test ! -f {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512 ||
         test "$(ls {{.RUN}}/batch-*.txt 2>/dev/null | wc -l)" -gt {{.BATCHES_MAX}} ||
         {{.CURL}} {{.URL}}/{{.TL}}/tlpkg/texlive.tlpdb.sha512 | cmp - {{.RUN}}/tl/tlpkg/texlive.tlpdb.sha512{{else}}echo "no TL subtree: no tlpdb to read back"{{end}}
+      # One page this run redrew, read back under both of its keys by key (no zone rule
+      # needed) and sized against the copy still in staging, the same one-byte ranged read as
+      # the sample. Both keys hold the same bytes, so one staged copy sizes both. The root has
+      # no slashless key. ${d} in braces: the INDEX that follows it opens with a letter, and
+      # $d{{.INDEX}} names a variable that does not exist; test -n makes that a stated
+      # failure, not a size of "". The slashless key is read only over http: the offline
+      # check serves the pages off a filesystem, which cannot hold both a/b and a/b/.
+      - >-
+        {{if .INDEX}}test ! -s {{.RUN}}/index-make.txt || { d=$(tail -1 {{.RUN}}/index-make.txt);
+        want=$(wc -c < "{{.STAGING}}/${d}{{.INDEX}}" | tr -d ' '); test -n "$want";
+        case "{{.URL}}" in http*) b=1 ;; *) b=0 ;; esac;
+        printf '%s\n' "$d" | awk -v i="{{.INDEX}}" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }'
+        | {{.ENCODE}} | while IFS="$(printf '\t')" read -r k u; do
+        got=$({{.CURL}} -r 0-0 -D - -o /dev/null "{{.URL}}/$u" | {{.SIZE}});
+        echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; }{{else}}echo "no INDEX: no directory pages to read back"{{end}}
+      # Cloudflare's HTML rewriters sit between R2 and the client, so what finds them is the
+      # object compared with what the domain serves. A size check is not enough: Automatic
+      # HTTPS Rewrites can lengthen a link by a byte while the parser it runs in collapses a
+      # newline inside the same tag, leaving the length unchanged and the bytes different.
+      # CANARY carries plain http:// links or mailto: addresses, which HTTPS Rewrites, Email
+      # Obfuscation and Rocket Loader each alter. It is read as libwww-perl, because Browser
+      # Integrity Check answers 403 to that client and every other mirror serves it; CURL's -f
+      # makes the 403 the failure. Skipped over file://, which has no bucket to read, and until
+      # the state holds the path.
+      - >-
+        {{if .CANARY}}h={{shellQuote .CANARY}}; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' {{.RUN}}/applied.txt 2>/dev/null);
+        case "{{.URL}}" in http*) ;; *) n= ;; esac;
+        if test -z "$n"; then echo "canary skipped ($h is not in the state, or the URL is not http)"; else
+        aws s3 cp {{.AWS_FLAGS}} "{{.S3}}/$h" {{.RUN}}/canary.html &&
+        {{.CURL}} -A "libwww-perl/6.67" "{{.URL}}/$h" | cmp - {{.RUN}}/canary.html &&
+        echo "$h: $n bytes, served to a libwww-perl client as the bucket holds them"; fi{{else}}echo "no CANARY: the edge's HTML rewriters and client filter go unchecked"{{end}}
+      # A stored Content-Encoding tells every client to unpack the file in flight, so a
+      # tarball arrives as a tar; GNU's mirror guidelines ask for none. The AWS CLI guesses
+      # ("application/x-tar", "gzip") for foo.tar.gz and stores only the first half; this is
+      # the check that a CLI bump never stores the second. The first non-empty .tar.gz in the
+      # state, read with no Accept-Encoding, must come back without one; an empty one answers
+      # the ranged read with 416, which CURL's -f would fail. Skipped over file:// and when
+      # the state holds no such file.
+      - >-
+        g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' {{.RUN}}/applied.txt 2>/dev/null);
+        case "{{.URL}}" in http*) ;; *) g= ;; esac;
+        if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else
+        u=$(printf '%s\n' "$g" | {{.ENCODE}} | cut -f2) && h=$({{.CURL}} -r 0-0 -D - -o /dev/null "{{.URL}}/$u") &&
+        if printf '%s' "$h" | grep -qi '^content-encoding:'; then echo "$g: served with a content-encoding" >&2; exit 1; fi &&
+        echo "$g: served with no content-encoding"; fi
       # A called task sees none of its caller's call vars, so the three a check reads are
       # handed on; in a run they are the values above, and an offline check passes its own.
       - {task: smoke-mirror, vars: {RUN: '{{.RUN}}', STAGING: '{{.STAGING}}', URL: '{{.URL}}'}}
@@ -591,6 +638,8 @@ tasks:
         EOF
       # prepare made RUN/tl, so the delta touched the signed subtree.
       - test ! -d {{.RUN}}/tl || echo "| Signature | $(test -f {{.RUN}}/tl/tlpkg/texlive.tlpdb && echo "tlpdb signed by {{.TL_KEY}}; $(test -f {{.RUN}}/tl/missing.txt && wc -l < {{.RUN}}/tl/missing.txt || echo 0) named containers missing from the bucket" || echo "the tlpdb did not verify") |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
+      # With INDEX set, what index drew and removed.
+      - test -z "{{.INDEX}}" || echo "| Directory pages | $(test -f {{.RUN}}/index-make.txt && wc -l < {{.RUN}}/index-make.txt || echo 0) redrawn under both keys, $(test -f {{.RUN}}/index-gone.txt && wc -l < {{.RUN}}/index-gone.txt || echo 0) keys removed |" >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}"
 
   retry:
     desc: Run CMD, retrying rsync's transport exit codes (5 10 12 30 35) five times with backoff and jitter; 23 (a path it could not read, a dangling symlink under -L) and 24 (file vanished mid-transfer) are successes
diff --git a/examples/rsync/Taskfile.yml b/examples/rsync/Taskfile.yml
index ab7a113..e3d01fc 100644
--- a/examples/rsync/Taskfile.yml
+++ b/examples/rsync/Taskfile.yml
@@ -12,6 +12,7 @@ vars:
   LIST_FLOOR: 1
   INDEX: '{{.HOST}}.directory.index.html'
   PAGE_FOOT: 'A mirror of rsync.example.org/pub/. <a href="https://example.org/issues">Report a problem</a>.'
+  CANARY: pub/index.html
 includes:
   toolbox:
     taskfile: ../../toolbox.yml
@@ -121,11 +122,25 @@ tasks:
       - diff -r {{.F}}/run-pages/want/staging {{.F}}/run-pages/staging
       - diff -r {{.F}}/run-pages/want/slash {{.F}}/run-pages/slash
       - cmp {{.F}}/run-pages/want/index-make.txt {{.F}}/run-pages/index-make.txt && cmp {{.F}}/run-pages/want/index-gone.txt {{.F}}/run-pages/index-gone.txt
+      # run-smoke: an hour whose only change is timestamp, whose one redrawn page is the root.
+      # smoke reads timestamp back and sizes it against the listing, then the root page under
+      # its INDEX key (a filesystem cannot hold the slashless key beside it). Over file:// the
+      # canary and the Content-Encoding check say they skipped.
+      - {task: smoke, vars: {RUN: '{{.F}}/run-smoke', STAGING: '{{.F}}/run-smoke/staging', URL: 'file://{{.F}}/run-smoke/staging'}}
+      # A page index drew that is not in staging cannot pass.
+      - '! task smoke-refuses 2>/dev/null'
       - 'echo "offline: every check passed"'
   due-refuses:
     desc: due-rule over offline's due fixture with RECONCILE R and RECONCILE_HOURS H; offline expects it to refuse
     cmds:
       - {task: due-rule, vars: {RUN: '{{.RUN}}/fixtures/due', RECONCILE: '{{.R | default "auto"}}', RECONCILE_HOURS: '{{.H | default 24}}'}}
+  smoke-refuses:
+    desc: smoke over run-smoke with the page index drew gone from staging, after offline copied the fixtures; offline expects it to fail
+    vars:
+      R: '{{.RUN}}/fixtures/run-smoke'
+    cmds:
+      - rm -f {{.R}}/staging/{{.INDEX}}
+      - {task: smoke, vars: {RUN: '{{.R}}', STAGING: '{{.R}}/staging', URL: 'file://{{.R}}/staging'}}
   verify-refuses:
     desc: verify over a batch naming a container the tlpdb does not describe, after offline staged run-tl; offline expects it to fail
     vars:
diff --git a/examples/rsync/fixtures/run-smoke/applied.txt b/examples/rsync/fixtures/run-smoke/applied.txt
new file mode 100644
index 0000000..d25ed5c
--- /dev/null
+++ b/examples/rsync/fixtures/run-smoke/applied.txt
@@ -0,0 +1,2 @@
+a/b/f.txt	0	2026/09/08 00:00:00
+timestamp	12	2026/09/08 00:00:00
diff --git a/examples/rsync/fixtures/run-smoke/index-make.txt b/examples/rsync/fixtures/run-smoke/index-make.txt
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/examples/rsync/fixtures/run-smoke/index-make.txt
@@ -0,0 +1 @@
+
diff --git a/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html b/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html
new file mode 100644
index 0000000..9d250ea
--- /dev/null
+++ b/examples/rsync/fixtures/run-smoke/staging/example.katoptra.org.directory.index.html
@@ -0,0 +1,7 @@
+<!DOCTYPE html><html><head><meta charset="utf-8"><title>Index of /</title>
+<base href="/">
+<style>body{font-family:monospace;margin:2em}td,th{text-align:left;padding:0 1.5em 0 0}</style></head><body>
+<h1>Index of /</h1><table><tr><th>Name</th><th>Last modified</th><th>Size</th></tr>
+<tr><td><a href="a/">a/</a></td><td></td><td></td></tr>
+<tr><td><a href="timestamp">timestamp</a></td><td>2026/09/08 00:00:00</td><td>12</td></tr>
+</table><p><a href="https://ctan.org/tex-archive/">This directory on ctan.org</a>. Sizes in bytes, times as CTAN lists them.</p></body></html>
diff --git a/examples/rsync/fixtures/run-smoke/staging/timestamp b/examples/rsync/fixtures/run-smoke/staging/timestamp
new file mode 100644
index 0000000..53e8aaa
--- /dev/null
+++ b/examples/rsync/fixtures/run-smoke/staging/timestamp
@@ -0,0 +1 @@
+20260908T00
diff --git a/examples/rsync/fixtures/run-smoke/upstream.txt b/examples/rsync/fixtures/run-smoke/upstream.txt
new file mode 100644
index 0000000..d25ed5c
--- /dev/null
+++ b/examples/rsync/fixtures/run-smoke/upstream.txt
@@ -0,0 +1,2 @@
+a/b/f.txt	0	2026/09/08 00:00:00
+timestamp	12	2026/09/08 00:00:00
diff --git a/examples/rsync/render.txt b/examples/rsync/render.txt
index 054bc94..d4c76b2 100644
--- a/examples/rsync/render.txt
+++ b/examples/rsync/render.txt
@@ -203,4 +203,14 @@ function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = s
   $2 == "416" { z = 1 }
   END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); echo "$k: $got bytes (listing: $want)"; test "$got" = "$want" || exit 1; done
 task: [smoke] echo "no TL subtree: no tlpdb to read back"
+task: [smoke] test ! -s /work/examples/rsync/.run/index-make.txt || { d=$(tail -1 /work/examples/rsync/.run/index-make.txt); want=$(wc -c < "/work/examples/rsync/staging/${d}example.katoptra.org.directory.index.html" | tr -d ' '); test -n "$want"; case "https://example.katoptra.org" in http*) b=1 ;; *) b=0 ;; esac; printf '%s\n' "$d" | awk -v i="example.katoptra.org.directory.index.html" -v b="$b" '{ print $0 i } b == 1 && $0 != "" { print substr($0, 1, length($0) - 1) }' | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
+    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | while IFS="$(printf '\t')" read -r k u; do got=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }
+  tolower($1) == "content-length:" { c = $2 }
+  $2 == "416" { z = 1 }
+  END { sub(/\r/, "", r); sub(/\r/, "", c); print (r != "" ? r : (z ? 0 : c)) }'); echo "$k: $got bytes (page: $want)"; test "$got" = "$want" || exit 1; done; }
+task: [smoke] h=pub/index.html; n=$(awk -F'\t' -v k="$h" '$1 == k { print $2 }' /work/examples/rsync/.run/applied.txt 2>/dev/null); case "https://example.katoptra.org" in http*) ;; *) n= ;; esac; if test -z "$n"; then echo "canary skipped ($h is not in the state, or the URL is not http)"; else aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 "s3://example/$h" /work/examples/rsync/.run/canary.html && curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -A "libwww-perl/6.67" "https://example.katoptra.org/$h" | cmp - /work/examples/rsync/.run/canary.html && echo "$h: $n bytes, served to a libwww-perl client as the bucket holds them"; fi
+task: [smoke] g=$(awk -F'\t' '$1 ~ /\.tar\.gz$/ && $2 > 0 { print $1; exit }' /work/examples/rsync/.run/applied.txt 2>/dev/null); case "https://example.katoptra.org" in http*) ;; *) g= ;; esac; if test -z "$g"; then echo "content-encoding check skipped (no non-empty .tar.gz in the state, or the URL is not http)"; else u=$(printf '%s\n' "$g" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i }
+function enc(s,  out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1)
+    out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2) && h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://example.katoptra.org/$u") && if printf '%s' "$h" | grep -qi '^content-encoding:'; then echo "$g: served with a content-encoding" >&2; exit 1; fi && echo "$g: served with no content-encoding"; fi
 task: [ping] test -z "$HEALTHCHECK_URL" || curl -fsS -m 10 --retry 3 -o /dev/null "$HEALTHCHECK_URL"