From ae21065d79d121ad936564e0b9d06956febe2ec8 Mon Sep 17 00:00:00 2001
From: "manuel.carrera" <manuel.carrera@workday.com>
Date: Thu, 29 Oct 2026 09:55:12 -0600
Subject: [PATCH 1/4] fix: Restore package.json manifests before release
 recovery publish

A failed `lerna publish` leaves every `package.json` modified (`gitHead` and resolved dependency
specifiers) because lerna only resets them after a fully successful publish. The release safety
net (`yarn publish:from-package`) then starts on that dirty tree, and
`lerna publish from-package` refuses to run with EUNCOMMIT, so it could never recover a release.

Reset modified `package.json` files before every lerna attempt in `utils/publish-packages.mjs`.
Add a spec that reproduces the dirty tree in a real temp git repo, and `RELEASE_FAILURES.md`, an
investigation log of the release failures seen so far.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 RELEASE_FAILURES.md                 | 119 ++++++++++++++++++++++++++++
 utils/publish-packages.mjs          |  28 +++++++
 utils/spec/publish-packages.spec.ts |  92 +++++++++++++++++++++
 3 files changed, 239 insertions(+)
 create mode 100644 RELEASE_FAILURES.md
 create mode 100644 utils/spec/publish-packages.spec.ts

diff --git a/RELEASE_FAILURES.md b/RELEASE_FAILURES.md
new file mode 100644
index 0000000000..6550f05d81
--- /dev/null
+++ b/RELEASE_FAILURES.md
@@ -0,0 +1,119 @@
+# Release failures — investigation log
+
+Running log of why the **Publish to npm** workflow (`.github/workflows/publish.yml` →
+`release.yml`) fails on `master`/`support`, what has been tried, and what is still unknown. Read this
+before "fixing" the release job again, and **append to it** when a new failure shows up.
+
+Entries are marked **Verified** (seen in logs or source) or **Unverified** (hypothesis). Keep that
+distinction.
+
+## How a patch release works
+
+1. `release.yml` calls `Workday/canvas-kit-actions/release@v2` (a separate repo). It runs
+   `lerna version`, updates the changelog, commits `chore: Release vX.Y.Z [skip release]`, tags,
+   pushes, builds, then runs `yarn lerna publish from-package --yes --dist-tag <tag>`.
+2. That step is `continue-on-error`. If it fails, `release.yml` runs the safety net
+   `yarn publish:from-package` ([`utils/publish-packages.mjs`](utils/publish-packages.mjs)), which
+   re-publishes whatever is still missing from npm, serially, with retries.
+3. If the safety net also fails, the job fails, `report-failure` posts to Slack, and the release is
+   **half published**: the bump commit and tag exist on the branch, but some packages are not on npm.
+
+The follow-up `Release vX.Y.Z [skip release]` runs that appear green afterwards are only the skip
+runs for the release commit. They do not publish anything. A green run after a red one does **not**
+mean the release was recovered.
+
+## Two stacked failures
+
+A red run is two independent problems, one after the other.
+
+### Layer 1 — the trigger: transient publish errors (not fixed, only worked around)
+
+During `lerna publish from-package` inside the release action, some packages fail and lerna carries
+on with the rest, then exits non-zero.
+
+- **Verified:** `lerna ERR! TLOG_CREATE_ENTRY_ERROR error creating tlog entry - (409) an equivalent
+  entry already exists in the transparency log` (Sigstore/Rekor, npm provenance via OIDC trusted
+  publishing). Seen in 15 Oct, 21 Oct, 26 Oct (x2), 29 Oct runs. Typically 1–7 of the 13 packages
+  fail; the rest publish.
+- **Verified:** a different transient error on 26 Oct (run 40349815957): GitHub's OIDC token
+  endpoint returned `504`, which npm surfaced as `E404 Not found`.
+- **Unverified:** why the 409 happens. One unrelated repo
+  ([BitGo/BitGoJS#9348](https://github.com/BitGo/BitGoJS/pull/9348)) attributes it to a failed registry
+  write leaving a Rekor entry behind, so npm's internal retry produces an identical entry. Failed
+  packages here took 8–20 s, which is consistent with an internal retry, but nothing in our logs
+  proves it. Nothing authoritative from npm or Sigstore was found.
+- We cannot fix this from this repo. The workaround is the retry loop in layer 2.
+
+### Layer 2 — why the safety net never recovered anything (fixed here)
+
+The safety net has not recovered a release on `master` since it was added. Two different bugs, in
+order:
+
+| Period | Safety-net error | Cause | Fix |
+| --- | --- | --- | --- |
+| 15 Oct – 26 Oct 15:29 | `MultipleProjectsWithSameNameError` from `lerna ls` (Nx) | `postpack` (`utils/publish.js post`) used a repo-root-relative glob, so the generated nameless `package.json` files were never deleted. Nx saw duplicate projects. | #4224 (`5cdbb558a`) |
+| 26 Oct 18:30 – now | `lerna ERR! EUNCOMMIT Working tree has uncommitted changes` listing all 13 `modules/*/package.json` | See below | this change |
+
+**EUNCOMMIT root cause (Verified in `node_modules/lerna/dist/commands/publish/index.js`, lerna 9.0.1):**
+
+- `lerna publish from-package` always calls `verifyWorkingTreeClean()` first (`detectFromPackage`)
+  and throws `EUNCOMMIT` on any dirty tree.
+- `execute()` writes `gitHead` and resolved dependency specifiers into **every** manifest it will
+  publish (`annotateGitHead` + `serializeChanges`) and restores them with `resetChanges()` only
+  **after** `publishPacked()` returns. If a package fails, the manifests are left dirty.
+- So the safety net starts on the tree the failed publish left behind and can never pass lerna's own
+  check. `EUNCOMMIT` is not classed as retryable in `publish-packages.mjs`, so it gave up at once, and
+  every retry would have re-dirtied the tree anyway.
+- This was masked until #4224, because the Nx error fired first.
+- Not directly observed: the CI log lists the modified files but not the diff, so "the diff is
+  `gitHead` plus resolved specifiers" is read from the lerna source, not from a log.
+
+**Fix:** `restoreLernaManifests()` in `utils/publish-packages.mjs` runs `git checkout --` on modified
+`package.json` files before every lerna attempt. The bump commit is already in `HEAD`, so this only
+discards lerna's temporary edits. Other dirty files are left alone so lerna still reports them.
+Covered by `utils/spec/publish-packages.spec.ts` (real temp git repo).
+
+**Not verified end-to-end:** the fix has unit coverage but has not yet run in CI against a real 409.
+Next time a release hits one, check that the `Publish remaining packages` step goes green and that
+all 13 packages show up on npm. Record the result below.
+
+## Run history (`publish.yml`, master/support, non-skip runs)
+
+| Date (UTC) | Branch | Run | Layer 1 | Layer 2 |
+| --- | --- | --- | --- | --- |
+| 15 Oct | master | 38781099215 | tlog 409 (2 pkgs) | Nx duplicate projects |
+| 21 Oct | master | 39669576846 | tlog 409 (5 pkgs) | Nx duplicate projects |
+| 22 Oct | support | 39850244173 | git exit 128 (not investigated) | none yet (#4218 added it) |
+| 26 Oct 15:10 | master | 40346670180 | tlog 409 (1 pkg) | Nx duplicate projects |
+| 26 Oct 15:29 | master | 40349815957 | OIDC 504 → E404 (1 pkg) | Nx duplicate projects |
+| 26 Oct 18:30 | support | 40378385152 | tlog 409 (1 pkg) | EUNCOMMIT |
+| 29 Oct 15:13 | master | 40899439848 | tlog 409 (7 pkgs) | EUNCOMMIT |
+
+Earlier attempts: #4150 (retry script + safety net), #4162 (commit message parsing in
+`publish.yml`), #4218 (safety net for `support`), #4224 (remove generated `package.json` after packing).
+
+## Recovering a half-published release
+
+Check what is on npm:
+
+```bash
+v=$(node -p 'require("./lerna.json").version')
+for p in codemod css docs labs-css labs-react mcp popup-stack preview-css preview-react react react-fonts styling styling-transform; do
+  echo "$p: $(npm view @workday/canvas-kit-$p@$v version 2>/dev/null || echo MISSING)"
+done
+```
+
+Then run **Publish to npm** manually (Actions → Run workflow) on `master` (or `support`) with
+release type `complete-unpublished`. It does not bump versions and publishes only what is missing.
+npm's registry can lag a minute or so after a publish (on 29 Oct `canvas-kit-mcp@16.2.12` was
+reported published and then listed as missing 50 s later), so re-check before concluding something
+is still missing.
+
+## Open questions
+
+- Why does Rekor return 409 for these publishes? Worth an issue against `npm/cli` with a failing run
+  link if it keeps happening.
+- Whether to drop or decouple provenance for patch releases. That removes Rekor from the path but
+  is a supply-chain decision (see `scorecard.yml`), so it needs a maintainer call.
+- Whether the release action (`canvas-kit-actions/release@v2`) should itself retry per package, which
+  would make the safety net a last resort instead of the normal path.
diff --git a/utils/publish-packages.mjs b/utils/publish-packages.mjs
index 55436538b1..851d7eadef 100644
--- a/utils/publish-packages.mjs
+++ b/utils/publish-packages.mjs
@@ -316,6 +316,31 @@ export function parsePublishedPackageLines(output) {
   return [...packages.values()];
 }
 
+/**
+ * Restores `package.json` manifests that a failed `lerna publish` left modified.
+ *
+ * Lerna rewrites every manifest it publishes (`gitHead`, resolved dependency specifiers) and only
+ * resets them after the whole publish succeeds. When a package fails mid-run (e.g. a Sigstore tlog
+ * 409), the manifests stay dirty and the next `lerna publish from-package` refuses to start with
+ * EUNCOMMIT. The release commit already holds the correct versions, so resetting to it is safe.
+ *
+ * Only `package.json` files are restored; any other uncommitted change is left alone so lerna's
+ * own working tree check still reports it.
+ *
+ * @param {string} [cwd]
+ */
+export async function restoreLernaManifests(cwd = REPO_ROOT) {
+  const {stdout} = await execFileAsync('git', ['diff', '--name-only', '--relative', '-z'], {cwd});
+  const manifests = stdout.split('\0').filter(file => path.basename(file) === 'package.json');
+
+  if (manifests.length === 0) {
+    return;
+  }
+
+  console.log(`Restoring ${manifests.length} package.json file(s) left modified by lerna publish.`);
+  await execFileAsync('git', ['checkout', '--', ...manifests], {cwd});
+}
+
 /**
  * Publish any workspace versions that are not yet on the registry.
  * `lerna publish from-package` skips versions that already exist, so this is
@@ -363,6 +388,9 @@ export async function publishFromPackage({
       `Publishing unpublished packages (attempt ${attempt}/${retries}) with dist-tag "${distTag}" and concurrency 1...`
     );
 
+    // The release action's own publish (and every failed attempt here) leaves manifests dirty.
+    await restoreLernaManifests();
+
     const result = await runStreaming('yarn', [
       'lerna',
       'publish',
diff --git a/utils/spec/publish-packages.spec.ts b/utils/spec/publish-packages.spec.ts
new file mode 100644
index 0000000000..62b5d83619
--- /dev/null
+++ b/utils/spec/publish-packages.spec.ts
@@ -0,0 +1,92 @@
+import {execFileSync} from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const publishPackagesScript = path.resolve(__dirname, '../publish-packages.mjs');
+
+/**
+ * Calls an exported function of `utils/publish-packages.mjs` in a child process with `repo` as the
+ * working directory. A child process keeps the ESM script out of the TypeScript program (`allowJs`
+ * is off) and mirrors how CI runs it.
+ */
+function callPublishPackages(fn: string, cwd: string) {
+  execFileSync(
+    process.execPath,
+    [
+      '--input-type=module',
+      '-e',
+      `import * as m from ${JSON.stringify(publishPackagesScript)}; await m.${fn}(process.cwd());`,
+    ],
+    {cwd, env: {...process.env, NODE_NO_WARNINGS: '1'}, stdio: 'pipe'}
+  );
+}
+
+function git(cwd: string, ...args: string[]) {
+  return execFileSync('git', args, {cwd, encoding: 'utf8'}).trim();
+}
+
+describe('publish-packages.mjs', () => {
+  describe('restoreLernaManifests', () => {
+    let repo: string;
+    const manifests = [
+      'package.json',
+      'modules/react/package.json',
+      'modules/styling/package.json',
+    ];
+
+    const read = (file: string) => fs.readFileSync(path.join(repo, file), 'utf8');
+
+    beforeEach(() => {
+      repo = fs.mkdtempSync(path.join(os.tmpdir(), 'ck-publish-packages-'));
+      git(repo, 'init', '--quiet');
+      git(repo, 'config', 'user.email', 'test@example.com');
+      git(repo, 'config', 'user.name', 'Test');
+
+      manifests.forEach(file => {
+        fs.mkdirSync(path.dirname(path.join(repo, file)), {recursive: true});
+        fs.writeFileSync(path.join(repo, file), '{"version": "1.0.0"}\n');
+      });
+      fs.writeFileSync(path.join(repo, 'README.md'), 'readme\n');
+      git(repo, 'add', '.');
+      git(repo, 'commit', '--quiet', '-m', 'initial');
+    });
+
+    afterEach(() => {
+      fs.rmSync(repo, {recursive: true, force: true});
+    });
+
+    // Regression test: a failed `lerna publish` leaves `gitHead` (and resolved dependency
+    // specifiers) in every manifest because it only resets them after a fully successful publish.
+    // `lerna publish from-package` then refuses to run with EUNCOMMIT.
+    it('should restore manifests that a failed lerna publish left modified', () => {
+      manifests.forEach(file => {
+        fs.writeFileSync(path.join(repo, file), '{"version": "1.0.0", "gitHead": "abc123"}\n');
+      });
+      expect(git(repo, 'status', '--porcelain')).not.toBe('');
+
+      callPublishPackages('restoreLernaManifests', repo);
+
+      expect(git(repo, 'status', '--porcelain')).toBe('');
+      manifests.forEach(file => {
+        expect(read(file)).toBe('{"version": "1.0.0"}\n');
+      });
+    });
+
+    it('should not touch changes to files that are not package.json manifests', () => {
+      fs.writeFileSync(path.join(repo, 'README.md'), 'changed\n');
+      fs.writeFileSync(path.join(repo, 'modules/react/package.json'), '{"gitHead": "abc123"}\n');
+
+      callPublishPackages('restoreLernaManifests', repo);
+
+      // Unexpected changes must still surface through lerna's own working tree check.
+      expect(read('README.md')).toBe('changed\n');
+      expect(git(repo, 'status', '--porcelain')).toBe('M README.md');
+    });
+
+    it('should do nothing when the working tree is clean', () => {
+      expect(() => callPublishPackages('restoreLernaManifests', repo)).not.toThrow();
+      expect(git(repo, 'status', '--porcelain')).toBe('');
+    });
+  });
+});

From 102bb930b724dce22b06ed2457fcf378c03adad4 Mon Sep 17 00:00:00 2001
From: "manuel.carrera" <manuel.carrera@workday.com>
Date: Thu, 29 Oct 2026 10:28:00 -0600
Subject: [PATCH 2/4] fix: Check npm for published versions without the
 CDN-cached document

`npm view name@version` reads the package-level document, which the registry CDN caches for
5 minutes (`cache-control: max-age=300`). Right after a publish it still reports the new version
as missing: in run 40909685326 the check listed packages as missing 54 s after lerna published them.
With the dirty-tree bug fixed, that would make the recovery loop retry until it throws "Exhausted"
even though everything is on npm.

Read the uncached per-version endpoint instead (200 = published, 404 = missing, anything else
throws). Log the 29 Oct manual re-run (which hit the same EUNCOMMIT) in RELEASE_FAILURES.md.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 RELEASE_FAILURES.md                 | 37 +++++++++++++--
 utils/publish-packages.mjs          | 36 +++++++++-----
 utils/spec/publish-packages.spec.ts | 74 ++++++++++++++++++++++++++++-
 3 files changed, 130 insertions(+), 17 deletions(-)

diff --git a/RELEASE_FAILURES.md b/RELEASE_FAILURES.md
index 6550f05d81..eb243ee586 100644
--- a/RELEASE_FAILURES.md
+++ b/RELEASE_FAILURES.md
@@ -77,6 +77,29 @@ Covered by `utils/spec/publish-packages.spec.ts` (real temp git repo).
 Next time a release hits one, check that the `Publish remaining packages` step goes green and that
 all 13 packages show up on npm. Record the result below.
 
+### Layer 3 — the registry check lies right after a publish (fixed here)
+
+**Verified** in run 40909685326: six packages published during attempt 1 (16:19:10–16:19:50), then
+the script's own check at 16:20:04 reported **all 12** as still missing, including `canvas-kit-codemod`,
+published 54 s earlier. The same happened on 29 Oct 15:21 (`canvas-kit-mcp` published, listed missing 50 s
+later).
+
+Cause (checked with `curl -I`): `npm view name@version` fetches the package-level document
+(`registry.npmjs.org/@workday%2fname`), which Cloudflare serves with `cache-control: public, max-age=300`
+(`cf-cache-status: HIT`). For up to 5 minutes it can omit a version that was just published. The
+per-version endpoint (`.../@workday%2fname/16.2.12`) is `cf-cache-status: DYNAMIC` (not cached).
+
+Why it matters once the dirty-tree bug is gone: lerna handles the stale view itself (it treats
+`E409` / `EPUBLISHCONFLICT` / "cannot publish over the previously published versions" as already
+published), but the script's own check would keep reporting packages as missing after lerna succeeded.
+It would retry up to its limit and then fail with `Exhausted ... attempts`, even though everything was on npm.
+
+**Fix:** `isPackageVersionOnNpm()` now reads the per-version endpoint with `fetch` (200 = published,
+404 = missing, anything else throws). Covered by a spec using a local fake registry.
+
+**Unverified:** the exact point at which the cached document refreshes, and whether lerna's `pacote`
+packument lookup is cached the same way (the E409 handling makes this harmless either way).
+
 ## Run history (`publish.yml`, master/support, non-skip runs)
 
 | Date (UTC) | Branch | Run | Layer 1 | Layer 2 |
@@ -88,6 +111,7 @@ all 13 packages show up on npm. Record the result below.
 | 26 Oct 15:29 | master | 40349815957 | OIDC 504 → E404 (1 pkg) | Nx duplicate projects |
 | 26 Oct 18:30 | support | 40378385152 | tlog 409 (1 pkg) | EUNCOMMIT |
 | 29 Oct 15:13 | master | 40899439848 | tlog 409 (7 pkgs) | EUNCOMMIT |
+| 29 Oct 16:18 | master | 40909685326 | tlog 409 (1 pkg), manual `complete-unpublished` | EUNCOMMIT on attempt 2 |
 
 Earlier attempts: #4150 (retry script + safety net), #4162 (commit message parsing in
 `publish.yml`), #4218 (safety net for `support`), #4224 (remove generated `package.json` after packing).
@@ -105,9 +129,16 @@ done
 
 Then run **Publish to npm** manually (Actions → Run workflow) on `master` (or `support`) with
 release type `complete-unpublished`. It does not bump versions and publishes only what is missing.
-npm's registry can lag a minute or so after a publish (on 29 Oct `canvas-kit-mcp@16.2.12` was
-reported published and then listed as missing 50 s later), so re-check before concluding something
-is still missing.
+Until the `restoreLernaManifests` fix is on the branch you run it from, a 409 during the first
+attempt still ends the job with `EUNCOMMIT`. Just re-run: every run starts from a clean checkout and
+only publishes what is still missing, so each run makes progress (29 Oct: 1 of 13, then 6 more).
+
+Check npm with the per-version endpoint (the `npm view` loop above can be up to 5 minutes stale
+right after a publish; see layer 3):
+
+```bash
+curl -s -o /dev/null -w '%{http_code}\n' "https://registry.npmjs.org/@workday%2fcanvas-kit-react/$v"  # 200 = published
+```
 
 ## Open questions
 
diff --git a/utils/publish-packages.mjs b/utils/publish-packages.mjs
index 851d7eadef..d264c3342e 100644
--- a/utils/publish-packages.mjs
+++ b/utils/publish-packages.mjs
@@ -24,6 +24,7 @@ const BUILD_MARKER = path.join(REPO_ROOT, 'modules', 'react', 'dist', 'es6', 'in
 const DEFAULT_RETRIES = 5;
 const RETRY_BASE_MS = 8000;
 const REGISTRY_CHECK_CONCURRENCY = 4;
+const NPM_REGISTRY = 'https://registry.npmjs.org';
 const ALLOWED_DIST_TAGS = new Set(['latest', 'support', 'next', 'prerelease-next']);
 
 /** @typedef {{distTag: string, retries: number, verifyOnly: boolean, requireUnpublished: boolean}} PublishArgs */
@@ -181,27 +182,36 @@ export async function getPublicPackages() {
 /**
  * Checks whether an exact package version already exists on the public npm registry.
  *
+ * Reads the per-version endpoint on purpose. `npm view` fetches the package-level document, which
+ * the registry CDN caches for 5 minutes (`cache-control: max-age=300`), so right after a publish it
+ * keeps reporting the new version as missing. The per-version endpoint is served uncached.
+ *
  * @param {string} name
  * @param {string} version
+ * @param {string} [registry]
  */
-export async function isPackageVersionOnNpm(name, version) {
+export async function isPackageVersionOnNpm(name, version, registry = NPM_REGISTRY) {
+  const url = `${registry}/${name.replace('/', '%2f')}/${encodeURIComponent(version)}`;
+
+  let response;
   try {
-    const {stdout} = await execFileAsync('npm', ['view', `${name}@${version}`, 'version'], {
-      timeout: 30000,
-    });
-    return stdout.trim() === version;
+    response = await fetch(url, {signal: AbortSignal.timeout(30000)});
   } catch (err) {
-    const execErr = /** @type {Error & {stdout?: string, stderr?: string}} */ (err);
-    const output = `${execErr.stdout ?? ''}\n${execErr.stderr ?? ''}\n${execErr.message}`;
+    // Timeout or network failure — do not misreport as "missing from npm".
+    throw new Error(`Could not check ${name}@${version} on npm: ${url}`, {cause: err});
+  }
 
-    // Version is actually missing from npm — treat as unpublished, not an operational error.
-    if (/E404|is not in this registry|No match found for version/.test(output)) {
-      return false;
-    }
+  if (response.ok) {
+    return true;
+  }
 
-    // Timeout, auth, or registry outage — do not misreport as "missing from npm".
-    throw new Error(`Could not check ${name}@${version} on npm:\n${output}`, {cause: err});
+  // Version is actually missing from npm — treat as unpublished, not an operational error.
+  if (response.status === 404) {
+    return false;
   }
+
+  // Registry outage or auth error — do not misreport as "missing from npm".
+  throw new Error(`Could not check ${name}@${version} on npm: HTTP ${response.status} from ${url}`);
 }
 
 /**
diff --git a/utils/spec/publish-packages.spec.ts b/utils/spec/publish-packages.spec.ts
index 62b5d83619..2a5ba14352 100644
--- a/utils/spec/publish-packages.spec.ts
+++ b/utils/spec/publish-packages.spec.ts
@@ -1,7 +1,12 @@
-import {execFileSync} from 'node:child_process';
+import {execFile, execFileSync} from 'node:child_process';
 import fs from 'node:fs';
+import http from 'node:http';
+import type {AddressInfo} from 'node:net';
 import os from 'node:os';
 import path from 'node:path';
+import {promisify} from 'node:util';
+
+const execFileAsync = promisify(execFile);
 
 const publishPackagesScript = path.resolve(__dirname, '../publish-packages.mjs');
 
@@ -26,7 +31,74 @@ function git(cwd: string, ...args: string[]) {
   return execFileSync('git', args, {cwd, encoding: 'utf8'}).trim();
 }
 
+/**
+ * Calls `isPackageVersionOnNpm(name, version, registry)` in a child process. Async so the fake
+ * registry in this process can answer while the child waits.
+ */
+async function isOnNpm(name: string, version: string, registry: string) {
+  const {stdout} = await execFileAsync(
+    process.execPath,
+    [
+      '--input-type=module',
+      '-e',
+      `import * as m from ${JSON.stringify(publishPackagesScript)};
+       console.log(JSON.stringify(await m.isPackageVersionOnNpm(${JSON.stringify(
+         name
+       )}, ${JSON.stringify(version)}, ${JSON.stringify(registry)})));`,
+    ],
+    {env: {...process.env, NODE_NO_WARNINGS: '1'}}
+  );
+  return JSON.parse(stdout);
+}
+
 describe('publish-packages.mjs', () => {
+  // Regression test: `npm view` reads the package-level document, which the registry CDN caches for
+  // 5 minutes (`cache-control: max-age=300`), so it kept reporting just-published versions as
+  // missing. The per-version endpoint is served uncached and is the source of truth.
+  describe('isPackageVersionOnNpm', () => {
+    let server: http.Server;
+    let registry: string;
+    const requested: string[] = [];
+
+    beforeEach(async () => {
+      requested.length = 0;
+      server = http.createServer((req, res) => {
+        requested.push(req.url ?? '');
+        if (req.url === '/@workday%2fpublished/1.0.0') {
+          res.writeHead(200, {'content-type': 'application/json'}).end('{"version":"1.0.0"}');
+        } else if (req.url === '/@workday%2fbroken/1.0.0') {
+          res.writeHead(503).end('unavailable');
+        } else {
+          res.writeHead(404).end('{"error":"version not found"}');
+        }
+      });
+      await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+      registry = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+    });
+
+    afterEach(async () => {
+      await new Promise(resolve => server.close(resolve));
+    });
+
+    it('should be true when the registry has the exact version', async () => {
+      expect(await isOnNpm('@workday/published', '1.0.0', registry)).toBe(true);
+    });
+
+    it('should query the per-version endpoint, not the cached package document', async () => {
+      await isOnNpm('@workday/published', '1.0.0', registry);
+
+      expect(requested).toEqual(['/@workday%2fpublished/1.0.0']);
+    });
+
+    it('should be false when the registry does not have the version', async () => {
+      expect(await isOnNpm('@workday/published', '2.0.0', registry)).toBe(false);
+    });
+
+    it('should fail, not report "missing", when the registry is unavailable', async () => {
+      await expect(isOnNpm('@workday/broken', '1.0.0', registry)).rejects.toThrow();
+    });
+  });
+
   describe('restoreLernaManifests', () => {
     let repo: string;
     const manifests = [

From 9873c2732456a69a159abc205d80cb5d80cb382c Mon Sep 17 00:00:00 2001
From: "manuel.carrera" <manuel.carrera@workday.com>
Date: Thu, 29 Oct 2026 11:01:51 -0600
Subject: [PATCH 3/4] fix: Decide release recovery from publish responses, not
 registry reads

The npm registry accepts a publish immediately, but every read path (`npm view`, the per-version
endpoint, lerna's own lookup) lags by 1-13 minutes afterwards. Run 40912333768 published all 6
remaining packages in attempt 1, then the script asked the registry, was told all of them were still
missing, and retried. Attempt 3 re-published `styling`, which was already on npm, and failed on a
Rekor 409; attempt 4 died with EUNCOMMIT. The release was complete after attempt 1.

- Revert 102bb930b: the per-version endpoint is not fresher than `npm view` right after a publish
  (it returned 404 for `preview-react` 9 minutes after it was published).
- Record which packages lerna confirmed (`success published` / `already published`) and never
  un-confirm them. Succeed when lerna exits 0 or every package is confirmed; retry only the rest.
- Retry when GitHub's OIDC token endpoint returns a 5xx (surfaced by npm as E404).
- Compare real paths in the main guard. `import.meta.url` is resolved, `process.argv[1]` is not, so
  a symlinked path silently skipped the script and exited 0.
- Rewrite RELEASE_FAILURES.md around the corrected model, with the measured lag and run history.
- Tests replay the real failure sequences against a fake yarn/npm.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 RELEASE_FAILURES.md                 | 167 ++++++++-------
 utils/publish-packages.mjs          | 118 ++++++-----
 utils/spec/publish-packages.spec.ts | 304 +++++++++++++++++++++-------
 3 files changed, 399 insertions(+), 190 deletions(-)

diff --git a/RELEASE_FAILURES.md b/RELEASE_FAILURES.md
index eb243ee586..45119d4294 100644
--- a/RELEASE_FAILURES.md
+++ b/RELEASE_FAILURES.md
@@ -22,9 +22,11 @@ The follow-up `Release vX.Y.Z [skip release]` runs that appear green afterwards
 runs for the release commit. They do not publish anything. A green run after a red one does **not**
 mean the release was recovered.
 
-## Two stacked failures
+## Three layers
 
-A red run is two independent problems, one after the other.
+A red run is several independent problems stacked on each other: a transient publish error
+(layer 1), a registry that lags behind publishes (layer 2), and a safety net that mishandled both
+(layer 3).
 
 ### Layer 1 — the trigger: transient publish errors (not fixed, only worked around)
 
@@ -42,17 +44,51 @@ on with the rest, then exits non-zero.
   write leaving a Rekor entry behind, so npm's internal retry produces an identical entry. Failed
   packages here took 8–20 s, which is consistent with an internal retry, but nothing in our logs
   proves it. Nothing authoritative from npm or Sigstore was found.
-- We cannot fix this from this repo. The workaround is the retry loop in layer 2.
+- **Verified:** the 409 also happens when a package that is **already on npm** is published again
+  (run 40912333768, attempt 3: `styling` had been published in attempt 1). So a 409 does not mean the
+  package is missing.
+- We cannot fix this from this repo. Provenance stays on (decision 29 Oct 2026); the retry loop in
+  layer 2 is the workaround.
 
-### Layer 2 — why the safety net never recovered anything (fixed here)
+### Layer 2 — the registry lags publishes, and the safety net trusted it (fixed here)
 
-The safety net has not recovered a release on `master` since it was added. Two different bugs, in
-order:
+**Verified, measured 29 Oct 2026 (run 40912333768):** a publish is accepted immediately, but reads
+lag behind it. Lerna logged `success published`, and the package became visible on npm (tarball
+`Last-Modified`) this much later:
 
-| Period | Safety-net error | Cause | Fix |
+| Package | Lerna said published | Visible on npm | Lag |
+| --- | --- | --- | --- |
+| `styling` | 16:36:32 | 16:37:54 | 1.4 min |
+| `react` | 16:36:41 | 16:42:59 | 6.3 min |
+| `preview-react` | 16:37:01 | 16:46:21 | 9.3 min |
+| `mcp` (run 40899439848) | 15:21:31 | 15:34:06 | 12.6 min |
+
+During the lag **every read path says the version is missing**: `npm view` (package document, also
+CDN-cached for 5 min: `cache-control: max-age=300`), the per-version endpoint
+(`registry.npmjs.org/<pkg>/<version>` returned 404 for `preview-react` 9 min after its publish) and
+lerna's own lookup. The write path is authoritative: at 16:37:48 the registry already answered
+"already published" for `preview-react`, 8.5 minutes before it was readable. The lag looked longer for
+larger packages (`mcp`, `react`, `preview-react`); that is a correlation, not a measured cause.
+
+**Consequence (run 40912333768):** attempt 1 published all 6 remaining packages and exited 0. The
+script then asked the registry, was told all 12 were still missing, and retried. Lerna's own lookup
+was equally stale, so attempt 3 re-published `styling`, which was already on npm, and hit a Rekor 409
+(see layer 1). That left the manifests dirty and attempt 4 died with `EUNCOMMIT`. The release was
+actually complete after attempt 1.
+
+**Rule: never decide that a package is "still missing" from a registry read made less than ~15
+minutes after a publish. Decide from the registry's response to the publish call itself.**
+
+### Layer 3 — bugs in the safety net itself (fixed here)
+
+The safety net (`utils/publish-packages.mjs`) had not recovered a release on `master` since it was
+added. Three independent bugs, in the order they surfaced:
+
+| Seen | Symptom | Cause | Fix |
 | --- | --- | --- | --- |
 | 15 Oct – 26 Oct 15:29 | `MultipleProjectsWithSameNameError` from `lerna ls` (Nx) | `postpack` (`utils/publish.js post`) used a repo-root-relative glob, so the generated nameless `package.json` files were never deleted. Nx saw duplicate projects. | #4224 (`5cdbb558a`) |
-| 26 Oct 18:30 – now | `lerna ERR! EUNCOMMIT Working tree has uncommitted changes` listing all 13 `modules/*/package.json` | See below | this change |
+| 26 Oct 18:30 – 29 Oct | `EUNCOMMIT Working tree has uncommitted changes` listing every `modules/*/package.json` | lerna leaves manifests modified after a failed publish (below) | `restoreLernaManifests()` |
+| 29 Oct 16:30 | Retried and failed after a publish that had fully succeeded | The loop used registry reads (layer 2) to decide what was missing | `parseConfirmedPackages()` |
 
 **EUNCOMMIT root cause (Verified in `node_modules/lerna/dist/commands/publish/index.js`, lerna 9.0.1):**
 
@@ -61,48 +97,41 @@ order:
 - `execute()` writes `gitHead` and resolved dependency specifiers into **every** manifest it will
   publish (`annotateGitHead` + `serializeChanges`) and restores them with `resetChanges()` only
   **after** `publishPacked()` returns. If a package fails, the manifests are left dirty.
-- So the safety net starts on the tree the failed publish left behind and can never pass lerna's own
-  check. `EUNCOMMIT` is not classed as retryable in `publish-packages.mjs`, so it gave up at once, and
-  every retry would have re-dirtied the tree anyway.
-- This was masked until #4224, because the Nx error fired first.
+- So the next run starts on the tree the failed publish left behind and can never pass lerna's own
+  check. It was masked until #4224 because the Nx error fired first.
 - Not directly observed: the CI log lists the modified files but not the diff, so "the diff is
   `gitHead` plus resolved specifiers" is read from the lerna source, not from a log.
 
-**Fix:** `restoreLernaManifests()` in `utils/publish-packages.mjs` runs `git checkout --` on modified
-`package.json` files before every lerna attempt. The bump commit is already in `HEAD`, so this only
-discards lerna's temporary edits. Other dirty files are left alone so lerna still reports them.
-Covered by `utils/spec/publish-packages.spec.ts` (real temp git repo).
-
-**Not verified end-to-end:** the fix has unit coverage but has not yet run in CI against a real 409.
-Next time a release hits one, check that the `Publish remaining packages` step goes green and that
-all 13 packages show up on npm. Record the result below.
-
-### Layer 3 — the registry check lies right after a publish (fixed here)
-
-**Verified** in run 40909685326: six packages published during attempt 1 (16:19:10–16:19:50), then
-the script's own check at 16:20:04 reported **all 12** as still missing, including `canvas-kit-codemod`,
-published 54 s earlier. The same happened on 29 Oct 15:21 (`canvas-kit-mcp` published, listed missing 50 s
-later).
-
-Cause (checked with `curl -I`): `npm view name@version` fetches the package-level document
-(`registry.npmjs.org/@workday%2fname`), which Cloudflare serves with `cache-control: public, max-age=300`
-(`cf-cache-status: HIT`). For up to 5 minutes it can omit a version that was just published. The
-per-version endpoint (`.../@workday%2fname/16.2.12`) is `cf-cache-status: DYNAMIC` (not cached).
-
-Why it matters once the dirty-tree bug is gone: lerna handles the stale view itself (it treats
-`E409` / `EPUBLISHCONFLICT` / "cannot publish over the previously published versions" as already
-published), but the script's own check would keep reporting packages as missing after lerna succeeded.
-It would retry up to its limit and then fail with `Exhausted ... attempts`, even though everything was on npm.
-
-**Fix:** `isPackageVersionOnNpm()` now reads the per-version endpoint with `fetch` (200 = published,
-404 = missing, anything else throws). Covered by a spec using a local fake registry.
-
-**Unverified:** the exact point at which the cached document refreshes, and whether lerna's `pacote`
-packument lookup is cached the same way (the E409 handling makes this harmless either way).
+**What the script does now** (`utils/publish-packages.mjs`, covered by `utils/spec/publish-packages.spec.ts`):
+
+1. Decides what to publish once, at the start, from registry reads (state is old by then).
+2. Before every lerna attempt, resets modified `package.json` files to `HEAD` (`git checkout --`).
+   The bump commit is already in `HEAD`; other dirty files are left alone so lerna still reports them.
+3. Records which packages the registry **confirmed** in lerna's output (`lerna success published ...`
+   or `Package is already published: ...`) and never un-confirms them.
+4. Succeeds when lerna exits 0 **or** every package is confirmed (a non-zero exit with everything
+   confirmed is a failed re-publish of a package that is already there). Otherwise retries
+   (`--retries`, default 5) only for transient errors and fails fast on anything else.
+5. Treats a 5xx from GitHub's OIDC token endpoint (`/idtoken/... 504`, surfaced as `E404`) as
+   transient.
+6. Compares real paths in its `main` guard. It used to compare `process.argv[1]` with the resolved
+   `import.meta.url`, so running it through a symlinked path silently did nothing and exited 0.
+
+**Known gap, not fixed here:** the release action (`canvas-kit-actions/release@v2`) publishes once.
+If that fails, the safety net publishes the rest and the job goes green, but the action's later steps
+(**Publish Storybook** to `gh-pages` and **Update Chromatic Baseline**) are skipped and never re-run,
+so a recovered release has no updated Storybook or Chromatic baseline. Verified skipped in run
+40899439848. A branch in `canvas-kit-actions` that made the action retry itself was written and
+dropped in favour of keeping the logic in this repo.
+
+**Not verified end-to-end:** all of this is covered by tests that replay the real failure sequences
+with a fake `yarn`/`npm`, but it has not yet run in CI against a real 409. Next time a release hits
+one, check that `Publish remaining packages` goes green and that all 13 packages appear on npm
+(allow ~15 minutes for reads to catch up). Record the result below.
 
 ## Run history (`publish.yml`, master/support, non-skip runs)
 
-| Date (UTC) | Branch | Run | Layer 1 | Layer 2 |
+| Date (UTC) | Branch | Run | Trigger (layer 1) | Safety net |
 | --- | --- | --- | --- | --- |
 | 15 Oct | master | 38781099215 | tlog 409 (2 pkgs) | Nx duplicate projects |
 | 21 Oct | master | 39669576846 | tlog 409 (5 pkgs) | Nx duplicate projects |
@@ -111,40 +140,38 @@ packument lookup is cached the same way (the E409 handling makes this harmless e
 | 26 Oct 15:29 | master | 40349815957 | OIDC 504 → E404 (1 pkg) | Nx duplicate projects |
 | 26 Oct 18:30 | support | 40378385152 | tlog 409 (1 pkg) | EUNCOMMIT |
 | 29 Oct 15:13 | master | 40899439848 | tlog 409 (7 pkgs) | EUNCOMMIT |
-| 29 Oct 16:18 | master | 40909685326 | tlog 409 (1 pkg), manual `complete-unpublished` | EUNCOMMIT on attempt 2 |
+| 29 Oct 16:15 | master | 40909685326 | manual `complete-unpublished`: published 6, then tlog 409 (1 pkg) | EUNCOMMIT on attempt 2 |
+| 29 Oct 16:30 | master | 40912333768 | manual `complete-unpublished`: published the last 6, then stale reads | retried a finished publish; 409 on re-publish; EUNCOMMIT |
+
+v16.2.12 took three runs and ~1.5 hours to complete (1 package, then 6, then 6), although the
+publishes themselves worked.
 
 Earlier attempts: #4150 (retry script + safety net), #4162 (commit message parsing in
 `publish.yml`), #4218 (safety net for `support`), #4224 (remove generated `package.json` after packing).
 
 ## Recovering a half-published release
 
-Check what is on npm:
-
-```bash
-v=$(node -p 'require("./lerna.json").version')
-for p in codemod css docs labs-css labs-react mcp popup-stack preview-css preview-react react react-fonts styling styling-transform; do
-  echo "$p: $(npm view @workday/canvas-kit-$p@$v version 2>/dev/null || echo MISSING)"
-done
-```
-
-Then run **Publish to npm** manually (Actions → Run workflow) on `master` (or `support`) with
-release type `complete-unpublished`. It does not bump versions and publishes only what is missing.
-Until the `restoreLernaManifests` fix is on the branch you run it from, a 409 during the first
-attempt still ends the job with `EUNCOMMIT`. Just re-run: every run starts from a clean checkout and
-only publishes what is still missing, so each run makes progress (29 Oct: 1 of 13, then 6 more).
+1. **Wait ~15 minutes after the last publish**, then check what is on npm. Reads lag (layer 2), so
+   an earlier check gives false "missing" answers:
 
-Check npm with the per-version endpoint (the `npm view` loop above can be up to 5 minutes stale
-right after a publish; see layer 3):
+   ```bash
+   v=$(node -p 'require("./lerna.json").version')
+   for p in codemod css docs labs-css labs-react mcp popup-stack preview-css preview-react react react-fonts styling styling-transform; do
+     echo "$p: $(npm view @workday/canvas-kit-$p@$v version 2>/dev/null || echo MISSING)"
+   done
+   ```
 
-```bash
-curl -s -o /dev/null -w '%{http_code}\n' "https://registry.npmjs.org/@workday%2fcanvas-kit-react/$v"  # 200 = published
-```
+2. Run **Publish to npm** manually (Actions → Run workflow) on `master` (or `support`) with release
+   type `complete-unpublished`. It does not bump versions and publishes only what is missing. Packages
+   it re-publishes by mistake get "already published", which counts as done.
+3. If the job fails, read the log before re-running: the packages that lerna logged
+   `success published` are done even if the job is red.
 
 ## Open questions
 
-- Why does Rekor return 409 for these publishes? Worth an issue against `npm/cli` with a failing run
-  link if it keeps happening.
-- Whether to drop or decouple provenance for patch releases. That removes Rekor from the path but
-  is a supply-chain decision (see `scorecard.yml`), so it needs a maintainer call.
-- Whether the release action (`canvas-kit-actions/release@v2`) should itself retry per package, which
-  would make the safety net a last resort instead of the normal path.
+- Why does Rekor return 409 for these publishes, including for re-publishes? Worth an issue against
+  `npm/cli` with a failing run link if it keeps happening.
+- Provenance: **kept on** (decision 29 Oct 2026). Turning it off (`NPM_CONFIG_PROVENANCE=false`) would
+  take Rekor out of the publish path, but gives up the attestation (see `scorecard.yml`).
+- Whether the release action should retry its own publish so the Storybook and Chromatic steps still
+  run after a transient failure (the known gap above).
diff --git a/utils/publish-packages.mjs b/utils/publish-packages.mjs
index d264c3342e..bb0ff8a7c5 100644
--- a/utils/publish-packages.mjs
+++ b/utils/publish-packages.mjs
@@ -22,9 +22,8 @@ const execFileAsync = promisify(execFile);
 const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const BUILD_MARKER = path.join(REPO_ROOT, 'modules', 'react', 'dist', 'es6', 'index.js');
 const DEFAULT_RETRIES = 5;
-const RETRY_BASE_MS = 8000;
+const RETRY_BASE_MS = Number(process.env.PUBLISH_RETRY_BASE_MS ?? 8000);
 const REGISTRY_CHECK_CONCURRENCY = 4;
-const NPM_REGISTRY = 'https://registry.npmjs.org';
 const ALLOWED_DIST_TAGS = new Set(['latest', 'support', 'next', 'prerelease-next']);
 
 /** @typedef {{distTag: string, retries: number, verifyOnly: boolean, requireUnpublished: boolean}} PublishArgs */
@@ -157,8 +156,13 @@ export function parsePublishArgs(argv) {
  * @param {string} output
  */
 export function isRetryablePublishFailure(output) {
-  return /TLOG_CREATE_ENTRY_ERROR|error creating tlog entry|an equivalent entry already exists|ECONNRESET|ETIMEDOUT|EAI_AGAIN|502 Bad Gateway|503 Service Unavailable/.test(
-    output
+  return (
+    /TLOG_CREATE_ENTRY_ERROR|error creating tlog entry|an equivalent entry already exists|ECONNRESET|ETIMEDOUT|EAI_AGAIN|502 Bad Gateway|503 Service Unavailable/.test(
+      output
+    ) ||
+    // GitHub's OIDC token endpoint failing (`.../idtoken/... 504`) surfaces from npm as a bare
+    // `E404 Not found`, which would otherwise look like a permanent error.
+    /\/idtoken\/\S*\s+5\d\d\b/.test(output)
   );
 }
 
@@ -182,36 +186,27 @@ export async function getPublicPackages() {
 /**
  * Checks whether an exact package version already exists on the public npm registry.
  *
- * Reads the per-version endpoint on purpose. `npm view` fetches the package-level document, which
- * the registry CDN caches for 5 minutes (`cache-control: max-age=300`), so right after a publish it
- * keeps reporting the new version as missing. The per-version endpoint is served uncached.
- *
  * @param {string} name
  * @param {string} version
- * @param {string} [registry]
  */
-export async function isPackageVersionOnNpm(name, version, registry = NPM_REGISTRY) {
-  const url = `${registry}/${name.replace('/', '%2f')}/${encodeURIComponent(version)}`;
-
-  let response;
+export async function isPackageVersionOnNpm(name, version) {
   try {
-    response = await fetch(url, {signal: AbortSignal.timeout(30000)});
+    const {stdout} = await execFileAsync('npm', ['view', `${name}@${version}`, 'version'], {
+      timeout: 30000,
+    });
+    return stdout.trim() === version;
   } catch (err) {
-    // Timeout or network failure — do not misreport as "missing from npm".
-    throw new Error(`Could not check ${name}@${version} on npm: ${url}`, {cause: err});
-  }
+    const execErr = /** @type {Error & {stdout?: string, stderr?: string}} */ (err);
+    const output = `${execErr.stdout ?? ''}\n${execErr.stderr ?? ''}\n${execErr.message}`;
 
-  if (response.ok) {
-    return true;
-  }
+    // Version is actually missing from npm — treat as unpublished, not an operational error.
+    if (/E404|is not in this registry|No match found for version/.test(output)) {
+      return false;
+    }
 
-  // Version is actually missing from npm — treat as unpublished, not an operational error.
-  if (response.status === 404) {
-    return false;
+    // Timeout, auth, or registry outage — do not misreport as "missing from npm".
+    throw new Error(`Could not check ${name}@${version} on npm:\n${output}`, {cause: err});
   }
-
-  // Registry outage or auth error — do not misreport as "missing from npm".
-  throw new Error(`Could not check ${name}@${version} on npm: HTTP ${response.status} from ${url}`);
 }
 
 /**
@@ -309,21 +304,32 @@ function formatPackageList(packages) {
 }
 
 /**
- * Parses `@scope/name@version` strings from Lerna publish output.
+ * Reads which `name@version` packages the registry confirmed in response to lerna's publish calls:
+ * either lerna published them, or the registry said they were already published.
+ *
+ * This is the only reliable evidence that a package is on npm. A publish is accepted immediately,
+ * but reads (`npm view`, the packument, tarballs) can lag by 1–13 minutes afterwards, so a read
+ * right after a publish may still say "missing". Packages that merely appear in a failure line are
+ * not confirmed.
  *
  * @param {string} output
- * @returns {{name: string, version: string}[]}
+ * @returns {Set<string>}
  */
-export function parsePublishedPackageLines(output) {
-  /** @type {Map<string, {name: string, version: string}>} */
-  const packages = new Map();
+export function parseConfirmedPackages(output) {
+  const text = output.replace(/\u001b\[[0-9;]*m/g, '');
+  const confirmed = new Set();
 
-  for (const match of output.matchAll(/@workday\/([a-z-]+)@(\d+\.\d+\.\d+(?:-[^\s'"]+)?)/g)) {
-    const name = `@workday/${match[1]}`;
-    packages.set(name, {name, version: match[2]});
+  // `lerna success published @workday/canvas-kit-react 16.2.12`
+  for (const match of text.matchAll(/lerna success published (\S+) (\d+\.\d+\.\d+\S*)/g)) {
+    confirmed.add(`${match[1]}@${match[2]}`);
   }
 
-  return [...packages.values()];
+  // `lerna WARN publish Package is already published: @workday/canvas-kit-react@16.2.12`
+  for (const match of text.matchAll(/Package is already published: (\S+)@(\d+\.\d+\.\d+\S*)/g)) {
+    confirmed.add(`${match[1]}@${match[2]}`);
+  }
+
+  return confirmed;
 }
 
 /**
@@ -393,6 +399,14 @@ export async function publishFromPackage({
 
   assertPackagesBuilt();
 
+  // Decide what is left from the registry's responses to our own publish calls, never from reads:
+  // right after a publish, `npm view` and lerna's own lookup still say "missing" for minutes, which
+  // used to send this loop into re-publishing packages that were already on npm (and failing on
+  // them with a Rekor 409).
+  const targets = unpublished;
+  const confirmed = new Set();
+  const getMissing = () => targets.filter(pkg => !confirmed.has(`${pkg.name}@${pkg.version}`));
+
   for (let attempt = 1; attempt <= retries; attempt++) {
     console.log(
       `Publishing unpublished packages (attempt ${attempt}/${retries}) with dist-tag "${distTag}" and concurrency 1...`
@@ -410,40 +424,42 @@ export async function publishFromPackage({
       '--concurrency=1',
     ]);
 
-    unpublished = await getUnpublishedPackages(packages);
+    parseConfirmedPackages(result.output).forEach(key => confirmed.add(key));
+    const missing = getMissing();
 
-    // Re-check the registry; Lerna may exit non-zero even after some packages landed.
-    if (unpublished.length === 0) {
-      console.log('All workspace package versions are on npm.');
+    // Exit 0 means lerna published, or saw on npm, everything it believed was missing. A non-zero
+    // exit with nothing left to confirm is a failed re-publish of a package that is already there.
+    if (result.code === 0 || missing.length === 0) {
+      console.log('All workspace package versions were published or confirmed on npm.');
       return packages;
     }
 
-    // Lerna can exit 0 while packages are still missing, or non-zero for retryable tlog errors.
-    const retryable = result.code === 0 || isRetryablePublishFailure(result.output);
-    if (!retryable) {
+    if (!isRetryablePublishFailure(result.output)) {
       throw new Error(
-        `Publish failed with a non-retryable error. Still missing:\n${formatPackageList(
-          unpublished
-        )}`
+        `Publish failed with a non-retryable error. Still missing:\n${formatPackageList(missing)}`
       );
     }
 
-    console.warn(`Retryable publish failure. Still missing:\n${formatPackageList(unpublished)}`);
+    console.warn(`Retryable publish failure. Still missing:\n${formatPackageList(missing)}`);
 
-    // Back off before the next from-package pass (which skips versions already on npm).
+    // Back off before the next from-package pass.
     if (attempt < retries) {
       await waitForRetry(attempt);
     }
   }
 
-  unpublished = await getUnpublishedPackages(packages);
   throw new Error(
-    `Exhausted ${retries} publish attempts. Still missing:\n${formatPackageList(unpublished)}`
+    `Exhausted ${retries} publish attempts. Still missing:\n${formatPackageList(getMissing())}`
   );
 }
 
-// Allow `node utils/publish-packages.mjs --dist-tag latest` from release workflows.
-if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+// Allow `node utils/publish-packages.mjs --dist-tag latest` from release workflows. Compare real
+// paths: `import.meta.url` is already resolved, so a symlinked path (e.g. macOS `/var` ->
+// `/private/var`) would otherwise skip this block and exit 0 without publishing anything.
+if (
+  process.argv[1] &&
+  fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url))
+) {
   Promise.resolve()
     .then(() => publishFromPackage(parsePublishArgs(process.argv.slice(2))))
     .catch(err => {
diff --git a/utils/spec/publish-packages.spec.ts b/utils/spec/publish-packages.spec.ts
index 2a5ba14352..9fbf676bb1 100644
--- a/utils/spec/publish-packages.spec.ts
+++ b/utils/spec/publish-packages.spec.ts
@@ -1,7 +1,5 @@
 import {execFile, execFileSync} from 'node:child_process';
 import fs from 'node:fs';
-import http from 'node:http';
-import type {AddressInfo} from 'node:net';
 import os from 'node:os';
 import path from 'node:path';
 import {promisify} from 'node:util';
@@ -31,74 +29,7 @@ function git(cwd: string, ...args: string[]) {
   return execFileSync('git', args, {cwd, encoding: 'utf8'}).trim();
 }
 
-/**
- * Calls `isPackageVersionOnNpm(name, version, registry)` in a child process. Async so the fake
- * registry in this process can answer while the child waits.
- */
-async function isOnNpm(name: string, version: string, registry: string) {
-  const {stdout} = await execFileAsync(
-    process.execPath,
-    [
-      '--input-type=module',
-      '-e',
-      `import * as m from ${JSON.stringify(publishPackagesScript)};
-       console.log(JSON.stringify(await m.isPackageVersionOnNpm(${JSON.stringify(
-         name
-       )}, ${JSON.stringify(version)}, ${JSON.stringify(registry)})));`,
-    ],
-    {env: {...process.env, NODE_NO_WARNINGS: '1'}}
-  );
-  return JSON.parse(stdout);
-}
-
 describe('publish-packages.mjs', () => {
-  // Regression test: `npm view` reads the package-level document, which the registry CDN caches for
-  // 5 minutes (`cache-control: max-age=300`), so it kept reporting just-published versions as
-  // missing. The per-version endpoint is served uncached and is the source of truth.
-  describe('isPackageVersionOnNpm', () => {
-    let server: http.Server;
-    let registry: string;
-    const requested: string[] = [];
-
-    beforeEach(async () => {
-      requested.length = 0;
-      server = http.createServer((req, res) => {
-        requested.push(req.url ?? '');
-        if (req.url === '/@workday%2fpublished/1.0.0') {
-          res.writeHead(200, {'content-type': 'application/json'}).end('{"version":"1.0.0"}');
-        } else if (req.url === '/@workday%2fbroken/1.0.0') {
-          res.writeHead(503).end('unavailable');
-        } else {
-          res.writeHead(404).end('{"error":"version not found"}');
-        }
-      });
-      await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
-      registry = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
-    });
-
-    afterEach(async () => {
-      await new Promise(resolve => server.close(resolve));
-    });
-
-    it('should be true when the registry has the exact version', async () => {
-      expect(await isOnNpm('@workday/published', '1.0.0', registry)).toBe(true);
-    });
-
-    it('should query the per-version endpoint, not the cached package document', async () => {
-      await isOnNpm('@workday/published', '1.0.0', registry);
-
-      expect(requested).toEqual(['/@workday%2fpublished/1.0.0']);
-    });
-
-    it('should be false when the registry does not have the version', async () => {
-      expect(await isOnNpm('@workday/published', '2.0.0', registry)).toBe(false);
-    });
-
-    it('should fail, not report "missing", when the registry is unavailable', async () => {
-      await expect(isOnNpm('@workday/broken', '1.0.0', registry)).rejects.toThrow();
-    });
-  });
-
   describe('restoreLernaManifests', () => {
     let repo: string;
     const manifests = [
@@ -161,4 +92,239 @@ describe('publish-packages.mjs', () => {
       expect(git(repo, 'status', '--porcelain')).toBe('');
     });
   });
+
+  /**
+   * Runs the whole script in a temp checkout with fake `yarn` (lerna) and `npm` commands, replaying
+   * what happened in real release runs.
+   *
+   * What the registry actually does, measured on 29 Oct 2026: a publish is accepted immediately (a
+   * second publish gets "already published"), but reads (`npm view`, the packument, tarballs) lag by
+   * 1–13 minutes. So after a publish the registry can still report the version as missing, and the
+   * only reliable evidence that a package is published is lerna's own response to the publish call.
+   */
+  describe('publishing', () => {
+    const packages = ['@workday/a', '@workday/b', '@workday/c'];
+    let root: string;
+    let state: string;
+
+    const lernaCalls = () => Number(fs.readFileSync(path.join(state, 'calls'), 'utf8'));
+
+    /**
+     * `publish` is the body of a fake `lerna publish from-package`. It gets the attempt number as
+     * `$1` and helpers: `ok <pkg>`, `already <pkg>`, `fail <pkg>`, `dirty` and `refuse_if_dirty`.
+     * `visible` lists the `name@version` strings that `npm view` can see.
+     */
+    function setup({publish, visible = []}: {publish: string; visible?: string[]}) {
+      root = fs.mkdtempSync(path.join(os.tmpdir(), 'ck-publishing-'));
+      state = fs.mkdtempSync(path.join(os.tmpdir(), 'ck-publishing-state-'));
+      const bin = path.join(state, 'bin');
+      fs.mkdirSync(bin);
+
+      // The script resolves its repo root from its own location, so run a copy inside the temp repo.
+      fs.mkdirSync(path.join(root, 'utils'));
+      fs.copyFileSync(publishPackagesScript, path.join(root, 'utils/publish-packages.mjs'));
+      fs.mkdirSync(path.join(root, 'modules/react/dist/es6'), {recursive: true});
+      fs.writeFileSync(path.join(root, 'modules/react/dist/es6/index.js'), '');
+      fs.writeFileSync(path.join(root, '.gitignore'), 'dist\n');
+      packages.forEach(name => {
+        const dir = path.join(root, 'modules', name.split('/')[1]);
+        fs.mkdirSync(dir, {recursive: true});
+        fs.writeFileSync(
+          path.join(dir, 'package.json'),
+          `{"name": "${name}", "version": "1.0.0"}\n`
+        );
+      });
+      git(root, 'init', '--quiet');
+      git(root, 'config', 'user.email', 'test@example.com');
+      git(root, 'config', 'user.name', 'Test');
+      git(root, 'add', '.');
+      git(root, 'commit', '--quiet', '-m', 'chore: Release v1.0.0 [skip release]');
+
+      fs.writeFileSync(
+        path.join(state, 'packages.json'),
+        JSON.stringify(packages.map(name => ({name, version: '1.0.0'})))
+      );
+      fs.writeFileSync(path.join(state, 'visible'), visible.join('\n') + '\n');
+      fs.writeFileSync(path.join(state, 'publish.sh'), publish);
+      fs.writeFileSync(
+        path.join(state, 'helpers.sh'),
+        `ok() { echo "lerna success published $1 1.0.0"; }
+already() { echo "lerna WARN publish Package is already published: $1@1.0.0"; }
+fail() {
+  echo "lerna WARN notice Package failed to publish: $1"
+  echo "lerna ERR! TLOG_CREATE_ENTRY_ERROR error creating tlog entry - (409) an equivalent entry already exists"
+}
+dirty() { for f in modules/*/package.json; do echo '{"gitHead": "abc123"}' > "$f"; done; }
+refuse_if_dirty() {
+  if [ -n "$(git status --porcelain)" ]; then
+    echo "lerna ERR! EUNCOMMIT Working tree has uncommitted changes"
+    exit 1
+  fi
+}
+`
+      );
+
+      fs.writeFileSync(
+        path.join(bin, 'yarn'),
+        `#!/bin/bash
+S="${state}"
+if [ "$*" = "-s lerna ls --json" ]; then cat "$S/packages.json"; exit 0; fi
+n=$(( $(cat "$S/calls" 2>/dev/null || echo 0) + 1 )); echo $n > "$S/calls"
+. "$S/helpers.sh"
+. "$S/publish.sh" $n
+`,
+        {mode: 0o755}
+      );
+      // \`npm view name@version version\`: only versions in the visible list exist.
+      fs.writeFileSync(
+        path.join(bin, 'npm'),
+        `#!/bin/bash
+if grep -qx "$2" "${state}/visible"; then echo "\${2##*@}"; exit 0; fi
+echo "npm error code E404" >&2; echo "npm error 404 No match found for version" >&2; exit 1
+`,
+        {mode: 0o755}
+      );
+
+      return bin;
+    }
+
+    async function run(bin: string, args: string[] = []) {
+      try {
+        const {stdout, stderr} = await execFileAsync(
+          process.execPath,
+          [path.join(root, 'utils/publish-packages.mjs'), '--dist-tag', 'latest', ...args],
+          {
+            cwd: root,
+            env: {
+              ...process.env,
+              PATH: `${bin}:${process.env.PATH}`,
+              PUBLISH_RETRY_BASE_MS: '0',
+              NODE_NO_WARNINGS: '1',
+            },
+          }
+        );
+        return {code: 0, output: stdout + stderr};
+      } catch (err) {
+        const e = err as {code: number; stdout: string; stderr: string};
+        return {code: e.code, output: e.stdout + e.stderr};
+      }
+    }
+
+    afterEach(() => {
+      fs.rmSync(root, {recursive: true, force: true});
+      fs.rmSync(state, {recursive: true, force: true});
+    });
+
+    // Run 40912333768: attempt 1 published all 6 packages and exited 0, but the registry still
+    // reported them as missing, so the script retried, re-published, hit a Rekor 409 and failed.
+    it('should succeed when lerna publishes everything, even if the registry still says missing', async () => {
+      const bin = setup({publish: 'for p in @workday/a @workday/b @workday/c; do ok $p; done'});
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(lernaCalls()).toBe(1);
+    });
+
+    // Run 40899439848: a 409 mid-publish leaves every manifest modified and lerna then refuses to
+    // run again with EUNCOMMIT.
+    it('should retry only after a failure, restoring the manifests lerna left modified', async () => {
+      const bin = setup({
+        publish: `refuse_if_dirty
+if [ "$1" = "1" ]; then ok @workday/a; ok @workday/b; fail @workday/c; dirty; exit 1; fi
+already @workday/a; already @workday/b; ok @workday/c`,
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(lernaCalls()).toBe(2);
+      expect(git(root, 'status', '--porcelain')).toBe('');
+    });
+
+    // Run 40912333768, attempt 3: re-publishing a package that is already on npm failed with a
+    // Rekor 409. That is not a missing package.
+    it('should not count a failed re-publish of a package lerna already published', async () => {
+      const bin = setup({
+        publish: `refuse_if_dirty
+if [ "$1" = "1" ]; then ok @workday/a; ok @workday/b; fail @workday/c; dirty; exit 1; fi
+fail @workday/a; fail @workday/b; ok @workday/c; dirty; exit 1`,
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(lernaCalls()).toBe(2);
+    });
+
+    it('should not retry when every package was confirmed but lerna still exited non-zero', async () => {
+      const bin = setup({
+        publish: 'ok @workday/a; already @workday/b; ok @workday/c; fail @workday/a; exit 1',
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(lernaCalls()).toBe(1);
+    });
+
+    it('should fail, naming the packages that were never confirmed, after the last attempt', async () => {
+      const bin = setup({
+        publish: 'refuse_if_dirty; ok @workday/a; ok @workday/b; fail @workday/c; dirty; exit 1',
+      });
+
+      const result = await run(bin, ['--retries', '3']);
+
+      expect(result.code).not.toBe(0);
+      expect(lernaCalls()).toBe(3);
+      expect(result.output).toMatch(
+        /Exhausted 3 publish attempts\. Still missing:\n@workday\/c@1\.0\.0/
+      );
+      expect(result.output).not.toMatch(/Still missing:[^]*@workday\/a@1\.0\.0/);
+    });
+
+    // Run 40349815957: GitHub's OIDC token endpoint returned 504 and npm surfaced it as E404.
+    it("should retry when GitHub's OIDC token endpoint fails", async () => {
+      const bin = setup({
+        publish: `refuse_if_dirty
+if [ "$1" = "1" ]; then
+  ok @workday/a; ok @workday/b
+  echo "lerna http fetch GET https://run-actions-1-azure-eastus.actions.githubusercontent.com/235//idtoken/f821b02e/3bf0a923?api-version=2.0&audience=npm%3Aregistry.npmjs.org 504"
+  echo "lerna WARN notice Package failed to publish: @workday/c"
+  echo "lerna ERR! E404 Not found"
+  dirty; exit 1
+fi
+ok @workday/c`,
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(lernaCalls()).toBe(2);
+    });
+
+    it('should fail fast on an error that is not transient', async () => {
+      const bin = setup({
+        publish: 'echo "lerna ERR! E403 You do not have permission to publish"; exit 1',
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).not.toBe(0);
+      expect(lernaCalls()).toBe(1);
+      expect(result.output).toMatch(/non-retryable/);
+    });
+
+    it('should not publish when every version is already visible on npm', async () => {
+      const bin = setup({
+        publish: 'ok @workday/a',
+        visible: packages.map(name => `${name}@1.0.0`),
+      });
+
+      const result = await run(bin);
+
+      expect(result.code).toBe(0);
+      expect(fs.existsSync(path.join(state, 'calls'))).toBe(false);
+    });
+  });
 });

From 0896b36baf0e5da045b4ea96d68a21bbdec252e6 Mon Sep 17 00:00:00 2001
From: "manuel.carrera" <manuel.carrera@workday.com>
Date: Thu, 29 Oct 2026 11:48:06 -0600
Subject: [PATCH 4/4] fix: Keep parsePublishedPackageLines exported for
 publish-canary.mjs

`utils/publish-canary.mjs` imports `parsePublishedPackageLines` from `publish-packages.mjs`. The
previous commit removed that export, so every canary publish would have failed at load time with a
missing-export error before doing any work. Restore it unchanged and add a spec that fails whenever
`publish-canary.mjs` imports a name `publish-packages.mjs` does not export.

Also remove RELEASE_FAILURES.md from this change.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 RELEASE_FAILURES.md                 | 177 ----------------------------
 utils/publish-packages.mjs          |  21 ++++
 utils/spec/publish-packages.spec.ts |  33 ++++++
 3 files changed, 54 insertions(+), 177 deletions(-)
 delete mode 100644 RELEASE_FAILURES.md

diff --git a/RELEASE_FAILURES.md b/RELEASE_FAILURES.md
deleted file mode 100644
index 45119d4294..0000000000
--- a/RELEASE_FAILURES.md
+++ /dev/null
@@ -1,177 +0,0 @@
-# Release failures — investigation log
-
-Running log of why the **Publish to npm** workflow (`.github/workflows/publish.yml` →
-`release.yml`) fails on `master`/`support`, what has been tried, and what is still unknown. Read this
-before "fixing" the release job again, and **append to it** when a new failure shows up.
-
-Entries are marked **Verified** (seen in logs or source) or **Unverified** (hypothesis). Keep that
-distinction.
-
-## How a patch release works
-
-1. `release.yml` calls `Workday/canvas-kit-actions/release@v2` (a separate repo). It runs
-   `lerna version`, updates the changelog, commits `chore: Release vX.Y.Z [skip release]`, tags,
-   pushes, builds, then runs `yarn lerna publish from-package --yes --dist-tag <tag>`.
-2. That step is `continue-on-error`. If it fails, `release.yml` runs the safety net
-   `yarn publish:from-package` ([`utils/publish-packages.mjs`](utils/publish-packages.mjs)), which
-   re-publishes whatever is still missing from npm, serially, with retries.
-3. If the safety net also fails, the job fails, `report-failure` posts to Slack, and the release is
-   **half published**: the bump commit and tag exist on the branch, but some packages are not on npm.
-
-The follow-up `Release vX.Y.Z [skip release]` runs that appear green afterwards are only the skip
-runs for the release commit. They do not publish anything. A green run after a red one does **not**
-mean the release was recovered.
-
-## Three layers
-
-A red run is several independent problems stacked on each other: a transient publish error
-(layer 1), a registry that lags behind publishes (layer 2), and a safety net that mishandled both
-(layer 3).
-
-### Layer 1 — the trigger: transient publish errors (not fixed, only worked around)
-
-During `lerna publish from-package` inside the release action, some packages fail and lerna carries
-on with the rest, then exits non-zero.
-
-- **Verified:** `lerna ERR! TLOG_CREATE_ENTRY_ERROR error creating tlog entry - (409) an equivalent
-  entry already exists in the transparency log` (Sigstore/Rekor, npm provenance via OIDC trusted
-  publishing). Seen in 15 Oct, 21 Oct, 26 Oct (x2), 29 Oct runs. Typically 1–7 of the 13 packages
-  fail; the rest publish.
-- **Verified:** a different transient error on 26 Oct (run 40349815957): GitHub's OIDC token
-  endpoint returned `504`, which npm surfaced as `E404 Not found`.
-- **Unverified:** why the 409 happens. One unrelated repo
-  ([BitGo/BitGoJS#9348](https://github.com/BitGo/BitGoJS/pull/9348)) attributes it to a failed registry
-  write leaving a Rekor entry behind, so npm's internal retry produces an identical entry. Failed
-  packages here took 8–20 s, which is consistent with an internal retry, but nothing in our logs
-  proves it. Nothing authoritative from npm or Sigstore was found.
-- **Verified:** the 409 also happens when a package that is **already on npm** is published again
-  (run 40912333768, attempt 3: `styling` had been published in attempt 1). So a 409 does not mean the
-  package is missing.
-- We cannot fix this from this repo. Provenance stays on (decision 29 Oct 2026); the retry loop in
-  layer 2 is the workaround.
-
-### Layer 2 — the registry lags publishes, and the safety net trusted it (fixed here)
-
-**Verified, measured 29 Oct 2026 (run 40912333768):** a publish is accepted immediately, but reads
-lag behind it. Lerna logged `success published`, and the package became visible on npm (tarball
-`Last-Modified`) this much later:
-
-| Package | Lerna said published | Visible on npm | Lag |
-| --- | --- | --- | --- |
-| `styling` | 16:36:32 | 16:37:54 | 1.4 min |
-| `react` | 16:36:41 | 16:42:59 | 6.3 min |
-| `preview-react` | 16:37:01 | 16:46:21 | 9.3 min |
-| `mcp` (run 40899439848) | 15:21:31 | 15:34:06 | 12.6 min |
-
-During the lag **every read path says the version is missing**: `npm view` (package document, also
-CDN-cached for 5 min: `cache-control: max-age=300`), the per-version endpoint
-(`registry.npmjs.org/<pkg>/<version>` returned 404 for `preview-react` 9 min after its publish) and
-lerna's own lookup. The write path is authoritative: at 16:37:48 the registry already answered
-"already published" for `preview-react`, 8.5 minutes before it was readable. The lag looked longer for
-larger packages (`mcp`, `react`, `preview-react`); that is a correlation, not a measured cause.
-
-**Consequence (run 40912333768):** attempt 1 published all 6 remaining packages and exited 0. The
-script then asked the registry, was told all 12 were still missing, and retried. Lerna's own lookup
-was equally stale, so attempt 3 re-published `styling`, which was already on npm, and hit a Rekor 409
-(see layer 1). That left the manifests dirty and attempt 4 died with `EUNCOMMIT`. The release was
-actually complete after attempt 1.
-
-**Rule: never decide that a package is "still missing" from a registry read made less than ~15
-minutes after a publish. Decide from the registry's response to the publish call itself.**
-
-### Layer 3 — bugs in the safety net itself (fixed here)
-
-The safety net (`utils/publish-packages.mjs`) had not recovered a release on `master` since it was
-added. Three independent bugs, in the order they surfaced:
-
-| Seen | Symptom | Cause | Fix |
-| --- | --- | --- | --- |
-| 15 Oct – 26 Oct 15:29 | `MultipleProjectsWithSameNameError` from `lerna ls` (Nx) | `postpack` (`utils/publish.js post`) used a repo-root-relative glob, so the generated nameless `package.json` files were never deleted. Nx saw duplicate projects. | #4224 (`5cdbb558a`) |
-| 26 Oct 18:30 – 29 Oct | `EUNCOMMIT Working tree has uncommitted changes` listing every `modules/*/package.json` | lerna leaves manifests modified after a failed publish (below) | `restoreLernaManifests()` |
-| 29 Oct 16:30 | Retried and failed after a publish that had fully succeeded | The loop used registry reads (layer 2) to decide what was missing | `parseConfirmedPackages()` |
-
-**EUNCOMMIT root cause (Verified in `node_modules/lerna/dist/commands/publish/index.js`, lerna 9.0.1):**
-
-- `lerna publish from-package` always calls `verifyWorkingTreeClean()` first (`detectFromPackage`)
-  and throws `EUNCOMMIT` on any dirty tree.
-- `execute()` writes `gitHead` and resolved dependency specifiers into **every** manifest it will
-  publish (`annotateGitHead` + `serializeChanges`) and restores them with `resetChanges()` only
-  **after** `publishPacked()` returns. If a package fails, the manifests are left dirty.
-- So the next run starts on the tree the failed publish left behind and can never pass lerna's own
-  check. It was masked until #4224 because the Nx error fired first.
-- Not directly observed: the CI log lists the modified files but not the diff, so "the diff is
-  `gitHead` plus resolved specifiers" is read from the lerna source, not from a log.
-
-**What the script does now** (`utils/publish-packages.mjs`, covered by `utils/spec/publish-packages.spec.ts`):
-
-1. Decides what to publish once, at the start, from registry reads (state is old by then).
-2. Before every lerna attempt, resets modified `package.json` files to `HEAD` (`git checkout --`).
-   The bump commit is already in `HEAD`; other dirty files are left alone so lerna still reports them.
-3. Records which packages the registry **confirmed** in lerna's output (`lerna success published ...`
-   or `Package is already published: ...`) and never un-confirms them.
-4. Succeeds when lerna exits 0 **or** every package is confirmed (a non-zero exit with everything
-   confirmed is a failed re-publish of a package that is already there). Otherwise retries
-   (`--retries`, default 5) only for transient errors and fails fast on anything else.
-5. Treats a 5xx from GitHub's OIDC token endpoint (`/idtoken/... 504`, surfaced as `E404`) as
-   transient.
-6. Compares real paths in its `main` guard. It used to compare `process.argv[1]` with the resolved
-   `import.meta.url`, so running it through a symlinked path silently did nothing and exited 0.
-
-**Known gap, not fixed here:** the release action (`canvas-kit-actions/release@v2`) publishes once.
-If that fails, the safety net publishes the rest and the job goes green, but the action's later steps
-(**Publish Storybook** to `gh-pages` and **Update Chromatic Baseline**) are skipped and never re-run,
-so a recovered release has no updated Storybook or Chromatic baseline. Verified skipped in run
-40899439848. A branch in `canvas-kit-actions` that made the action retry itself was written and
-dropped in favour of keeping the logic in this repo.
-
-**Not verified end-to-end:** all of this is covered by tests that replay the real failure sequences
-with a fake `yarn`/`npm`, but it has not yet run in CI against a real 409. Next time a release hits
-one, check that `Publish remaining packages` goes green and that all 13 packages appear on npm
-(allow ~15 minutes for reads to catch up). Record the result below.
-
-## Run history (`publish.yml`, master/support, non-skip runs)
-
-| Date (UTC) | Branch | Run | Trigger (layer 1) | Safety net |
-| --- | --- | --- | --- | --- |
-| 15 Oct | master | 38781099215 | tlog 409 (2 pkgs) | Nx duplicate projects |
-| 21 Oct | master | 39669576846 | tlog 409 (5 pkgs) | Nx duplicate projects |
-| 22 Oct | support | 39850244173 | git exit 128 (not investigated) | none yet (#4218 added it) |
-| 26 Oct 15:10 | master | 40346670180 | tlog 409 (1 pkg) | Nx duplicate projects |
-| 26 Oct 15:29 | master | 40349815957 | OIDC 504 → E404 (1 pkg) | Nx duplicate projects |
-| 26 Oct 18:30 | support | 40378385152 | tlog 409 (1 pkg) | EUNCOMMIT |
-| 29 Oct 15:13 | master | 40899439848 | tlog 409 (7 pkgs) | EUNCOMMIT |
-| 29 Oct 16:15 | master | 40909685326 | manual `complete-unpublished`: published 6, then tlog 409 (1 pkg) | EUNCOMMIT on attempt 2 |
-| 29 Oct 16:30 | master | 40912333768 | manual `complete-unpublished`: published the last 6, then stale reads | retried a finished publish; 409 on re-publish; EUNCOMMIT |
-
-v16.2.12 took three runs and ~1.5 hours to complete (1 package, then 6, then 6), although the
-publishes themselves worked.
-
-Earlier attempts: #4150 (retry script + safety net), #4162 (commit message parsing in
-`publish.yml`), #4218 (safety net for `support`), #4224 (remove generated `package.json` after packing).
-
-## Recovering a half-published release
-
-1. **Wait ~15 minutes after the last publish**, then check what is on npm. Reads lag (layer 2), so
-   an earlier check gives false "missing" answers:
-
-   ```bash
-   v=$(node -p 'require("./lerna.json").version')
-   for p in codemod css docs labs-css labs-react mcp popup-stack preview-css preview-react react react-fonts styling styling-transform; do
-     echo "$p: $(npm view @workday/canvas-kit-$p@$v version 2>/dev/null || echo MISSING)"
-   done
-   ```
-
-2. Run **Publish to npm** manually (Actions → Run workflow) on `master` (or `support`) with release
-   type `complete-unpublished`. It does not bump versions and publishes only what is missing. Packages
-   it re-publishes by mistake get "already published", which counts as done.
-3. If the job fails, read the log before re-running: the packages that lerna logged
-   `success published` are done even if the job is red.
-
-## Open questions
-
-- Why does Rekor return 409 for these publishes, including for re-publishes? Worth an issue against
-  `npm/cli` with a failing run link if it keeps happening.
-- Provenance: **kept on** (decision 29 Oct 2026). Turning it off (`NPM_CONFIG_PROVENANCE=false`) would
-  take Rekor out of the publish path, but gives up the attestation (see `scorecard.yml`).
-- Whether the release action should retry its own publish so the Storybook and Chromatic steps still
-  run after a transient failure (the known gap above).
diff --git a/utils/publish-packages.mjs b/utils/publish-packages.mjs
index bb0ff8a7c5..d412430e9d 100644
--- a/utils/publish-packages.mjs
+++ b/utils/publish-packages.mjs
@@ -303,6 +303,27 @@ function formatPackageList(packages) {
   return packages.map(pkg => `${pkg.name}@${pkg.version}`).join('\n');
 }
 
+/**
+ * Parses `@scope/name@version` strings from Lerna publish output.
+ *
+ * Used by `publish-canary.mjs`. It matches every mention of a package, including failures, so it
+ * says which packages lerna attempted, not which ones were published (see `parseConfirmedPackages`).
+ *
+ * @param {string} output
+ * @returns {{name: string, version: string}[]}
+ */
+export function parsePublishedPackageLines(output) {
+  /** @type {Map<string, {name: string, version: string}>} */
+  const packages = new Map();
+
+  for (const match of output.matchAll(/@workday\/([a-z-]+)@(\d+\.\d+\.\d+(?:-[^\s'"]+)?)/g)) {
+    const name = `@workday/${match[1]}`;
+    packages.set(name, {name, version: match[2]});
+  }
+
+  return [...packages.values()];
+}
+
 /**
  * Reads which `name@version` packages the registry confirmed in response to lerna's publish calls:
  * either lerna published them, or the registry said they were already published.
diff --git a/utils/spec/publish-packages.spec.ts b/utils/spec/publish-packages.spec.ts
index 9fbf676bb1..b7df8bc504 100644
--- a/utils/spec/publish-packages.spec.ts
+++ b/utils/spec/publish-packages.spec.ts
@@ -30,6 +30,39 @@ function git(cwd: string, ...args: string[]) {
 }
 
 describe('publish-packages.mjs', () => {
+  // Regression test: `publish-canary.mjs` imports helpers from this module. Dropping one of them
+  // makes every canary publish fail at load time, before any publish logic runs.
+  describe('exports used by publish-canary.mjs', () => {
+    it('should export every name that publish-canary.mjs imports', () => {
+      const canary = fs.readFileSync(path.resolve(__dirname, '../publish-canary.mjs'), 'utf8');
+      const match = /import\s*\{([^}]*)\}\s*from\s*'\.\/publish-packages\.mjs'/.exec(canary);
+      if (!match) {
+        throw new Error('publish-canary.mjs no longer imports from ./publish-packages.mjs');
+      }
+      const imported = match[1]
+        .split(',')
+        .map(name => name.trim())
+        .filter(Boolean);
+
+      const exported = JSON.parse(
+        execFileSync(
+          process.execPath,
+          [
+            '--input-type=module',
+            '-e',
+            `import * as m from ${JSON.stringify(
+              publishPackagesScript
+            )}; console.log(JSON.stringify(Object.keys(m)));`,
+          ],
+          {encoding: 'utf8', env: {...process.env, NODE_NO_WARNINGS: '1'}}
+        )
+      );
+
+      expect(imported.length).toBeGreaterThan(0);
+      expect(exported).toEqual(expect.arrayContaining(imported));
+    });
+  });
+
   describe('restoreLernaManifests', () => {
     let repo: string;
     const manifests = [