Report the share of SDK code that comes from Snipsync

.github/workflows/notify-automation-failures.yml

@@ -19,6 +19,7 @@ on:
       - Delete Visual Tests Reports
       - Warm Build Cache
       - Environment config drift
+      - Snipsync Coverage
     types:
       - completed
 
@@ -27,9 +28,9 @@ permissions:
 
 jobs:
   notify:
-    # `Check Metrics Against SDKs` and `Environment config drift` also run on
-    # pull_request; exclude those runs so this only reports invocations that have
-    # nowhere else to surface, such as the scheduled ones.
+    # `Check Metrics Against SDKs`, `Environment config drift`, and `Snipsync
+    # Coverage` also run on pull_request; exclude those runs so this only reports
+    # invocations that have nowhere else to surface, such as the scheduled ones.
     if: >-
       github.event.workflow_run.event != 'pull_request' &&
       contains(fromJSON('["failure", "timed_out", "startup_failure"]'), github.event.workflow_run.conclusion)

.github/workflows/snipsync-coverage.yml

@@ -0,0 +1,113 @@
+name: Snipsync Coverage
+
+# Reports the share of SDK code lines in docs/ that Snipsync pulls from a
+# sample repository, as opposed to code written into the page by hand. This is
+# informational, not a merge gate: the job never fails on its numbers.
+#
+# On a pull request it compares the merge result with the base it merges into,
+# writes the numbers to the job summary, and keeps one PR comment up to date.
+# It only posts that comment when the SDK line counts change, so PRs that don't
+# touch code samples stay quiet. On main it writes the numbers to the job
+# summary. The numbers depend only on the files in docs/, so the full trend is
+# rebuilt from git history with `yarn report:snipsync-coverage --history`
+# rather than stored anywhere.
+
+on:
+  pull_request:
+    paths:
+      - "docs/**"
+      - "bin/code-blocks.js"
+      - "bin/report-snipsync-coverage.js"
+      - ".github/workflows/snipsync-coverage.yml"
+  push:
+    branches: [main]
+    paths:
+      - "docs/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: snipsync-coverage-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  snipsync-coverage:
+    name: Report Snipsync coverage
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          # On a pull request HEAD is the merge commit, and HEAD^1 is the base
+          # it merges into, so the comparison covers exactly this PR's changes.
+          fetch-depth: 2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "24"
+
+      - name: Report coverage
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+        run: |
+          if [ "$EVENT_NAME" = "pull_request" ]; then
+            base="$(git rev-parse HEAD^1)"
+            node bin/report-snipsync-coverage.js --base "$base" --ref HEAD --markdown > report.md
+            node bin/report-snipsync-coverage.js --base "$base" --ref HEAD --json > report.json
+          else
+            node bin/report-snipsync-coverage.js --markdown > report.md
+          fi
+          cat report.md
+          cat report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        # A pull request from a fork gets a read-only token and can't comment.
+        # The numbers are still in the job summary.
+        continue-on-error: true
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- snipsync-coverage -->';
+            const { changed } = JSON.parse(fs.readFileSync('report.json', 'utf8'));
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              per_page: 100,
+            });
+            const existing = comments.find(
+              (comment) => comment.user?.login === 'github-actions[bot]' && comment.body?.includes(marker),
+            );
+
+            // Most pull requests don't change any code samples. Don't post a
+            // comment saying so unless an earlier push changed them.
+            if (!changed && !existing) {
+              core.info('SDK code line counts are unchanged and there is no earlier comment; skipping.');
+              return;
+            }
+
+            const body = `${marker}\n${fs.readFileSync('report.md', 'utf8')}`;
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              core.info(`Updated the Snipsync coverage comment (${existing.id}).`);
+            } else {
+              const { data: comment } = await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+              core.info(`Created the Snipsync coverage comment (${comment.id}).`);
+            }

AGENTS.md

@@ -156,6 +156,9 @@ Adding or moving pages usually requires:
 - Prefer code extracted from CI-enabled sample repos via [Snipsync](https://github.com/temporalio/snipsync).
 - Snippets are wrapped in `<!--SNIPSTART id-->` / `<!--SNIPEND-->`. Edit the **source repo** named inside the wrapper,
   then run `yarn snipsync`.
+- `yarn report:snipsync-coverage` reports the share of SDK code lines that come from Snipsync. On a pull request, the
+  Snipsync Coverage workflow comments when the change moves that number. Replacing a synced block with hand-written code
+  lowers it. See [UTILITIES.md](./readme/UTILITIES.md#snipsync-coverage).
 
 ## Pull requests
 

bin/code-blocks.js

@@ -0,0 +1,109 @@
+// Finds the fenced code blocks on a docs page and records which ones Snipsync
+// manages. Shared by bin/report-snipsync-coverage.js and the checkers that
+// compile hand-written samples.
+
+// Snipsync wraps synced code in either an HTML comment or an MDX comment.
+const SNIPSTART = /^\s*(?:<!--|\{\/\*)\s*SNIPSTART\b/;
+const SNIPEND = /^\s*(?:<!--|\{\/\*)\s*SNIPEND\b/;
+
+const FENCE_OPEN = /^(\s*)(`{3,}|~{3,})\s*([^\s`{]*)/;
+const FENCE_CLOSE = /^\s*(`{3,}|~{3,})\s*$/;
+
+// The line Snipsync writes above a synced block when source links are on:
+// [path/to/file.go](https://github.com/owner/repo/blob/ref/path/to/file.go)
+const SOURCE_LINK = /^\s*\[[^\]]*\]\(https:\/\/github\.com\/([^/\s)]+)\/([^/\s)]+)\/blob\//;
+
+// The closing token of a comment that opens on this line and doesn't close,
+// or null. Fenced code inside an unclosed comment is never rendered.
+function openComment(line) {
+  const rest = line.replace(/<!--.*?-->/g, '').replace(/\{\/\*.*?\*\/\}/g, '');
+  if (rest.includes('<!--')) return '-->';
+  if (rest.includes('{/*')) return '*/}';
+  return null;
+}
+
+function dedent(line, indent) {
+  let i = 0;
+  while (i < indent && (line[i] === ' ' || line[i] === '\t')) i++;
+  return line.slice(i);
+}
+
+// Every fenced code block on a page, with the 1-based line of its first line
+// of code and whether Snipsync manages it. Code inside a list item or a JSX
+// component is indented to match its fence; that indentation is removed.
+//
+// A synced block also carries `origin`, the `owner/repo` from the source link
+// Snipsync wrote above it, or null when the wrapper turns source links off.
+function extractCodeBlocks(source) {
+  const lines = source.split('\n');
+  const blocks = [];
+  let fence = null;
+  let snipsync = false;
+  let origin = null;
+  let comment = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    if (fence) {
+      const close = line.match(FENCE_CLOSE);
+      if (close && close[1][0] === fence.marker[0] && close[1].length >= fence.marker.length) {
+        blocks.push({
+          lang: fence.lang,
+          line: fence.line,
+          code: fence.body.join('\n'),
+          snipsync: fence.snipsync,
+          origin: fence.origin,
+        });
+        fence = null;
+      } else {
+        fence.body.push(dedent(line, fence.indent));
+      }
+      continue;
+    }
+
+    if (comment) {
+      if (line.includes(comment)) comment = null;
+      continue;
+    }
+
+    if (SNIPSTART.test(line)) {
+      snipsync = true;
+      origin = null;
+      continue;
+    }
+    if (SNIPEND.test(line)) {
+      snipsync = false;
+      origin = null;
+      continue;
+    }
+
+    if (snipsync) {
+      const link = line.match(SOURCE_LINK);
+      if (link) {
+        origin = `${link[1]}/${link[2]}`;
+        continue;
+      }
+    }
+
+    const open = line.match(FENCE_OPEN);
+    if (open) {
+      fence = {
+        marker: open[2],
+        indent: open[1].length,
+        lang: open[3].toLowerCase(),
+        line: i + 2,
+        body: [],
+        snipsync,
+        origin: snipsync ? origin : null,
+      };
+      continue;
+    }
+
+    comment = openComment(line);
+  }
+
+  return blocks;
+}
+
+module.exports = { SNIPSTART, SNIPEND, extractCodeBlocks };