Add advisory checker for hand-written TypeScript samples

.github/workflows/check-code-samples.yml

@@ -0,0 +1,183 @@
+name: Check Code Samples
+
+# Shared by the per-language sample checks, such as Check TypeScript Samples
+# and Check Python Samples. Each of those sets its own triggers and calls this
+# with its checker. See bin/code-samples.js for what the checkers share.
+#
+# This is advisory. Findings never fail the job: the SDK is fetched at its
+# latest release, so a release can break a sample without any docs change,
+# and a page may deliberately show an older API. On pull requests it annotates
+# the changed pages. On the default branch it opens a tracking issue per
+# language, updates it while findings remain, and closes it once they're gone.
+# The job fails only when the check can't run at all, for example when the
+# package registry is unreachable.
+
+on:
+  workflow_call:
+    inputs:
+      language:
+        description: The language name used in the summary and the tracking issue, such as Python.
+        required: true
+        type: string
+      checker:
+        description: The checker script, such as bin/check-python-samples.js. Its tests are the same path with .test.js.
+        required: true
+        type: string
+      baseline:
+        description: The checker's baseline file.
+        required: true
+        type: string
+      yarn-script:
+        description: The package.json script that runs the checker, named in the tracking issue.
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    name: Check samples against the SDK
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+    env:
+      LANGUAGE: ${{ inputs.language }}
+      CHECKER: ${{ inputs.checker }}
+      BASELINE: ${{ inputs.baseline }}
+      YARN_SCRIPT: ${{ inputs.yarn-script }}
+      ISSUE_TITLE: Hand-written ${{ inputs.language }} samples need fixes
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          # Pull requests diff against their base to find the changed pages.
+          # (0 is falsy in an expression, so the condition is written this way.)
+          fetch-depth: ${{ github.event_name != 'pull_request' && 1 || 0 }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '24'
+          cache: yarn
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --ignore-scripts
+
+      - name: Test the checker
+        run: node --test bin/code-samples.test.js "${CHECKER%.js}.test.js"
+
+      # A pull request that changes the checker gets a full scan. Otherwise it
+      # gets only the pages it changes, so it isn't annotated with findings on
+      # pages it didn't touch.
+      - name: Choose the pages to check
+        id: scope
+        env:
+          EVENT: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if [ "$EVENT" != "pull_request" ]; then
+            echo "mode=full" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          changed=$(git diff --name-only --diff-filter=d "$BASE_SHA"...HEAD)
+          if echo "$changed" | grep -qxF -e "$CHECKER" -e "${CHECKER%.js}.test.js" -e "$BASELINE" \
+              -e bin/code-samples.js -e bin/code-samples.test.js \
+            || echo "$changed" | grep -qE '^\.github/workflows/check-[a-z-]*samples\.yml$'; then
+            echo "mode=full" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          pages=$(echo "$changed" | grep -E '^docs/.*\.mdx$' | tr '\n' ' ' || true)
+          if [ -z "$pages" ]; then
+            echo "mode=none" >> "$GITHUB_OUTPUT"
+          else
+            echo "mode=pages" >> "$GITHUB_OUTPUT"
+            echo "pages=$pages" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Check the samples
+        id: check
+        if: steps.scope.outputs.mode != 'none'
+        env:
+          PAGES: ${{ steps.scope.outputs.pages }}
+        run: |
+          set -f
+          set +e
+          # --github adds a workflow command per finding, which annotates the
+          # page as tee echoes it. The report proper is everything else.
+          node "$CHECKER" --github $PAGES 2>&1 | tee output.txt
+          status=${PIPESTATUS[0]}
+          set -e
+
+          grep -v '^::' output.txt > report.txt || true
+
+          # 0 is clean and 2 is findings. Anything else means the check could
+          # not run, usually because a package download failed.
+          if [ "$status" -ne 0 ] && [ "$status" -ne 2 ]; then
+            echo "::error::The $LANGUAGE sample check could not run. See the log above."
+            exit "$status"
+          fi
+
+          if [ "$status" -eq 2 ]; then
+            echo "findings=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "findings=false" >> "$GITHUB_OUTPUT"
+          fi
+
+          {
+            echo "### $LANGUAGE samples (informational; does not block merge)"
+            echo
+            echo '```'
+            cat report.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Open or update the tracking issue
+        if: >-
+          github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch &&
+          steps.check.outputs.findings == 'true'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          {
+            echo "\`$CHECKER\` found hand-written $LANGUAGE samples in \`docs/\` that"
+            echo "use SDK APIs the latest SDK release doesn't have."
+            echo
+            echo '```'
+            cat report.txt
+            echo '```'
+            echo
+            echo "To resolve, fix the sample, or record the finding with a note in"
+            echo "\`$BASELINE\` if the page shows that API on purpose."
+            echo "Reproduce locally with \`yarn $YARN_SCRIPT\`."
+            echo
+            echo "_Updated by [${GITHUB_WORKFLOW}](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})._"
+          } > issue-body.md
+
+          number=$(gh issue list --state open --search "in:title \"$ISSUE_TITLE\"" --json number --jq '.[0].number // empty')
+
+          if [ -n "$number" ]; then
+            gh issue edit "$number" --body-file issue-body.md
+            echo "Updated issue #$number."
+          else
+            gh issue create --title "$ISSUE_TITLE" --body-file issue-body.md
+          fi
+
+      - name: Close the tracking issue
+        if: >-
+          github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch &&
+          steps.check.outputs.findings == 'false'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          number=$(gh issue list --state open --search "in:title \"$ISSUE_TITLE\"" --json number --jq '.[0].number // empty')
+
+          if [ -n "$number" ]; then
+            gh issue close "$number" --comment "The checker no longer finds anything to fix."
+          fi

.github/workflows/check-python-samples.yml

@@ -0,0 +1,42 @@
+name: Check Python Samples
+
+# Type-checks the hand-written Python samples in docs/ with Pyright against
+# the latest temporalio release on PyPI, and reports SDK classes, functions,
+# modules, and arguments that the samples use but the package doesn't have.
+# Advisory; the steps are shared with the other languages in
+# check-code-samples.yml.
+
+on:
+  schedule:
+    # Mondays at 11:30 UTC.
+    - cron: '30 11 * * 1'
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'docs/develop/python/**'
+      - 'bin/check-python-samples.js'
+      - 'bin/check-python-samples.test.js'
+      - 'bin/python-samples-baseline.json'
+      - 'bin/code-samples.js'
+      - 'bin/code-samples.test.js'
+      - '.github/workflows/check-python-samples.yml'
+      - '.github/workflows/check-code-samples.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: check-python-samples-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    permissions:
+      contents: read
+      issues: write
+    uses: ./.github/workflows/check-code-samples.yml
+    with:
+      language: Python
+      checker: bin/check-python-samples.js
+      baseline: bin/python-samples-baseline.json
+      yarn-script: check:py-samples

.github/workflows/check-typescript-samples.yml

@@ -0,0 +1,42 @@
+name: Check TypeScript Samples
+
+# Type-checks the hand-written TypeScript and JavaScript samples in docs/
+# against the latest published @temporalio packages, and reports SDK methods,
+# properties, and exports that the samples use but the packages don't have.
+# The same pass checks docs links written in code comments. Advisory; the
+# steps are shared with the other languages in check-code-samples.yml.
+
+on:
+  schedule:
+    # Mondays at 11:00 UTC.
+    - cron: '0 11 * * 1'
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'docs/develop/typescript/**'
+      - 'bin/check-typescript-samples.js'
+      - 'bin/check-typescript-samples.test.js'
+      - 'bin/typescript-samples-baseline.json'
+      - 'bin/code-samples.js'
+      - 'bin/code-samples.test.js'
+      - '.github/workflows/check-typescript-samples.yml'
+      - '.github/workflows/check-code-samples.yml'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: check-typescript-samples-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    permissions:
+      contents: read
+      issues: write
+    uses: ./.github/workflows/check-code-samples.yml
+    with:
+      language: TypeScript
+      checker: bin/check-typescript-samples.js
+      baseline: bin/typescript-samples-baseline.json
+      yarn-script: check:ts-samples

.github/workflows/notify-automation-failures.yml

@@ -15,6 +15,8 @@ on:
       - Update Custom Role Permissions
       - Update SDK Versions
       - Check Metrics Against SDKs
+      - Check TypeScript Samples
+      - Check Python Samples
       - CLI Docs Update
       - Delete Visual Tests Reports
       - Warm Build Cache
@@ -28,12 +30,13 @@ permissions:
 
 jobs:
   notify:
-    # `Check Metrics Against SDKs`, `Environment config drift`, and `Snipsync
-    # Coverage` also run on pull_request; exclude those runs so this only reports
-    # invocations that have nowhere else to surface, such as the scheduled ones.
+    # `Check Metrics Against SDKs`, the sample checks, `Environment config
+    # drift`, and `Snipsync Coverage` also run on pull_request; exclude those
+    # runs so this only reports invocations that have nowhere else to surface,
+    # such as the scheduled ones.
     if: >-
-      github.event.workflow_run.event != 'pull_request' &&
-      contains(fromJSON('["failure", "timed_out", "startup_failure"]'), github.event.workflow_run.conclusion)
+      github.event.workflow_run.event != 'pull_request' && contains(fromJSON('["failure", "timed_out",
+      "startup_failure"]'), github.event.workflow_run.conclusion)
     runs-on: ubuntu-latest
     steps:
       - name: Post failure to Slack

AGENTS.md

@@ -185,6 +185,8 @@ yarn check:metrics       # SDK metrics reference against itself; runs in CI on P
 yarn check:metrics:sdks  # SDK metrics reference against the SDK sources; advisory, clones the SDK repos
 yarn check:orphans       # docs pages Docusaurus renders but no sidebar entry links to; not yet wired into CI
 yarn check:redirects     # vercel.json redirect rules that can't work as written; runs in CI on PRs
+yarn check:py-samples    # hand-written Python samples against the published SDK package; advisory
+yarn check:ts-samples    # hand-written TypeScript samples against the published SDK packages; advisory
 ```
 
 `yarn check:metrics:sdks` reports metrics an SDK defines but the page omits. When one is deliberately left undocumented,
@@ -200,6 +202,12 @@ belongs in `bin/orphan-pages-baseline.json` with a note, rather than being silen
 containing `#`, a `:param` with no `/` before it, a rule shadowed by an earlier one, or a destination the site doesn't
 serve. Fix the rule. A known, accepted exception belongs in `bin/redirect-baseline.json` with a note.
 
+`yarn check:ts-samples` and `yarn check:py-samples` type-check the TypeScript and Python code blocks that aren't
+synced by Snipsync against the latest SDK release, and report classes, methods, exports, and arguments the SDK doesn't
+have. The TypeScript check also reports docs links in code comments that don't resolve. Pass page paths to check only
+those pages. A sample that shows an API on purpose belongs in that language's `bin/*-samples-baseline.json` with a
+note.
+
 Vale linting (style). Requires Vale 3.20+ (CI already runs 3.20.0; upgrade a local install with
 `brew upgrade vale`) — `vale/styles/Std` needs it for its nested rule directories.