From dee9d87cab33b1cafa56ba2f20c2178743d8c924 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Mon, 9 Nov 2026 19:33:10 +0700 Subject: [PATCH] ci(release): filter release line pushes by path and drop the merge queue handling --- .claude/rules/ci.md | 16 ++-- .github/workflows/ci-lint.yml | 26 +----- .github/workflows/ci.yml | 49 +++-------- .github/workflows/ios-tests.yml | 2 +- .github/workflows/macos-tests.yml | 2 +- AGENTS.md | 2 +- apple/RELEASING.md | 5 +- apple/scripts/release/test_version.py | 105 +++++++++++++++++++++++- apple/scripts/release/version.py | 63 +++++++++++--- linux/docs/testing.md | 2 +- scripts/ci/detect-changed-paths.sh | 14 +--- scripts/ci/test_detect_changed_paths.py | 22 ++--- 12 files changed, 195 insertions(+), 113 deletions(-) diff --git a/.claude/rules/ci.md b/.claude/rules/ci.md index 18a87ba5e2..edd57b351d 100644 --- a/.claude/rules/ci.md +++ b/.claude/rules/ci.md @@ -16,9 +16,9 @@ paths: ## Required checks -- main requires `CI`, the last job of `ci.yml`, and merges through GitHub's merge queue, which builds up to five pull requests at once, each on top of those ahead of it, and squashes. Strict is off. Admins can bypass in pull_request mode. +- main requires `CI`, the last job of `ci.yml`. There is no merge queue and strict is off: a pull request squash-merges on its own green run, and the main push tests what changed again. Admins can bypass in pull_request mode. - `release/*` requires `CI` and `PR title` (ruleset 25435242). `release/0.82` was cut before `ci.yml`: it keeps its own workflow files and the old per-workflow checks (ruleset 23957966). -- `PR title` (`pr-title.yml`) is advisory on main: the reviewer reads the squash subject. It also runs the changelog check, since a title edit starts no CI run; on main the merge group's Lint job runs it again with the current title. +- `PR title` (`pr-title.yml`) is advisory on main: the reviewer reads the squash subject. It also runs the changelog check, since a title edit starts no CI run. - `CI` fails when a job it needs failed or was cancelled, or when the plan selected a job that did not succeed. A job the plan left out is skipped, which passes. - Tags never move (ruleset 25308039, no bypass). Only admins create `release/*` and the `v*`, `ios-v*`, `linux-v*`, `windows-v*`, `libs-*` and `plugin-*` tags. @@ -26,16 +26,16 @@ paths: - Open the CI run and read the red job under it. `CI` only lists the results. - PR CI tests the merge with main as of the trigger. After main is fixed, merge main in instead of re-running. -- A merge group runs `CI` again on the squash of the pull request onto main; a red one drops the pull request from the queue. - Artifacts, annotations and summaries: `.claude/skills/fix-issue/references/verification-apple.md`, Reading CI. ## What runs where -- `ci.yml` runs on every PR, every merge group, pushes to main and `release/**`, and by dispatch. `Plan` runs the detector per suite, then calls each workflow when its suite is true: `ci-lint.yml` (always; SwiftLint with `swiftlint`, the HANA helper with `hana`), `macos-tests.yml` (`macos`, plugins with `plugins`), `ios-tests.yml` (`ios`, TableProOracle with `oracle`), `build-linux.yml` (`linux-docs`, docs job only unless `linux`; `linux-deps` adds the minimum Rust version, the aarch64 Flatpak and the advisory database), `linux-packages.yml` (`packages`, x86_64), `core.yml` (`core`) and `windows.yml` (`windows`). +- `ci.yml` runs on every PR, pushes to main and `release/**`, and by dispatch. `Plan` runs the detector per suite, then calls each workflow when its suite is true: `ci-lint.yml` (always; SwiftLint with `swiftlint`, the HANA helper with `hana`), `macos-tests.yml` (`macos`, plugins with `plugins`), `ios-tests.yml` (`ios`, TableProOracle with `oracle`), `build-linux.yml` (`linux-docs`, docs job only unless `linux`; `linux-deps` adds the minimum Rust version, the aarch64 Flatpak and the advisory database), `linux-packages.yml` (`packages`, x86_64), `core.yml` (`core`) and `windows.yml` (`windows`). - `build-linux.yml` on a change runs the fast job, the default server releases, the x86_64 Flatpak Devel build and its tests, translations and supply chain; no job waits for the fast job. `extended` adds every other server release, the minimum Rust version, the aarch64 Flatpak, paths with spaces and the advisory database. The em dash check warns and never fails. - A `release/**` push, or a PR into one, runs no Linux, core or Windows job: Linux ships from main or `linux-X.Y`, and the core and the Windows app have no release line. -- The base is a PR's base SHA, a merge group's base SHA, or on a main push the head SHA of the last green CI push run on main. No such run, or a dispatch, runs everything. A main push runs no suite when its parent is that last green commit and the commit itself already passed `CI` in a merge group. A dispatch never counts as main's last green run or for a tag. -- Concurrency in `ci.yml`: a PR keeps its newest run, main pushes run one at a time and a newer pending run replaces the older, and each merge group, `release/**` commit and dispatch has its own group. `linux-packages.yml` runs only when called or dispatched, under its caller's concurrency. +- The base is a PR's base SHA, or on a push the head SHA of the last green CI push run on the same branch. No such run, as on the first push of a release line, or a dispatch, runs everything. A dispatch never counts as a branch's last green run or for a tag. +- A release PR changes only the changelog, which no suite reads, so its merge runs Lint only. `version.py check` then takes each test job from the line's last run that passed it (`apple/RELEASING.md`, Branches). +- Concurrency in `ci.yml`: a PR keeps its newest run, main pushes run one at a time and a newer pending run replaces the older, and each `release/**` commit and dispatch has its own group. `linux-packages.yml` runs only when called or dispatched, under its caller's concurrency. - `nightly.yml` is the one schedule, at 19:17 UTC: every suite in full on main, with core's Arm64 SSH tunnel tests and `build-linux.yml` extended, then `linux-packages.yml` with the Flatpak repository (arm64 too on Sundays, the second cron line), and the Release Build job, which builds and packages the app and every registry plugin as a tag does, unsigned, on `xcode-27`. No tag waits for it. GitHub mails a red run to whoever last edited the schedule. - macOS Tests runs the Unit and UI plans in full, with no shard, parallel, iteration or retry flag. - `core.yml` runs the core on Windows x64 and Arm64 and on macOS, where the same job builds and tests the Swift bindings, and SQL Server's integrated login on Windows x64. The SSH tunnel tests through Windows' OpenSSH Server run on x64; Arm64, whose image installs the server in about 20 minutes, runs them nightly. @@ -44,8 +44,8 @@ paths: ## Change detector -- `scripts/ci/detect-changed-paths.sh [--base ] [--head ] [--ref ] ` prints `true` or `false`. No base prints true; an all-zero base diffs from the merge base with `origin/main`; `--ref refs/heads/release/*` prints true. -- Suites: `macos`, `plugins`, `ios`, `oracle`, `hana`, `swiftlint`, `linux` (`linux/`, `core/` and the Cargo workspace files at the top; no Markdown but `linux/CHANGELOG.md`, `linux/docs/regression-tests.md` and `linux/data/`), `linux-docs`, `linux-deps` (Cargo manifests and lock, toolchain, `deny.toml`, the Flatpak manifests, the Debian build dependencies), `packages` (Cargo manifests and lock, toolchain, Meson files, `linux/build-aux/` but its tests, `po/LINGUAS`; no Markdown), `core` (`core/`, `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, the FFI Swift package and what builds it; no Markdown), `windows` (`windows/`, `core/`, `Cargo.toml`, `Cargo.lock` and `rust-toolchain.toml` at the top; no Markdown), `release` (what the Release Build job reads) and `shipped` (`release` without `.github/`; `version.py check` reads it). +- `scripts/ci/detect-changed-paths.sh [--base ] [--head ] ` prints `true` or `false`. No base prints true; an all-zero base diffs from the merge base with `origin/main`. +- Suites: `macos`, `plugins`, `ios`, `oracle`, `hana`, `swiftlint`, `linux` (`linux/`, `core/` and the Cargo workspace files at the top; no Markdown but `linux/CHANGELOG.md`, `linux/docs/regression-tests.md` and `linux/data/`), `linux-docs`, `linux-deps` (Cargo manifests and lock, toolchain, `deny.toml`, the Flatpak manifests, the Debian build dependencies), `packages` (Cargo manifests and lock, toolchain, Meson files, `linux/build-aux/` but its tests, `po/LINGUAS`; no Markdown), `core` (`core/`, `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, the FFI Swift package and what builds it; no Markdown), `windows` (`windows/`, `core/`, `Cargo.toml`, `Cargo.lock` and `rust-toolchain.toml` at the top; no Markdown), `release` (what the Release Build job reads) and `shipped` (`release` without `.github/`; `version.py check` reads it, and `macos` and `ios` for a test job a tag's run left out). - Every suite but `shipped` watches `ci.yml` and its own workflow. A path a suite newly reads joins its pattern in the same change. ## Runners diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index 0f97eac36d..e742097778 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -109,35 +109,17 @@ jobs: "$RUNNER_TEMP/release-venv/bin/towncrier" build --draft --dir linux --version 0.0.0 # pr-title.yml runs the same check on a title edit, which starts no CI run. The title is read now, not from the - # event, so a merge group or a re-run after a retitle sees the current one. + # event, so a re-run after a retitle sees the current one. - name: Check only a release pull request edits a CHANGELOG.md + if: github.event_name == 'pull_request' env: - EVENT: ${{ github.event_name }} PR_NUMBER: ${{ github.event.pull_request.number }} - GROUP_REF: ${{ github.event.merge_group.head_ref }} - GROUP_BASE: ${{ github.event.merge_group.base_sha }} REPOSITORY: ${{ github.repository }} GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - case "$EVENT" in - pull_request) - number="$PR_NUMBER" - base="$(git rev-parse HEAD^1)" - ;; - merge_group) - number="${GROUP_REF##*/pr-}" - number="${number%%-*}" - base="$GROUP_BASE" - ;; - *) - echo "Not a pull request or a merge group, so nothing to check." - exit 0 - ;; - esac - case "$number" in '' | *[!0-9]*) echo "::error::no pull request number for this $EVENT"; exit 1 ;; esac - title="$(gh api "repos/$REPOSITORY/pulls/$number" --jq .title)" - scripts/ci/check-changelog-edits.sh "$base" "$title" + title="$(gh api "repos/$REPOSITORY/pulls/$PR_NUMBER" --jq .title)" + scripts/ci/check-changelog-edits.sh "$(git rev-parse HEAD^1)" "$title" # shellcheck cannot tell a library function from a command, so a missing source is exit 127 # at release time (v0.67.1). diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f93596d214..bb34f44284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,14 +4,12 @@ name: CI on: pull_request: - merge_group: - types: [checks_requested] push: branches: [main, 'release/**'] workflow_dispatch: # A pull request keeps only its newest run. main runs one at a time, and a newer pending run replaces the older one. -# Each merge group, each release/** commit and each dispatch gets its own group, so none is replaced. +# Each release/** commit and each dispatch gets its own group, so none is replaced. concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.run_id) || github.ref == 'refs/heads/main' && github.ref || format('{0}-{1}', github.ref, github.sha) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -46,57 +44,39 @@ jobs: fetch-depth: 0 persist-credentials: false - # A merge group compares with the commit it lands on. A main push compares with the last commit a CI push run - # passed on, so a commit whose run was replaced or failed is tested again; none, one on HEAD itself, or a - # dispatch runs everything. A main push runs no suite when that commit is its parent and the merge queue already - # passed CI at this exact commit, since nothing between them went untested. + # A push compares with the last commit a CI push run on its own branch passed on, so a commit whose run was + # replaced or failed is tested again, and a release line's changelog commit runs Lint only. None, as on the first + # push of a release line, one on HEAD itself, or a dispatch runs everything. - name: Find the base id: base env: EVENT: ${{ github.event_name }} PR_BASE: ${{ github.event.pull_request.base.sha }} - GROUP_BASE: ${{ github.event.merge_group.base_sha }} - REF: ${{ github.ref }} + BRANCH: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} run: | set -euo pipefail base="" - tested=false - head="$(git rev-parse HEAD)" case "$EVENT" in pull_request) base="$PR_BASE" ;; - merge_group) base="$GROUP_BASE" ;; push) - if [ "$REF" = refs/heads/main ]; then - passed="$(gh api "repos/$REPOSITORY/actions/workflows/ci.yml/runs?branch=main&event=push&status=success&per_page=1" \ - --jq '.workflow_runs[0].head_sha // ""')" - if [ -n "$passed" ] && [ "$passed" != "$head" ] && git merge-base --is-ancestor "$passed" HEAD; then - base="$passed" - fi - if [ -n "$base" ] && [ "$base" = "$(git rev-parse HEAD^1)" ]; then - queued="$(gh api "repos/$REPOSITORY/actions/workflows/ci.yml/runs?event=merge_group&head_sha=$head&status=success&per_page=1" \ - --jq '.total_count')" - [ "$queued" = 0 ] || tested=true - fi + passed="$(gh api "repos/$REPOSITORY/actions/workflows/ci.yml/runs?branch=$BRANCH&event=push&status=success&per_page=1" \ + --jq '.workflow_runs[0].head_sha // ""')" + if [ -n "$passed" ] && [ "$passed" != "$(git rev-parse HEAD)" ] && git merge-base --is-ancestor "$passed" HEAD; then + base="$passed" fi ;; esac - if [ "$tested" = true ]; then - echo "The merge queue passed CI at $head, so no suite runs again." - else - echo "Base: ${base:-none, so every suite runs}" - fi + echo "Base: ${base:-none, so every suite runs}" echo "base=$base" >> "$GITHUB_OUTPUT" - echo "tested=$tested" >> "$GITHUB_OUTPUT" - # --ref makes a release/** push run every suite. Linux ships from main or linux-X.Y, and the core and the Windows - # app have no release line, so a release/** push or a pull request into one skips them. + # Linux ships from main or linux-X.Y, and the core and the Windows app have no release line, so a release/** push + # or a pull request into one skips them. - name: Decide which suites run id: decide env: BASE: ${{ steps.base.outputs.base }} - TESTED: ${{ steps.base.outputs.tested }} REF: ${{ github.ref }} BASE_REF: ${{ github.base_ref }} run: | @@ -105,10 +85,7 @@ jobs: case "$REF" in refs/heads/release/*) release_line=true ;; esac case "$BASE_REF" in release/*) release_line=true ;; esac for suite in macos plugins ios oracle hana swiftlint linux linux-docs linux-deps packages core windows; do - verdict=false - if [ "$TESTED" != true ]; then - verdict="$(scripts/ci/detect-changed-paths.sh --base "$BASE" --ref "$REF" "$suite")" - fi + verdict="$(scripts/ci/detect-changed-paths.sh --base "$BASE" "$suite")" case "$suite" in linux | linux-docs | linux-deps | packages | core | windows) [ "$release_line" = false ] || verdict=false ;; esac diff --git a/.github/workflows/ios-tests.yml b/.github/workflows/ios-tests.yml index 6488c51716..2eee442713 100644 --- a/.github/workflows/ios-tests.yml +++ b/.github/workflows/ios-tests.yml @@ -70,7 +70,7 @@ jobs: CODE_SIGNING_ALLOWED=NO COMPILER_INDEX_STORE_ENABLE=NO COMPILATION_CACHE_ENABLE_CACHING=NO \ | xcbeautify --renderer github-actions - # Saved by a push or the nightly only: a cache saved on a pull request or merge group ref is visible to it alone. + # Saved by a push or the nightly only: a cache saved on a pull request ref is visible to it alone. - name: Save Swift package checkouts if: (github.event_name == 'push' || github.event_name == 'schedule') && steps.tests.outcome == 'success' && steps.spm.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/macos-tests.yml b/.github/workflows/macos-tests.yml index 9dd0cb1674..6a480c3843 100644 --- a/.github/workflows/macos-tests.yml +++ b/.github/workflows/macos-tests.yml @@ -41,7 +41,7 @@ jobs: -project TablePro.xcodeproj -scheme TablePro \ -clonedSourcePackagesDirPath ~/.spm-cache -skipPackagePluginValidation - # Saved by a push or the nightly only: a cache saved on a pull request or merge group ref is visible to it alone. + # Saved by a push or the nightly only: a cache saved on a pull request ref is visible to it alone. - name: Save Swift package checkouts if: (github.event_name == 'push' || github.event_name == 'schedule') && steps.prepare.outputs.cache-hit != 'true' uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/AGENTS.md b/AGENTS.md index 587a6f4fb5..8ef7bc03ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ Native database clients with a native UI per platform: `apple/` holds the macOS - The PR title is the squash commit subject, `type(scope): description`. The body follows `.github/pull_request_template.md`. - A rename or signature change updates every caller and test in the same commit. -- Agents merge on green with `gh pr merge --auto`: main's merge queue squashes and refuses `--squash`; a `release/*` pull request takes `--auto --squash`. They create a `release/*` branch or a tag only when the maintainer says to release, and use `--admin` only on the maintainer's word. +- Agents merge on green with `gh pr merge --auto --squash`, into main or a `release/*` line. They create a `release/*` branch or a tag only when the maintainer says to release, and use `--admin` only on the maintainer's word. ## CI diff --git a/apple/RELEASING.md b/apple/RELEASING.md index 9275ea5a00..58487e0583 100644 --- a/apple/RELEASING.md +++ b/apple/RELEASING.md @@ -31,6 +31,9 @@ Such a change stops the release until a fix lands on `main`. - `release/X.Y` is cut from `main` at beta.1 and closes when X.(Y+1).0 ships. A later security or data-loss fix for its users ships as a critical patch on the current line. - Every fix lands on `main` first. A pick is one fix per PR, never a `feat` or a PluginKit bump. A patch takes only a regression from the previous stable, a crash, a plugin that fails to load, a security fix or data loss. - `release/0.82` predates the move into `apple/` and follows the `RELEASING.md` on that branch: paths at the root, no `ci.yml`, and a `verify` that waits up to 240 min for green macOS Tests, iOS and Package Tests and Release Build push runs covering the tag. A pick from `main` relies on rename detection. +- A line push runs the suites whose paths changed since the line's last green CI push run; the first push of a line runs them all. A release PR's merge changes only the changelog, so it runs Lint only. +- `version.py check` accepts a test job the tag's run left out when the job passed at an ancestor on the line and its suite has not changed since. +- `release/0.86` and older lines run every suite on every push, so a tag had its own test runs (#3584). That re-ran 77 to 114 min of tests after each release PR, so later lines filter like `main`. ```bash git fetch origin && git switch -c backport/-X.Y origin/release/X.Y @@ -112,7 +115,7 @@ Once a stable or patch publishes, cherry-pick its release commit to `main`, and git fetch origin && git switch -c forward/vX.Y.Z origin/main git cherry-pick -x && git push -u origin HEAD gh pr create --base main --title "chore(release): vX.Y.Z" --body "Forward-port of #." -gh pr merge --auto +gh pr merge --auto --squash ``` - A newer open line gets the same PR from `origin/release/`, with `--base release/`. diff --git a/apple/scripts/release/test_version.py b/apple/scripts/release/test_version.py index ebe571a432..d4066777bc 100644 --- a/apple/scripts/release/test_version.py +++ b/apple/scripts/release/test_version.py @@ -23,8 +23,8 @@ } MAIN_CHANGELOG = "# Changelog\n\n## [0.81.0] - 2026-10-16\n\n### Fixed\n\n- A bug. (#1)\n" DETECTOR_SOURCE = Path(__file__).resolve().parents[3] / version.DETECTOR -# What the fake detector in CheckTests calls shipped; AppleTreeTests runs the real one. -SHIPPED_PREFIXES = ("TablePro/", "Plugins/") +# What the fake detector in CheckTests calls each suite; AppleTreeTests runs the real one. +SUITE_PREFIXES = {"shipped": ("TablePro/", "Plugins/"), "macos": ("TablePro/", "Plugins/"), "ios": ("Mobile/",)} class GrammarTests(unittest.TestCase): @@ -77,11 +77,13 @@ def test_parse_refuses_a_bad_tag(self): class FakeGitHub: - def __init__(self, releases=(), runs=None, jobs=None): + def __init__(self, releases=(), runs=None, jobs=None, passed=None): self._releases = list(releases) self._runs = runs or {} self._jobs = jobs or {} + self._passed = passed or {} self.run_queries = [] + self.passed_queries = [] def releases(self): return self._releases @@ -90,6 +92,10 @@ def push_runs(self, sha, branch): self.run_queries.append((sha, branch)) return self._runs.get((sha, branch), []) + def passed_push_runs(self, branch): + self.passed_queries.append(branch) + return self._passed.get(branch, []) + def job_conclusions(self, run_id, job): return self._jobs.get((run_id, job), []) @@ -99,6 +105,18 @@ def green_on(sha, branch="release/0.82"): return {(sha, branch): [run]}, {("ci", job): ["success"] for job in version.TEST_JOBS} +# A green CI push run whose plan ran only the test jobs in ran; the others' calling jobs are skipped. +def planned(run_id, sha, ran, branch="release/0.82"): + run = version.Run(run_id, version.CI_WORKFLOW, "completed", "success", f"https://ci.example/{run_id}") + jobs = {} + for job in version.TEST_JOBS: + if job in ran: + jobs[(run_id, job)] = ["success"] + else: + jobs[(run_id, job.split(" / ")[0])] = ["skipped"] + return {(sha, branch): [run]}, jobs + + def with_ci(runs, **changes): return {key: [replace(run, **changes) if run.path == version.CI_WORKFLOW else run for run in found] for key, found in runs.items()} @@ -149,7 +167,7 @@ def setUp(self): def changed(self, base, head, suite): self.diffs.append((base, head, suite)) paths = self.repo.git("diff-tree", "-r", "--name-only", base, head).splitlines() - return any(path.startswith(SHIPPED_PREFIXES) for path in paths) + return any(path.startswith(SUITE_PREFIXES[suite]) for path in paths) def check(self, tag, github): return version.check(tag, self.git, github, changed=self.changed) @@ -212,6 +230,80 @@ def test_a_failed_run_refuses_the_tag(self): self.assertEqual(self.check("v0.82.0-beta.1", github), [f"CI ended '{conclusion}' at https://ci.example/ci"]) + def line_runs(self, *planned_runs): + runs, jobs, passed = {}, {}, [] + for run_id, sha, ran in planned_runs: + more_runs, more_jobs = planned(run_id, sha, ran) + runs.update(more_runs) + jobs.update(more_jobs) + passed.insert(0, (run_id, sha)) + return runs, jobs, {"release/0.82": passed} + + def patch_after(self): + release_commit = self.repo.commit("chore(release): v0.82.1", MAIN_CHANGELOG.replace( + "## [0.81.0]", "## [0.82.1] - 2026-10-27\n\n## [0.82.0] - 2026-10-20\n\n## [0.81.0]")) + self.repo.line("0.82", release_commit) + self.repo.git("tag", "v0.82.1", release_commit) + return release_commit + + def test_a_release_commit_takes_each_test_job_from_its_last_pass(self): + pick = self.repo.commit("fix: a macOS fix", source="TablePro/Fix.swift") + release_commit = self.patch_after() + both = tuple(version.TEST_JOBS) + runs, jobs, passed = self.line_runs(("cut", self.cut, both), ("pick", pick, ("macos / macOS Tests",)), + ("release", release_commit, ())) + github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0", False)], runs=runs, jobs=jobs, passed=passed) + self.assertEqual(self.check("v0.82.1", github), []) + self.assertEqual(self.diffs, [(pick, "v0.82.1", "macos"), (self.cut, "v0.82.1", "ios")]) + + def test_a_left_out_job_refuses_a_change_to_its_suite_since_it_last_passed(self): + self.repo.commit("fix: a pick whose run failed", source="TablePro/Fix.swift") + release_commit = self.patch_after() + runs, jobs, passed = self.line_runs(("cut", self.cut, tuple(version.TEST_JOBS)), + ("release", release_commit, ())) + github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0", False)], runs=runs, jobs=jobs, passed=passed) + self.assertEqual(self.check("v0.82.1", github), [ + "macos / macOS Tests ran no tests at https://ci.example/release (job: none)", + "ios / iOS and Package Tests ran no tests at https://ci.example/release (job: none)", + f"v0.82.1 changes the macos suite since {self.cut}, where macos / macOS Tests last passed on release/0.82", + ]) + + def test_a_left_out_job_needs_a_pass_at_an_ancestor_on_the_line(self): + self.repo.git("checkout", "--quiet", "-b", "side", self.base) + side = self.repo.commit("off the line") + self.repo.git("checkout", "--quiet", "main") + release_commit = self.patch_after() + both = tuple(version.TEST_JOBS) + runs, jobs, passed = self.line_runs(("side", side, both), ("cut", self.cut, ()), + ("release", release_commit, ())) + passed["release/0.82"].append(("gone", "f" * 40)) + jobs.update({("gone", job): ["success"] for job in both}) + main_runs, main_jobs = planned("main", self.base, both, branch="main") + jobs.update(main_jobs) + github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0", False)], runs={**runs, **main_runs}, jobs=jobs, + passed={**passed, "main": [("main", self.base)]}) + self.assertEqual(self.check("v0.82.1", github), [ + "macos / macOS Tests ran no tests at https://ci.example/release (job: none)", + "ios / iOS and Package Tests ran no tests at https://ci.example/release (job: none)", + "no earlier CI push run on release/0.82 ran macos / macOS Tests at an ancestor of v0.82.1", + "no earlier CI push run on release/0.82 ran ios / iOS and Package Tests at an ancestor of v0.82.1", + ]) + self.assertEqual(github.passed_queries, ["release/0.82"]) + self.assertEqual(self.diffs, []) + + def test_a_failed_test_job_beside_a_left_out_one_is_not_excused(self): + release_commit = self.patch_after() + runs, jobs, passed = self.line_runs(("cut", self.cut, tuple(version.TEST_JOBS)), + ("release", release_commit, ())) + del jobs[("release", "macos")] + jobs[("release", "macos / macOS Tests")] = ["skipped"] + github = FakeGitHub(releases=[("v0.81.0", False), ("v0.82.0", False)], runs=runs, jobs=jobs, passed=passed) + self.assertEqual(self.check("v0.82.1", github), [ + "macos / macOS Tests ran no tests at https://ci.example/release (job: skipped)", + "ios / iOS and Package Tests ran no tests at https://ci.example/release (job: none)", + ]) + self.assertEqual(github.passed_queries, []) + def test_a_run_of_another_workflow_does_not_count(self): self.repo.git("tag", "v0.82.0-beta.1", self.cut) old = version.Run("old", ".github/workflows/macos-tests.yml", "completed", "success", "https://ci.example/old") @@ -541,6 +633,7 @@ def run(command, **_): calls.append(command) path = command[command.index("--jq") - 1] outputs = { + "/workflows/ci.yml/runs?": "21\tabc\n22\tdef\n", "/releases": "v0.82.0\tfalse\nplugin-oracle-v1.0.0\ttrue\n", "/runs?": ("11\t.github/workflows/ci.yml\tcompleted\tsuccess\thttps://ci.example/11\n" "12\t.github/workflows/ci.yml\tin_progress\t\thttps://ci.example/12\n"), @@ -556,8 +649,12 @@ def run(command, **_): version.Run("12", ".github/workflows/ci.yml", "in_progress", "", "https://ci.example/12"), ]) self.assertEqual(github.job_conclusions("11", "macos / macOS Tests"), ["success", ""]) + self.assertEqual(github.passed_push_runs("release/0.83"), [("21", "abc"), ("22", "def")]) self.assertIn("--paginate", calls[0]) self.assertIn("--paginate", calls[2]) + self.assertIn("--paginate", calls[3]) + self.assertIn("repos/Owner/Repo/actions/workflows/ci.yml/runs?branch=release/0.83&event=push&status=success" + "&per_page=100", calls[3]) self.assertIn('select(.name == "macos / macOS Tests")', calls[2][-1]) self.assertIn("repos/Owner/Repo/actions/runs?head_sha=abc&event=push&branch=release/0.83&per_page=100", calls[1]) diff --git a/apple/scripts/release/version.py b/apple/scripts/release/version.py index a77fefb478..26cd5e9805 100755 --- a/apple/scripts/release/version.py +++ b/apple/scripts/release/version.py @@ -22,8 +22,8 @@ GRAMMAR = re.compile(r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-beta\.(0|[1-9][0-9]*))?") CI_WORKFLOW = ".github/workflows/ci.yml" -# A job of a called workflow is named " / ". -TEST_JOBS = ("macos / macOS Tests", "ios / iOS and Package Tests") +# A job of a called workflow is named " / ". The plan runs each when its suite changed. +TEST_JOBS = {"macos / macOS Tests": "macos", "ios / iOS and Package Tests": "ios"} DETECTOR = "scripts/ci/detect-changed-paths.sh" SHIPPED = "shipped" FRAGMENTS = "changelog.d/" @@ -165,6 +165,12 @@ def push_runs(self, sha: str, branch: str) -> List[Run]: ".workflow_runs[] | [.id, .path, .status, .conclusion // \"\", .html_url] | @tsv") return [Run(*line.split("\t")) for line in lines if line] + def passed_push_runs(self, branch: str) -> List[Tuple[str, str]]: + lines = self._api(f"repos/{self.repository}/actions/workflows/{Path(CI_WORKFLOW).name}/runs?branch={branch}" + "&event=push&status=success&per_page=100", + ".workflow_runs[] | [.id, .head_sha] | @tsv", paginate=True) + return [(run_id, sha) for run_id, sha in (line.split("\t") for line in lines if line)] + def job_conclusions(self, run_id: str, job: str) -> List[str]: return self._api(f"repos/{self.repository}/actions/runs/{run_id}/jobs?filter=latest&per_page=100", f".jobs[] | select(.name == \"{job}\") | .conclusion // \"\"", paginate=True) @@ -301,25 +307,60 @@ def fragment_errors(paths: List[str], version: Version) -> List[str]: return [f"{version.tag} still has {', '.join(fragments)} in {FRAGMENTS}; assemble again on the line head"] -def run_errors(github: GitHub, run: Run) -> List[str]: +# The plan skips the ci.yml job that calls a test job, so the test job itself never appears in the run. +def left_out(github: GitHub, run: Run, job: str) -> bool: + return github.job_conclusions(run.id, job.split(" / ")[0]) == ["skipped"] + + +# A job the plan left out counts when its own suite has not changed since it last passed on the line. Not shipped: a +# macOS-only pick leaves iOS last passing before it, and no test job reads the release scripts in shipped. +def untested_errors(git: Git, github: GitHub, changed: Callable[[str, str, str], bool], tag: str, commit: str, + branch: str, jobs: List[str]) -> List[str]: + ancestors = [(run_id, sha) for run_id, sha in github.passed_push_runs(branch) + if sha != commit and git.exists(sha) and git.is_ancestor(sha, commit)] + errors = [] + for job in jobs: + suite = TEST_JOBS[job] + passed = next((sha for run_id, sha in ancestors if github.job_conclusions(run_id, job) == ["success"]), None) + if passed is None: + errors.append(f"no earlier CI push run on {branch} ran {job} at an ancestor of {tag}") + elif changed(passed, tag, suite): + errors.append(f"{tag} changes the {suite} suite since {passed}, where {job} last passed on {branch}") + return errors + + +def run_errors(github: GitHub, run: Run, untested: Callable[[List[str]], List[str]]) -> List[str]: if run.status != "completed": return [f"CI is {run.status} at {run.url}: re-run Verify once it is green"] if run.conclusion != "success": return [f"CI ended '{run.conclusion}' at {run.url}"] errors = [] + skipped = [] for job in TEST_JOBS: conclusions = github.job_conclusions(run.id, job) - if conclusions != ["success"]: - errors.append(f"{job} ran no tests at {run.url} (job: {', '.join(conclusions) or 'none'})") - return errors + if conclusions == ["success"]: + continue + errors.append(f"{job} ran no tests at {run.url} (job: {', '.join(conclusions) or 'none'})") + if not conclusions and left_out(github, run, job): + skipped.append(job) + if not errors or len(skipped) < len(errors): + return errors + reasons = untested(skipped) + return errors + reasons if reasons else [] -def push_run_errors(github: GitHub, sha: str, line: str) -> List[str]: +def push_run_errors(git: Git, github: GitHub, changed: Callable[[str, str, str], bool], tag: str, + line: str) -> List[str]: branch = f"release/{line}" - runs = [run for run in github.push_runs(sha, branch) if run.path == CI_WORKFLOW] + commit = git.commit(tag) + runs = [run for run in github.push_runs(commit, branch) if run.path == CI_WORKFLOW] if not runs: - return [f"no CI push run on {branch} at {sha}"] - return [error for run in runs for error in run_errors(github, run)] + return [f"no CI push run on {branch} at {commit}"] + + def untested(jobs: List[str]) -> List[str]: + return untested_errors(git, github, changed, tag, commit, branch, jobs) + + return [error for run in runs for error in run_errors(github, run, untested)] def check(tag: str, git: Git, github: GitHub, changed: Callable[[str, str, str], bool]) -> List[str]: @@ -340,7 +381,7 @@ def check(tag: str, git: Git, github: GitHub, changed: Callable[[str, str, str], errors += fragment_errors(git.list_tree(tag, FRAGMENTS), version) if errors: return errors - return push_run_errors(github, git.commit(tag), version.line) + return push_run_errors(git, github, changed, tag, version.line) def main(argv: Optional[List[str]] = None) -> int: diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 62717b636e..14493583fe 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -593,7 +593,7 @@ Left for a machine with a GPU and a HiDPI screen: the GL and Vulkan renderers on `.github/workflows/core.yml` builds the core crates for Windows x64 and Arm64 and macOS, runs the unit tests of `tablepro-core` and the SQL Server driver there, with the driver's `kerberos` feature on macOS, where it links the system's GSS framework, and runs the SSPI login tests above. -`.github/workflows/ci.yml` calls `build-linux.yml` for a pull request, a merge group and a push to `main`, never on a `release/**` line: the baseline, which is the fast job, the default server releases, the x86_64 Flatpak Devel build with its tests, translations and supply chain. A change to a dependency, the toolchain or a manifest adds the minimum Rust version, the aarch64 Flatpak and the advisory database. A change to what the packages declare or how they build also calls `linux-packages.yml`, x86_64 only. A change to Markdown alone, other than `CHANGELOG.md` and `docs/regression-tests.md`, runs only the Documentation links job. `.github/workflows/nightly.yml` runs the rest every night: every other server release, the minimum Rust version, both Flatpak architectures, paths with spaces, the advisory database, and the packages with the Flatpak repository, with arm64 on Sundays. The required check is `CI` (`.claude/rules/ci.md`). +`.github/workflows/ci.yml` calls `build-linux.yml` for a pull request and a push to `main`, never on a `release/**` line: the baseline, which is the fast job, the default server releases, the x86_64 Flatpak Devel build with its tests, translations and supply chain. A change to a dependency, the toolchain or a manifest adds the minimum Rust version, the aarch64 Flatpak and the advisory database. A change to what the packages declare or how they build also calls `linux-packages.yml`, x86_64 only. A change to Markdown alone, other than `CHANGELOG.md` and `docs/regression-tests.md`, runs only the Documentation links job. `.github/workflows/nightly.yml` runs the rest every night: every other server release, the minimum Rust version, both Flatpak architectures, paths with spaces, the advisory database, and the packages with the Flatpak repository, with arm64 on Sundays. The required check is `CI` (`.claude/rules/ci.md`). ## Coverage diff --git a/scripts/ci/detect-changed-paths.sh b/scripts/ci/detect-changed-paths.sh index ad1ea12262..67a7833ccd 100755 --- a/scripts/ci/detect-changed-paths.sh +++ b/scripts/ci/detect-changed-paths.sh @@ -2,19 +2,19 @@ set -euo pipefail # Prints "true" when a suite has to run for the changes from to , "false" when it can skip. -# Usage: detect-changed-paths.sh [--base ] [--head ] [--ref ] +# Usage: detect-changed-paths.sh [--base ] [--head ] # Suites: macos, plugins, ios, oracle, hana, release, shipped, swiftlint, linux, linux-docs, linux-deps, packages, core, # windows. +# No suite reads apple/CHANGELOG.md or apple/changelog.d/, so a release PR runs Lint only and its tag takes +# the tests of an earlier commit on the line (apple/scripts/release/version.py, untested_errors). BASE="" HEAD_REV="HEAD" -REF="" SUITE="" while [ $# -gt 0 ]; do case "$1" in --base) BASE="${2:-}"; shift 2 ;; --head) HEAD_REV="${2:-}"; shift 2 ;; - --ref) REF="${2:-}"; shift 2 ;; -*) echo "detect-changed-paths.sh: unknown flag $1" >&2; exit 2 ;; *) SUITE="$1"; shift ;; esac @@ -135,14 +135,6 @@ case "$SUITE" in exit 2 ;; esac -# Every suite runs on a release line: a tag's verify needs its commit's own runs, and a release PR -# changes only the changelog. -case "$REF" in - refs/heads/release/*) - echo "true" - exit 0 ;; -esac - # A new branch has no previous commit, so it is compared with the point where it left main. if [ -n "$BASE" ] && [ -z "${BASE//0/}" ]; then BASE="$(git merge-base "$HEAD_REV" origin/main 2> /dev/null || true)" diff --git a/scripts/ci/test_detect_changed_paths.py b/scripts/ci/test_detect_changed_paths.py index 8185b24ea6..6e818df03a 100644 --- a/scripts/ci/test_detect_changed_paths.py +++ b/scripts/ci/test_detect_changed_paths.py @@ -50,14 +50,12 @@ def commit(self, paths): self.git("commit", "-q", "-m", "change") return self.git("rev-parse", "HEAD") - def detect(self, suite, base=None, head=None, ref=None): + def detect(self, suite, base=None, head=None): args = ["bash", "scripts/ci/detect-changed-paths.sh"] if base is not None: args += ["--base", base] if head is not None: args += ["--head", head] - if ref is not None: - args += ["--ref", ref] result = subprocess.run(args + [suite], cwd=self.repo, env=self.env, capture_output=True, text=True) return result.returncode, result.stdout.strip() @@ -499,28 +497,20 @@ def test_shipped_is_the_release_build_without_its_ci(self): with self.subTest(path=path): self.assertEqual(self.verdict_after("shipped", path), "false") - def test_a_release_branch_runs_every_suite(self): + def test_a_release_pull_request_runs_no_suite(self): # The ios suite reads its shared paths out of the iOS project spec. spec = self.repo / "apple/TableProMobile/project.yml" spec.parent.mkdir(parents=True) spec.write_text("sources:\n - path: ../Packages/TableProCore\n", encoding="utf-8") - base = self.commit([]) - self.git("update-ref", "refs/remotes/origin/main", base) + base = self.commit(["apple/changelog.d/3500.fixed.md"]) + self.git("rm", "-q", "apple/changelog.d/3500.fixed.md") self.commit(["apple/CHANGELOG.md"]) for suite in [ - "macos", "plugins", "ios", "oracle", "hana", "release", "swiftlint", "linux", "linux-docs", "linux-deps", - "packages", "core", "windows", + "macos", "plugins", "ios", "oracle", "hana", "release", "shipped", "swiftlint", "linux", "linux-docs", + "linux-deps", "packages", "core", "windows", ]: with self.subTest(suite=suite): self.assertEqual(self.detect(suite, base), (0, "false")) - self.assertEqual(self.detect(suite, base, ref="refs/heads/release/0.83"), (0, "true")) - self.assertEqual(self.detect(suite, ZERO, ref="refs/heads/release/0.83"), (0, "true")) - - def test_main_and_pull_request_refs_keep_the_path_filter(self): - self.commit(["apple/CHANGELOG.md"]) - for ref in ["refs/heads/main", "refs/pull/12/merge", "refs/heads/backport/12-0.83"]: - with self.subTest(ref=ref): - self.assertEqual(self.detect("macos", self.seed, ref=ref), (0, "false")) def test_head_ends_the_diff(self): change = self.commit(["apple/TablePro/App.swift"])