diff --git a/.github/qualify-rocm/FEED.md b/.github/qualify-rocm/FEED.md index ce67f4584108..f5ce39034477 100644 --- a/.github/qualify-rocm/FEED.md +++ b/.github/qualify-rocm/FEED.md @@ -6,13 +6,20 @@ external dashboard can poll. This file is that contract. A copy is kept on the branch as `qualification/SCHEMA.md`, refreshed on every publish, so the data and its description never drift apart. Treat it as versioned (`schema_version`). +The layout and field names match the +[lemonade-sdk/llamacpp-rocm feed](https://github.com/lemonade-sdk/llamacpp-rocm/blob/qualification-data/qualification/SCHEMA.md), +so one dashboard can read both. Where the two repositories' archives really +differ, the values differ too; +[Differences from lemonade-sdk/llamacpp-rocm](#differences-from-lemonade-sdkllamacpp-rocm) +lists every one. + ## Why a branch and not the releases The records are already attached to each release, but that copy cannot serve a dashboard: -- A release only exists when something shipped. The dashboard needs every run, - including the ones whose ROCm archives were withheld. +- A release only holds what shipped. The dashboard needs every run, including + the ones whose ROCm archives were withheld. - Reading history out of releases means one API call per release, and the Releases API allows only 60 unauthenticated requests an hour. @@ -41,73 +48,160 @@ The dashboard's one regular fetch is `.../qualification/index.json`. ## One row per archive A release run builds six ROCm archives — Windows and Linux for each of ROCm -7.14, 10.0 and 10.1 — and each is qualified on its own. Each one gets its own -row, because an archive is what a user downloads and what a bug report names. -Rows from the same run share `run_id` and `tag`; group on those to show a run as -a whole. +7.14, 10.0 and 10.1 (`ROCM_VERSIONS` in `release.yml`) — and each is qualified +on its own. Each one gets its own row, because an archive is what a user +downloads and what a bug report names. Rows from the same run share `run_id`; +group on it to show a run as a whole. + +Every archive the run was meant to build gets a row, including one whose build +failed (see [Placeholder rows](#placeholder-rows)). A run therefore always has +six rows. ## `build_id` and re-runs ``` -build_id = "--rocm-" e.g. 39775718636-win-rocm-10.1 +build_id = "--rocm-" e.g. 41550569320-win-rocm-10.1 ``` -Unique per archive per run. Publishing is an upsert keyed on `build_id`: -re-running the release job replaces that run's rows and files instead of adding -new ones. `run_attempt` records which attempt the row came from. +Unique per archive per run. Publishing is an upsert keyed on `build_id`. A +re-run of a run keeps its `run_id`, so it replaces that run's rows and its +`reports/` files instead of adding new ones; the latest attempt's values win. +`run_attempt` records which attempt published the rows. A new run has a new +`run_id` and adds new rows. ## Which runs appear -Runs of `.github/workflows/release.yml` that reach its `release` job: the -scheduled runs, and manual runs with `create_release` ticked. Pass or fail, -every record that job writes is published. +Runs of `.github/workflows/release.yml` whose `release` job runs: the scheduled +runs, and manual runs with `create_release` ticked. Pass or fail, every one of +them is published. The records normally come from the `release` job; if it +stopped before writing them, `publish-qualification` builds them from the +Tier 0 and Tier 1 reports instead. Not in the feed: -- **Pull requests**, and manual runs without `create_release`. They run the - checks, but never reach the step that turns them into records. -- **An archive whose build failed.** There was nothing to check, so there is no - record and no row. A run normally has six rows; fewer means a build leg - failed (see `run_url`). -- **A run that failed before qualifying anything**, for example because no - artifact at all was built. +- **Pull requests**, and manual runs without `create_release`. +- **A run that was cancelled.** +- **A run whose `resolve-upstream` job failed.** No llama.cpp commit was + pinned, so nothing was built. -## `index.json` +## Example `index.json` ```jsonc { - "schema_version": 1, + "schema_version": 2, "repo": "lemonade-sdk/llama.cpp", - "updated_at": "2026-10-22T14:51:14Z", // generated_at of the newest row - "builds": [ // newest run first + "updated_at": "2026-11-03T03:40:49Z", // the most recent generated_at of any row + "builds": [ // newest run (highest run_id) first + // ... the other rows of run 41550569320 ... + + // A placeholder: this archive's build failed, so there was nothing to check. + { + "build_id": "41550569320-win-rocm-10.1", + "tag": "b12062", // llama.cpp build number; set even though nothing shipped + "upstream_sha": "29c720eaf72fcf8e4e61a56a88b994d92668261d", + "artifact": "llama-b12062-bin-win-rocm-10.1-x64.zip", // the name it would have had + "os": "win", + "rocm_version": "10.1", + "gfx_target": null, + "hardware_validated": false, + "gfx": { "requested": 0, "present": 0, "missing": [] }, // no archive, so nothing was audited + "generated_at": "2026-11-03T03:40:49Z", + "run_id": "41550569320", + "run_attempt": 1, + "run_url": "https://github.com/lemonade-sdk/llama.cpp/actions/runs/41550569320", + "event": "schedule", + "build_result": "fail", // the windows-rocm (10.1.0) job failed + "tests": { "passed": 0, "failed": 0, "warned": 0, "skipped": 0, "missing": 15, "not_applicable": 2, "total": 17 }, + "qualified": false, + "version_qualified": false, + "published": false, + "overall": "fail", + "tier0": "missing", // should have run, but there was no archive + "tier1": "missing", + "blocked_by": ["build failed"], + "blocking_tests": [], + "results": { + "T0.1": "missing", "T0.2": "missing", "T0.3": "missing", "T0.4": "missing", + "T0.5": "missing", "T0.6": "missing", "T0.7": "missing", "T0.8": "missing", + "T0.9": "missing", "T0.10": "not_applicable", "T0.11": "not_applicable", "T0.12": "missing", + "T1.1": "missing", "T1.2": "missing", "T1.3": "missing", "T1.4": "missing", "T1.5": "missing" + }, + "report_url": "reports/41550569320-win-rocm-10.1.json" + }, + + // The Linux archive of the same version, from the same run: it passed every + // check, but did not ship, because its Windows counterpart above never built. + { + "build_id": "41550569320-ubuntu-rocm-10.1", + "tag": "b12062", + "upstream_sha": "29c720eaf72fcf8e4e61a56a88b994d92668261d", + "artifact": "llama-b12062-bin-ubuntu-rocm-10.1-x64.tar.gz", + "os": "ubuntu", + "rocm_version": "10.1", + "gfx_target": "gfx1151", + "hardware_validated": true, + "gfx": { "requested": 27, "present": 27, "missing": [] }, + "generated_at": "2026-11-03T03:40:49Z", + "run_id": "41550569320", + "run_attempt": 1, + "run_url": "https://github.com/lemonade-sdk/llama.cpp/actions/runs/41550569320", + "event": "schedule", + "build_result": "pass", + "tests": { "passed": 14, "failed": 0, "warned": 0, "skipped": 1, "missing": 0, "not_applicable": 2, "total": 17 }, + "qualified": true, // this archive passed its own checks + "version_qualified": false, // its Windows counterpart has no record + "published": false, // so ROCm 10.1 was withheld from b12062 + "overall": "pass", + "tier0": "pass", + "tier1": "pass", + "blocked_by": [], + "blocking_tests": [], + "results": { + "T0.1": "pass", "T0.2": "pass", "T0.3": "pass", "T0.4": "pass", + "T0.5": "pass", "T0.6": "pass", "T0.7": "pass", "T0.8": "pass", + "T0.9": "skip", "T0.10": "not_applicable", "T0.11": "not_applicable", // dropped before the post-merge checks + "T0.12": "pass", + "T1.1": "pass", "T1.2": "pass", "T1.3": "pass", "T1.4": "pass", "T1.5": "pass" + }, + "report_url": "reports/41550569320-ubuntu-rocm-10.1.json" + }, + + // ... the rows of earlier runs ... + + // An archive that passed and shipped in release b12003. { - "build_id": "39775718636-win-rocm-10.1", - "tag": "b11819", // llama.cpp build number = the release tag - "upstream_sha": "0693dbfb915f33233818373b5f6969396658d02c", - "artifact": "llama-b11819-bin-win-rocm-10.1-x64.zip", + "build_id": "40977810710-win-rocm-10.1", + "tag": "b12003", // llama.cpp build number = the release tag + "upstream_sha": "36a0e2ffc0b352b774f8d71c43668b75b3061ef5", + "artifact": "llama-b12003-bin-win-rocm-10.1-x64.zip", "os": "win", // win | ubuntu "rocm_version": "10.1", // currently 7.14 | 10.0 | 10.1 - "gfx_target": "gfx1151", // the target tested on real hardware, or null - "hardware_validated": true, // did Tier 1 run on a GPU + "gfx_target": "gfx1151", // the one gfx target Tier 1 ran on; null when Tier 1 did not run + "hardware_validated": true, // Tier 1 ran on a real GPU (see tier1 for how it went) "gfx": { "requested": 20, "present": 20, "missing": [] }, - "generated_at": "2026-10-22T14:51:14Z", // ISO-8601 UTC, the time axis - "run_id": "39775718636", - "run_attempt": 1, - "run_url": "https://github.com/lemonade-sdk/llama.cpp/actions/runs/39775718636", + "generated_at": "2026-10-30T03:47:31Z", // ISO-8601 UTC, when this row's record was written + "run_id": "40977810710", + "run_attempt": 1, // the attempt that published this row + "run_url": "https://github.com/lemonade-sdk/llama.cpp/actions/runs/40977810710", "event": "schedule", // schedule | workflow_dispatch - "tests": { "passed": 13, "failed": 0, "warned": 1, "skipped": 3, "missing": 0, "total": 17 }, - "qualified": true, // this archive passed its checks - "version_qualified": true, // both archives of this ROCm version qualified - "overall": "warn", // pass | warn | fail - "blocked_by": [], // e.g. ["tier0=fail"] - "blocking_tests": [], // e.g. ["T0.3"] + "build_result": "pass", // pass | fail | cancelled | skipped | unknown + "tests": { "passed": 13, "failed": 0, "warned": 1, "skipped": 3, "missing": 0, "not_applicable": 0, "total": 17 }, + "qualified": true, // passed every gating check required of it: Tier 0, plus + // Tier 1 if it ran. Not whether it shipped. + "version_qualified": true, // both archives of ROCm 10.1 exist and qualified + "published": true, // this run attached it to release b12003 + "overall": "warn", // pass | warn | fail; fail exactly when qualified is false + "tier0": "warn", // pass | warn | fail | missing + "tier1": "pass", // pass | warn | fail | missing + "blocked_by": [], // why it did not qualify, e.g. ["tier0=fail"] + "blocking_tests": [], // the gating tests that failed, e.g. ["T0.3"] "results": { "T0.1": "warn", "T0.2": "pass", "T0.3": "pass", "T0.4": "pass", "T0.5": "pass", "T0.6": "skip", "T0.7": "skip", "T0.8": "pass", "T0.9": "skip", "T0.10": "pass", "T0.11": "pass", "T0.12": "pass", "T1.1": "pass", "T1.2": "pass", "T1.3": "pass", "T1.4": "pass", "T1.5": "pass" }, - "report_url": "reports/39775718636-win-rocm-10.1.json" + "report_url": "reports/40977810710-win-rocm-10.1.json" } ] } @@ -116,54 +210,154 @@ Not in the feed: ### Field notes - **`results`** has one entry per test: T0.1–T0.12 and T1.1–T1.5, as listed in - [README.md](README.md). Values are `pass | fail | warn | skip | missing`. A - test that did not run at all is `missing`. A test id added to the suite later - appears here as soon as it runs, so the failure-frequency and heatmap views - can be built from `index.json` alone. + [README.md](README.md). A test id added to the suite later appears here as + soon as it runs, so the failure-frequency and heatmap views can be built from + `index.json` alone. The values: + - `pass`: the check ran and passed. + - `fail`: the check ran and found a problem. It blocks the archive only if + the test is gating; T0.9, T1.4 and T1.5 never block. + - `warn`: the check ran and found something worth recording that does not + block, e.g. upstream test programs shipped in the archive (T0.1). + - `skip`: the check ran but could not apply, e.g. T0.6 and T0.7 on Windows + (Linux-only checks). + - `missing`: the check should have run for this archive and did not. For + Tier 1 this does not block the archive (see `qualified`), but it is still a + gap: the archive was not tested on a GPU. + - `not_applicable`: the check was never going to run for this archive. + T0.10 and T0.11 check the final archive inside the release job, and only + run on archives still in the release at that point. An archive dropped + before then — because the other archive of its ROCm version, or the + Windows CPU backend, was missing — or never built has them + `not_applicable`. - **`tests`** counts the values in `results`, so - `passed + failed + warned + skipped + missing == total` always holds. -- **`missing` is not `failed`.** This is the one deliberate difference from the - vllm-rocm feed, which counts missing tests as failed. Here, Tier 1 needs a - machine with an AMD GPU; when none ran it, all five T1 tests are `missing`, - `hardware_validated` is `false`, and the archive can still pass - (`qualified: true`). Showing that as a failure would paint a shipped archive - red. Use `qualified`, not the counts, to decide pass or fail. -- **`qualified`** is the verdict for this one archive: Tier 0 ran and did not - fail, and Tier 1 did not fail if it ran. A `warn` does not block, and tests - marked non-gating in README.md never affect it. -- **`version_qualified`** is the verdict the release acts on. A ROCm version is - withheld from the release unless its Windows **and** Linux archives both - qualify, so an archive can have `qualified: true` and still not ship because - its counterpart failed. `null` means that run did not record the decision. - (A version can also be withheld because one of its archives failed to build; - that archive then has no row at all.) -- **`blocked_by`** gives the reason per tier (`tier0=fail`, `tier0 missing`, - `tier1=fail`); **`blocking_tests`** lists the release-blocking tests that - failed. Both are empty when the archive qualified. + `passed + failed + warned + skipped + missing + not_applicable == total` + always holds. Use `qualified`, not the counts, to decide pass or fail. +- **`qualified`** is the verdict for this one archive: Tier 0 ran and no gating + Tier 0 check failed, and no gating Tier 1 check failed if Tier 1 ran. A + `warn` does not block, and non-gating tests never affect it. It does not say + whether the archive shipped (`published`) or ran on a GPU (`tier1`, + `hardware_validated`). +- **`version_qualified`** is specific to this repository. It is `true` when + both archives of this ROCm version — Windows **and** Linux — exist and + qualified. A version ships only as a pair, so an archive can have + `qualified: true` and `version_qualified: false` because its counterpart + failed a check or never built. `null` means the run recorded no decision. +- **`published`** says whether this run attached this archive to the release + `tag`. It can be `false` even when `version_qualified` is `true`: the Windows + ROCm archive also needs the Windows CPU backend merged into it, and an upload + can fail. Two runs built from the same upstream commit share a tag and a + release; the later one replaces that release's files with its own, so + `published` describes what this run shipped, not what the release holds + today. +- **`overall`** is `fail` exactly when `qualified` is `false`. Otherwise it is + the worst result of the gating tests that ran: `warn` if any of them warned, + else `pass`. +- **`tier0` / `tier1`** are each tier's result: `pass`, `warn` or `fail` when + the tier ran, and `missing` when it did not. +- **`tag`** is the llama.cpp build number this run pinned, which is also the + release tag. Every row of a run has it, whether or not anything shipped. +- **`build_result`** is how the archive's build job (`ubuntu-24-rocm` or + `windows-rocm`) ended: `pass`, `fail` (including a timeout), `cancelled`, + `skipped`, or `unknown` when the run's job list could not be read. The three + `windows-rocm` builds stop together: when one fails, GitHub cancels the + others, so they show `cancelled`. +- **`run_attempt`** is the attempt of the run that published the row. It is the + same on every row of a run. +- **`generated_at`** is when the row's record was written. A re-run writes it + again, later than any run that started after it, so order runs by `run_id` + (rows are already newest-run-first) rather than by `generated_at`. +- **`blocked_by`** gives the reasons the archive did not qualify. It is empty + when it qualified. The values: + - `tier0=fail`: a gating Tier 0 check failed; + - `tier0 missing`: no Tier 0 report; + - `tier1=fail`: a gating Tier 1 check failed; + - on placeholder rows only: `build failed`, `build cancelled`, `build skipped` + or `no report`. +- **`blocking_tests`** lists the gating tests that failed; empty when none did. - **`gfx`** summarises the check that every gfx target was compiled into the GPU backend (T0.3): how many were requested, how many are present, and the names of any that are missing. -- **`gfx_target` / `hardware_validated`** say which single target was actually - tested on a GPU. Every other target in `gfx.requested` was only checked for - presence, never run. Don't present those as tested. +- **`gfx_target`** is the one gfx target Tier 1 actually ran on, and `null` + when Tier 1 did not run. **`hardware_validated`** is `true` when Tier 1 ran on + a real GPU, whether or not it then passed; `tier1` gives the outcome. Every + other target in `gfx.requested` was only checked for presence, never run. + Don't present those as tested. - **`report_url`** is relative to the base URL; it holds every check's `status`, `gating`, `error` and `details`. +### Placeholder rows + +An archive with no record at all gets a placeholder row. This happens when its +build failed, so there was nothing to check, or when no check reported on it. +In a placeholder row: + +- every test is `missing` except T0.10 and T0.11, which are `not_applicable`; + `tier0` and `tier1` are `missing`; `qualified`, `version_qualified` and + `published` are `false`; and `overall` is `fail`; +- `blocked_by` follows `build_result`: `build failed`, `build cancelled`, + `build skipped`, or `no report` when the build passed or its result is + `unknown`; +- `gfx` is all zero and empty: there is no binary, and the target list lives in + the build job; +- `gfx_target` is `null` and `hardware_validated` is `false`; `tag` and + `artifact` are the ones the archive would have had. + +The first row of the [`index.json`](#indexjson) example is a placeholder. + ## `reports/.json` -The `qualification-.json` record exactly as `aggregate.py` wrote it -(`kind: "qualification_record"`): the `build` metadata, `overall`, `promoted`, -`promotion.blocked_by`, raw `counts`, the `gfx` audit, and `tiers` with every -test. The same content is attached to the release when the run published one. -Its own `build.build_id` (`@`) differs from the feed's -`build_id`; use the feed's. +The `qualification-.json` record exactly as `aggregate.py` wrote it (`kind: "qualification_record"`): +- the `build` metadata, +- `overall`, +- `promoted`, +- `promotion.blocked_by`, +- raw `counts`, +- the `gfx` audit, and +- `tiers` with every test. + +The same content is attached to the release. Its own `build.build_id` (`@`) differs from the feed's `build_id`; use the feed's. + +For a placeholder row it is a placeholder (`kind: "qualification_placeholder"`) with the same top-level fields: +- `build` metadata, +- `overall: "fail"`, +- `promoted: false`, +- `promotion.blocked_by`, +- `build_result` (in the feed's vocabulary), +- the empty `gfx` block, +- zero `counts` and +- empty `tiers`. ## `latest/-rocm-.json` The newest full record for one archive (for example `latest/win-rocm-10.1.json`), -for status tiles that want the current state without scanning the index. It is -replaced only by a newer record, so re-running an old release cannot move it -backwards. +for status tiles that want the current state without scanning the index. A +placeholder is written here too, so a tile shows a failing build instead of the +last good one. It is replaced only by a record from the same run or a newer one +(a higher `run_id`), so re-running an old run cannot move it backwards. + +## Differences from lemonade-sdk/llamacpp-rocm + +That feed has one archive per GPU family, with ROCm bundled inside; this one +has one archive per ROCm version, with ROCm installed separately by the user. +Everything else in the row means the same. + +| Field | this repository | lemonade-sdk/llamacpp-rocm | +|---|---|---| +| `schema_version` | `2` | `1` (same row shape) | +| `build_id` | `--rocm-` | `--rocm-` | +| rows per run | 6 | 14 | +| `tag` | llama.cpp build number; set on every row | that repository's release tag; `null` unless `published` | +| `artifact` | always the release name | the untagged CI name unless `published` | +| `os` | `win` \| `ubuntu` | `windows` \| `ubuntu` | +| `rocm_version` | `7.14` \| `10.0` \| `10.1`, one per archive | full nightly, e.g. `10.3.0a20261102`, same for the whole run | +| `gfx_family` | — | the GPU family the archive serves | +| `version_qualified` | both archives of a ROCm version exist and qualified | not present (a release there is all or nothing) | +| `tier1` | Tier 1 runs on every archive; `missing` does not block | Tier 1 runs only on gfx1151 (`not_applicable` elsewhere); `missing` blocks | +| `not_applicable` | T0.10, T0.11 on an archive dropped before the release checks | T1.x outside gfx1151; T0.11 on a run with no release | +| `blocked_by` | no `tier1 missing`, no `T0.11 missing` | also `tier1 missing`, `T0.11 missing` | +| `results` keys | T0.1–T0.12, T1.1–T1.5 | T0.1–T0.3, T0.5–T0.9, T0.11, T0.13, T1.1–T1.6 | +| placeholder `gfx` | all zero | requested targets, all in `missing` | +| `latest/` file | `-rocm-.json` | `-rocm-.json` | ## Versioning and growth @@ -171,6 +365,15 @@ Bump `schema_version` on any breaking change to `index.json` or the row shape. Adding a field is not breaking. The dashboard should read `schema_version` and show a notice, rather than break, on a version it does not know. -`index.json` grows by about six rows per run, roughly a dozen a day. That is +Version 2 replaced version 1 because the meaning of existing fields changed: +`version_qualified` now also counts an archive that never built, `overall` is +`fail` whenever `qualified` is `false`, and `missing` now only means "should +have run". It also added `published`, `build_result`, `tier0`, `tier1`, the +`not_applicable` result, and placeholder rows, and orders runs by `run_id`. A +version 1 feed is not mixed with version 2 rows: the first version 2 publish +starts a fresh `index.json`, `reports/` and `latest/`. The version 1 files stay +in the branch's git history. + +`index.json` grows by six rows per run, roughly a dozen a day. That is comfortable to poll for a long time, especially compressed. If it ever is not, older rows will move into per-month files under a new `schema_version`. diff --git a/.github/qualify-rocm/README.md b/.github/qualify-rocm/README.md index 2f52ef5c5650..26515f56d28c 100644 --- a/.github/qualify-rocm/README.md +++ b/.github/qualify-rocm/README.md @@ -221,8 +221,10 @@ the CI logs have expired. The release job's records are also published, pass or fail, to the `qualification-data` branch by `publish_qualification.py`, so a dashboard can -follow the history of every run without reading each release. The layout and -every field are described in [FEED.md](FEED.md). +follow the history of every run without reading each release. An archive whose +build failed gets a placeholder row, so a missing archive shows up as a failure +rather than a gap. The layout and every field are described in +[FEED.md](FEED.md). ## Out of scope diff --git a/.github/qualify-rocm/aggregate.py b/.github/qualify-rocm/aggregate.py index 8fbc60866413..ba5c19d74f46 100644 --- a/.github/qualify-rocm/aggregate.py +++ b/.github/qualify-rocm/aggregate.py @@ -11,12 +11,19 @@ a consumer installs exactly one of them, so a problem in one is no reason to withhold another. +With --operating-systems and --rocm-versions, an archive that should exist but +has no report at all (usually because its build failed) also blocks its version: +the version cannot ship without it, so its other archive must not be reported +as publishable either. + Within one archive: - the static checks must be present and must pass. - the GPU checks must pass IF they ran. An archive with no GPU report is recorded as not hardware-validated rather than being failed -- each build covers 20 or more gfx targets and only one of them has test hardware, so the record states which is which instead of implying they are all equal. +An archive that is not promoted always has overall "fail", even when the reason +is a tier that never reported rather than a failed check. With --fail-on-no-promote the exit code is non-zero if any version failed. The release job uses that to drop only the affected ROCm versions, leaving every @@ -25,6 +32,7 @@ Usage: python3 aggregate.py --fragments-dir ./fragments --tag bNNNNN \ --records-dir ./qualification --output-summary summary.json \ + --operating-systems ubuntu,win --rocm-versions 7.14,10.0,10.1 \ --fail-on-no-promote """ @@ -53,6 +61,11 @@ def load_fragments(fragments_dir): return fragments +def split_list(text): + """Split a comma or whitespace separated list into clean names.""" + return [name for part in (text or "").split(",") for name in part.split() if name] + + def group_key(fragment): build = fragment.get("build", {}) return (build.get("os") or "?", build.get("rocm_version") or "?") @@ -109,7 +122,7 @@ def gfx_summary(record): } -def write_step_summary(records, promote_all): +def write_step_summary(records, promote_all, missing=()): path = os.environ.get("GITHUB_STEP_SUMMARY") if not path: return @@ -128,6 +141,8 @@ def write_step_summary(records, promote_all): f"| `{build.get('artifact')}` | {t0} | {t1} | " f"{len(gfx['present_in_binary'])}/{len(gfx['requested'])} | {hw} | " f"{'yes' if record['promoted'] else 'NO'} |") + for os_name, rocm_version in missing: + lines.append(f"| {os_name} ROCm {rocm_version} | no report | | | | NO |") problems = [] for record in records: @@ -154,16 +169,28 @@ def main(): help="write one qualification-.json per archive here") parser.add_argument("--output-summary", default=None, help="write the combined decision here") + parser.add_argument("--operating-systems", default=None, + help="the operating systems every ROCm version is built for, e.g. ubuntu,win") + parser.add_argument("--rocm-versions", default=None, + help="the ROCm versions this run builds, e.g. 7.14,10.0,10.1") parser.add_argument("--fail-on-no-promote", action="store_true") args = parser.parse_args() + # Every archive the run should have produced. Without the lists, only the archives that reported can be judged. + expected = [] + if args.operating_systems and args.rocm_versions: + expected = [(o, v) for v in split_list(args.rocm_versions) + for o in split_list(args.operating_systems)] + fragments = load_fragments(args.fragments_dir) if not fragments: print(f"::error::No tier fragments found in {args.fragments_dir}. The " "qualification jobs produced nothing, so nothing can be vouched for.") - if args.fail_on_no_promote: - sys.exit(1) - return + # With no expected list there is no verdict to write, and the release job then withholds every ROCm version. + if not expected: + if args.fail_on_no_promote: + sys.exit(1) + return grouped = {} for fragment in fragments: @@ -178,6 +205,9 @@ def main(): promote, reasons = decide(record) record["promoted"] = promote record["promotion"] = {"blocked_by": reasons} + # A tier that never reported fails no test, so the test rollup alone could still say "pass". + if not promote: + record["overall"] = report.STATUS_FAIL record["gfx"] = gfx_summary(record) records.append(record) @@ -207,18 +237,26 @@ def main(): for record in records: version = record["build"].get("rocm_version") or "?" by_version.setdefault(version, []).append(record) - blocked = sorted(v for v, group in by_version.items() - if not all(r["promoted"] for r in group)) - promote_all = not blocked - - write_step_summary(records, promote_all) + blocked = {v for v, group in by_version.items() + if not all(r["promoted"] for r in group)} + # An archive with no report at all cannot ship, so neither can its version. + missing = [(o, v) for o, v in expected if (o, v) not in grouped] + for os_name, rocm_version in missing: + print(f"::error::No qualification report at all for {os_name} ROCm {rocm_version}.") + blocked.add(rocm_version) + blocked = sorted(blocked) + promote_all = bool(records) and not blocked + + write_step_summary(records, promote_all, missing) if args.output_summary: + os.makedirs(os.path.dirname(os.path.abspath(args.output_summary)), exist_ok=True) with open(args.output_summary, "w", encoding="utf-8") as handle: json.dump({ "tag": args.tag, "all_versions_publishable": promote_all, "blocked_rocm_versions": blocked, + "missing_archives": [f"{o}-rocm-{v}" for o, v in missing], "artifacts": [{"artifact": r["build"].get("artifact"), "rocm_version": r["build"].get("rocm_version"), "publishable": r["promoted"], diff --git a/.github/qualify-rocm/publish_qualification.py b/.github/qualify-rocm/publish_qualification.py index 3b8cb2ce55eb..c0595261dbd2 100644 --- a/.github/qualify-rocm/publish_qualification.py +++ b/.github/qualify-rocm/publish_qualification.py @@ -1,29 +1,34 @@ #!/usr/bin/env python3 """ -Publish the qualification records from one release run to the dashboard feed -on the `qualification-data` branch. +Publish the qualification records from one release run to the dashboard feed on the `qualification-data` branch. -Reads the records `aggregate.py` wrote (one qualification-.json per -archive, plus qualification-summary.json) and upserts them into --data-dir, -which is the `qualification/` folder of a checkout of that branch: +Reads the records `aggregate.py` wrote (one qualification-.json per archive, plus qualification-summary.json) and upserts them into --data-dir, which is the `qualification/` folder of a checkout of that branch: - index.json one compact row per archive per run, newest first + index.json one compact row per archive per run, newest run first reports/.json the full record, for drill-down latest/-rocm-.json the newest full record for that archive build_id is "--rocm-". -Publishing the same run again (a re-run of the release job) replaces its rows and files -rather than duplicating them, so this script is safe to run any number of times. +Publishing the same run again (a re-run) replaces its rows and files rather than duplicating them, so this script is safe to run any number of times. -A test the record does not contain is reported as "missing" and counted separately -- NOT folded into "failed". -A missing Tier 1 is a legitimate state: the archive was not tested on hardware, and aggregate.py lets it ship. -Counting it as a failure would show a published archive as red. +Every archive the run was meant to build gets a row. +An archive with no record (its build failed / no check ever reported on it) gets a placeholder row and a placeholder report saying so. Therefore, a failing build shows up in the feed instead of silently leaving a gap. + +A test the record does not contain is reported as either: + - "not_applicable": it is never run for this archive -- the post-merge checks T0.10 and T0.11 on an archive that never reached them (it was dropped from the release before they ran); or + - "missing": it should have run and did not. +Neither is folded into "failed"; `qualified` is the verdict. + +A feed written by an older schema_version is not mixed with this one: the first publish starts a fresh index.json, reports/ and latest/. +The old files stay in the branch's git history. The contract this writes is FEED.md, copied to the branch as SCHEMA.md. Usage: - python3 publish_qualification.py --records-dir ./qualification \ - --data-dir ./data/qualification --event schedule + python3 publish_qualification.py --records-dir ./records \ + --data-dir ./data/qualification --operating-systems ubuntu,win \ + --rocm-versions 7.14,10.0,10.1 --tag b12062 --upstream-sha \ + --jobs-file jobs.jsonl --published-artifacts "" --post-merge-checked "" """ import argparse @@ -31,27 +36,54 @@ import json import os import re +import shutil import sys -SCHEMA_VERSION = 1 +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import aggregate # noqa: E402 +import report # noqa: E402 + +SCHEMA_VERSION = 2 # Every test the suite runs, in the order the README lists them. -# A test absent from a record shows up as "missing", so the dashboard always has the same columns to draw. +# A test absent from a record shows up as "missing" or "not_applicable", so the dashboard always has the same columns to draw. # A test id not listed here is still published, never dropped. TIER_TESTS = { "tier0": ["T0.1", "T0.2", "T0.3", "T0.4", "T0.5", "T0.6", "T0.7", "T0.8", "T0.9", "T0.10", "T0.11", "T0.12"], "tier1": ["T1.1", "T1.2", "T1.3", "T1.4", "T1.5"], } -KNOWN_TESTS = [test_id for ids in TIER_TESTS.values() for test_id in ids] +# Run by the release job on the archives still in the release after its completeness check. +POST_MERGE_TESTS = {"T0.10", "T0.11"} STATUS_MISSING = "missing" +STATUS_NOT_APPLICABLE = "not_applicable" SUMMARY_FILE = "qualification-summary.json" -# build_id becomes a filename on the data branch, so it may only contain -# characters that cannot escape the folder it is written to. +# build_result is GitHub's conclusion for the archive's build job, in the feed's own vocabulary. +# Any other conclusion (failure, timed_out, startup_failure, ...) is "fail". +BUILD_PASS = "pass" +BUILD_FAIL = "fail" +BUILD_UNKNOWN = "unknown" +BUILD_RESULTS = {"success": BUILD_PASS, "cancelled": "cancelled", "skipped": "skipped"} + +# Why a placeholder row has no record, by its build_result. +PLACEHOLDER_REASONS = { + BUILD_FAIL: "build failed", + "cancelled": "build cancelled", + "skipped": "build skipped", +} +PLACEHOLDER_NO_REPORT = "no report" + +# build_id becomes a filename on the data branch, so it may only contain characters that cannot escape the folder it is written to. _SAFE_ID = re.compile(r"^[A-Za-z0-9._-]+$") +# The ROCm build jobs in release.yml. GitHub names a matrix job after its matrix values, the ROCm version first: +# "windows-rocm (10.1.0, gfx1010;..., x64, https://...)". Long names are cut short with "...", so only the first value is read. +_BUILD_JOBS = {"ubuntu-24-rocm": "ubuntu", "windows-rocm": "win"} +_BUILD_JOB = re.compile(r"^(?P[a-z0-9-]+) \((?P[^,)]+)") + def _int(value): try: @@ -68,6 +100,15 @@ def _test_order(test_id): return (sys.maxsize, sys.maxsize, test_id or "") +def _run_order(run_id): + """GitHub run ids only grow, so they order runs by when they started. + + generated_at cannot: a re-run writes its records later than a newer run did. + """ + number = _int(run_id) + return number if number is not None else -1 + + def load_records(records_dir): """Return (records, summary) from the folder aggregate.py wrote.""" records = [] @@ -86,30 +127,75 @@ def load_records(records_dir): return records, summary -def build_id_for(record): - build = record.get("build", {}) - run_id = build.get("run_id") - os_name = build.get("os") - rocm_version = build.get("rocm_version") +def load_build_results(jobs_file): + """Map (os, rocm_version) -> result of its build job. + + jobs_file holds one {"name": ..., "conclusion": ...} object per line, as + `gh api .../jobs --jq '.jobs[] | {name, conclusion}'` prints them. + """ + results = {} + if not jobs_file: + return results + try: + with open(jobs_file, encoding="utf-8") as handle: + lines = handle.read().splitlines() + except OSError as exc: + print(f"::warning::could not read {jobs_file}: {exc}; build_result will be {BUILD_UNKNOWN!r}") + return results + for line in lines: + line = line.strip() + if not line: + continue + try: + job = json.loads(line) + except ValueError: + print(f"::warning::skipping unreadable job line {line!r}") + continue + match = _BUILD_JOB.match(job.get("name") or "") + if not match or match.group("job") not in _BUILD_JOBS: + continue + # "10.1.0" -> "10.1", the form the archive names and records use. + version = ".".join(match.group("first").strip().split(".")[:2]) + conclusion = job.get("conclusion") + results[(_BUILD_JOBS[match.group("job")], version)] = ( + BUILD_RESULTS.get(conclusion, BUILD_FAIL) if conclusion else BUILD_UNKNOWN) + return results + + +def make_build_id(run_id, os_name, rocm_version): if not (run_id and os_name and rocm_version): raise SystemExit( - f"::error::record for {build.get('artifact')!r} has no run_id, os or " - "rocm_version, so it cannot be given a build_id") + f"::error::cannot form a build_id from run_id={run_id!r} os={os_name!r} " + f"rocm_version={rocm_version!r}") build_id = f"{run_id}-{os_name}-rocm-{rocm_version}" if not _SAFE_ID.match(build_id): raise SystemExit(f"::error::refusing unsafe build_id {build_id!r}") return build_id -def collect_results(record): +def absent_status(record, test_id, post_merge_checked): + """What a test the record does not contain is reported as.""" + if test_id in POST_MERGE_TESTS and record.get("build", {}).get("artifact") not in post_merge_checked: + return STATUS_NOT_APPLICABLE + return STATUS_MISSING + + +def tier_status(record, tier): + """pass | warn | fail for a tier that reported, otherwise missing.""" + reported = record.get("tiers", {}).get(tier) + return reported.get("status") if reported else STATUS_MISSING + + +def collect_results(record, post_merge_checked): """Per-test status across the known suite, plus any test it does not list.""" by_id = {} for tier in record.get("tiers", {}).values(): for test in tier.get("tests", []): by_id.setdefault(test.get("id"), test) - results = {test_id: by_id[test_id]["status"] if test_id in by_id else STATUS_MISSING - for test_id in KNOWN_TESTS} + results = {test_id: by_id[test_id]["status"] if test_id in by_id + else absent_status(record, test_id, post_merge_checked) + for ids in TIER_TESTS.values() for test_id in ids} for test_id in sorted((t for t in by_id if t not in results), key=_test_order): results[test_id] = by_id[test_id]["status"] @@ -119,17 +205,47 @@ def collect_results(record): return results, blocking -def build_entry(record, build_id, summary, run_url, event): +def make_placeholder(os_name, rocm_version, build_result, tag, upstream_sha, + run_id, run_attempt, generated_at): + """A record-shaped stand-in for an archive that has no record.""" + build = report.build_meta( + artifact=aggregate.published_name(tag, os_name, rocm_version), + os_name=os_name, + rocm_version=rocm_version, + llamacpp_tag=tag, + upstream_sha=upstream_sha, + run_id=run_id, + run_attempt=str(run_attempt) if run_attempt is not None else None, + ) + return { + "schema_version": report.SCHEMA_VERSION, + "kind": "qualification_placeholder", + "generated_at": generated_at, + "build": build, + "overall": report.STATUS_FAIL, + "promoted": False, + "counts": {report.STATUS_PASS: 0, report.STATUS_FAIL: 0, + report.STATUS_WARN: 0, report.STATUS_SKIP: 0, "total": 0}, + "tiers": {}, + "promotion": {"blocked_by": [PLACEHOLDER_REASONS.get(build_result, PLACEHOLDER_NO_REPORT)]}, + "build_result": build_result, + # No archive, so nothing was audited. The gfx target list lives in the build job. + "gfx": { + "requested": [], + "present_in_binary": [], + "missing_from_binary": [], + "hardware_validated": [], + }, + } + + +def build_entry(record, build_id, build_result, version_qualified, published, + post_merge_checked, run_url, event, run_attempt): """Derive the compact, dashboard-facing index row from a full record.""" build = record.get("build", {}) - results, blocking = collect_results(record) + results, blocking = collect_results(record, post_merge_checked) statuses = list(results.values()) - version_qualified = None - if summary is not None: - blocked = summary.get("blocked_rocm_versions") or [] - version_qualified = build.get("rocm_version") not in blocked - gfx = record.get("gfx", {}) return { "build_id": build_id, @@ -147,20 +263,26 @@ def build_entry(record, build_id, summary, run_url, event): }, "generated_at": record.get("generated_at"), "run_id": build.get("run_id"), - "run_attempt": _int(build.get("run_attempt")), + # The attempt that published this row, the same for every row of the run. + "run_attempt": run_attempt if run_attempt is not None else _int(build.get("run_attempt")), "run_url": run_url, "event": event, + "build_result": build_result, "tests": { "passed": statuses.count("pass"), "failed": statuses.count("fail"), "warned": statuses.count("warn"), "skipped": statuses.count("skip"), "missing": statuses.count(STATUS_MISSING), + "not_applicable": statuses.count(STATUS_NOT_APPLICABLE), "total": len(statuses), }, "qualified": bool(record.get("promoted")), "version_qualified": version_qualified, + "published": published, "overall": record.get("overall"), + "tier0": tier_status(record, "tier0"), + "tier1": tier_status(record, "tier1"), "blocked_by": record.get("promotion", {}).get("blocked_by", []), "blocking_tests": blocking, "results": results, @@ -174,26 +296,42 @@ def write_json(path, data): handle.write("\n") +def start_fresh_if_older(data_dir): + """Drop a feed written by an older schema_version instead of mixing rows of two shapes.""" + index_path = os.path.join(data_dir, "index.json") + if not os.path.exists(index_path): + return + with open(index_path, encoding="utf-8") as handle: + version = json.load(handle).get("schema_version") + if (_int(version) or 0) >= SCHEMA_VERSION: + return + print(f"The feed holds schema_version {version}; starting a fresh schema_version " + f"{SCHEMA_VERSION} feed. The old files stay in the branch's git history.") + os.remove(index_path) + for folder in ("reports", "latest"): + shutil.rmtree(os.path.join(data_dir, folder), ignore_errors=True) + + def update_latest(path, record): - """Write the record unless the file already holds a newer one. + """Write the record unless the file already holds one from a newer run. - Re-publishing an old run (a late re-run) must not roll the status tile - back to an older result. + Re-running an old run must not roll the status tile back to an older + result; a re-run of the same run replaces it. """ if os.path.exists(path): try: with open(path, encoding="utf-8") as handle: - current = json.load(handle).get("generated_at") or "" - except (OSError, ValueError): - current = "" - if current > (record.get("generated_at") or ""): + current = json.load(handle).get("build", {}).get("run_id") + except (OSError, ValueError, AttributeError): + current = None + if _run_order(current) > _run_order(record.get("build", {}).get("run_id")): return False write_json(path, record) return True def upsert_index(index_path, repo, entries): - """Replace-or-append each entry by build_id, newest first.""" + """Replace-or-append each entry by build_id, newest run first.""" if os.path.exists(index_path): with open(index_path, encoding="utf-8") as handle: index = json.load(handle) @@ -205,12 +343,12 @@ def upsert_index(index_path, repo, entries): builds.extend(entries) # Newest run first; the archives within one run in a fixed, readable order. builds.sort(key=lambda b: b.get("build_id") or "") - builds.sort(key=lambda b: b.get("generated_at") or "", reverse=True) + builds.sort(key=lambda b: _run_order(b.get("run_id")), reverse=True) return { "schema_version": SCHEMA_VERSION, "repo": repo or index.get("repo"), - "updated_at": builds[0].get("generated_at") if builds else None, + "updated_at": max((b.get("generated_at") or "" for b in builds), default="") or None, "builds": builds, } @@ -221,29 +359,83 @@ def main(): help="folder holding the qualification-*.json records aggregate.py wrote") parser.add_argument("--data-dir", required=True, help="the qualification/ folder of the qualification-data branch") + parser.add_argument("--operating-systems", required=True, + help="the operating systems every ROCm version is built for, e.g. ubuntu,win") + parser.add_argument("--rocm-versions", required=True, + help="the ROCm versions this run builds, e.g. 7.14,10.0,10.1") + parser.add_argument("--tag", required=True, help="the llama.cpp build number this run pinned") + parser.add_argument("--upstream-sha", help="the llama.cpp commit this run pinned") + parser.add_argument("--jobs-file", + help="this run's jobs, one {name, conclusion} JSON object per line") + parser.add_argument("--published-artifacts", default="", + help="the ROCm archives the release job uploaded, space separated") + parser.add_argument("--post-merge-checked", default="", + help="the ROCm archives the post-merge checks ran on, space separated") parser.add_argument("--event", default=os.environ.get("GITHUB_EVENT_NAME"), help="what triggered the run, e.g. schedule or workflow_dispatch") parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY"), help="owner/name, used to build run_url") parser.add_argument("--server-url", default=os.environ.get("GITHUB_SERVER_URL", "https://github.com")) + parser.add_argument("--run-id", default=os.environ.get("GITHUB_RUN_ID")) + parser.add_argument("--run-attempt", default=os.environ.get("GITHUB_RUN_ATTEMPT")) args = parser.parse_args() + expected = [(o, v) for v in aggregate.split_list(args.rocm_versions) + for o in aggregate.split_list(args.operating_systems)] + published_artifacts = set(args.published_artifacts.split()) + post_merge_checked = set(args.post_merge_checked.split()) + run_attempt = _int(args.run_attempt) + records, summary = load_records(args.records_dir) - if not records: - raise SystemExit(f"::error::no qualification records in {args.records_dir}; nothing to publish") if summary is None: print(f"::warning::no {SUMMARY_FILE} in {args.records_dir}; version_qualified will be null") + blocked_versions = set((summary or {}).get("blocked_rocm_versions") or []) + build_results = load_build_results(args.jobs_file) + if args.jobs_file and not build_results: + print(f"::warning::no ROCm build jobs found in {args.jobs_file}; build_result will be {BUILD_UNKNOWN!r}") + + # Placeholders share the run's timestamp, so every row of a run shows the same time. + run_time = max((r.get("generated_at") or "" for r in records), default="") or report.now_iso() + covered = set() + items = [] # (record, build_id, is_placeholder) + for record in records: + build = record.get("build", {}) + key = (build.get("os"), build.get("rocm_version")) + covered.add(key) + items.append((record, make_build_id(build.get("run_id") or args.run_id, *key), False)) + for os_name, rocm_version in expected: + if (os_name, rocm_version) in covered: + continue + build_result = build_results.get((os_name, rocm_version), BUILD_UNKNOWN) + record = make_placeholder(os_name, rocm_version, build_result, args.tag, args.upstream_sha, + args.run_id, run_attempt, run_time) + items.append((record, make_build_id(args.run_id, os_name, rocm_version), True)) + + if not items: + raise SystemExit(f"::error::no qualification records in {args.records_dir} and no " + "expected archives; nothing to publish") + + start_fresh_if_older(args.data_dir) os.makedirs(os.path.join(args.data_dir, "reports"), exist_ok=True) os.makedirs(os.path.join(args.data_dir, "latest"), exist_ok=True) entries = [] - for record in records: + for record, build_id, is_placeholder in items: build = record["build"] - build_id = build_id_for(record) - run_url = (f"{args.server_url}/{args.repo}/actions/runs/{build['run_id']}" + run_url = (f"{args.server_url}/{args.repo}/actions/runs/{build.get('run_id') or args.run_id}" if args.repo else None) - entry = build_entry(record, build_id, summary, run_url, args.event) + build_result = build_results.get((build.get("os"), build.get("rocm_version")), BUILD_UNKNOWN) + # An archive without a record can never let its version qualify. + if is_placeholder: + version_qualified = False + elif summary is None: + version_qualified = None + else: + version_qualified = build.get("rocm_version") not in blocked_versions + published = not is_placeholder and build.get("artifact") in published_artifacts + entry = build_entry(record, build_id, build_result, version_qualified, published, + post_merge_checked, run_url, args.event, run_attempt) entries.append(entry) write_json(os.path.join(args.data_dir, "reports", f"{build_id}.json"), record) @@ -254,13 +446,15 @@ def main(): tests = entry["tests"] print(f"{build_id}: {tests['passed']}/{tests['total']} passed, " f"{tests['failed']} failed, {tests['missing']} missing, " - f"qualified={entry['qualified']} version_qualified={entry['version_qualified']}" + f"build={build_result} qualified={entry['qualified']} " + f"version_qualified={entry['version_qualified']} published={entry['published']}" + f"{' (placeholder: ' + ', '.join(entry['blocked_by']) + ')' if is_placeholder else ''}" f"{'' if moved else ' (latest/ kept: it already holds a newer run)'}") index_path = os.path.join(args.data_dir, "index.json") index = upsert_index(index_path, args.repo, entries) write_json(index_path, index) - print(f"Published {len(entries)} record(s); {len(index['builds'])} build(s) in the index") + print(f"Published {len(entries)} row(s); {len(index['builds'])} build(s) in the index") if __name__ == "__main__": diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59f18ef6b211..29b39244b3b6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,11 @@ concurrency: cancel-in-progress: true env: + # The ROCm archives every run should produce: each version, for each operating system. + # The qualification records and the feed use these to notice an archive that never got built. + # Keep in step with the ubuntu-24-rocm / windows-rocm matrices and the "Check release artifacts" step. + ROCM_VERSIONS: "7.14,10.0,10.1" + ROCM_OPERATING_SYSTEMS: "ubuntu,win" BRANCH_NAME: ${{ github.head_ref || github.ref_name }} CMAKE_ARGS: "-DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_TOOLS=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON" @@ -1384,6 +1389,11 @@ jobs: - qualify-tier0 - qualify-tier1 + # For publish-qualification: which ROCm archives the post-merge checks ran on, and which this run uploaded. + outputs: + post_merge_checked: ${{ steps.post_merge.outputs.archives }} + published_rocm: ${{ steps.published_rocm.outputs.archives }} + steps: - name: Clone id: checkout @@ -1629,9 +1639,11 @@ jobs: run: | shopt -s nullglob mkdir -p fragments + checked="" for archive in release/llama-${TAG}-bin-ubuntu-rocm-*.tar.gz \ release/llama-${TAG}-bin-win-rocm-*.zip; do base=$(basename "$archive") + checked="${checked:+$checked }$base" if [[ "$base" == *-win-rocm-* ]]; then os=win; else os=ubuntu; fi ver=$(sed -E 's/.*-rocm-([0-9]+\.[0-9]+)-x64\..*/\1/' <<<"$base") echo "::group::Post-merge — $base" @@ -1640,7 +1652,9 @@ jobs: --output "fragments/tier0-postmerge-$os-rocm-$ver.json" || true echo "::endgroup::" done + echo "archives=$checked" >> "$GITHUB_OUTPUT" + # An archive with no report at all (its build failed) blocks its ROCm version too. - name: Qualify ROCm release artifacts id: qualify continue-on-error: true @@ -1653,6 +1667,8 @@ jobs: --tag "$TAG" \ --records-dir ./qualification \ --output-summary ./qualification/qualification-summary.json \ + --operating-systems "$ROCM_OPERATING_SYSTEMS" \ + --rocm-versions "$ROCM_VERSIONS" \ --fail-on-no-promote # Hand the records to publish-qualification, which adds them to the dashboard feed. @@ -1722,6 +1738,63 @@ jobs: cp -v "$record" release/ done + # The release notes link only the files this run is about to attach. + # A family dropped above (incomplete, or a ROCm version that did not qualify) gets no link. + # Written to a file because it is used twice: to create the release, and to rewrite the notes once the upload succeeded. + - name: Write release notes + uses: actions/github-script@v8 + env: + TAG_NAME: ${{ steps.tag.outputs.name }} + RELEASE_SUMMARY: ${{ steps.release_summary.outputs.value }} + with: + script: | + const fs = require('fs'); + const tag = process.env.TAG_NAME; + const { owner, repo } = context.repo; + const files = fs.readdirSync('./release'); + const attached = new Set(files); + const url = (file) => `https://github.com/${owner}/${repo}/releases/download/${tag}/${file}`; + const link = (label, file) => attached.has(file) ? [`- [${label}](${url(file)})`] : []; + const any = (pattern) => files.some((file) => pattern.test(file)); + const versions = process.env.ROCM_VERSIONS.split(',').map((v) => v.trim()).filter(Boolean); + const rocm = (os, v) => os === 'win' + ? `llama-${tag}-bin-win-rocm-${v}-x64.zip` + : `llama-${tag}-bin-ubuntu-rocm-${v}-x64.tar.gz`; + const cudaNote = '(replace XX with your GPU compute capability)'; + + const linux = [ + ...versions.flatMap((v) => link(`Ubuntu x64 (ROCm ${v})`, rocm('ubuntu', v))), + ...(any(new RegExp(`^llama-${tag}-ubuntu-cuda-sm_\\d+-x64\\.tar\\.xz$`)) + ? ['- Ubuntu x64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-x64.tar.xz` ' + cudaNote] : []), + ...(any(new RegExp(`^llama-${tag}-ubuntu-cuda-sm_\\d+-arm64\\.tar\\.xz$`)) + ? ['- Ubuntu arm64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-arm64.tar.xz` ' + cudaNote] : []), + ...files + .map((file) => file.match(new RegExp(`^llama-${tag}-bin-ubuntu-openvino-(.+)-x64\\.tar\\.gz$`))) + .filter(Boolean) + .flatMap((m) => link(`Ubuntu x64 (OpenVINO ${m[1]})`, m[0])), + ]; + const windows = [ + ...versions.flatMap((v) => link(`Windows x64 (ROCm ${v})`, rocm('win', v))), + ...(any(new RegExp(`^llama-${tag}-windows-cuda-sm_\\d+-x64\\.7z$`)) + ? ['- Windows x64 (CUDA): `llama-' + tag + '-windows-cuda-sm_XX-x64.7z` ' + cudaNote] : []), + ]; + // A ROCm version ships as a Windows + Linux pair or not at all. + const withheld = versions.filter((v) => !attached.has(rocm('ubuntu', v)) || !attached.has(rocm('win', v))); + const plural = withheld.length > 1; + + const body = [ + '
', '', process.env.RELEASE_SUMMARY, '', '
', '', + ...(linux.length ? ['**Linux:**', ...linux, ''] : []), + ...(windows.length ? ['**Windows:**', ...windows, ''] : []), + ...(withheld.length + ? [`ROCm ${withheld.join(', ')} ${plural ? 'are' : 'is'} not in this release: ` + + `${plural ? 'they' : 'it'} did not build, or did not pass the release checks. ` + + 'The attached `qualification-*.json` records list the failing checks.'] + : []), + ].join('\n').trimEnd() + '\n'; + fs.writeFileSync('release-notes.md', body); + core.info(body); + # Get the release for this tag, creating it if it does not exist yet. # This is idempotent: if a previous run created the release but failed # partway through uploading (e.g. a transient GitHub outage), re-running @@ -1731,10 +1804,10 @@ jobs: uses: actions/github-script@v8 env: TAG_NAME: ${{ steps.tag.outputs.name }} - RELEASE_SUMMARY: ${{ steps.release_summary.outputs.value }} with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | + const fs = require('fs'); const tag = process.env.TAG_NAME; const { owner, repo } = context.repo; let release; @@ -1743,27 +1816,7 @@ jobs: core.info(`Reusing existing release ${tag} (id ${release.id})`); } catch (e) { if (e.status !== 404) throw e; - const body = [ - '
', - '', - process.env.RELEASE_SUMMARY, - '', - '
', - '', - '**Linux:**', - `- [Ubuntu x64 (ROCm 7.14)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-rocm-7.14-x64.tar.gz)`, - `- [Ubuntu x64 (ROCm 10.0)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-rocm-10.0-x64.tar.gz)`, - `- [Ubuntu x64 (ROCm 10.1)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-rocm-10.1-x64.tar.gz)`, - '- Ubuntu x64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-x64.tar.xz` (replace XX with your GPU compute capability)', - '- Ubuntu arm64 (CUDA): `llama-' + tag + '-ubuntu-cuda-sm_XX-arm64.tar.xz` (replace XX with your GPU compute capability)', - `- [Ubuntu x64 (OpenVINO 2026.0)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-ubuntu-openvino-2026.0-x64.tar.gz)`, - '', - '**Windows:**', - `- [Windows x64 (ROCm 7.14)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-win-rocm-7.14-x64.zip)`, - `- [Windows x64 (ROCm 10.0)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-win-rocm-10.0-x64.zip)`, - `- [Windows x64 (ROCm 10.1)](https://github.com/${owner}/${repo}/releases/download/${tag}/llama-${tag}-bin-win-rocm-10.1-x64.zip)`, - '- Windows x64 (CUDA): `llama-' + tag + '-windows-cuda-sm_XX-x64.7z` (replace XX with your GPU compute capability)', - ].join('\n'); + const body = fs.readFileSync('release-notes.md', 'utf8'); try { release = (await github.rest.repos.createRelease({ owner, repo, tag_name: tag, body })).data; core.info(`Created release ${tag} (id ${release.id})`); @@ -1866,6 +1919,38 @@ jobs: core.info(`Done: ${uploaded} uploaded, ${skipped} already present, ${files.length} total.`); + # For the feed: the ROCm archives this run attached to the release. + - name: Record published ROCm archives + id: published_rocm + if: steps.upload_release.outcome == 'success' + env: + TAG: ${{ steps.tag.outputs.name }} + run: | + shopt -s nullglob + archives="" + for archive in release/llama-${TAG}-bin-ubuntu-rocm-*.tar.gz release/llama-${TAG}-bin-win-rocm-*.zip; do + archives="${archives:+$archives }$(basename "$archive")" + done + echo "archives=$archives" >> "$GITHUB_OUTPUT" + echo "Published ROCm archives: ${archives:-none}" + + # A release reused from an earlier run of the same tag still carries that run's notes. + # The upload above has just made its files match this run, so make the notes match too. + - name: Update release notes + if: steps.upload_release.outcome == 'success' + uses: actions/github-script@v8 + env: + RELEASE_ID: ${{ steps.release.outputs.id }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const { owner, repo } = context.repo; + await github.rest.repos.updateRelease({ + owner, repo, release_id: Number(process.env.RELEASE_ID), + body: fs.readFileSync('release-notes.md', 'utf8'), + }); + - name: Fail if any artifacts were missing if: steps.check_artifacts.outputs.missing != '0' run: | @@ -1880,7 +1965,7 @@ jobs: # --------------------------------------------------------------------------- # Publish this run's ROCm qualification records to the qualification-data branch, where the dashboard polls them. - # Every record is published irrespective of it passing or failing. + # Every record is published irrespective of it passing or failing, and every ROCm archive the run should have built gets a row, including one whose build failed. # A feed that only held the runs that shipped would show nothing but successes. # The format is described in .github/qualify-rocm/FEED.md. # --------------------------------------------------------------------------- @@ -1889,7 +1974,8 @@ jobs: # nothing about the ROCm records it wrote, so publish after either outcome. # Skipped along with the release job on pull requests and on dispatches # without create_release so neither produces records. - if: ${{ always() && (needs.release.result == 'success' || needs.release.result == 'failure') }} + # Without resolve-upstream there is no build number and nothing was built. + if: ${{ always() && needs.resolve-upstream.result == 'success' && (needs.release.result == 'success' || needs.release.result == 'failure') }} needs: - resolve-upstream @@ -1899,6 +1985,7 @@ jobs: permissions: contents: write # to push to the qualification-data branch + actions: read # to read each ROCm build job's result steps: - name: Checkout qualification suite @@ -1906,6 +1993,7 @@ jobs: with: path: lemonade-ci + # Written by the release job when it got as far as qualifying the ROCm archives. - name: Download qualification records continue-on-error: true uses: actions/download-artifact@v7 @@ -1913,19 +2001,65 @@ jobs: name: qualification-records path: ./records - - name: Publish to qualification-data branch + - name: Download qualification fragments + continue-on-error: true + uses: actions/download-artifact@v7 + with: + pattern: qualify-fragments-* + path: ./fragments + merge-multiple: true + + # If the release job stopped before writing records, build them here from the Tier 0 and Tier 1 reports. + - name: Collect qualification records env: - GH_TOKEN: ${{ github.token }} TAG: ${{ needs.resolve-upstream.outputs.tag }} run: | set -euo pipefail shopt -s nullglob - records=(records/qualification-*.json) - if [ ${#records[@]} -eq 0 ]; then - echo "::warning::The release job produced no qualification records, so there is nothing to publish." + records=(records/qualification-llama-*.json) + if [ ${#records[@]} -gt 0 ]; then + echo "Using the ${#records[@]} record(s) the release job wrote." exit 0 fi + echo "The release job wrote no records; building them from the Tier 0 and Tier 1 reports." + rm -rf records + mkdir -p records fragments + python3 lemonade-ci/.github/qualify-rocm/aggregate.py \ + --fragments-dir ./fragments \ + --tag "$TAG" \ + --records-dir ./records \ + --output-summary ./records/qualification-summary.json \ + --operating-systems "$ROCM_OPERATING_SYSTEMS" \ + --rocm-versions "$ROCM_VERSIONS" + + # build_result for every row, and the reason for an archive that has no record. + - name: Read build job results + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + for attempt in 1 2 3; do + if gh api --paginate \ + "repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=latest&per_page=100" \ + --jq '.jobs[] | {name, conclusion}' > jobs.jsonl; then + grep -E '"name":"(ubuntu-24-rocm|windows-rocm) ' jobs.jsonl || true + exit 0 + fi + echo "Could not read the run's jobs (attempt $attempt); retrying" + sleep $((attempt * 5)) + done + echo "::warning::Could not read the run's jobs; build_result will be 'unknown'." + : > jobs.jsonl + - name: Publish to qualification-data branch + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.resolve-upstream.outputs.tag }} + UPSTREAM_SHA: ${{ needs.resolve-upstream.outputs.sha }} + PUBLISHED_ROCM: ${{ needs.release.outputs.published_rocm }} + POST_MERGE_CHECKED: ${{ needs.release.outputs.post_merge_checked }} + run: | + set -euo pipefail git config --global user.name "github-actions[bot]" git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" REPO_URL="https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git" @@ -1951,7 +2085,14 @@ jobs: python3 lemonade-ci/.github/qualify-rocm/publish_qualification.py \ --records-dir records \ - --data-dir data/qualification + --data-dir data/qualification \ + --operating-systems "$ROCM_OPERATING_SYSTEMS" \ + --rocm-versions "$ROCM_VERSIONS" \ + --tag "$TAG" \ + --upstream-sha "$UPSTREAM_SHA" \ + --jobs-file jobs.jsonl \ + --published-artifacts "$PUBLISHED_ROCM" \ + --post-merge-checked "$POST_MERGE_CHECKED" # Keep the format description next to the data it describes. cp lemonade-ci/.github/qualify-rocm/FEED.md data/qualification/SCHEMA.md