From 65616a8cde6a581661ac3734378da78a9fcbe5ed Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sat, 3 Oct 2026 20:56:04 +0800 Subject: [PATCH 1/8] feat(w2): validate full Qwen3 frontend and runtime contracts --- .github/workflows/llm-deploy.yml | 164 +++++- docs/llm-deploy-v1.0/W1/README.md | 8 +- ...47\350\241\214\350\256\241\345\210\222.md" | 2 +- ...42\346\265\213\346\226\271\346\241\210.md" | 4 +- docs/llm-deploy-v1.0/W1/interfaces.md | 21 +- docs/llm-deploy-v1.0/W1/risks.md | 2 +- docs/llm-deploy-v1.0/W2/README.md | 262 ++++++++++ ...14\346\224\266\346\212\245\345\221\212.md" | 125 +++++ ...21\345\270\203\350\257\264\346\230\216.md" | 27 + ...00\345\217\221\350\256\241\345\210\222.md" | 10 +- probes/w2_backend_ops/README.md | 45 ++ probes/w2_backend_ops/cases.py | 174 +++++++ probes/w2_backend_ops/run.py | 258 ++++++++++ probes/w2_qwen3_parse/README.md | 87 ++++ probes/w2_qwen3_parse/audit.py | 417 +++++++++++++++ probes/w2_qwen3_parse/run.py | 367 +++++++++++++ probes/w2_qwen3_parse/validation.py | 427 +++++++++++++++ probes/w2_qwen3_small/README.md | 7 + probes/w2_runtime/README.md | 59 +++ probes/w2_runtime/corpus.json | 487 ++++++++++++++++++ probes/w2_runtime/run.py | 379 ++++++++++++++ probes/w2_runtime_model/README.md | 42 ++ probes/w2_runtime_model/run.py | 402 +++++++++++++++ pyproject.toml | 1 + scratchv/backend/tensor_c_codegen.py | 9 + scratchv/runtime/llm_inputs.py | 179 +++++++ scratchv/runtime/qwen3_tokenizer.py | 223 ++++++++ scratchv/runtime/riscv_tensor.py | 68 ++- scripts/check_w2_ci_results.py | 38 ++ scripts/run_w2_acceptance.py | 447 ++++++++++++++++ tests/test_llm_inputs.py | 228 ++++++++ tests/test_qwen3_frontend_patterns.py | 56 ++ tests/test_qwen3_full_audit.py | 375 ++++++++++++++ tests/test_qwen3_full_parse.py | 422 +++++++++++++++ tests/test_qwen3_full_structure.py | 327 ++++++++++++ tests/test_qwen3_tokenizer.py | 221 ++++++++ tests/test_riscv_tensor_runtime.py | 90 +++- tests/test_tensor_c_codegen.py | 42 ++ tests/test_w2_acceptance.py | 296 +++++++++++ tests/test_w2_backend_ops_gate.py | 220 ++++++++ tests/test_w2_ci_results.py | 49 ++ tests/test_w2_runtime_gate.py | 192 +++++++ tests/test_w2_runtime_metadata.py | 103 ++++ tests/test_w2_runtime_model.py | 252 +++++++++ tests/test_w2_runtime_reporting.py | 120 +++++ 45 files changed, 7695 insertions(+), 39 deletions(-) create mode 100644 docs/llm-deploy-v1.0/W2/README.md create mode 100644 "docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" create mode 100644 "docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" create mode 100644 probes/w2_backend_ops/README.md create mode 100644 probes/w2_backend_ops/cases.py create mode 100644 probes/w2_backend_ops/run.py create mode 100644 probes/w2_qwen3_parse/README.md create mode 100644 probes/w2_qwen3_parse/audit.py create mode 100644 probes/w2_qwen3_parse/run.py create mode 100644 probes/w2_qwen3_parse/validation.py create mode 100644 probes/w2_runtime/README.md create mode 100644 probes/w2_runtime/corpus.json create mode 100644 probes/w2_runtime/run.py create mode 100644 probes/w2_runtime_model/README.md create mode 100644 probes/w2_runtime_model/run.py create mode 100644 scratchv/runtime/llm_inputs.py create mode 100644 scratchv/runtime/qwen3_tokenizer.py create mode 100644 scripts/check_w2_ci_results.py create mode 100644 scripts/run_w2_acceptance.py create mode 100644 tests/test_llm_inputs.py create mode 100644 tests/test_qwen3_frontend_patterns.py create mode 100644 tests/test_qwen3_full_audit.py create mode 100644 tests/test_qwen3_full_parse.py create mode 100644 tests/test_qwen3_full_structure.py create mode 100644 tests/test_qwen3_tokenizer.py create mode 100644 tests/test_w2_acceptance.py create mode 100644 tests/test_w2_backend_ops_gate.py create mode 100644 tests/test_w2_ci_results.py create mode 100644 tests/test_w2_runtime_gate.py create mode 100644 tests/test_w2_runtime_metadata.py create mode 100644 tests/test_w2_runtime_model.py create mode 100644 tests/test_w2_runtime_reporting.py diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index 7052655..492177c 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -6,12 +6,14 @@ name: LLM Deploy v1.0 # one tests the deployment project. A failure here should not read as "the # compiler is broken". # -# The PR job executes small numeric gates. A separate manual/nightly job -# downloads and validates the pinned full-model release with ORT. +# PRs execute small numeric gates and a separate complete frontend audit. +# Both are mandatory for the W2 acceptance result. Full-model ORT remains +# manual/nightly; frontend parsing never claims full IR execution. # # docs:interfaces docs:risks # probe:qemu-matmul probe:small-transformer probe:qwen3-onnx -# numeric:ir-qwen3-small +# numeric:ir-qwen3-small frontend:parse-qwen3 +# unit:frontend-ops unit:backend-ops unit:runtime integration:runtime-model # # Later weeks' gates (numeric:ir-full-qwen3 at W3, numeric:qemu-full-qwen3 at # W4, e2e:qemu-paris at W5, release:gate at W8) are NOT stubbed here. They get @@ -24,10 +26,14 @@ on: - 'probes/**' - 'scripts/check_llm_deploy_docs.py' - 'scripts/check_w1_repro_env.py' + - 'scripts/run_w2_acceptance.py' + - 'scripts/check_w2_ci_results.py' - 'export_qwen3_onnx.py' - 'scratchv/**' - 'requirements/**' + - 'pyproject.toml' - 'tests/test_qwen3_small*.py' + - 'tests/test_qwen3_full*.py' - 'tests/test_tensor*.py' - 'tests/test_riscv_tensor*.py' - 'tests/test_optimizer*.py' @@ -35,6 +41,10 @@ on: - 'tests/test_qwen_artifact_selection.py' - 'tests/test_w1_qwen3*.py' - 'tests/test_w1_repro_env.py' + - 'tests/test_qwen3_frontend_patterns.py' + - 'tests/test_qwen3_tokenizer.py' + - 'tests/test_llm_inputs.py' + - 'tests/test_w2_*.py' - '.github/workflows/llm-deploy.yml' schedule: # Nightly, off the hour: 19:23 UTC = 03:23 北京时间. @@ -50,11 +60,12 @@ env: PROBE_TRANSFORMER: probes/w1_tiny_transformer/run.py PROBE_QWEN3_EXPORT: probes/w1_qwen3_export/run.py PROBE_QWEN3_SMALL: probes/w2_qwen3_small/run.py + PROBE_QWEN3_PARSE: probes/w2_qwen3_parse/run.py QWEN3_PY: output/qwen3-probe-venv/bin/python jobs: llm-deploy: - name: "LLM Deploy v1.0 / W1" + name: "LLM Deploy v1.0 / W1 + W2" runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -108,7 +119,15 @@ jobs: - name: Tensor compiler and runtime regressions run: | - python3 -m pytest tests/test_tensor_c_codegen.py tests/test_tensor_compiler.py tests/test_riscv_tensor_runtime.py tests/test_optimizer_numeric_semantics.py tests/test_qwen_artifact_selection.py tests/test_w1_qwen3_export.py tests/test_qwen3_small_riscv_gate.py tests/test_w1_repro_env.py -q + python3 -m pytest \ + tests/test_tensor_c_codegen.py tests/test_tensor_compiler.py \ + tests/test_riscv_tensor_runtime.py tests/test_optimizer_numeric_semantics.py \ + tests/test_qwen_artifact_selection.py tests/test_w1_qwen3_export.py tests/test_qwen3_small_riscv_gate.py \ + tests/test_w1_repro_env.py tests/test_qwen3_full_structure.py \ + tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py \ + tests/test_llm_inputs.py tests/test_w2_backend_ops_gate.py \ + tests/test_w2_runtime_gate.py tests/test_w2_runtime_metadata.py \ + tests/test_w2_runtime_reporting.py tests/test_w2_acceptance.py tests/test_w2_ci_results.py -q - name: Repeated Linux timeout cleanup regression timeout-minutes: 5 @@ -123,6 +142,18 @@ jobs: --junit-xml="output/process-cleanup-repeats/attempt-$attempt.xml" done + - name: "unit:frontend-ops" + id: frontend_ops + run: | + python3 -m pytest tests/test_qwen3_frontend_patterns.py -q \ + --junit-xml=output/w2-frontend-ops.xml + + - name: "unit:backend-ops" + id: backend_ops + timeout-minutes: 15 + run: | + python3 probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops + # ── probe:small-transformer ─────────────────────────────────────── - name: "probe:small-transformer" id: transformer @@ -151,6 +182,34 @@ jobs: "$QWEN3_PY" -X utf8 scripts/check_w1_repro_env.py \ --require-clean --output-dir output/w1-repro-env + - name: Runtime unit regressions in pinned environment + run: | + "$QWEN3_PY" -m pytest \ + tests/test_llm_inputs.py tests/test_qwen3_tokenizer.py \ + tests/test_w2_runtime_gate.py tests/test_w2_runtime_metadata.py \ + tests/test_w2_runtime_reporting.py tests/test_w2_runtime_model.py \ + -q --junit-xml=output/w2-runtime-tests.xml + + - name: "unit:runtime" + id: runtime + timeout-minutes: 10 + run: | + "$QWEN3_PY" probes/w2_runtime/run.py --mode download \ + --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime + + - name: "integration:runtime-model" + id: runtime_model + timeout-minutes: 15 + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: | + "$QWEN3_PY" probes/w2_runtime_model/run.py \ + --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime-model + - name: "numeric:ir-qwen3-small" id: qwen3_small timeout-minutes: 15 @@ -210,6 +269,39 @@ jobs: "$QWEN3_PY" probes/w2_qwen3_small/riscv.py \ --model-dir output/qwen3-small --output-dir output/qwen3-riscv + - name: Summarize W2 unit gates + if: always() + env: + FRONTEND_OUTCOME: ${{ steps.frontend_ops.outcome }} + BACKEND_OUTCOME: ${{ steps.backend_ops.outcome }} + RUNTIME_OUTCOME: ${{ steps.runtime.outcome }} + RUNTIME_MODEL_OUTCOME: ${{ steps.runtime_model.outcome }} + run: | + { + echo "## W2 frontend, backend and host runtime" + echo "unit:frontend-ops: $FRONTEND_OUTCOME" + echo "unit:backend-ops: $BACKEND_OUTCOME" + echo "unit:runtime: $RUNTIME_OUTCOME" + echo "integration:runtime-model: $RUNTIME_MODEL_OUTCOME" + for report in output/w2-backend-ops/report.md output/w2-runtime/report.md output/w2-runtime-model/report.md; do + if [ -f "$report" ]; then cat "$report"; fi + done + echo "These gates do not execute full-model IR/QEMU or text generation." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload W2 unit evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: w2-unit-gates-output + path: | + output/w2-backend-ops/ + output/w2-runtime/ + output/w2-runtime-model/report.* + output/w2-frontend-ops.xml + output/w2-runtime-tests.xml + if-no-files-found: ignore + - name: Upload RISC-V tensor evidence if: always() uses: actions/upload-artifact@v4 @@ -352,3 +444,65 @@ jobs: name: qwen3-full-onnx-probe-output path: output/qwen3-full-probe/ if-no-files-found: ignore + + + full-qwen3-frontend: + name: "W2 / complete Qwen3 frontend" + # Required on PRs as well as manual/nightly runs. No full-model ORT or IR inference. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Install pinned frontend audit dependencies + run: | + python3 -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" + python3 -m pip check + - name: "frontend:parse-qwen3" + id: full_frontend + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + run: | + set -euo pipefail + python3 "$PROBE_QWEN3_PARSE" --mode download \ + --model-dir output/qwen3-full-model \ + --output-dir output/qwen3-parse --timeout 1200 + - name: Summarize complete Qwen3 frontend gate + if: always() + env: + PROBE_OUTCOME: ${{ steps.full_frontend.outcome }} + run: | + { + echo "## frontend:parse-qwen3" + echo "Outcome: $PROBE_OUTCOME" + if [ -f output/qwen3-parse/report.md ]; then + cat output/qwen3-parse/report.md + fi + echo "Complete graph and binding audit only; no full IR or QEMU numerical claim." + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload complete Qwen3 frontend evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: qwen3-full-frontend-parse-output + path: output/qwen3-parse/ + if-no-files-found: ignore + + w2-acceptance: + name: "W2 / required acceptance" + needs: [llm-deploy, full-qwen3-frontend] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Require actual success from every W2 job + env: + W2_JOB_RESULTS: ${{ toJSON(needs) }} + run: python3 scripts/check_w2_ci_results.py >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/llm-deploy-v1.0/W1/README.md b/docs/llm-deploy-v1.0/W1/README.md index ecc1738..661d413 100644 --- a/docs/llm-deploy-v1.0/W1/README.md +++ b/docs/llm-deploy-v1.0/W1/README.md @@ -2,7 +2,7 @@ 本指南用于另一名成员从目标提交重新生成参考结果、编译产物并运行探测。当前已知本地结果见 [执行计划](W1-执行计划.md)，接口候选见 [interfaces.md](interfaces.md)，风险见 [risks.md](risks.md)。**Mastttttter 已于 2026-10-03 完成 `df18b02f` 的两层数值链路独立复现并本人确认；团队接口确认和 W1 出口评审仍待完成。** 原始证据见 [独立复现报告](https://github.com/ScratchV-Compiler/ScratchV/pull/91#issuecomment-5958157573)。该确认归属所列提交，不自动覆盖后续修复。 -本次 PR #91 集成此前本地收尾修复、复现预检和 QEMU 时长记录；历史本地结果与待决策问题见 [收尾与审查报告](W1-本地收尾与审查报告.md)。本次提交的实际测试、CI 与精确 SHA 以 PR 说明和对应运行报告为准，不用历史结果替代。 +PR #91 已合并，最终 head 为 `5903c1381d57f24062de7395f24385a59e279325`，合并提交为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。报告落盘失败和源码身份归属修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)；发布前整合工作树记录见 [本轮修复与验收报告](../W2/W1修复与W2本地验收报告.md)。人工确认和完整模型第二人复现继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。下方原 PR 修复数据均为历史记录。 所有命令在仓库根目录执行。先记录 `git rev-parse HEAD` 和 `git status --short`；干净checkout便于关联提交，存在本地改动则保留diff并明确标记。不要使用第一人的预生成参考数组代替自己的运行。下述输出目录用完后保留，下一轮使用不同名称，不删除旧证据。 @@ -12,7 +12,7 @@ ## 0. 确定复现版本与领取任务 -`5ea22ecc7fd314025f6453c556dbb3e3f16a3175` 是已有 Linux CI 证据的**历史基线**，不包含本次新增的环境预检和时长汇总。复现本次更新时，应先从 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91) 确认维护者/作者约定的完整 head SHA，再获取 PR head 并核对；PR 分支可能继续更新，不能仅记录分支名。 +`5ea22ecc7fd314025f6453c556dbb3e3f16a3175` 是较早的 Linux CI **历史基线**。复现已合并 W1 时可使用下述固定合并 SHA；若复现本次尚未提交的修复，则必须在该基线上另取得完整增量文件，并核对验收报告中的源码哈希，不能把仅检出的合并提交称为已包含本地修复。 在新目录获取本次版本的示例（需已安装 Git 和 Git LFS，PowerShell/bash 均可逐条执行）： @@ -20,11 +20,11 @@ git clone --no-checkout https://github.com/ScratchV-Compiler/ScratchV.git ScratchV-w1-repro cd ScratchV-w1-repro git lfs install --local -git fetch origin pull/91/head +git fetch origin 3bb88e81498d6d9293aed70e25a4e5a2104465d0 git rev-parse FETCH_HEAD ``` -先确认输出与约定的 PR head SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 +先确认输出与约定的合并 SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 ```bash git checkout --detach FETCH_HEAD diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" index ac6abc8..4814a0f 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" @@ -3,7 +3,7 @@ > 上游：[开发计划.md](../开发计划.md) §4.3 W1。负责人：E2 主导，全员参与。 > 状态（2026-10-03）：**Mastttttter 已完成 `df18b02f` 的合成图、MatMul、真实两层 IR/QEMU 独立复现并本人确认。完整模型第二人复现、团队确认与 W1 出口评审仍待完成。** [原始报告](https://github.com/ScratchV-Compiler/ScratchV/pull/91#issuecomment-5958157573)保留首轮 1 项失败、3 项跳过及后续复跑；本次修复退出状态误判与产物路径缺口，新提交验证单独记录。下文 `5ea22ec`、`545e696` CI 均为历史证据。 > 本文记录实现事实和待办，不代表已召开冻结会议。命令与签字模板见 [独立复现指南](README.md)。 -> 本次将暂留的本地收尾与仿真计时改动集成到 PR #91；上述远端结果只对应所列提交，新增改动须另行验证，不继承远端 PASS。 +> PR #91 已合并为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。本地继续补齐报告发布与源码身份修复，结果见 [本轮报告](../W2/W1修复与W2本地验收报告.md)。W1 报告修复单独发布于 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付；人工确认和完整模型第二人复现由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 继续跟踪，旧 CI 不覆盖本地增量。 ## 1. 目标和验收边界 diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" index 8112c1f..465338c 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" @@ -1,9 +1,9 @@ # W1 探测方案与结果记录 > 上游：[开发计划.md](../开发计划.md) §4.3 W1、[执行计划](W1-执行计划.md)。 -> 更新：2026-10-02。**`5ea22ec` 主 CI 和小模型 Linux 部署任务已通过；完整 ONNX 重型证据来自验证源码未变的 `545e696`。独立复现和团队确认待完成**。 +> 更新：2026-10-03。**PR #91 已合并；Mastttttter 已确认 `df18b02f` 的两层独立复现。完整模型第二人复现、团队确认和出口评审仍待完成**。下文 `5ea22ec`、`545e696` 等远端结果均是历史证据。 > 完整命令和验收模板见 [README](README.md)。保留各探测的范围，不用替代模型或解析成功冒充原数值门槛。 -> 本次将暂留的本地收尾与仿真计时改动集成到 PR #91；新增结果与上述远端基线分开记录，不能把旧 CI 结果当作新增修改的验证。 +> 合并后的本地修复与 W2 增量见 [本轮验收报告](../W2/W1修复与W2本地验收报告.md)；其中 W1 报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付，远端 CI 以对应提交实际执行为准。人工待办继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。 ## 0. 总览 diff --git a/docs/llm-deploy-v1.0/W1/interfaces.md b/docs/llm-deploy-v1.0/W1/interfaces.md index dedf913..bfdf308 100644 --- a/docs/llm-deploy-v1.0/W1/interfaces.md +++ b/docs/llm-deploy-v1.0/W1/interfaces.md @@ -1,7 +1,7 @@ # 接口基线 v1.1：待团队冻结确认 > 上游：[开发计划.md](../开发计划.md) §2.2、§4.3 W1。 -> 状态（2026-10-02）：**按当前实现核对的候选基线，团队确认未完成**。 +> 状态（2026-10-03）：**按当前实现核对的候选基线，团队确认未完成**。PR #91 已合并；W2 运行时扩展与 W1 报告修复分开交付，D1–D8 不因此自动通过。 > `docs:interfaces` 只检查文档结构；“有版本号”不等于冻结会议已通过。正式决议见 §7。 ## 0. 四类接口与路线 @@ -13,7 +13,7 @@ | 三 | 后端 | `CompilerDriver(backend="tensor-c")` → 显式张量循环 C → Zig/LLVM → RV64GC ELF | | 四 | 运行时 | 指针数组模型 ABI；QEMU virt 裸机原始输入装载与 UART 输出协议 | -上述新路径已经本地及 `5ea22ec` 的 Linux 数值验证；本次收尾与仿真计时改动一起集成到 PR #91；新增改动须以对应新提交的检查为准，不能继承该提交的 CI PASS。原 `riscv` 标量选择器及旧 `llvm` 路径不由本探测证明；旧选择器的 MatMul 占位/张量与 FP32 覆盖缺口仍保留。不能将“仓库有 LLVM 后端”或 CNN standalone 可运行等同于 Qwen FP32 已支持。 +上述新路径已有本地及历史提交的 Linux 数值验证，收尾与仿真计时已随 PR #91 合并。合并后的本地修复与 W2 扩展须以本轮报告及未来对应提交的检查为准，不能继承旧 CI PASS。原 `riscv` 标量选择器及旧 `llvm` 路径不由本探测证明；旧选择器的 MatMul 占位/张量与 FP32 覆盖缺口仍保留。不能将“仓库有 LLVM 后端”或 CNN standalone 可运行等同于 Qwen FP32 已支持。 ## 1. 冻结与变更规则 @@ -107,11 +107,19 @@ int scratchv_run(const void *const inputs[], void *output); 当前 QEMU 使用 virt/TCG、512 MiB RAM、无 BIOS/OS，自带启动汇编和链接脚本。输入通过 raw loader 写入保留区域（地址 `0x9c000000`，容量 64 MiB），输出通过 UART 二进制帧返回。帧包含标识、状态、长度、校验和和结束标记；解析拒绝截断/损坏/异常状态，超时失败并清理本次子进程。该传输是现有探测协议，不应误写为通用 Linux FFI 或 mmap。 -### 5.3 后续运行时边界 +### 5.3 W2 Host 运行时辅助接口 -Tokenizer、采样、生成循环和完整模型权重装载仍待实现/验收。当前小模型权重静态嵌入；完整 0.6B 外部分片合计 2,384,201,728 bytes（约 2.22 GiB），不应直接套用 512 MiB guest 配置。W4 需设计代码与权重分离，明确名称/偏移/dtype/shape 描述、装载策略、容量和地址重叠检查、生命周期及分块输出；选择 Linux mmap 时须另建 Linux guest/user-mode 路线，裸机不具备该系统调用。完整 logits 对照仍是原数值门槛，后续生成专用“最后有效位置 logits”接口需独立验收，不能悄悄替换原输出。 +`scratchv/runtime/qwen3_tokenizer.py` 的 `Qwen3Tokenizer.from_directory` 读取已校验的官方本地配置，提供编码/解码和特殊 token 信息，不下载权重、不依赖 Torch/Transformers、不渲染聊天模板。基础词表、含 added tokens 的实际 ID 集合与模型 logits 宽度是不同契约；Qwen3-0.6B 分别为 151643、151669 和 151936。未知或空余模型 ID 解码时显式报错。普通文本编码不自动加入 BOS/EOS；NFC 规范化遵循官方 tokenizer JSON。 -固定长度的 host 生成循环须选择最后一个有效 token 的 logits，不能选择右侧 padding 的最后物理位置。ID 范围、mask、空输入、EOS 和有效长度达到 256 的行为需要显式测试；超长输入不静默截断。上述生成策略是后续实现要求，不代表当前探测已经提供生成 API。 +`scratchv/runtime/llm_inputs.py` 提供 `prepare_inputs`、`build_attention_mask`、`last_valid_logits`、`greedy_next_token` 和 `generation_stop_reason`。输入维持 batch=1、L=256、INT64 IDs、FP32 加性 mask；空/超长输入失败，prompt 中出现 pad ID 不影响有效长度。greedy 选择最后有效位置、平局取最小 ID，拒绝错误形状/dtype和非有限输出。停止判断区分生成 EOS、上下文容量和生成预算，prompt 中的 EOS 不自行停止。 + +这些接口是 W2 辅助能力，未改变模型 ABI 或交付生成循环。验收以 [运行时探测](../../../probes/w2_runtime/README.md) 的固定语料与官方 fast/slow 对照为准；不替代 D1–D8 团队确认。 + +### 5.4 后续运行时边界 + +生成循环和完整模型权重装载仍待实现/验收。当前小模型权重静态嵌入；完整 0.6B 外部分片合计 2,384,201,728 bytes（约 2.22 GiB），不应直接套用 512 MiB guest 配置。W4 需设计代码与权重分离，明确名称/偏移/dtype/shape 描述、装载策略、容量和地址重叠检查、生命周期及分块输出；选择 Linux mmap 时须另建 Linux guest/user-mode 路线，裸机不具备该系统调用。完整 logits 对照仍是原数值门槛，后续生成专用“最后有效位置 logits”接口需独立验收，不能悄悄替换原输出。 + +固定长度的 host 生成循环须复用已验收的最后有效位置、mask、ID 范围、空输入、EOS 和长度边界辅助函数，不能选择右侧 padding 的最后物理位置或静默截断。后续仍需将这些能力与完整模型前向组合验证；当前探测未提供端到端生成 API。 ## 6. 能力与剩余缺口 @@ -122,7 +130,8 @@ Tokenizer、采样、生成循环和完整模型权重装载仍待实现/验收 | tensor-c / RV64 运行器 | 本地及 `5ea22ec` 的[Linux 部署任务](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)通过，含 28 次两层 QEMU 执行；该提交主 CI 也通过 | E3/E4/E5：第二人独立复现、后续修改的受影响门禁；不外推完整模型 | | 完整模型 IR 数值与容量 | 未由小模型验收 | E2/E4/E5：逐步扩大配置及预训练权重验证 | | 完整 ONNX 门禁 | 历史导出、本地 verify，以及 `545e696` 的[Linux download/ORT 重型任务](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767)均通过；三项验证源码至 `5ea22ec` 未变 | E1/E5：第二人复现及受影响门禁；不冒称重新导出或 `5ea22ec` 重跑重型任务 | -| Tokenizer / 生成 / 大权重加载 | 当前探测不覆盖 | E4：W2 基础运行时、W4 权重装载、W5 生成集成及专项验收 | +| Tokenizer / mask / greedy | W2 基础模块与独立门禁已交付，结果按当前提交检查 | E4/E5：第二人复现，后续集成验收 | +| 生成 / 大权重加载 | 当前探测不覆盖 | E4：W4 权重装载、W5 生成集成及专项验收 | ## 7. 团队待决议与确认 diff --git a/docs/llm-deploy-v1.0/W1/risks.md b/docs/llm-deploy-v1.0/W1/risks.md index 26def2c..acd810a 100644 --- a/docs/llm-deploy-v1.0/W1/risks.md +++ b/docs/llm-deploy-v1.0/W1/risks.md @@ -1,7 +1,7 @@ # 风险清单 v1.1 候选 > 上游：[开发计划.md](../开发计划.md) §5。 -> 状态（2026-10-02）：**已按本地实测和 `5ea22ec` Linux CI 回填；责任人及处置路线待团队确认。本次收尾与仿真计时改动集成到 PR #91，验证须对应新提交，不能继承旧 CI PASS**。 +> 状态（2026-10-03）：**PR #91 已合并，责任人及处置路线仍待团队确认**。下文旧 SHA 的数值结果是历史证据；合并后的本地修复与 W2 新结果见 [本轮报告](../W2/W1修复与W2本地验收报告.md)。人工待办由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪，不能继承旧 CI PASS。 > 对应 `docs:risks` 的结构检查：21 条风险，每条有触发信号和 Plan B。自动检查不替代人工评审。 ## 0. 阅读约定 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md new file mode 100644 index 0000000..e233014 --- /dev/null +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -0,0 +1,262 @@ +# W2：完整前端解析、基础后端与 Host 运行时 + +本目录记录开发计划中的 W2 验收：完整 28 层 Qwen3-0.6B 的前端解析与审计、四类模型模式回归、基础算子 QEMU，以及 Tokenizer/输入/mask/greedy。已有真实两层模型的 IR/ORT/QEMU 门禁继续保留。 + +完整图的范围仍止于前端验收。完整 IR 与 ORT 的数值对照、真实维度预训练子图、权重独立装载、完整 QEMU 前向和生成循环仍是后续工作。本轮也不自动关闭 W1 的第二人复现、接口确认和出口评审。 + +**当前交付状态（2026-10-03）**：本次按 W1/W2 拆分为独立分支。W2 直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含另一 W1 分支的两层报告/参考测试修复。当前分支的范围、重新执行的证据及 PR 归属见 [分支验收与发布说明](W2-分支验收与发布说明.md)。下方原本地整合记录及历史 CI 按各自源码归属，不代表本分支已通过远端 CI。 + +## W2 独立技术验收入口 + +使用已有的 Python 3.12 固定 CPU 环境（[依赖](../../../requirements/qwen3-small-probe.txt)）、Zig 0.14.1、QEMU RV64，以及固定完整 ONNX 和官方 Tokenizer 资产。在仓库或完整源码副本根目录执行： + +```powershell +$w2Python = 'D:/path/to/qwen3-venv/Scripts/python.exe' +& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` + --python $w2Python ` + --output-dir output/w2-acceptance-new ` + --tokenizer-dir D:/path/to/qwen3-tokenizer ` + --model-dir D:/path/to/qwen3-0.6b-onnx ` + --cc D:/path/to/zig.exe ` + --qemu D:/path/to/qemu-system-riscv64.exe +$LASTEXITCODE +``` + +Linux 使用相同参数，将解释器和工具替换为本机路径即可。程序不安装依赖；默认两个资产目录均使用 `verify` 离线校验。无资产时可显式传 `--tokenizer-mode download` 或 `--model-mode download`，后者需要下载约 1.24 GB 的完整模型发布包。所有结果使用本次全新的输出目录，不能复用已有目录，即使为空。 + +| 执行顺序 | 独立门禁 | 要求 | +|---|---|---| +| 1 | `frontend` | 22 项 RMSNorm/RoPE/SwiGLU/GQA 模式单测，不接受 skip | +| 2 | `backend` | 16 个小图 × none/all，32 次实际 RV64 QEMU | +| 3 | `runtime` | 官方 Tokenizer 20 组/480 对照，以及 5 组输入、mask、采样与停止条件 | +| 4 | `runtime-model` | 官方文本 ID 直入完整词表的真实两层小配置，Torch/ORT/IR none/all 单步接口集成 | +| 5 | `small-ir` | 从固定种子重新导出原两层普通/诊断图，7 组输入、29 检查点与不变性验证 | +| 6 | `small-qemu` | 使用本轮新导出的模型与报告，28 次两层实际 QEMU | +| 7 | `full-parse` | 固定 28 层完整图解析、逐节点/权重绑定审计与 IR verifier | + +只有七项全部实际执行并通过才返回 **PASS / 退出 0**。`--gates frontend backend` 等子集支持开发期间定位，但最多返回 **PARTIAL / 退出 2**；失败或阻断返回 **FAIL / 退出 1**。small-qemu 自动包含 small-ir，runtime-model 自动包含 runtime。默认每门禁截止 1800 秒、每次 QEMU 截止 180 秒、完整解析 worker 截止 300 秒，可用 `--gate-timeout`、`--qemu-timeout`、`--parse-timeout` 明确调整。 + +总报告在 `output/w2-acceptance-new/report.json/md/html`，每项有独立目录和 `logs/.log`。总入口核对退出码、报告身份、必要计数、实际输出和源码指纹；运行期间源文件变化也会失败。报告记录 HEAD/dirty diff（可用时）及本次源码 SHA256；没有 `.git` 的源码包允许运行，但必须明确标为未知 Git 身份，不能借用父目录仓库的提交。 + +新 [单步模型集成探测](../../../probes/w2_runtime_model/README.md) 保留完整词表 151936，hidden32/FFN96/两层随机权重。原文本 ID 不裁剪、不取模、不重新映射。逐元素检查完整 logits（含 padding），校验最后有效位置、原始 greedy ID 和官方解码；不要求随机权重生成有意义的文本。本地会保存较大的参考数组（约 614 MiB），CI 仅上传摘要报告，不上传这些参考数组或权重。该探测不执行完整 0.6B、完整词表 QEMU 或多步生成。 + +当前本地 workflow 已加入单步集成、完整 28 层解析必跑任务和汇总检查。`W2 / required acceptance` 要求轻量/数值任务与完整解析任务均为 success；失败、取消、跳过、缺失结果均不通过。完整 ONNX 的 ORT 前向仍是独立的手动/夜间 W1 任务。配置随本 W2 分支发布，远端 CI 以对应提交的实际执行为准；分支保护规则未变更。源码隔离复现共用本机 Python/工具链时，只能证明不依赖旧源码目录与旧产物，不能代替第二人或另一台机器复现。 + +统一验收的 Linux 超时路径会先发 SIGTERM，让运行器清理独立会话内的编译器、QEMU 或解析 worker，再执行进程组兜底清理。Windows 继续使用进程树/Job 清理。Linux 专属进程集成回归在 Windows 上明确跳过，发布前须在 Linux 实跑；不能用 Windows 结果替代它。 + +## 本轮继续交付与最终实测（2026-10-02，本地未提交） + +本轮新增完整词表的单步模型接口集成、七项统一验收入口及回归，并补齐原两层 IR 门禁的报告原子写入：MD/HTML 或 JSON 发布失败时必须返回失败，保留原数值错误，不遗留成功 JSON。此前四条报告路径的修复继续保留。 + +从当前工作树复制 **553 个源码文件**到全新的 `output/w2-independent-source-v2`，包含未提交改动，不复制 `.git`、历史 output、Python 环境或模型产物；从该目录真实运行统一入口，最终 **7/7 PASS，退出 0**。固定 ONNX/Tokenizer 资产重新验证哈希，Python 环境与 Zig/QEMU 工具仍共用本机现有安装。导入路径确认来自新副本，模型重新导出、ELF 重新编译、所有结果重新执行；总报告的 **134 项源码指纹与最终工作树一致**。 + +总证据：`output/w2-independent-source-v2/output/w2-acceptance/report.json/md/html`；每项明细、误差位置、日志和实际命令位于同一目录下对应子目录。快照来源及逐文件哈希在 `output/w2-independent-source-v2-manifest.json`，外层日志在 `output/w2-independent-acceptance-v2.log`。这些本机产物不进入版本库。 + +| 门禁 | 最终实测 | +|---|---| +| 前端四模式 | **22/22** 通过，未跳过 | +| 基础后端 | **32/32 QEMU**；最大绝对误差 **2.384185791015625e-7** | +| 运行时 | **20/20** 官方语料、**480/480** 编码/解码对照、**5/5** 输入与采样场景 | +| 单步模型集成 | **2 个原始文本、8 次模型调用**（2 Torch + 2 ORT + 4 IR）；完整输出 `[1,256,151936]`；最大绝对误差 **2.8312206268310547e-7**，采样 ID/解码一致 | +| 两层 IR | **7 组输入、29 检查点、6 项不变性**全部通过；所有数值比较最大绝对误差 **1.9073486328125e-6** | +| 两层 QEMU | **28/28**，普通/诊断图及 none/all 全通过；普通 logits 最大误差 **3.129243850708008e-7**，诊断检查点最大误差 **1.6689300537109375e-6** | +| 完整前端 | **28/28 层、7,847 节点、8,161 value、2,148 绑定**全部通过；5,910 条 IR 指令、verifier 无错误；绑定数据 **2,384,216,772 bytes** | + +统一流程 **264.340 秒**；两层 QEMU 进程累计 **157.642 秒**，其门禁全流程 **172.264 秒**；完整解析门禁 **9.538 秒**，其中 parser **2.031 秒**，worker 峰值 RSS **5,064,372,224 bytes（约 4.72 GiB）**。QEMU 计时包含进程启动、guest 运算和输出传输，诊断图包含检查点工作；这些是单次主机观察，不是纯推理性能，也不能据此声称提速。 + +首次源码副本实跑保存在 `output/w2-independent-source`，总结果正确为 FAIL：前端 pytest 临时目录的父目录未创建，导致 22 个 fixture setup 错误，其余六项通过。修复为使用统一输出根下的临时目录，新增真实 pytest 子进程回归后从新副本完整重跑，没有复用第一次的 PASS。另修复源码包向上查找 Git 的身份错误：运行时、两层 IR 和完整解析入口检查自身 `.git`；缺失时保留源码指纹并明确未知提交。最终实跑不依赖 `GIT_CEILING_DIRECTORIES` 等外部补救设置。 + +专项回归先执行 **631 passed / 0 skipped**（`output/w2-closeout-combined-tests.xml`）。最终修正上述入口后，对受影响两组分别复验 **72 passed / 0 skipped**（`output/w2-source-identity-all-tests.xml`）与 **47 passed / 0 skipped**（`output/w2-acceptance-tests-d.xml`）；批次重叠，不相加。工作流 YAML、25 段 Bash 语法、文档链接和 diff 格式检查通过。本轮未执行全仓测试或远端 CI，未提交、未推送、未更新 PR #91。 + +本轮补齐的是 W2 的独立技术验收和单步接口集成。第二人独立复现、团队接口确认以及发布后对应源码的远端 CI 仍需完成；完整预训练模型的 IR 数值、真实维度后端、独立权重装载和多步生成仍属于后续阶段。 + +## 前一轮：拆分后的本地 Review（2026-10-02） + +本轮审查完整解析、四模式前端、基础后端、Tokenizer、输入/mask/greedy，以及本地分支携带的两层 QEMU 修复。发现一类 P2 验收漏洞并修复四条报告路径：必需的报告写入失败时，可能留下可解析的 `passed=true` JSON。它会使仅消费 artifact 的脚本误判，不能以数值计算正确为理由忽略。 + +| 入口 | 复现与修复 | +|---|---| +| `probes/w2_backend_ops/run.py` | 原来先写 PASS JSON，随后 MD/HTML 失败；改为保存视图结果后发布 JSON，保存失败使门禁非零退出，保留原错误 | +| `probes/w2_qwen3_parse/run.py` | 同类先发布成功状态问题；保留验证阶段、failure_context 和主错误，报告保存异常单列，JSON 最后原子发布 | +| `probes/w2_runtime/run.py` | 虽然原来已最后写 JSON，完整内容写入后关闭/flush 抛异常仍可留下 PASS；改为同目录临时文件成功关闭后原子替换，失败不发布成功 JSON | +| `probes/w2_qwen3_small/riscv.py` | 既有两层 QEMU 门禁存在同类问题；视图与 JSON 原子写入，JSON 最后发布，保留原执行错误与时间证据。此修复只在本地，未同步 PR #91 | + +新增 **30 项**失败注入回归，覆盖 MD/HTML/JSON 无法保存、完整写入后报错、JSON 发布失败、原探测已失败及 CLI 状态。没有修改数值算法、精度阈值或模型配置。在本轮实际核对的路径和样本中，未发现新的前端/MatMul/Tokenizer 计算错误；这不构成完整模型数值正确性的证明。 + +| 本轮验证 | 结果与证据 | +|---|---| +| W2 与两层报告相关专项 | **396 passed / 0 skipped**；`output/w2-review-combined-tests.xml` | +| 前端模式、C 后端与 RISC-V 运行时相关回归 | **138 passed / 0 skipped**；`output/w2-review-compiler-regression.xml`。与上一行有重叠，不相加 | +| 基础后端实际 RV64 QEMU | **32/32**；对 ORT/独立 NumPy 最大绝对误差均 **2.384185791015625e-7**；流程 **23.588 秒**，QEMU 累计 **5.800 秒**；`output/w2-review-backend-real/report.json` | +| 官方 Tokenizer 离线探测 | **20/20** 语料、**480/480** 对照、**5/5** 输入与采样场景；流程 **3.739 秒**；`output/w2-review-runtime-offline-fixed/report.json` | + +已核对上述后端 30 项、运行时 5 项源码 SHA256 与修复后代码一致。额外 500 组随机 Unicode/特殊 token 文本编码和 4,000 次解码对照也通过；它们是本轮探索性检查，不算新增 CI 用例。本轮未重跑完整 28 层模型、整两层 QEMU 或全仓测试，也没有远端 CI；历史结果仍按原 SHA 归属。计时为单次观察，不用于声称性能改善。 + +### 可以继续的工作，按优先级 + +1. **W2 交付收尾**：第二人用自己的环境执行统一验收入口；未来按用户指示整理独立 W2 PR，明确哪些两层门禁修复应单独回补 W1。当前 PR #91 的绿色状态不是本地 W2 的验收；同机源码副本复现也不能代替第二人验收。 +2. **真实维度与预训练子图**：先验证投影 MatMul、RMSNorm、RoPE、Attention、SwiGLU，保留 Torch/ORT/IR 和适当小子图的 QEMU 误差。固定源配置为 hidden=1024、head_dim=128、16 个 Q head/8 个 KV head、FFN=3072。现有两层缩小配置和 3-token GQA case 不能证明这些长累加维度满足误差要求。优先建立基线，再决定 MatMul 分块/布局优化。 +3. **参数化中等模型与逐层定位（W3）**：将两层配置和固定 29 检查点推广成配置驱动，跑计划中的六层中等模型，记录首个偏差、最大绝对误差、耗时与峰值内存。Attention 可先沿用现有分解 IR，融合不是正确性前置。 +4. **完整 IR 内存准备（W3）**：当前解释器会复制 initializer，并在 `values` 中保留中间结果，没有最后使用后回收。先测权重、活跃张量与 RSS，设计只读权重复用和按生命周期回收；再执行完整 IR/ORT，沿用计划中的 `<1e-4` 门槛。 +5. **完整后端与生成集成（W4/W5）**：当前常量/工作区默认各 256 MiB，QEMU 为 512 MiB，权重展开为 C 字面量；仅真实 embedding 就约 593.5 MiB。需要代码与权重分离、装载 ABI、地址布局和容量验证，再接完整前向与生成循环。Host helper 与完整词表两层随机模型的单步调用现已集成；真实预训练模型调用、逐 token 循环、chat template 与流式解码仍未完成。 + +建议下一轮从第 2 项的真实维度数值基线开始，同时准备第 3 项的可配置探测，避免直接以完整模型试跑替代可定位的小范围验证。 + +## W2 基础能力补齐（2026-10-02，历史提交 ff193c7） + +该次在已通过完整解析和两层数值探测的 `dfc9437` 上补齐三个独立入口，曾更新 PR #91，现已按 W1/W2 拆分要求撤出该 PR 并保留在本地。表中“通过”必须读取对应提交的报告或 CI；本页不把旧提交或跳过项计作当前本地修改通过。 + +| W2 门禁 | 当前入口与范围 | +|---|---| +| `unit:frontend-ops` | `tests/test_qwen3_frontend_patterns.py`：RMSNorm、RoPE、SwiGLU、GQA 四类模式，含错误 epsilon、旋转符号、门控分支与 KV 重复顺序反例 | +| `unit:backend-ops` | `probes/w2_backend_ops/run.py`：16 小图，MatMul/逐元素/Softmax/RMSNorm/RoPE/SwiGLU/GQA 七类；none/all 共 32 次实际 RV64 QEMU；独立 NumPy 公式和 ORT 双参考 | +| `unit:runtime` | `probes/w2_runtime/run.py`：固定官方七份 tokenizer/配置文件，20 组语料、fast/slow 双参考与 golden IDs/解码文本；mask/最后有效位置/greedy/EOS/容量/预算 | +| `numeric:ir-small` | 既有 `probes/w2_qwen3_small/run.py`：真实两层缩小配置随机权重，继续验收 IR 对照；另保留两层 QEMU 门禁 | +| `frontend:parse-qwen3` | 既有 `probes/w2_qwen3_parse/run.py`：完整 28 层固定产物解析与绑定审计；手动/定时完整模型 CI 实际执行 | + +具体命令见 [基础后端门禁](../../../probes/w2_backend_ops/README.md) 和 [运行时门禁](../../../probes/w2_runtime/README.md)。生产新增 `scratchv/runtime/llm_inputs.py` 与 `scratchv/runtime/qwen3_tokenizer.py`；Tokenizer 只依赖可选 `tokenizers`，不在生产模块引入 Torch/Transformers。纯文本编码、完整模型输入准备与单步采样为可复用函数，尚未组合成端到端生成循环。 + +交叉 review 修复了非默认 prefix/special-token 配置被静默接受而与官方语义不同、缺省 unk token 偏离参考的问题；门禁也增加词表/特殊 ID 元数据与默认参数对照，并修正报告写入失败遗留 PASS 的路径。官方 NFC 规范化是保留行为，不把组合 Unicode 归一化误判成编译器错误。未知或空余 logits 行解码时失败，不静默丢弃或改变 greedy 候选。 + +本次 CI 增加明确的三个门禁、固定环境单测、独立 Summary 与 `w2-unit-gates-output` artifact。运行时下载约 15.2 MiB 的 tokenizer/config/许可证，按固定 revision 和既有 manifest SHA256 校验，不下载模型权重。源码和测试文件进入 Git，模型、下载缓存、ELF 与本地 output 不进入 Git。 + +### 本轮本地验收 + +| 验证 | 结果 | 本地证据 | +|---|---|---| +| 新增专项测试（固定 CPU 环境） | **228 passed / 0 skipped**，含四类前端模式 22 项、后端门禁 18 项、运行时 188 项 | `output/w2-final-unit-tests.xml` | +| 全仓回归（通用开发环境） | **2,732 passed / 41 skipped / 0 failed** | `output/w2-final-full-ready-tests.xml` | +| 基础后端真实 RV64 QEMU | **32 / 32** 通过；对 ORT 与独立 NumPy 参考的最大绝对误差均为 **2.384185791015625e-7** | `output/w2-backend-ops-final/report.json/md/html` | +| 官方 Tokenizer 下载与运行时门禁 | **20 / 20** 语料、**480 / 480** 比较、**5 / 5** 输入与采样场景通过 | `output/w2-runtime-final-download/report.json/md/html` | +| 同一执行身份离线复验 | 禁用 Hugging Face 在线访问，复用经过哈希校验的本地资源；上述 **20 / 480 / 5** 全部通过 | `output/w2-runtime-final-offline-same-user/report.json/md/html` | + +41 个跳过项包括本机缺少 `qemu-riscv32` 的 38 项，以及通用环境没有 Torch/Tokenizers 的 3 个模块；新 Tokenizer 模块已在上述固定 CPU 环境执行。两个既有 Torch 模块在本轮未本地重跑，PR CI 的固定环境继续执行原两层数值探测。各批次测试有重叠，不能相加成通过总数。 + +首次全仓运行因子进程未获得 LLVM 工具路径失败，修正测试进程的 PATH 后得到上表结果；首次跨执行身份的离线复验因 Windows 目录读取权限失败，同一身份复验通过。失败报告仍保留，未通过修改数值门槛或跳过检查消除失败。 + +后端探测全流程 **22.194 秒**，32 次 QEMU 进程墙钟累计 **5.660 秒**；Tokenizer 下载门禁 **14.824 秒**，同一身份离线复验 **4.198 秒**。均为本机单次观察，不是性能改善结论。已核对后端 30 项、运行时 5 项源码指纹与本次源码一致。仓库指定的本地 L2 harness 文件缺失，未宣称执行；本轮以实际专项、全仓和真实探测结果交付。 + +## 完整前端解析的本地复现 + +详细环境、Windows/Linux 命令、下载模式和报告说明见 [探测入口 README](../../../probes/w2_qwen3_parse/README.md)。所有命令在仓库根目录执行；使用 Python 3.12 和固定 NumPy/ONNX/ORT/protobuf 环境，无需 torch、Zig 或 QEMU。 + +已有完整模型时： + +```bash +python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +``` + +PowerShell 可使用实际虚拟环境解释器： + +```powershell +& .\output\qwen3-parse-venv\Scripts\python.exe -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +$LASTEXITCODE +``` + +没有完整模型时，将 `verify` 改成 `download`，仍使用同一固定 W1 manifest。模型必须包含真实 external-data 分片。每次使用新的报告目录，失败时保留已有文件与日志。 + +## 如何判断本阶段完成 + +| 项目 | 应保留的证据 | +|---|---| +| 源码身份 | checkout SHA、是否有本地改动、实际 diff 与新增文件 | +| 固定模型身份 | 发布产物和 external-data 文件校验结果 | +| 完整结构与层覆盖 | 本次结构审计摘要及 28 层检查结果 | +| 实际节点转换 | `nodes.json` 与审计摘要 | +| initializer 绑定 | `bindings.json` 与审计摘要 | +| 共享 IR | `ir.txt` 与 IR 验证器结果 | +| 本次执行 | 命令、退出码、`report.json/md/html`、实测时间及内存（如平台可测） | +| CI | 对应提交和实际 run URL；未执行时明确写未执行 | + +退出 0 和 `passed=true` 必须来自本次执行。轻量 fixture 测试、历史 ORT 通过或一个可打开的报告文件都不足以代替完整图审计。失败发生较早时，某些明细可能尚未产生；按实际阶段和错误定位，不将缺失记录补成成功。 + +本地 W2 workflow 现在让 PR、手动和定时任务均执行独立的 `full-qwen3-frontend` 完整解析任务，无须等待完整 ONNX 的 ORT 前向。该任务下载约 1.24 GB 的固定发布包，解析时需约 5 GB RSS；手动/夜间同时运行 ORT 任务时，两台 runner 会各自下载。汇总任务拒绝失败或跳过的依赖任务。PR #91 已合并；未来 W2 远端结果须关联其独立提交和实际 run，不能继承旧提交的 PASS。 + +## 完整解析 Review 与实测（2026-10-02，历史提交 dfc9437） + +该次同时复核本地完整解析门禁和 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91)。核对时 PR head 为 `df18b02f152878b1ea838bbf6073def01d367456`，base 为 `d1c2b5f04cad817acf0b2f14d3bfa4ff56dbfa11`，当时差异涉及 52 个文件。审查覆盖编译入口、IR 优化、tensor-c 后端、RISC-V 运行时、数值探测、CI 和复现说明。下述完整解析与 Review 修复曾在 `dfc9437` 纳入 PR #91，现已撤出该 PR 并保留于本地；当时实际执行源码由报告中的 SHA256 确定。 + +### 完整解析门禁修复 + +| 问题 | 修复与验证 | +|---|---| +| 属性审计复用生产 parser 的属性解码，可能让相同错误在转换和审计两端同时通过 | 直接从原 ONNX protobuf 独立读取属性，拒绝重复属性；增加错误 Transpose 轴和错误 epsilon 的反例 | +| 未检查函数参数名称与常量标记 | 对照原 ONNX 输入校验名称、顺序、shape/dtype，并拒绝被标记为常量的输入 | +| 标量辅助 `LOAD_CONST` 校验不完整 | 对 initializer 和 Constant 两条路径核对 opcode、目标 Value、空操作数、shape/dtype、常量标记、常量值及指令属性 | +| 审计时隐式 `np.asarray` 可能掩盖错误的绑定类型 | 要求实际绑定是 `numpy.ndarray`；list 等错误类型直接失败，合法只读 ndarray 仍接受 | +| 日志写入失败可能覆盖 worker 原始错误 | stdout/stderr 独立尝试保存；保留原错误、阶段及退出码，附加 `log_errors`；成功路径缺少日志也不能报 PASS | +| 文档链接指向被忽略的本地输出，干净 checkout 中无法访问 | 改为明确的本地输出路径说明，并增加模拟缺少 output 目录的文档链接回归测试 | + +进一步修正失败明细写入的同类问题：保存 `nodes.partial.json` 等失败证据时，写入异常记录到 `evidence_errors`，不覆盖原始解析或绑定错误；成功路径缺少必需证据仍失败。supervisor 在启动 worker 前保存 `attempt`，记录命令、模型路径、解释器、checkout 和源码指纹，便于定位过早超时、worker 未能启动等情况；该字段不代表验证已经执行。 + +### PR #91 修复 + +| 问题 | 修复与验证 | +|---|---| +| tensor-c 接受与 IR 标量常量值不同的外部绑定，可能与解释器行为不一致 | 校验绑定与常量声明一致；增加 FP32/INT32/INT64 错值、错误向量形状和合法标量执行的回归测试 | +| 编译或 QEMU 失败时，保存日志的异常可能掩盖编译错误、超时或 guest 错误 | 独立保存两路日志；保留主错误及原耗时，并附加日志错误。覆盖编译失败/成功与 QEMU 超时/失败/成功等组合 | +| RISC-V 报告不足以确定本次 ORT/IR 对照的实际执行环境和完整源码身份 | 顶层 `environment` 记录当前进程，`export_evidence.environment` 保留导出时环境；源码指纹覆盖 parser、IR、优化、后端和运行时，共 30 项 | + +未发现本次审查范围内需要额外修改的 MatMul 实现问题；下列结果只证明已测试输入和路径。未调整精度门槛、模型权重或 D8 等团队待确认决策。 + +### 修复后的实测 + +| 验证 | 当前结果 | 本地证据 | +|---|---|---| +| 全仓回归 | 2,563 passed / 40 skipped / 0 failed | `output/review-fix-full-tests.xml`、同名 `.log` | +| 固定 CPU 环境最终相关测试 | 286 passed / 0 skipped，包含全部 114 项完整解析测试 | `output/review-fix-pinned-tests.xml`、同名 `.log` | +| 真实 Qwen3 结构两层小模型重新导出与 PyTorch/ORT/IR 对照 | 7 / 7 组输入通过；全部数值比较最大绝对误差 `1.9073486328125e-6` | `output/review-fix-qwen3-small/report.json` | +| RISC-V QEMU 两层探测 | 7 组输入 × 普通/诊断图 × none/all 优化，共 28 / 28 执行通过；QEMU 对照最大绝对误差 `1.6689300537109375e-6`，门槛 `atol=1e-5, rtol=0` | `output/review-fix-qwen3-riscv/report.json`、`report.md` | +| QEMU MatMul | 7 / 7 通过，最大绝对误差 `2.384185791015625e-7` | `output/review-fix-matmul/report.json` | +| 完整 Qwen3 前端解析 | 28 / 28 层；7,847 节点、23 类算子；5,910 条 IR；2,148 项绑定；IR verifier 0 问题 | `output/review-fix-full-qwen3-final/report.json`、`report.md`、`report.html` | + +全仓回归中的 38 个 RV32 执行测试因缺少 `qemu-riscv32` 跳过，另有 2 个依赖 torch 的模块因通用开发环境不含 torch 而跳过；最终固定 CPU 环境相关测试已覆盖这两个模块。全仓测试收集后补充了 3 项失败证据回归测试，均包含在最后的 286 项测试内；没有把不同批次或重叠测试相加为一个通过总数。RV64 tensor 真实运行时集成测试和上述实际 QEMU 探测已执行。 + +本次 QEMU 进程墙钟时间累计 **160.450 秒**，探测全流程 **174.634 秒**，4 个成功构建累计 **6.586 秒**。普通图每次平均约 **1.143 / 1.147 秒**（none/all），诊断图约 **10.138 / 10.494 秒**。QEMU 时间包含进程启动、guest 计算、输出打包/UART 和退出；诊断图还包含中间张量输出，不能当作纯前向耗时或真实硬件性能。 + +完整模型本次纯解析 **3.636 秒**，入口至验证及日志保存结束 **12.772 秒**，worker 峰值 RSS **5,064,331,264 bytes，约 4.72 GiB**。完整模型输出静态契约为 FP32 `[1,256,151936]`，绑定共 **2,384,216,772 bytes**；这些是解析与绑定审计结果，本次未执行完整模型数值前向。以上耗时均为本机单次运行，期间存在并行测试，不用于声称性能改善。 + +已核对完整解析报告的 20 项、RISC-V 报告的 30 项源码 SHA256 与当前源码一致；完整解析的 `nodes.json`、`bindings.json`、`ir.txt` 的文件大小和 SHA256 均与报告一致。测试后仅补充说明文档，执行源码未再修改。输出目录被 Git 忽略，仅在本机保存；他人复现需运行入口生成自己的报告。 + +最终还实跑了 0.1 秒截止和模型缺失两个失败路径：均正确退出 1，分别记录 `timeout`、`hashes` 阶段，且保留启动前的 20 项源码指纹；报告在 `output/review-fix-final-expected-timeout/` 和 `output/review-fix-final-expected-missing/`。文档检查包含尚未跟踪的新文档，共检查 97 个文件、196 个相对链接，0 个失效；LLM workflow YAML 与 19 段 shell 命令语法检查通过，`git diff --check` 通过。 + +当前没有已确认且尚未修复的本轮阻断性问题。完整 IR/ORT 数值与容量验收、完整模型 QEMU 前向、权重独立装载、W1 第二人复现和接口签字仍未由本次工作完成。本页实测均为本地证据，远端 CI 状态及 run 链接以 PR 描述和 Checks 为准。 + +## 首次交付结果（2026-10-02，历史记录） + +首次交付工作在独立本地分支 `codex/w2-full-qwen3-parse`，基于 PR #91 的 `df18b02f152878b1ea838bbf6073def01d367456`。实际工作目录为 `D:/cyq/code/ScratchV/output/w1-pr89-review`；当时修改未暂存、未提交、未推送，尚未更新 PR #91。上层原工作目录中的已有改动未纳入本轮。 + +首次交付的完整实物执行退出 0，结果 **PASS**。本地证据保存在 `output/w2-full-qwen3-parse-complete/`：`report.md`、`report.json`、`nodes.json`、`bindings.json` 和 `ir.txt`。这些是该次 checkout 的本地输出，`output/` 不进入版本库；其他机器需执行入口重新生成，不能依赖这些历史路径存在。 + +| 验收项 | 首次交付实测 | +|---|---| +| 固定模型 | Qwen3-0.6B，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 / L=256 / 无 KV Cache | +| 完整层链 | 28 / 28 层，含最终归一化与共享 embedding / LM Head | +| ONNX 节点 | 7,847 / 7,847；23 类基础算子；全部可达 logits | +| ONNX value | 8,161 项静态 shape / dtype 审计通过 | +| 实际 IR | 5,910 条指令；IR verifier 0 个问题 | +| 真实绑定 | 312 个 initializer + 1,836 个 Constant，共 2,148 项；逐项内容哈希通过 | +| 绑定字节数 | 2,384,216,772 bytes | +| 合法别名 | 112 个 Identity，均核对实际对象映射 | +| 返回值 | `logits`，FP32 `[1,256,151936]` | +| 纯解析耗时 | 2.493 秒 | +| 全入口耗时 | 10.533 秒，包含检查、解析、审计与子进程开销 | +| 验证进程峰值 RSS | 5,063,430,144 bytes，约 4.72 GiB，Windows PeakWorkingSetSize | + +耗时来自单次本地运行，不是平均值或性能承诺；本轮没有做解析性能优化。报告记录运行环境、实际源码 SHA256 和产物哈希。工作树有本地改动，不能将此结果仅归属于基准 HEAD。 + +首次交付的轻量测试 **91 / 91 通过**，在固定 CPU 环境和现有开发环境各执行一次，分别记录于 `output/w2-gates-pinned-delivery.xml` 和 `output/w2-gates-general-delivery.xml`。测试含真实两层 fixture 经文件哈希、结构检查、生产 parser、IR verifier 和绑定审计的贯通，以及漏节点、错权重、错 shape/dtype、外部数据损坏、解析异常、缺报告与超时等反例。当时全仓回归 **2,521 passed / 40 skipped**，记录于 `output/w2-regression-final.xml`；首次交付最后修正后重跑上述全部 91 项相关测试。当前修复后的验证以本页上方结果为准；跳过项不计为通过。 + +实际进程反例也已执行：模型缺失在 `hashes` 阶段退出 1；0.1 秒截止测试在 `timeout` 阶段退出 1，实测总耗时约 0.184 秒，均保存 FAIL 报告。记录分别位于 `output/w2-expected-missing-model/` 和 `output/w2-expected-timeout/`。 + +首次交付新增三个探测模块、三组测试与复现文档，并更新本地 LLM CI 配置。没有发现这份完整固定图需要新增前端算子，当时未修改通用 ONNX parser 或编译后端。首次互审修正了验收实现中的两个重点问题：控制参数必须从原 ONNX 独立求值，避免解析器错误自证通过；超时日志写入失败、进程异常退出及 verifier 失败时应尽量保留原始错误和节点证据。后续 Review 的更多修复见本页上方。 + +本阶段已具备完整前端解析的本地验收证据；远端 CI、第二人复现、完整模型 IR 数值及完整模型 QEMU 前向尚未由本轮验证。 + +## 后续衔接 + +1. 前端审计通过后，依据实际图和权重规模建立完整 IR 数值与峰值内存基线，逐步扩大预训练模型验证范围。 +2. 设计代码与权重分离、权重绑定 ABI 和内存布局；当前 C 常量展开及小模型 guest 配置不能直接承担完整 0.6B 模型。 +3. 完整 IR 数值验收通过后，再验证完整 QEMU 单次前向；随后将已有 Tokenizer、输入准备、最后有效 token logits、采样和停止条件组合成生成循环，并验证真实文本输出。 + +前端解析通过不能跳过这些阶段，也不能据此宣布完整 Qwen3 已在 ScratchV 上运行。开发范围见 [总开发计划](../开发计划.md)，W1 人工验收状态见 [W1 独立复现记录](../W1/README.md)。 diff --git "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" new file mode 100644 index 0000000..e08da37 --- /dev/null +++ "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -0,0 +1,125 @@ +# W1 修复与 W2 本地验收报告 + +日期：2026-10-03。**历史整合工作树报告**：记录的是发布前 W1+W2 合并在本地的验证；其中“未提交”等为当时状态。后来按 W1/W2 拆分独立发布，当前 W2 分支结果见 [分支验收与发布说明](W2-分支验收与发布说明.md)，不能将下述组合源码哈希当作当前分支哈希。 + +## 1. 结论与源码位置 + +W1 的报告可靠性与源码归属修复已补入；W2 已在 PR #91 合并基线上整合，并重新执行七项技术验收，结果 **7/7 PASS**。完整图的通过范围是前端解析、结构和权重绑定审计；完整 0.6B 的 IR 数值、QEMU 前向与生成循环仍未完成。本报告不替代团队冻结接口、第二人复现或远端 CI。 + +本轮源码在 `D:/cyq/code/ScratchV/output/w1-w2-local-closeout`，本地分支 `codex/w1-w2-local-closeout`。基线是 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，所有增量均未提交。原 `output/w1-pr89-review` 的 W2 修改、`output/w1-pr91-repro-fixes` 和最外层工作目录的代码保持原样。 + +下述 `output/...` 均相对于本轮源码目录。它们是本机证据，不是仓库发布资产；复现者应使用新的目录生成自己的报告。 + +## 2. W1 补完了什么 + +| 修复 | 原问题 | 当前行为与主要文件 | +|---|---|---| +| 两层报告完整发布 | MD/HTML 写入失败或 JSON 发布失败可能留下成功证据 | 视图先写、JSON 最后原子发布；失败返回非零，尽力保存 FAIL，保留原始数值错误。`probes/w2_qwen3_small/run.py`、`riscv.py` | +| 源码与环境归属 | 无 `.git` 的源码副本可能向父目录查找 Git，误认其他 checkout；QEMU 对照环境可能与导出环境混淆 | 无本目录元数据时明确未知 Git 身份，保留源码 SHA256；分别记录当前执行与导出环境，并补齐 parser/IR/优化/运行时指纹 | +| 回归覆盖 | 成功路径测试不足以验证失败时的证据 | 故障注入覆盖写入、关闭、替换失败与原始错误保留；补源码副本及当前环境归属回归 | +| 原有算子用例与固定 ORT 环境兼容 | ORT 1.22.1 无 DOUBLE Cos 内核，ReduceMean 的 omitted axes + noop=1 路径报内核错误 | 三项用例显式使用 ONNX 官方 ReferenceEvaluator，继续实际运行生产 parser/IR，ReduceMean 再检查与输入完全相等；其余用例继续 ORT 对照，不捕获或忽略数值错误 | +| 共享后端与运行时防错 | 外部绑定可能覆盖标量 IR 常量；日志 I/O 失败可能遮蔽原编译/QEMU 错误 | `tensor_c_codegen.py` 校验常量绑定一致；`riscv_tensor.py` 独立保存两路日志，保留主错误和耗时；一并整合之前 W2 分支暂存的修复 | +| 文档状态 | 仍使用 PR 未合并、旧提交或“所有第二人复现待做”的描述 | 固定已合并基线，保留 `df18b02f` 第二人两层复现事实，区分完整模型复现与接口签字待办 | + +在整合 W2 前，W1 四个源码/测试文件的专项测试已执行 **120 passed / 0 skipped**，证据为 `output/w1-fix-tests-v2.xml`。W1 独立检查点在 `output/w1-fix-checkpoint/`：包含补丁、四文件快照与哈希，便于后续按 W1/W2 分拆提交。这次没有重新打开或更新已合并的 PR #91。 + +最终共享修复的建议拆分补丁为 `output/w1-final-review/w1-fixes.patch`，含上述报告/归属、标量绑定、运行时清理/日志及 ORT 用例兼容的 10 个源码/测试文件，配有 `manifest.json`；它是供审阅的本地补丁，不是已暂存或已提交的改动。 + +最后还单独实跑原 W1 入口：MatMul **7/7**（4×4 小数最大误差 `1.4901161193847656e-8`，七例最大 `2.384185791015625e-7`）；PR #89 合成两层图的 IR vs ORT 最大误差约 `7.153e-7`，ORT vs 独立 NumPy 约 `1.097e-6`。证据在 `output/w1-matmul-final/`、`output/w1-transformer-final/`。完整 0.6B ONNX 的 ORT 前向本轮没有重跑，其历史验收仍按原记录归属。 + +W1 人工出口继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪：团队 D1–D8 接口决议、风险处置责任与 Plan B、完整模型第二人复现和最终出口评审。本轮没有代签或关闭这些事项。 + +## 3. W2 工作与开发计划逐项对照 + +| 开发计划要求 | 当前实现与本轮证据 | 状态边界 | +|---|---|---| +| `unit:frontend-ops`：RMSNorm/RoPE/SwiGLU/GQA | 22 项真实 ONNX 模式测试全部通过 | 通过基础算子组合解析，未新增同名融合 IR 指令 | +| `unit:backend-ops`：基础算子 QEMU | MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA，7 类/16 小图 × none/all，共 32 次通过 | 超过表中六类最低覆盖；仅证明这些 shape 与输入 | +| `numeric:ir-small`：两层 IR vs ORT | 真实 Qwen3 类的两层缩小配置，7 输入、29 检查点、6 项不变性通过；none/basic/all 优化对照 | 随机固定权重，不是预训练权重切片 | +| Attention IR 定义 | 候选接口文档明确基础算子组合、Q/K 独立 RMSNorm、全维 RoPE、GQA 分组与加性 mask 语义 | 当前不强制新增融合 ATTENTION opcode；这项接口决议仍需团队确认 | +| `unit:runtime`：Tokenizer、greedy、mask | 固定官方 Tokenizer 20 组语料/480 对照，5 组输入、mask、采样与停止条件通过 | 往返要求按官方 tokenizer 的特殊 token/规范化语义校验，不声称任意文本字节级恒等 | +| `frontend:parse-qwen3`：完整 28 层 | 固定 FP32/L=256 产物完整解析；7,847 节点、8,161 value、2,148 绑定、5,910 IR 指令通过审计，verifier 无问题 | 不代表完整 IR 数值或 QEMU 已运行 | +| 单步运行时与模型接口集成 | 2 个文本、完整词表 151936；2 Torch + 2 ORT + 4 IR，共 8 次调用；原始 token ID、最后有效位置、greedy ID 与解码一致 | hidden32/FFN96 两层随机小配置；未执行完整词表 QEMU 或多步生成 | +| W1 数值链路不退化 | 两层普通/诊断图 × none/all × 7 输入，28 次真实 QEMU 全通过 | 原 W1 的 20 次 Linux 超时回归与 3 项显式产物集成 CI 配置继续保留 | +| CI 全绿 | 本地配置增加完整 28 层解析 PR 必跑任务与汇总检查，拒绝跳过/失败/缺失 | **尚未提交，所以没有本轮远端 CI 通过结论** | + +本轮整合原先仅本地保存的完整解析、后端和运行时工作，新增修复 CI 验收缺口与嵌套进程超时清理。生产后端还保留审查所得的标量常量绑定一致性检查，拒绝调用方用不同绑定覆盖 IR 常量；日志写入失败保留原编译/QEMU 错误与耗时。这些此前 W2 分支的修复均已在本轮基线上重新验证。 + +## 4. 本轮重新执行的验收数据 + +总入口：`scripts/run_w2_acceptance.py`。总报告：`output/w2-acceptance-20261003/report.json`，另有同名 Markdown/HTML。每项包含自己的原始报告、日志、命令、输出数组、模型或 ELF 身份。总入口核对 **136 项源码指纹**，拒绝旧目录、缺失门禁、skip、非有限误差及运行中代码变化。 + +| 门禁 | 结果 | 最大绝对误差 | 顶层观测耗时 | +|---|---|---:|---:| +| frontend | 22/22 | 不适用 | 1.157 秒 | +| backend | 32/32 QEMU | `2.384185791015625e-7` | 23.769 秒 | +| runtime | 20/20、480/480、5/5 | 离散结果精确对照 | 4.905 秒 | +| runtime-model | 2 文本、8 次调用 | `2.8312206268310547e-7` | 14.889 秒 | +| small-ir | 7 输入/29 检查点/6 不变性 | `1.9073486328125e-6` | 6.971 秒 | +| small-qemu | 28/28 QEMU | 普通图 `3.129243850708008e-7`；诊断图 `1.6689300537109375e-6` | 153.658 秒 | +| full-parse | 28/28 层、逐节点/绑定审计通过 | 本项不执行数值前向 | 11.111 秒 | + +数值门槛保持 `max_abs < 1e-5`、`rtol=0`，没有放宽精度、裁剪 token ID 或用历史产物替代本次执行。small-qemu 报告也包含主机 IR 参考，其全报告最大误差 `1.9073486328125e-6` 不应误写成 QEMU 自身误差。 + +整轮 **216.597 秒（约 3 分 37 秒）**。基础后端 32 次 QEMU 进程墙钟累计 **5.545 秒**；两层 28 次累计 **140.312 秒**，4 次 C→RV64 编译累计 **6.189 秒**。两层普通图平均约 1.06/1.07 秒（none/all），诊断图约 8.94/8.97 秒；诊断输出多，不能当作纯前向速度。 + +完整前端 worker 的纯解析 **2.579 秒**，验证流程 **10.818 秒**，峰值 RSS **5,064,474,624 bytes，约 4.72 GiB**；绑定数据 **2,384,216,772 bytes**。这些是本机单次观察，不是优化前后基准对比，也不等同真实硬件性能。 + +## 5. Review、回归及 CI 检查 + +本轮 Review 先修复两个新发现的问题： + +1. 完整解析原来只在手动/夜间任务运行，PR 绿色无法证明 W2 全部门禁。现在 `full-qwen3-frontend` 在匹配路径的 PR 上必跑；`w2-acceptance` 对所有依赖要求 success。对应 21 项结果校验测试覆盖失败、跳过、取消、缺失和畸形 JSON。分支保护设置未修改。 +2. 顶层探测的 Linux 超时直接杀进程组，可能遗留 `_run_process` 创建的独立编译器/QEMU 会话。现在先 SIGTERM 触发运行器清理，再强制清理原组；增加实际进程中断回归和 Linux 嵌套会话回归。Windows 路径沿用 Job/进程树处理。 + +扩大回归时还修复了上文三项 ORT 参考内核兼容问题；相关四个测试文件最终 **251 passed / 0 skipped**，证据 `output/review-regression-fixes.xml`。没有修改编译器计算算法或数值门槛来适应参考执行器。 + +超时与 CI 专项结果 **128 passed / 1 skipped**，证据 `output/timeout-ci-tests.xml`；跳过的是 Linux 专属嵌套会话集成用例，本机 Windows 无法执行，不能计作通过。W2 集成/报告/CI 结果校验前一轮为 **111 passed / 0 skipped**，证据 `output/w2-integration-tests.xml`。各批次有重叠，不能相加。 + +最终 `tests/` 全量回归 **3020 passed / 39 skipped / 0 failed，177.66 秒**，证据为 `output/final-all-tests-v4.xml` 和同名 `.log`。39 个 skip 明确分为：本机无 `qemu-riscv32` 的 38 个旧 RV32 用户态执行测试，以及 1 个 Linux 专属嵌套会话清理测试。真实 RV64 tensor 运行时测试、三项显式产物优化集成及本轮所有可在 Windows 执行的 W1/W2 测试均已执行；不能把 39 个 skip 写成通过。 + +最终工作流 YAML、29 段 Bash 语法、文档相对链接、W1 四类接口/21 条风险及 Plan B 的结构检查、`git diff --check` 通过；详见 `output/final-static-review.json`。七项验收记录的 136 项源码指纹仍与最终执行代码一致。报告及状态文档在数值执行结束后补写，不改变该数值结果对应的执行代码。项目约定的 `.claude/harness/verify/run.py` 不存在，未宣称运行 L2 harness。 + +失败尝试也保留：W1 首次测试因自定义 pytest 临时目录的父目录尚未创建而产生 82 个 setup error，创建目录后 120 项通过；全仓第一次在收集阶段发现固定模型环境缺少可选 `llvmlite`。后者使用已有开发环境的 llvmlite 0.49.0 与 tinyfive 1.0.0 的本地副本补入仅本轮测试的路径，未改固定模型环境或全局安装。 + +中间全仓尝试 `final-all-tests-v2` 为 228 failed / 2791 passed / 40 skipped：224 项是 LLVM 工具未发现，3 项是上述 ORT 参考限制，1 项文档链接失败发生在报告文件尚未写完时。`v3` 显式指定 llvm-mca 后为 5 failed / 3014 passed / 40 skipped，剩余 5 项均因 clang/ld.lld 未从 PATH 找到。最终通过 `output/run_local_regressions.py` 在 Python 进程内配置路径，并确认三项 LLVM 工具实际可运行；不把这些中间失败隐藏或计算成通过。 + +## 6. 尚待完成与后续工作 + +- 本轮 Linux 嵌套会话清理集成回归、修改后的 GitHub workflow 和完整解析 runner 的实际容量，需要发布后取得对应源码的 Linux CI 证据。源码/配置本地通过不等于远端已通过。 +- W2 需要第二人用自己的环境和全新输出目录运行七项统一验收；W1 已确认的是旧 SHA 的两层链路，不能自动覆盖这次新增修复或 W2。 +- W1 接口冻结与风险责任决议保持候选，继续按 Issue #92 讨论；不要因技术门禁通过代替团队确认。 +- 后续按 W3 先做真实维度、预训练权重子图和更深层 IR 数值/内存基线，再做权重与代码分离、权重加载 ABI、内存复用和完整模型 QEMU 前向；最后接多步生成。原 `backend=riscv` 标量选择器并未因此支持大模型，本路线使用 `tensor-c`。 + +## 7. 本机复现命令 + +在本轮源码目录执行，输出目录必须是全新的。下面使用现有工具与固定模型资产，不自动安装或下载。 + +```powershell +$w2Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' +& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` + --python $w2Python ` + --output-dir output/w2-acceptance-review-new ` + --tokenizer-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` + --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` + --cc D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/zig-python/ziglang/zig.exe ` + --qemu D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/qemu/qemu-system-riscv64.exe +$LASTEXITCODE +``` + +人工首先检查本报告、七项总报告和文件清单，再决定后续拆分提交方式。本轮没有创建、修改或关闭 GitHub PR/Issue。 + +## 8. 修改文件审阅顺序 + +| 范围 | 主要文件 | +|---|---| +| W1 两层证据可靠性 | `probes/w2_qwen3_small/run.py`、`riscv.py` 及对应 gate 单测 | +| 编译/运行时基础修复 | `scratchv/backend/tensor_c_codegen.py`、`scratchv/runtime/riscv_tensor.py` 及对应单测 | +| 固定 ORT 测试兼容 | `tests/test_onnx_qwen_numeric_ops.py`、`tests/test_onnx_shape_ops.py` | +| 完整图审计 | `probes/w2_qwen3_parse/{run,validation,audit}.py`、三组 `test_qwen3_full_*.py` | +| 基础后端与前端模式 | `probes/w2_backend_ops/`、`tests/test_qwen3_frontend_patterns.py`、后端 gate 测试 | +| Host 运行时与单步模型集成 | `scratchv/runtime/{qwen3_tokenizer,llm_inputs}.py`、`probes/w2_runtime/`、`probes/w2_runtime_model/` 及对应测试 | +| 验收入口与 CI | `scripts/run_w2_acceptance.py`、`scripts/check_w2_ci_results.py`、`.github/workflows/llm-deploy.yml` 及对应测试 | +| 文档与依赖声明 | W1/W2 指南、执行/探测计划、接口/风险候选、总开发计划、各 probe README、`pyproject.toml` 的可选 tokenizer 依赖 | + +最终全文件清单与哈希保存在 `output/final-review-manifest.json`；完整已跟踪文件差异在 `output/final-tracked.patch`，新增文件保留在上表目录。模型、参考数组、ELF、可选测试依赖副本均在被忽略目录，未纳入源码差异。 diff --git "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" new file mode 100644 index 0000000..4e95d1e --- /dev/null +++ "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" @@ -0,0 +1,27 @@ +# W2 独立分支验收与发布说明 + +W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。本分支直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含独立 W1 PR 中的六个两层报告/参考测试文件。两部分可分别审阅和合并。共享后端的标量绑定、运行时日志和取消清理加固随 W2 交付。 + +## 范围 + +- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和权重绑定审计。 +- RMSNorm/RoPE/SwiGLU/GQA 前端模式，七类基础后端的 16 小图/32 次 QEMU。 +- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层小配置的单步模型集成。 +- 七项统一验收入口，完整前端 PR 必跑任务，以及拒绝失败、取消、跳过、缺失的 CI 汇总。 +- 标量 IR 常量外部绑定一致性；日志写入失败保留主错误；取消时清理独立编译器/QEMU 会话。 + +本分支 Windows 固定 CPU 环境重新执行：**629 passed / 1 skipped**（仅 Linux 嵌套会话清理用例跳过）；**七项统一验收 7/7 PASS**，实际 32 次基础后端 + 28 次两层 QEMU 全通过。完整图为 28 层、7,847 节点、2,148 绑定、5,910 条 IR，verifier 与逐项审计通过。运行时为 20 组语料/480 对照/5 场景；完整词表单步集成为 2 文本、8 次调用。136 项源码指纹与发布前代码一致。 + +总入口耗时 251.494 秒，两层 QEMU 进程累计 154.125 秒，基础后端 QEMU 累计 6.194 秒；并行验收时的本机单次观察，不用作性能对比。精度最大误差：基础后端 `2.384185791015625e-7`，单步集成 `2.8312206268310547e-7`，两层 IR `1.9073486328125e-6`，两层目标诊断 `1.6689300537109375e-6`，均小于 `1e-5`。 + +另外，以本分支新生成的模型显式执行 none/basic/all 三项产物优化集成，**3/3 通过、0 跳过**，证据为 `output/w2-publish-artifact.xml`。 + +证据在 `output/w2-publish-tests.xml`、`output/w2-publish-acceptance/report.json/md/html` 及各子目录。工作流 YAML、29 段 Bash、文档链接及 diff 格式检查通过。本分支没有重跑全仓 tests；原整合工作树 3020 项通过属于下述历史记录，不能用其代替本分支验证。固定 ORT 环境下三个原有算子参考用例的兼容修复归属 PR #93。项目的本地 L2 harness 文件缺失，未声称运行。此前 [W1 修复与 W2 本地验收报告](W1修复与W2本地验收报告.md) 记录的是两部分合在一起的历史工作树，不应冒充本分支的精确源码证据。 + +## 复现与限制 + +运行 [统一入口](README.md) 的七项门禁；每次使用新的输出目录。源文件哈希、模型/ELF 身份、原始命令、误差和仿真墙钟时间均保存在生成的 JSON/Markdown/HTML 中。 + +完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合，无新增融合 opcode。第二人复现、接口冻结、完整模型权重独立装载和生成循环仍需后续完成。 + +Linux 嵌套会话清理用例在 Windows 上明确跳过；CI 配置已经提供入口，但只有对应提交的实际远端执行才能证明 Linux 通过。完整 ONNX ORT 前向继续由独立手动/夜间任务验证。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 3f84dec..030275d 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -7,7 +7,7 @@ **周期**：8 周 **验收方式**：CI 门禁驱动 -**实施基线更新（2026-10-02）**：本文保留 v1 的范围、周次目标和数值门槛，按提交 `5ea22ec` 已验证的实现校正路线。当前小模型采用 `tensor-c → Zig/LLVM → RV64GC/LP64D → QEMU virt 裸机`，接口仍为候选基线，未代替团队冻结决议。W1 技术探测已通过，第二人独立复现、接口/风险确认和出口评审仍待完成，详见 [W1 执行计划](W1/W1-执行计划.md) 和 [独立复现指南](W1/README.md)。后文 W2–W8 是待验收计划，不因小模型成功自动完成。 +**实施基线更新（2026-10-03）**：本文保留 v1 的范围、周次目标和数值门槛。PR #91 已合并为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，小模型采用 `tensor-c → Zig/LLVM → RV64GC/LP64D → QEMU virt 裸机`。W1 已有 `df18b02f` 两层链路的第二人独立复现；完整模型第二人复现、接口/风险确认和出口评审仍待完成，详见 [W1 执行计划](W1/W1-执行计划.md) 和 [独立复现指南](W1/README.md)。合并后的 W1 报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 单独交付；发布前整合工作树实测与限制见 [本轮验收报告](W2/W1修复与W2本地验收报告.md)。W2–W8 按各自出口验收，不因小模型成功自动完成。 --- @@ -47,7 +47,7 @@ ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译 ### 2.1 整体数据流 -下图是完整模型的目标数据流。当前已实现的是 Python host 装载输入、启动裸机 QEMU 和读取 UART 结果；Tokenizer、采样、token 循环与完整模型仍待交付。图中的 FFI 表示模型调用边界：host 通过 QEMU 装载/传输访问 guest，不能在宿主机直接调用 RISC-V 函数。实际模型 ABI 为 `int scratchv_run(const void *const inputs[], void *output)`，详见 [候选接口](W1/interfaces.md)。 +下图是完整模型的目标数据流。当前已实现 Python host 装载输入、启动裸机 QEMU 和读取 UART 结果；W2 另提供固定官方 Tokenizer、输入/mask、最后有效位置 greedy 及停止边界辅助接口，见 [W2 交付记录](W2/README.md)。完整 token 循环与完整模型仍待交付。图中的 FFI 表示模型调用边界：host 通过 QEMU 装载/传输访问 guest，不能在宿主机直接调用 RISC-V 函数。实际模型 ABI 为 `int scratchv_run(const void *const inputs[], void *output)`，详见 [候选接口](W1/interfaces.md)。 ``` ┌─────────────────────────────────────────────────────────────────┐ @@ -77,11 +77,11 @@ ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译 | 阶段 | 已验证能力（两层小模型范围） | 后续需验收 / 扩展 | 负责模块 | |---|---|---|---| -| 前端 | Qwen3 基础算子解析、属性/shape/dtype 和权重绑定 | 完整 28 层导出图逐节点与权重验收；模式识别按优化需要添加 | `scratchv/frontend/` | +| 前端 | Qwen3 基础算子解析；固定完整 28 层图的属性/shape/dtype、逐节点转换和权重绑定已验收 | 预训练完整图数值；模式融合按优化需要添加 | `scratchv/frontend/` | | IR | 共享 SSA、verifier、NumPy 解释器可表达基础 Attention/RoPE/SwiGLU 图 | 完整 IR 数值与峰值内存；融合节点不是正确性前置 | `scratchv/ir/`、`scratchv/verification/` | | 优化 | none/basic/all 后 IR 对照；修复 FP32 舍入和 SSA 问题 | 更大配置回归；融合 Attention 等作为可选优化逐项验收 | `scratchv/optimizer/` | | 后端 | tensor-c 显式 FP32 张量循环，经 Zig/LLVM 生成 RV64GC 机器码 | 大权重与代码分离、完整容量、MatMul 分块/布局；旧标量选择器缺口另行决定是否补齐 | `scratchv/backend/` | -| 运行时 | 固定输入装载、裸机 QEMU、UART 张量输出与错误处理 | Tokenizer、采样、生成循环、完整权重装载与分块输出 | `scratchv/runtime/` | +| 运行时 | 固定输入装载、裸机 QEMU、UART 张量输出与错误处理；Tokenizer/mask/greedy/停止边界辅助接口 | 生成循环、完整权重装载与分块输出、端到端集成 | `scratchv/runtime/` | ### 2.3 Qwen3-0.6B 关键配置 @@ -188,6 +188,8 @@ Host 端循环： | **CI 验收** | `unit:frontend-ops`：四类模型模式所用基础算子及边界用例通过
`unit:backend-ops`：基础算子 QEMU 单测，误差 < 1e-5
`numeric:ir-small`：2 层小模型 IR vs ORT，误差 < 1e-5
`unit:runtime`：固定语料 tokenizer 编码/解码与官方配置一致，greedy、最后有效位置和掩码正确
`frontend:parse-qwen3`：完整 Qwen3 解析无错误，28 层和权重绑定完整 | | **出口标准** | 小模型 IR 误差 < 1e-5；完整 Qwen3 解析无缺失算子；CI 全绿 | +**W2 实施补记**：已新增四模式前端独立回归、七类基础后端的 16 个小图/32 次 QEMU 门禁，以及官方 tokenizer fast/slow/golden 对照和 host 输入/采样门禁。具体命令、实测与 CI 状态见 [W2 交付记录](W2/README.md)。Tokenizer 会遵循官方 NFC 规范化，“往返正确”按官方输出一致验收；模型 151936 个 logits 行不等于 151669 个可解码 token。融合 Attention 不是本轮正确性前置，完整 IR 数值仍按 W3 单独验收。 + #### W3：完整 Qwen3 IR 数值验证（硬门槛） | 项目 | 内容 | diff --git a/probes/w2_backend_ops/README.md b/probes/w2_backend_ops/README.md new file mode 100644 index 0000000..a535d14 --- /dev/null +++ b/probes/w2_backend_ops/README.md @@ -0,0 +1,45 @@ +# W2 前端模式与基础后端算子验收 + +`unit:frontend-ops` 明确检查 RMSNorm、RoPE、SwiGLU、GQA 四种分解图模式。 +这些模式由现有 ONNX 基础算子表达，不声明已有独立融合 Attention/RMSNorm IR。 + +```powershell +python -m pytest tests/test_qwen3_frontend_patterns.py -q +``` + +`unit:backend-ops` 从真实 ONNX 图走 production parser、优化器、tensor-c 后端， +交叉编译为 RV64GC ELF，然后在 QEMU `virt` 裸机执行。16 个用例分别运行 +`none`、`all` 两个优化级别，共 32 次目标执行；不下载模型权重。 + +```powershell +python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc path/to/zig --qemu path/to/qemu-system-riscv64 +python -m pytest tests/test_w2_backend_ops_gate.py -q +``` + +Linux 可省略工具路径，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 +使用现有 `requirements/qwen3-small-probe.txt` 中的 NumPy/ONNX/ORT 环境即可， +本探测不需要 PyTorch。输出目录须不存在或为空；依赖、工具、编译、执行或 +比较失败均非零退出，不将跳过当通过。 + +| 类别 | 用例与针对的问题 | +|---|---| +| MatMul | 4×4、四维 batch 广播、向量 dot 标量返回 | +| 逐元素 | Add/Sub/Mul/Div 串联，标量与尾维广播 | +| Softmax | axis=-1/0/1，大 logits 稳定性及非末轴 | +| RMSNorm | hidden/Q/K 的不同 head 数，全零/极小输入、权重、1e-6/1e-3 epsilon | +| RoPE | 全 head_dim rotate_half、Q/K head 广播、position=0/1/255 及非零起点 | +| SwiGLU | 两组不同 gate/up 权重，SiLU 仅作用于 gate，再经 down 投影 | +| GQA | KV 连续重复顺序、FP32 最小有限值因果 mask、attention context，重复比 2 与 MHA 比 1 | + +NumPy 参考使用独立数学公式及双精度中间值，最后转 FP32；ORT 单独执行 ONNX。 +IR 与 QEMU 都同时对照这两种参考，要求 shape、dtype 一致、全部有限，且 +`max_abs < 1e-5`、`rtol=0`。前端单测还注入错误 epsilon、旋转符号、门控分支、 +KV head 顺序，确认用例可以发现这些语义错误。 + +`report.json`、`report.md`、`report.html` 含逐例误差、失败阶段、环境及源码哈希， +还有编译命令、ELF 哈希和每次 QEMU 进程实测墙钟时长。 +`qemu_process_wall_seconds` 包括启动、guest 计算、UART 输出与退出，不是 guest +纯计算时间或周期数。每例同时保存模型、输入、NumPy/ORT/IR/QEMU 张量和工具日志， +可定位第一次不一致发生于参考、IR 还是后端。 + +门禁验证静态 FP32 小图，不代表完整 0.6B 的内存容量、性能或完整模型 QEMU 前向已通过。 diff --git a/probes/w2_backend_ops/cases.py b/probes/w2_backend_ops/cases.py new file mode 100644 index 0000000..49d182b --- /dev/null +++ b/probes/w2_backend_ops/cases.py @@ -0,0 +1,174 @@ +"""Small, deterministic ONNX graphs with independent NumPy formula references. + +These are semantic pattern tests, not fused RMSNorm/RoPE/SwiGLU/GQA operators. +The graph uses the same decomposed primitives accepted by the Qwen3 frontend. +""" +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import onnx +from onnx import TensorProto as T, helper, numpy_helper + + +@dataclass(frozen=True) +class Case: + name: str + family: str + model: onnx.ModelProto + feed: dict[str, np.ndarray] + expected: np.ndarray + purpose: str + + +def graph(name, family, nodes, feed, expected, purpose, constants=None): + expected = np.asarray(expected, dtype=np.float32) + model = helper.make_model(helper.make_graph( + nodes, name, + [helper.make_tensor_value_info(key, T.FLOAT, value.shape) for key, value in feed.items()], + [helper.make_tensor_value_info("y", T.FLOAT, expected.shape)], + [numpy_helper.from_array(np.asarray(value), key) for key, value in (constants or {}).items()], + ), opset_imports=[helper.make_opsetid("", 18)], ir_version=10) + onnx.checker.check_model(model) + return Case(name, family, model, feed, expected, purpose) + + +def rmsnorm_case(kind="hidden", epsilon=1e-6): + shape = (1, 3, 16) if kind == "hidden" else ((1, 4, 3, 8) if kind == "q" else (1, 2, 3, 8)) + x = np.linspace(-0.003, 0.004, np.prod(shape), dtype=np.float32).reshape(shape) + x[..., 0] = 0 + x[..., 0, :] = 0 + weight = np.linspace(0.4, 1.6, shape[-1], dtype=np.float32) + # Accumulate independently in double precision, then round only the result. + expected = (x.astype(np.float64) / np.sqrt(np.mean(x.astype(np.float64) ** 2, + axis=-1, keepdims=True) + np.float32(epsilon))) * weight + nodes = [helper.make_node("Pow", ["x", "two"], ["square"]), + helper.make_node("ReduceMean", ["square", "last"], ["variance"], keepdims=1), + helper.make_node("Add", ["variance", "epsilon"], ["stabilized"]), + helper.make_node("Sqrt", ["stabilized"], ["denominator"]), + helper.make_node("Div", ["x", "denominator"], ["normalized"]), + helper.make_node("Mul", ["normalized", "weight"], ["y"])] + return graph(f"rmsnorm_{kind}_eps_{epsilon:g}", "rmsnorm", nodes, {"x": x}, expected, + "Normalize the last dimension with learned weights; small inputs expose epsilon changes", + {"two": np.array(2, np.int64), "last": np.array([-1], np.int64), + "epsilon": np.array(epsilon, np.float32), "weight": weight}) + + +def rope_case(positions=(0, 1, 255), heads=4): + dim = 8 + x = np.linspace(-1.1, 1.2, heads * len(positions) * dim, dtype=np.float32).reshape(1, heads, -1, dim) + position = np.array(positions, np.float32).reshape(1, -1, 1) + inv = (1.0 / 1e6 ** (np.arange(0, dim, 2, dtype=np.float64) / dim)).astype(np.float32) + # Direct complex rotation of each pair of halves, not graph Slice/Concat. + angle = np.asarray(positions, np.float64)[:, None] * inv.astype(np.float64)[None, :] + first, second = x[..., :dim // 2].astype(np.float64), x[..., dim // 2:].astype(np.float64) + expected = np.empty_like(x) + expected[..., :dim // 2] = first * np.cos(angle) - second * np.sin(angle) + expected[..., dim // 2:] = second * np.cos(angle) + first * np.sin(angle) + nodes = [helper.make_node("Mul", ["position", "inv_freq"], ["angle"]), + helper.make_node("Concat", ["angle", "angle"], ["full_angle"], axis=-1), + helper.make_node("Cos", ["full_angle"], ["cos_3d"]), + helper.make_node("Sin", ["full_angle"], ["sin_3d"]), + helper.make_node("Unsqueeze", ["cos_3d", "head_axis"], ["cos"]), + helper.make_node("Unsqueeze", ["sin_3d", "head_axis"], ["sin"]), + helper.make_node("Slice", ["x", "zero", "half", "last"], ["first"]), + helper.make_node("Slice", ["x", "half", "end", "last"], ["second"]), + helper.make_node("Neg", ["second"], ["negative_second"]), + helper.make_node("Concat", ["negative_second", "first"], ["rotated"], axis=-1), + helper.make_node("Mul", ["x", "cos"], ["real"]), + helper.make_node("Mul", ["rotated", "sin"], ["imaginary"]), + helper.make_node("Add", ["real", "imaginary"], ["y"])] + return graph(f"rope_h{heads}_p{'_'.join(map(str, positions))}", "rope", nodes, + {"x": x, "position": position}, expected, + "Full head dimension rotate_half with broadcast positions, including nonzero offsets", + {"inv_freq": inv.reshape(1, 1, -1), "head_axis": np.array([1], np.int64), + "zero": np.array([0], np.int64), "half": np.array([4], np.int64), + "end": np.array([8], np.int64), "last": np.array([-1], np.int64)}) + + +def swiglu_case(seed=11): + rng = np.random.default_rng(seed) + x = rng.normal(size=(1, 3, 4)).astype(np.float32) + weights = {name: (rng.normal(size=shape) / 4).astype(np.float32) for name, shape in + (("gate_weight", (4, 7)), ("up_weight", (4, 7)), ("down_weight", (7, 4)))} + gate = x.astype(np.float64) @ weights["gate_weight"].astype(np.float64) + up = x.astype(np.float64) @ weights["up_weight"].astype(np.float64) + expected = ((gate / (1 + np.exp(-gate))) * up) @ weights["down_weight"].astype(np.float64) + nodes = [helper.make_node("MatMul", ["x", "gate_weight"], ["gate"]), + helper.make_node("MatMul", ["x", "up_weight"], ["up"]), + helper.make_node("Sigmoid", ["gate"], ["activation"]), + helper.make_node("Mul", ["gate", "activation"], ["silu"]), + helper.make_node("Mul", ["silu", "up"], ["hidden"]), + helper.make_node("MatMul", ["hidden", "down_weight"], ["y"])] + return graph(f"swiglu_seed_{seed}", "swiglu", nodes, {"x": x}, expected, + "Different gate/up projections, SiLU on gate only, and down projection", weights) + + +def gqa_case(repeats=2): + rng = np.random.default_rng(131) + kv_heads, length, dim = 2, 3, 4 + q = (rng.normal(size=(1, kv_heads * repeats, length, dim)) / 2).astype(np.float32) + k = (rng.normal(size=(1, kv_heads, length, dim)) / 2).astype(np.float32) + v = (np.arange(kv_heads * length * dim).reshape(1, kv_heads, length, dim) / 13).astype(np.float32) + mask = np.where(np.tril(np.ones((length, length), dtype=bool)), + 0, np.finfo(np.float32).min).astype(np.float32) + expected = np.empty_like(q) + # Explicit head mapping detects tile-vs-repeat mistakes; masked softmax is + # calculated independently in double precision for every query head. + for head in range(q.shape[1]): + kv = head // repeats + scores = q[0, head].astype(np.float64) @ k[0, kv].astype(np.float64).T / np.sqrt(dim) + mask + probabilities = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probabilities /= probabilities.sum(axis=-1, keepdims=True) + expected[0, head] = probabilities @ v[0, kv].astype(np.float64) + nodes = [] + for name in ("k", "v"): + nodes.extend([helper.make_node("Unsqueeze", [name, "repeat_axis"], [f"{name}_5d"]), + helper.make_node("Expand", [f"{name}_5d", "expanded_shape"], [f"{name}_expanded"]), + helper.make_node("Reshape", [f"{name}_expanded", "repeated_shape"], [f"{name}_repeated"])]) + nodes.extend([helper.make_node("Transpose", ["k_repeated"], ["kt"], perm=[0, 1, 3, 2]), + helper.make_node("MatMul", ["q", "kt"], ["unscaled"]), + helper.make_node("Mul", ["unscaled", "scale"], ["scores"]), + helper.make_node("Add", ["scores", "mask"], ["masked"]), + helper.make_node("Softmax", ["masked"], ["probabilities"], axis=-1), + helper.make_node("MatMul", ["probabilities", "v_repeated"], ["y"])]) + return graph(f"gqa_repeat_{repeats}", "gqa", nodes, {"q": q, "k": k, "v": v}, expected, + "Contiguous KV head repetition, causal mask and attention context, including MHA ratio 1", + {"repeat_axis": np.array([2], np.int64), "scale": np.array(1 / np.sqrt(dim), np.float32), + "mask": mask, "expanded_shape": np.array([1, kv_heads, repeats, length, dim], np.int64), + "repeated_shape": np.array([1, kv_heads * repeats, length, dim], np.int64)}) + + +def build_cases(): + rng = np.random.default_rng(314159) + cases = [] + for name, left, right in (("matrix", (4, 4), (4, 4)), + ("batch_broadcast", (2, 1, 3, 5), (1, 4, 5, 2)), + ("vector_dot", (5,), (5,))): + a, b = [(rng.normal(size=shape) / 3).astype(np.float32) for shape in (left, right)] + cases.append(graph(f"matmul_{name}", "matmul", [helper.make_node("MatMul", ["a", "b"], ["y"])], + {"a": a, "b": b}, a.astype(np.float64) @ b.astype(np.float64), + "Matrix, batched broadcasting and scalar dot-product outputs")) + x = np.linspace(-2, 2, 24, dtype=np.float32).reshape(2, 3, 4) + bias = np.array([0.1, -0.3, 0.7, 1.1], np.float32) + denominator = np.array(0.75, np.float32) + cases.append(graph("elementwise_broadcast", "elementwise", [ + helper.make_node("Add", ["x", "bias"], ["sum"]), + helper.make_node("Sub", ["sum", "denominator"], ["difference"]), + helper.make_node("Mul", ["difference", "bias"], ["product"]), + helper.make_node("Div", ["product", "denominator"], ["y"])], + {"x": x, "bias": bias, "denominator": denominator}, + (x.astype(np.float64) + bias - denominator) * bias / denominator, + "Scalar and trailing-dimension broadcast through Add/Sub/Mul/Div")) + for axis in (-1, 0, 1): + data = np.arange(24, dtype=np.float32).reshape(2, 3, 4) + 1000 + expected = np.exp(data.astype(np.float64) - data.max(axis=axis, keepdims=True)) + expected /= expected.sum(axis=axis, keepdims=True) + cases.append(graph(f"softmax_axis_{axis}", "softmax", [ + helper.make_node("Softmax", ["x"], ["y"], axis=axis)], {"x": data}, expected, + "Numerically stable softmax with large logits and non-last dimensions")) + cases.extend([rmsnorm_case("hidden"), rmsnorm_case("q"), rmsnorm_case("k", 1e-3), + rope_case(), rope_case((7, 19, 255), heads=2), + swiglu_case(), swiglu_case(23), gqa_case(), gqa_case(1)]) + return cases diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py new file mode 100644 index 0000000..c801a31 --- /dev/null +++ b/probes/w2_backend_ops/run.py @@ -0,0 +1,258 @@ +"""W2 operator gate: independent formulas -> ORT -> IR -> tensor C -> RV64 QEMU. + +All seven families and both optimization levels are mandatory. Missing tools, +nonfinite tensors, shape/type changes or max_abs >= 1e-5 fail the gate. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +from html import escape +import json +import math +from pathlib import Path +import platform +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import numpy as np +import onnx +import onnxruntime as ort + +from probes.w2_backend_ops.cases import build_cases +from probes.w2_qwen3_small.diagnostics import tensor_diff +from scratchv.analysis.ir_verifier import verify_ir +from scratchv.compiler import CompilerConfig, CompilerDriver +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.pass_manager import create_optimization_pass_manager +from scratchv.runtime.riscv_tensor import build_riscv_tensor, discover_toolchain, run_riscv_tensor +from scratchv.verification.ir_interpreter import IRInterpreter + +ATOL = 1e-5 +LEVELS = ("none", "all") +FAMILIES = ("matmul", "elementwise", "softmax", "rmsnorm", "rope", "swiglu", "gqa") +FRONTEND_FAMILIES = ("rmsnorm", "rope", "swiglu", "gqa") + + +def require_valid(program): + passed, issues = verify_ir(program) + if not passed: + raise ValueError("; ".join(str(issue) for issue in issues)) + + +def reference_case(case, path): + onnx.save(case.model, path) + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + expected = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]).run(None, case.feed)[0] + parser = ONNXParser() + program = parser.parse(str(path)) + require_valid(program) + return expected, parser, program + + +def interpret(program, bindings, feed, level): + manager = create_optimization_pass_manager(level) + manager.before_pass = lambda p, value: require_valid(value) + manager.after_pass = lambda p, value: require_valid(value) + optimized = manager.run_pipeline(copy.deepcopy(program)).data + require_valid(optimized) + return IRInterpreter(optimized).run(feed, initializers=bindings).return_value + + +def evidence(): + sources = [Path(__file__), ROOT / "probes/w2_backend_ops/cases.py", + ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "scratchv/compiler.py", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", + ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/pass_manager.py", + ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/verification/ir_interpreter.py", + ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/backend/tensor_c_codegen.py", ROOT / "scratchv/runtime/riscv_tensor.py", + *sorted((ROOT / "scratchv/analysis").glob("*.py")), + *sorted((ROOT / "scratchv/optimizer").glob("*.py"))] + return {"environment": {"python": platform.python_version(), "executable": sys.executable, + "platform": platform.platform(), "numpy": np.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__}, + "source_sha256": {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sources}} + + +def run_probe(out, report, *, cc=None, qemu=None, timeout=120): + cases = build_cases() + if (not cases or len({case.name for case in cases}) != len(cases) + or set(case.family for case in cases) != set(FAMILIES)): + raise ValueError("Cases must have unique names and cover every required operator family") + report.update(evidence(), stage="toolchain", families=list(FAMILIES), levels=list(LEVELS), + planned_executions=len(cases) * len(LEVELS), cases=[]) + tools = discover_toolchain(cc=cc, qemu=qemu) + report["toolchain"] = {"cc": list(tools.cc), "qemu": tools.qemu} + report["stage"] = "execution" + for case in cases: + folder = out / case.name + folder.mkdir() + row = {"name": case.name, "family": case.family, "purpose": case.purpose, + "passed": False, "stage": "reference", "executions": []} + report["cases"].append(row) + try: + path = folder / "model.onnx" + expected, parser, program = reference_case(case, path) + np.savez(folder / "inputs.npz", **case.feed) + np.save(folder / "numpy.npy", case.expected) + np.save(folder / "ort.npy", expected) + row["model_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + row["input_sha256"] = {name: hashlib.sha256(array.tobytes()).hexdigest() + for name, array in case.feed.items()} + row["ort_vs_numpy"] = tensor_diff(expected, case.expected, ATOL) + if not row["ort_vs_numpy"]["passed"]: + raise ValueError("ORT disagrees with independent NumPy formula") + for level in LEVELS: + target = folder / level + target.mkdir() + execution = {"optimization": level, "passed": False, "stage": "ir", + "qemu_process_wall_seconds": None} + row["executions"].append(execution) + try: + ir_output = interpret(program, parser.initializers, case.feed, level) + np.save(target / "ir.npy", ir_output) + execution["ir_vs_ort"] = tensor_diff(ir_output, expected, ATOL) + execution["ir_vs_numpy"] = tensor_diff(ir_output, case.expected, ATOL) + if not all(execution[key]["passed"] for key in ("ir_vs_ort", "ir_vs_numpy")): + raise ValueError("IR disagrees with ORT or independent NumPy formula") + execution["stage"] = "compile" + driver = CompilerDriver(CompilerConfig(backend="tensor-c", optimize_level=level, verify_ir=True)) + result = driver.compile(str(path), str(target / "model.c")) + if not result.success: + raise RuntimeError("; ".join(result.errors)) + executable = build_riscv_tensor(driver.tensor_artifact, target / "build", tools) + execution.update(elf_sha256=executable.elf_sha256, compile_command=executable.compile_command, + compile_seconds=executable.compile_seconds, + workspace_bytes=executable.workspace_bytes, tool_versions=executable.tool_versions, + stage="qemu") + actual = run_riscv_tensor(executable, case.feed, target / "run", timeout=timeout) + execution.update(qemu_process_wall_seconds=actual.elapsed_s, command=actual.command, stage="numeric") + np.save(target / "qemu.npy", actual.output) + execution["qemu_vs_ort"] = tensor_diff(actual.output, expected, ATOL) + execution["qemu_vs_numpy"] = tensor_diff(actual.output, case.expected, ATOL) + execution["passed"] = all(execution[key]["passed"] for key in ("qemu_vs_ort", "qemu_vs_numpy")) + execution["stage"] = "complete" + except Exception as exc: + execution["error"] = f"{type(exc).__name__}: {exc}" + if getattr(exc, "elapsed_s", None) is not None: + execution.update(qemu_process_wall_seconds=exc.elapsed_s, + command=getattr(exc, "command", None), + status=getattr(exc, "status", "runtime_error")) + print(f"[{case.name}/{level}] {'PASS' if execution['passed'] else 'FAIL'} " + f"max_abs={execution.get('qemu_vs_ort', {}).get('max_abs')} " + f"stage={execution['stage']}", flush=True) + row["passed"] = len(row["executions"]) == len(LEVELS) and all(item["passed"] for item in row["executions"]) + row["stage"] = "complete" + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + executions = [execution for case in report["cases"] for execution in case["executions"]] + report["completed_executions"] = sum(item["stage"] == "complete" for item in executions) + report["passed_executions"] = sum(item["passed"] for item in executions) + measured = [item["qemu_process_wall_seconds"] for item in executions if item["qemu_process_wall_seconds"] is not None] + report["qemu_process_wall_seconds"] = {"measured_count": len(measured), + "total": sum(measured) if measured else None} + report["passed"] = (len(executions) == report["planned_executions"] + and all(case["passed"] for case in report["cases"])) + report["stage"] = "complete" + + +def _report_markdown(report): + payload = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + lines = ["# W2 backend operator gate", "", f"Result: {'PASS' if report['passed'] else 'FAIL'}", + "", "Strict max_abs < 1e-5; rtol=0; exact shape/dtype; finite values.", + "", "| Case | Level | Result | QEMU vs ORT max_abs | QEMU process seconds |", + "|---|---|---|---|---|"] + for case in report.get("cases", []): + if not case["executions"]: + lines.append(f"| {case['name']} | not started | FAIL | - | - |") + for item in case["executions"]: + lines.append(f"| {case['name']} | {item['optimization']} | {'PASS' if item['passed'] else 'FAIL'} | " + f"{item.get('qemu_vs_ort', {}).get('max_abs')} | {item['qemu_process_wall_seconds']} |") + lines.extend(["", "Timing includes QEMU startup, guest execution, UART transfer and exit; not guest cycles.", + "", "Full diagnostics, commands, environment and source hashes:", "", "```json", payload.rstrip(), "```", ""]) + return "\n".join(lines) + + +def write_reports(out, report): + """Keep primary errors and publish JSON only after required evidence succeeds.""" + failures = [] + + def write(name, content, *, atomic=False): + path = out / name + pending = out / (name + ".tmp") if atomic else path + try: + pending.write_text(content, encoding="utf-8") + if atomic: + pending.replace(path) + except OSError as exc: + failures.append(f"{name}: {type(exc).__name__}: {exc}") + finally: + if atomic: + try: + pending.unlink(missing_ok=True) + except OSError: + pass + + def record_failure(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete backend gate evidence") + + def views(): + markdown = _report_markdown(report) + write("report.md", markdown) + write("report.html", "W2 backend operators" + "

W2 backend operators

" + escape(markdown) + "
") + + views() + if failures: + record_failure() + views() + # Write and close a sibling temporary file before publishing JSON. A late + # write/close failure must not leave a valid PASS artifact for CI consumers. + before_json = len(failures) + write("report.json", json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n", atomic=True) + if len(failures) != before_json: + record_failure() + views() + if failures: + print("[unit:backend-ops] FAIL; report write errors: " + "; ".join(failures), file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--output-dir", type=Path, default=ROOT / "output/w2-backend-ops") + cli.add_argument("--cc") + cli.add_argument("--qemu") + cli.add_argument("--timeout", type=float, default=120) + args = cli.parse_args(argv) + if not math.isfinite(args.timeout) or args.timeout <= 0: + cli.error("--timeout must be finite and positive") + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + cli.error("--output-dir must be empty") + out.mkdir(parents=True, exist_ok=True) + report = {"passed": False, "stage": "setup", "backend": "ScratchV tensor-c -> RV64GC -> QEMU", + "atol": ATOL, "rtol": 0, "timeout_seconds": args.timeout} + start = time.perf_counter() + try: + run_probe(out, report, cc=args.cc, qemu=args.qemu, timeout=args.timeout) + except Exception as exc: + report["error"] = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + report["seconds"] = time.perf_counter() - start + write_reports(out, report) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_qwen3_parse/README.md b/probes/w2_qwen3_parse/README.md new file mode 100644 index 0000000..23a69e7 --- /dev/null +++ b/probes/w2_qwen3_parse/README.md @@ -0,0 +1,87 @@ +# W2：完整 Qwen3 ONNX 前端解析探测 + +入口 `probes/w2_qwen3_parse/run.py` 面向固定发布的完整 28 层 Qwen3-0.6B 图，验证 ScratchV 前端解析与结构、节点及权重绑定审计。使用 W1 已固定的 FP32、batch=1、L=256、无 KV Cache 的 ONNX 产物和 [manifest](../w1_qwen3_export/manifest.json)。 + +本入口不执行完整模型 IR，不生成完整模型 RISC-V，不运行 QEMU，也不重新导出模型或比较 PyTorch/ORT 数值。完整模型的 ORT 执行继续由 [W1 导出门禁](../w1_qwen3_export/README.md) 负责；两项门禁分别报告结果。 + +## 准备环境 + +使用 Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 + +Windows PowerShell： + +```powershell +py -3.12 -m venv output/qwen3-parse-venv +$parsePython = ".\output\qwen3-parse-venv\Scripts\python.exe" +& $parsePython -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" +& $parsePython -m pip check +& $parsePython -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +$LASTEXITCODE +``` + +Linux shell： + +```bash +python3.12 -m venv output/qwen3-parse-venv +. output/qwen3-parse-venv/bin/activate +python -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" +python -m pip check +python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify \ + --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +echo $? +``` + +所有命令在仓库根目录执行。`--model-dir` 必填，应指向同目录保存 `model.onnx` 和三个真实 external-data 分片的完整产物，不能只有 ONNX 文件或 LFS pointer。`verify` 是默认模式，不访问网络；不存在完整模型时，将命令中的 `--mode verify` 改成 `--mode download`，由入口获取并校验 W1 固定发布产物。不要用新导出或其他 revision 的文件替换固定产物。 + +`--output-dir` 默认是 `output/qwen3-parse`，必须不存在或为空，每次使用新的目录以保留失败日志与首次报告。`--timeout` 默认 300 秒，限制整个验证子进程，包含文件检查、解析、审计，以及 download 模式的下载；慢网络应据实调整此上限。它是超时设置，不是执行耗时或纯解析耗时。 + +固定 ZIP 约 1.24 GB，解包后需要数 GiB 磁盘空间，完整权重解析也需要数 GiB 内存。解析通过不表示当前 tensor-c 的常量上限或 QEMU 内存布局能够装载完整模型；不要从小型 fixture 的内存消耗推断完整图容量。 + +## 验收范围与证据 + +检查顺序及具体失败阶段以 `report.json` 为准。完整门禁覆盖固定产物身份、完整图结构与 28 层覆盖、实际 ONNX→共享 IR 解析、节点转换与 initializer 绑定审计，以及共享 IR 验证器检查。节点数量、算子统计、绑定数量和实际耗时应读取本次报告，不使用旧报告代替执行。 + +结构检查沿实际输入依赖追踪模型，而不是只统计节点名中的层号： + +- 核对全部 initializer 的名字、shape、dtype，以及两个静态输入和 logits 输出。 +- 核对每层投影和 RMSNorm、Q/K 归一化、attention 的缩放/掩码/Softmax/V/O 路径、RoPE 位置依赖和 GQA 扩展。 +- 核对 SwiGLU、两个残差连接和完整层顺序，最后归一化连接共享 LM Head。 +- 要求全部节点和 initializer 都参与返回 logits 的依赖，避免断开的装饰节点被当作完整模型。 + +这些是固定图的结构规则，不是对算子数值实现或所有可能 ONNX 导出形式的证明。 + +转换审计记录实际 parser 的每个节点，核对 IR 指令区间、opcode、输入 Value、输出映射和静态属性，允许合法的 Constant 绑定与 Identity 别名而不把它们误报为漏转换。形状、切片和轴等控制参数从原 ONNX 小型控制子图独立求值，不复用被检查解析器的常量缓存。所有 ONNX value 的静态 shape/dtype、全局张量集合与唯一 RETURN 也会核对。权重和 Constant 绑定逐张量比较 shape、dtype 与内容 SHA256；外部分片按块读取校验，不为审计再保存一套权重。该审计采用明确支持的基础算子转换规则，未知转换方式应报错，由新增规则和测试后再接受。 + +| 文件 | 用途 | +|---|---| +| `report.json` | 机器可读的本次结果、阶段、来源与审计摘要 | +| `report.md`、`report.html` | 便于 review 的摘要和可浏览报告 | +| `nodes.json` | ONNX 节点与转换审计证据 | +| `bindings.json` | initializer / IR 绑定审计证据 | +| `ir.txt` | 本次解析得到的共享 IR 文本 | +| `worker.stdout`、`worker.stderr`、`worker-progress.json` | 子进程日志和最后进度，用于定位超时或异常退出 | + +报告目录不包含权重分片。CI 上传该目录，不上传模型目录。失败可能发生在报告明细生成之前，应结合报告与命令日志判断缺失文件，不能把缺失当作通过。 + +`attempt` 保存 supervisor 启动前的命令、输入路径、解释器与源码身份；它用于定位早期失败，不代表 worker 已初始化或验证通过。`log_errors` 和 `evidence_errors` 分别保存附加日志、失败明细的写入问题，不覆盖原始解析/审计错误。成功路径的必需证据或日志无法保存时仍失败。 + +`seconds` 记录 supervisor 溯源开始至验证与日志保存结束的耗时，不含随后读取最后进度与生成最终三种报告的时间；各阶段和纯解析耗时另行记录。峰值 RSS 覆盖验证子进程，不包含 supervisor 父进程。平台无法提供的测量值保持未测得。超时后只报告已经保存的进度，不能将 300 秒上限当作实际解析耗时或补全缺失的峰值内存。 + +成功需要命令退出 0 且本次 `report.json` 的 `passed` 为 true。失败时保留退出码、失败阶段、错误、输入模型身份和已经产生的明细。记录实际 checkout SHA；有未提交修改时还应保存完整 diff 和新增文件，单独记录 HEAD 不能确定所执行代码。 + +`passed=true` 只证明这份固定产物完成本门禁的前端解析与审计。它不证明完整 IR 数值正确、优化后语义等价、完整权重部署、文本生成或目标硬件性能。下一阶段仍需逐步开展完整模型 IR/ORT 数值对照和容量验证。 + +## 轻量测试与 CI + +```bash +python -m pip install "pytest==9.1.1" +python -m pytest tests/test_qwen3_full_structure.py tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py -q +``` + +PowerShell 将上面命令中的 `python` 替换为 `& $parsePython`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 + +[LLM 工作流](../../.github/workflows/llm-deploy.yml) 的 PR 路径接入上述三组测试，并在独立 `full-qwen3-frontend` job 以 `--mode download` 实跑完整固定图。手动/定时同样运行该解析任务。解析有独立 Summary，并以 `always()` 上传 `output/qwen3-parse/`，artifact 名为 `qwen3-full-frontend-parse-output`。下载约 1.24 GB，解析 worker 峰值 RSS 约 5 GB；runner 的实际容量与耗时仍须远端验证。 + +`w2-acceptance` 汇总任务要求数值任务和完整解析任务均实际成功，失败、取消、跳过或缺失都不能通过。手动/定时的 `full-qwen3-onnx` job 独立保留 W1 完整 ONNX ORT 验证；解析不依赖它成功。并行的两个完整模型任务会各自下载资产。 + +PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存本地实跑与 Review 修复结果，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 diff --git a/probes/w2_qwen3_parse/audit.py b/probes/w2_qwen3_parse/audit.py new file mode 100644 index 0000000..cc8a891 --- /dev/null +++ b/probes/w2_qwen3_parse/audit.py @@ -0,0 +1,417 @@ +"""Read-only evidence for complete ONNX -> IR translation and weight binding. + +This deliberately audits the fixed, single-output Qwen graph. It does not run +the model or claim numerical equivalence. External tensors are hashed in chunks +without loading another copy of the complete weights. +""" + +from __future__ import annotations + +import hashlib +import math +from pathlib import Path, PureWindowsPath + +import numpy as np +import onnx + +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.ir.types import DataType, OpCode +from probes.w2_qwen3_parse.validation import make_control_constant_reader + + +class AuditError(ValueError): + """Translation or binding evidence is incomplete or inconsistent.""" + + +_TYPES = {1: DataType.FLOAT32, 6: DataType.INT32, + 7: DataType.INT64, 11: DataType.FLOAT64} +_CHUNK_BYTES = 1024 * 1024 + + +def _require(condition, message): + if not condition: + raise AuditError(message) + + +def _value_record(value): + return {"ir_name": value.name, "shape": list(value.shape), + "dtype": value.dtype.value} + + +class AuditedONNXParser(ONNXParser): + """The production parser, with per-node observations rather than new lowering.""" + + def __init__(self, progress=None): + super().__init__() + self.progress = progress + self.node_records = [] + self.current_context = {"stage": "not_started"} + self._node_instructions = [] + + def parse(self, model_path): + self.node_records = [] + self._node_instructions = [] + self.current_context = {"stage": "parse", "model_path": str(model_path)} + if self.progress: + self.progress(dict(self.current_context)) + program = super().parse(model_path) + self.current_context = {"stage": "parsed", "node_count": len(self.node_records)} + if self.progress: + self.progress(dict(self.current_context)) + return program + + def _translate_node(self, node, output_names): + index = len(self.node_records) + block = self.builder.current_block + start = len(block.instructions) + self.current_context = {"stage": "translate_node", "node_index": index, + "node_name": node.name, "op_type": node.op_type, + "inputs": list(node.input), "outputs": list(node.output)} + if self.progress: + self.progress(dict(self.current_context)) + record = {**self.current_context, "ir_start": start, "status": "running", + "input_values": {name: _value_record(self._value_map[name]) + for name in node.input if name in self._value_map}} + try: + super()._translate_node(node, output_names) + except Exception as exc: + record.update(status="failed", error=f"{type(exc).__name__}: {exc}") + self.node_records.append(record) + raise + instructions = block.instructions[start:] + record.update(status="translated", ir_end=len(block.instructions), + output_values={name: _value_record(self._value_map[name]) + for name in node.output if name in self._value_map}, + instructions=[{"opcode": inst.opcode.value, + "dest": inst.dest.name if inst.dest else None, + "operands": [value.name for value in inst.operands], + "attrs": dict(inst.attrs)} for inst in instructions]) + self.node_records.append(record) + self._node_instructions.append(instructions) + + +def _array_hash(data): + """Hash logical C-order little-endian bytes with bounded temporary buffers.""" + digest = hashlib.sha256() + array = np.asarray(data) + little_dtype = array.dtype.newbyteorder("<") + if array.flags.c_contiguous and array.dtype == little_dtype: + view = memoryview(array.reshape(-1)).cast("B") + for offset in range(0, len(view), _CHUNK_BYTES): + digest.update(view[offset:offset + _CHUNK_BYTES]) + else: + iterator = np.nditer(array, flags=["external_loop", "buffered", "zerosize_ok"], + op_flags=["readonly"], op_dtypes=[little_dtype], + order="C", buffersize=max(1, _CHUNK_BYTES // array.itemsize)) + for chunk in iterator: + digest.update(chunk.tobytes(order="C")) + return digest.hexdigest() + + +def _external_hash(tensor, model_dir, expected_bytes): + entries = list(tensor.external_data) + fields = {item.key: item.value for item in entries} + _require(len(fields) == len(entries), f"{tensor.name}: duplicate external-data keys") + location = fields.get("location", "") + windows = PureWindowsPath(location) + _require(bool(location) and "\x00" not in location and not windows.drive + and not windows.is_absolute() and not Path(location).is_absolute(), + f"{tensor.name}: invalid external-data location") + root = Path(model_dir).resolve() + path = (root / location.replace("\\", "/")).resolve() + _require(path.is_relative_to(root) and path.is_file(), + f"{tensor.name}: external data missing or outside model directory: {location}") + for key in ("offset", "length"): + if key in fields: + _require(fields[key].isascii() and fields[key].isdecimal(), + f"{tensor.name}: invalid external-data {key}") + offset = int(fields.get("offset", 0)) + size = path.stat().st_size + length = int(fields.get("length", size - offset)) + _require(offset <= size and length == expected_bytes and offset + length <= size, + f"{tensor.name}: external-data range/length mismatch") + digest = hashlib.sha256() + remaining = length + with path.open("rb") as stream: + stream.seek(offset) + while remaining: + chunk = stream.read(min(_CHUNK_BYTES, remaining)) + _require(bool(chunk), f"{tensor.name}: truncated external data") + digest.update(chunk) + remaining -= len(chunk) + return digest.hexdigest(), {"location": location, "offset": offset, "length": length} + + +def _tensor_spec(tensor, model_dir): + _require(tensor.data_type in _TYPES, f"{tensor.name}: unsupported tensor dtype") + dtype = np.dtype(onnx.helper.tensor_dtype_to_np_dtype(tensor.data_type)) + shape = tuple(tensor.dims) + _require(all(dim >= 0 for dim in shape), f"{tensor.name}: negative tensor dimension") + size = math.prod(shape) * dtype.itemsize + if tensor.data_location == onnx.TensorProto.EXTERNAL: + sha, external = _external_hash(tensor, model_dir, size) + else: + sha = _array_hash(onnx.numpy_helper.to_array(tensor)) + external = None + return dtype, shape, size, sha, external + + +def _onnx_attributes(node): + """Read source protobuf directly, independently of production lowering.""" + attributes = {attribute.name: onnx.helper.get_attribute_value(attribute) + for attribute in node.attribute} + _require(len(attributes) == len(node.attribute), f"{node.name}: duplicate ONNX attributes") + return attributes + + +def _constant_spec(node, model_dir): + attrs = _onnx_attributes(node) + _require(len(attrs) == 1, f"{node.name}: Constant requires exactly one attribute") + key, data = next(iter(attrs.items())) + if key == "value": + return _tensor_spec(data, model_dir) + if key in ("value_int", "value_ints"): + array = np.asarray(data, dtype=np.int64) + elif key in ("value_float", "value_floats"): + array = np.asarray(data, dtype=np.float32) + else: + raise AuditError(f"{node.name}: unsupported Constant attribute {key}") + return array.dtype, array.shape, array.nbytes, _array_hash(array), None + + +def _same_literal(left, right): + if left == right: + return left != 0 or math.copysign(1, left) == math.copysign(1, right) + return (isinstance(left, float) and isinstance(right, float) + and math.isnan(left) and math.isnan(right)) + + +def _audit_scalar_load(instruction, data, dtype, context): + """Cover auxiliary LOAD_CONST results which have no ONNX value-map entry.""" + destination = instruction.dest + _require(instruction.opcode == OpCode.LOAD_CONST and destination is not None + and not instruction.operands and not instruction.target + and destination.dtype == dtype and destination.shape == () + and destination.is_constant is True + and _same_literal(destination.const_value, data.item()) + and set(instruction.attrs) == {"value"} + and _same_literal(instruction.attrs.get("value"), data.item()), + f"{context}: scalar instruction metadata mismatch") + + +def _expected_lowering(node, parser, read_control): + """Independent structural contract for the operations in the fixed graph.""" + attrs = _onnx_attributes(node) + names = [name for name in node.input if name] + args = [parser._value_map[name] for name in names] + op = node.op_type + ir_attrs = {} + operands = args + if op in {"Abs", "Add", "Cast", "Cos", "Div", "Exp", "Gather", "MatMul", + "Mul", "Neg", "Pow", "Reciprocal", "Relu", "Sigmoid", "Sin", + "Softmax", "Sqrt", "Sub", "Transpose", "Concat", "Reshape", + "Expand", "Unsqueeze", "Slice", "ReduceMean"}: + opcode = "reduce_mean" if op == "ReduceMean" else op.lower() + else: + raise AuditError(f"{node.name}: no independent lowering contract for {op}") + + def integers(index): + # Never ask the parser/IR/cache to certify its own shape evaluation. + # This reader resolves the original ONNX graph with bounded NumPy ops. + name = node.input[index] + array = read_control(name) + _require(array.ndim == 1 and array.dtype.kind in "iu", + f"{node.name}: {name} is not a constant integer vector") + return tuple(int(value) for value in array) + + if op == "Cast": + _require(set(attrs) == {"to"} and attrs["to"] in _TYPES, + f"{node.name}: unsupported Cast attributes") + # CAST encodes its target in dest.dtype, not an instruction attribute. + ir_attrs = {} + elif op in ("Gather", "Softmax"): + ir_attrs = {"axis": attrs.get("axis", 0 if op == "Gather" else -1)} + elif op == "MatMul": + a, b = args + if len(a.shape) == len(b.shape) == 2 and all(d > 0 for d in a.shape + b.shape): + ir_attrs = {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1]} + elif op in ("Reshape", "Expand"): + _require(not attrs.get("allowzero", 0), f"{node.name}: unsupported allowzero") + ir_attrs = {"shape": integers(1)} + operands = args[:1] + elif op == "Unsqueeze": + ir_attrs = {"axes": integers(1) if len(args) > 1 else tuple(attrs["axes"])} + operands = args[:1] + elif op == "Slice": + for index, key in enumerate(("starts", "ends", "axes", "steps"), 1): + if len(node.input) > index and node.input[index]: + ir_attrs[key] = integers(index) + elif key in attrs: + ir_attrs[key] = tuple(attrs[key]) + operands = args[:1] + elif op == "ReduceMean": + axes = integers(1) if len(args) > 1 else attrs.get("axes") + if not axes and attrs.get("noop_with_empty_axes", 0): + return None, args[:1], {} + ir_attrs = {"keepdims": bool(attrs.get("keepdims", 1))} + if axes is not None: + ir_attrs["axes"] = tuple(axes) + operands = args[:1] + elif op == "Transpose" and "perm" in attrs: + ir_attrs = {"perm": tuple(attrs["perm"])} + elif op == "Concat": + ir_attrs = {"axis": attrs["axis"]} + return opcode, operands, ir_attrs + + +def audit_parsed_model(model, parser, program, model_dir): + """Raise AuditError on any gap; otherwise return serializable evidence.""" + graph = model.graph + _require(len(graph.output) == 1, "fixed-graph audit requires exactly one graph output") + _require(program is parser.builder.program, "program is not the observed parser result") + _require(len(program.functions) == 1 and len(program.functions[0].blocks) == 1, + "fixed-graph audit requires one function and one basic block") + function = program.functions[0] + instructions = function.blocks[0].instructions + _require(len(parser.node_records) == len(graph.node) + and len(parser._node_instructions) == len(graph.node), + "translated node count does not cover the ONNX graph") + read_control = make_control_constant_reader(model) + inferred = onnx.shape_inference.infer_shapes(model, strict_mode=True) + metadata = {value.name: value.type.tensor_type for value in + [*inferred.graph.input, *inferred.graph.value_info, *inferred.graph.output]} + expected_names = {value.name for value in [*graph.input, *graph.initializer]} + initializer_names = {value.name for value in graph.initializer} + _require(len(initializer_names) == len(graph.initializer), "duplicate initializer names") + def progress(stage, **context): + parser.current_context = {"stage": stage, **context} + if parser.progress: + parser.progress(dict(parser.current_context)) + + bindings = [] + for value in graph.initializer: + progress("audit_source_tensor", tensor_name=value.name, source="initializer") + bindings.append((value.name, "initializer", _tensor_spec(value, model_dir))) + for node in graph.node: + _require(len(node.output) == 1 and bool(node.output[0]), + f"{node.name}: audit requires one nonempty node output") + _require(node.output[0] not in expected_names, f"{node.name}: duplicate ONNX definition") + expected_names.add(node.output[0]) + if node.op_type == "Constant": + progress("audit_source_tensor", tensor_name=node.output[0], source="Constant") + bindings.append((node.output[0], "Constant", _constant_spec(node, model_dir))) + _require(set(parser._value_map) == expected_names, + "ONNX -> IR value mapping has missing or unexpected names") + _require(graph.output[0].name in expected_names, + "graph output does not have an ONNX definition") + for name in expected_names - initializer_names: + info = metadata.get(name) + _require(info is not None and info.HasField("shape") + and all(dim.HasField("dim_value") for dim in info.shape.dim), + f"{name}: missing static ONNX shape") + _require(info.elem_type in _TYPES, f"{name}: unsupported ONNX dtype") + value = parser._value_map[name] + _require(value.shape == tuple(dim.dim_value for dim in info.shape.dim) + and value.dtype == _TYPES[info.elem_type], f"{name}: IR shape/dtype mismatch") + + globals_by_name = {value.name: value for value in program.global_values} + binding_names = {name for name, _, _ in bindings} + _require(len(globals_by_name) == len(program.global_values) + and set(globals_by_name) == binding_names, + "IR global definitions are missing, duplicated, or unexpected") + _require(set(parser.initializers) == binding_names, + "initializer bindings are missing or unexpected") + tensor_rows = [] + for name, source, (dtype, shape, size, expected_sha, external) in bindings: + progress("audit_bound_tensor", tensor_name=name, source=source) + array = parser.initializers[name] + _require(isinstance(array, np.ndarray), f"{name}: binding must be a NumPy array") + value = globals_by_name[name] + _require(value is parser._value_map[name], f"{name}: global/value mapping identity mismatch") + _require(array.dtype == dtype and array.shape == shape and value.shape == shape + and value.dtype == _TYPES[onnx.helper.np_dtype_to_tensor_dtype(dtype)], + f"{name}: tensor binding shape/dtype mismatch") + _require(value.is_constant == (len(shape) == 0), f"{name}: scalar metadata mismatch") + if len(shape) == 0: + _require(_same_literal(value.const_value, array.item()), f"{name}: scalar value mismatch") + actual_sha = _array_hash(array) + _require(actual_sha == expected_sha, f"{name}: bound tensor content SHA256 mismatch") + tensor_rows.append({"name": name, "source": source, "shape": list(shape), + "dtype": str(dtype), "bytes": size, "sha256": actual_sha, + "external": external}) + expected_params = [parser._value_map[value.name] for value in graph.input + if value.name not in initializer_names] + _require(len(function.params) == len(expected_params) + and all(a is b for a, b in zip(function.params, expected_params)), + "function parameters do not match ONNX inputs") + expected_param_names = [value.name for value in graph.input if value.name not in initializer_names] + _require([value.name for value in function.params] == expected_param_names, + "function parameter names do not match ONNX input bindings") + _require(not any(value.is_constant for value in function.params), + "function parameters must not be constants") + + scalar_initializers = [value for value in graph.initializer if len(value.dims) == 0] + cursor = len(scalar_initializers) + for inst, tensor in zip(instructions[:cursor], scalar_initializers): + data = parser.initializers[tensor.name] + _audit_scalar_load(inst, data, parser._value_map[tensor.name].dtype, + f"{tensor.name}: scalar initializer") + alias_rows = [] + for index, (node, record, observed) in enumerate(zip( + graph.node, parser.node_records, parser._node_instructions)): + progress("audit_node", node_index=index, node_name=node.name, op_type=node.op_type) + context = f"node #{index} {node.name} ({node.op_type})" + _require(record.get("status") == "translated" and record.get("node_index") == index + and record.get("node_name") == node.name and record.get("op_type") == node.op_type + and record.get("inputs") == list(node.input) and record.get("outputs") == list(node.output), + f"{context}: node observation mismatch") + end = record.get("ir_end") + _require(record.get("ir_start") == cursor and isinstance(end, int) + and end == cursor + len(observed) + and all(a is b for a, b in zip(instructions[cursor:end], observed)) + and len(instructions[cursor:end]) == len(observed), + f"{context}: emitted IR range changed or has a gap") + cursor = end + output = parser._value_map[node.output[0]] + if node.op_type == "Constant": + _require(not node.input, f"{context}: Constant has inputs") + array = parser.initializers[node.output[0]] + _require(len(observed) == (1 if array.ndim == 0 else 0), + f"{context}: Constant emitted unexpected instructions") + if observed: + _audit_scalar_load(observed[0], array, output.dtype, f"{context}: Constant") + continue + if node.op_type == "Identity": + _require(len(node.input) == 1, f"{context}: invalid Identity inputs") + opcode, operands, ir_attrs = None, [parser._value_map[node.input[0]]], {} + else: + opcode, operands, ir_attrs = _expected_lowering(node, parser, read_control) + if opcode is None: + _require(not observed and output is operands[0], f"{context}: incorrect alias binding") + alias_rows.append({"onnx_name": node.output[0], "input": node.input[0], + "ir_name": output.name}) + else: + _require(len(observed) == 1, f"{context}: node did not emit exactly one instruction") + inst = observed[0] + _require(inst.opcode.value == opcode and inst.dest is output, + f"{context}: wrong IR opcode or output binding") + _require(len(inst.operands) == len(operands) + and all(a is b for a, b in zip(inst.operands, operands)), + f"{context}: wrong IR dependency bindings") + _require(inst.attrs == ir_attrs, f"{context}: IR attributes mismatch") + _require(len(instructions) == cursor + 1, "unexpected instructions outside node ranges and RETURN") + returns = [inst for inst in instructions if inst.opcode == OpCode.RETURN] + output_name = graph.output[0].name + _require(len(returns) == 1 and instructions[-1] is returns[0] + and len(returns[0].operands) == 1 + and returns[0].operands[0] is parser._value_map[output_name], + "actual RETURN does not bind the unique ONNX graph output") + progress("audit_complete", node_count=len(graph.node), binding_count=len(bindings)) + return {"passed": True, "node_count": len(graph.node), "value_count": len(expected_names), + "ir_instruction_count": len(instructions), "global_count": len(globals_by_name), + "initializer_count": len(graph.initializer), + "constant_count": len(bindings) - len(graph.initializer), + "binding_count": len(bindings), "binding_bytes": sum(row["bytes"] for row in tensor_rows), + "tensor_bindings": tensor_rows, "alias_count": len(alias_rows), "aliases": alias_rows, + "return": {"onnx_name": output_name, **_value_record(parser._value_map[output_name])}, + "node_records": parser.node_records} diff --git a/probes/w2_qwen3_parse/run.py b/probes/w2_qwen3_parse/run.py new file mode 100644 index 0000000..1cf2366 --- /dev/null +++ b/probes/w2_qwen3_parse/run.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Audit complete, pinned Qwen3 ONNX -> ScratchV IR parsing and weight binding. + +This gate does not execute the full IR, generate RISC-V code, or run QEMU. +Large-model work runs in a bounded child process; failure preserves reports. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +from html import escape +import json +import math +from pathlib import Path +import subprocess +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w1_qwen3_export import run as artifacts + +GATE = "frontend:parse-qwen3" +SCOPE = ("Complete pinned 28-layer Qwen3 ONNX parsing, graph connectivity, IR verification " + "and actual weight/constant bindings. No full-model IR numerical comparison, " + "code generation or QEMU execution.") + + +def write_json(path, value): + path = Path(path) + # Readers must not observe a half-written progress or result file. + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + + +def source_fingerprints(): + files = [*sorted(Path(__file__).parent.glob("*.py")), + ROOT / "probes/w1_qwen3_export/run.py", ROOT / "probes/w1_qwen3_export/manifest.json", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", + ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/runtime/riscv_tensor.py", + *sorted((ROOT / "scratchv/analysis").glob("*.py"))] + return {p.relative_to(ROOT).as_posix(): artifacts.sha256(p) for p in files} + + +def checkout_evidence(): + result = {"head": None, "status": "unknown", "clean": None} + if not (ROOT / ".git").exists(): + # Worktrees have a .git file; source archives have neither a file nor + # a directory here and must not inherit another checkout's identity. + result["reason"] = "Source directory has no .git metadata" + return result + try: + for key, arguments in (("head", ["rev-parse", "HEAD"]), + ("status", ["status", "--porcelain=v1", "--untracked-files=all"])): + process = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=10, + check=True) + result[key] = process.stdout.rstrip("\r\n") + result["clean"] = not bool(result["status"]) + except (OSError, subprocess.SubprocessError) as exc: + result.update(head=None, status="unknown", clean=None, + error=f"{type(exc).__name__}: {exc}", + reason="Checkout metadata unavailable") + return result + + +def initial_report(mode): + return {"schema_version": 1, "gate": GATE, "passed": False, "stage": "initialization", + "created_at": datetime.now(timezone.utc).isoformat(), "mode": mode, "scope": SCOPE, + "model_id": artifacts.MANIFEST["model_id"], "revision": artifacts.MANIFEST["revision"], + "stages_seconds": {}} + + +def save_failure_evidence(path, value, report): + """A secondary evidence failure must not replace the active validation error.""" + try: + write_json(path, value) + except (OSError, TypeError, ValueError) as exc: + report.setdefault("evidence_errors", []).append( + f"{path.name}: {type(exc).__name__}: {exc}") + + +def run_validation(model_dir, out, report, *, mode="verify", manifest=None, progress=None): + """Run real parsing. Small test fixtures may explicitly supply a manifest. + + The CLI always uses the committed full-model manifest and architecture. + This function never replaces parsing with a handler inventory or ORT run. + """ + import onnx + from scratchv.analysis.ir_verifier import verify_ir + from probes.w2_qwen3_parse.audit import AuditedONNXParser, audit_parsed_model + from probes.w2_qwen3_parse.validation import audit_graph_structure + + manifest = artifacts.MANIFEST if manifest is None else manifest + model_dir, out = Path(model_dir).resolve(), Path(out).resolve() + report["model_dir"] = str(model_dir) + + def stage(name, operation): + report["stage"] = name + if progress: + progress({"stage": name}) + started = time.perf_counter() + try: + return operation() + finally: + report["stages_seconds"][name] = time.perf_counter() - started + + if mode == "download": + report["acquisition"] = stage("acquire", lambda: artifacts.acquire_model(model_dir, manifest=manifest)) + report["hashes"] = stage("hashes", lambda: artifacts.verify_files(model_dir, manifest)) + report["onnx_structure"] = stage("onnx-structure", lambda: artifacts.inspect_model(model_dir, report["hashes"])) + model = stage("graph-load", lambda: onnx.load(str(model_dir / "model.onnx"), load_external_data=False)) + report["graph"] = stage("graph-connectivity", lambda: audit_graph_structure(model)) + if report["graph"].get("passed") is not True: + raise ValueError("Complete Qwen3 graph connectivity did not pass") + parser = AuditedONNXParser(progress=progress) + try: + program = stage("scratchv-parse", lambda: parser.parse(str(model_dir / "model.onnx"))) + except Exception: + report["failure_context"] = dict(parser.current_context) + report["nodes_completed"] = sum(row.get("status") == "translated" for row in parser.node_records) + save_failure_evidence(out / "nodes.partial.json", parser.node_records, report) + raise + report["ir"] = {"functions": len(program.functions), "globals": len(program.global_values), + "blocks": sum(len(f.blocks) for f in program.functions), + "instructions": sum(len(b.instructions) for f in program.functions for b in f.blocks), + "operators": dict(sorted(Counter(i.opcode.value for f in program.functions + for b in f.blocks for i in b.instructions).items()))} + (out / "ir.txt").write_text(program.dump(), encoding="utf-8") + write_json(out / "nodes.json", parser.node_records) + verified, issues = stage("ir-verifier", lambda: verify_ir(program, stage="full-qwen3-after-parse")) + report["ir_verifier"] = {"passed": verified, "issue_count": len(issues), + "issues": [str(issue) for issue in issues]} + if not verified: + raise ValueError("ScratchV IR verifier rejected the complete model; see ir_verifier.issues") + try: + audit = stage("ir-and-bindings", lambda: audit_parsed_model(model, parser, program, model_dir)) + except Exception: + report["failure_context"] = dict(parser.current_context) + save_failure_evidence(out / "nodes.json", parser.node_records, report) + raise + nodes = audit.pop("node_records", parser.node_records) + bindings = audit.pop("tensor_bindings") + write_json(out / "nodes.json", nodes) + write_json(out / "bindings.json", bindings) + report["audit"] = audit + if audit.get("passed") is not True: + raise ValueError("IR/weight binding audit did not pass") + report["evidence"] = {name: {"sha256": artifacts.sha256(out / name), + "bytes": (out / name).stat().st_size} + for name in ("nodes.json", "bindings.json", "ir.txt")} + report["passed"] = True + report["stage"] = "complete" + + +def worker_main(args): + report = initial_report(args.mode) + out = args.output_dir.resolve() + started = time.perf_counter() + last_checkpoint = -math.inf + + def checkpoint(context): + nonlocal last_checkpoint + now = time.perf_counter() + # Stage boundaries are always recorded; frequent node progress is sampled. + if set(context) == {"stage"} or now - last_checkpoint >= 0.5: + write_json(out / "worker-progress.json", {"context": context, + "elapsed_seconds": now - started, "peak_memory": artifacts.peak_memory()}) + last_checkpoint = now + + try: + report["stage"] = "environment" + checkpoint({"stage": "environment"}) + report["environment"] = artifacts.environment() + report["checkout"] = checkout_evidence() + report["source_sha256"] = source_fingerprints() + run_validation(args.model_dir, out, report, mode=args.mode, progress=checkpoint) + except Exception as exc: + report["passed"] = False + report["error"] = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + report["validation_seconds"] = time.perf_counter() - started + report["peak_memory"] = artifacts.peak_memory() + report["peak_memory"]["scope"] = "Validation worker process, including loading and real ScratchV parsing; parent excluded" + write_json(out / "worker-report.json", report) + return 0 if report["passed"] else 1 + + +def worker_command(args): + return [sys.executable, "-B", "-X", "utf8", str(Path(__file__).resolve()), "--worker", + "--mode", args.mode, "--model-dir", str(args.model_dir.resolve()), + "--output-dir", str(args.output_dir.resolve())] + + +def launch_worker(args): + from scratchv.runtime.riscv_tensor import _run_process + return _run_process(worker_command(args), cwd=ROOT, timeout=args.timeout) + + +def number(value): + return "未测得" if value is None else f"{value:.3f}" + + +def save_worker_logs(out, report, stdout, stderr): + """Keep reporting the validation result even when a log file is locked.""" + for name, content in (("worker.stdout", stdout), ("worker.stderr", stderr)): + try: + (out / name).write_bytes(content or b"") + except OSError as exc: + report.setdefault("log_errors", []).append(f"{name}: {type(exc).__name__}: {exc}") + if report.get("log_errors") and report.get("passed"): + report.update(passed=False, stage="log-write", error="Could not preserve validation worker logs") + + +def _render_reports(report): + graph, audit, ir = report.get("graph", {}), report.get("audit", {}), report.get("ir", {}) + layers = graph.get("layers", []) + peak = report.get("peak_memory", {}) + lines = ["# 完整 Qwen3 ONNX → ScratchV IR 解析验收", "", + f"Result: **{'PASS' if report['passed'] else 'FAIL'}**；阶段：`{report['stage']}`", "", + "只验证完整解析、结构、IR 和实际权重绑定；不证明完整 IR 数值或 QEMU 前向通过。", "", + f"- 模型：`{report['model_id']}`，revision `{report['revision']}`。", + f"- 源码 HEAD：`{report.get('checkout', {}).get('head', 'unavailable')}`；" + f"工作树 clean：`{report.get('checkout', {}).get('clean', 'unavailable')}`。", + "- dirty 工作树的结果须结合 source_sha256 和本地 diff，不能只归属 HEAD。", + f"- 完整层链：{len(layers) if isinstance(layers, list) else layers} 层；" + f"ONNX 节点：{report.get('onnx_structure', {}).get('operators') and sum(report['onnx_structure']['operators'].values()) or '未完成'}。", + f"- IR：{ir.get('instructions', '未完成')} 条指令、{ir.get('globals', '未完成')} 个 globals；" + f"verifier：{report.get('ir_verifier', {}).get('passed', '未完成')}。", + f"- Supervisor 总耗时：{number(report.get('seconds'))} 秒；" + f"实际解析：{number(report.get('stages_seconds', {}).get('scratchv-parse'))} 秒。", + f"- 验证进程峰值 RSS：{peak.get('bytes', '未测得')} bytes；{peak.get('method', '')}。", "", + "| 阶段 | 耗时（秒） |", "|---|---:|"] + lines += [f"| {name} | {number(seconds)} |" for name, seconds in report.get("stages_seconds", {}).items()] + if report.get("error"): + lines += ["", "## 失败信息", "", report["error"], "", + "```json", json.dumps(report.get("failure_context", report.get("last_progress", {})), + ensure_ascii=False, indent=2), "```"] + if report.get("log_errors"): + lines += ["", "子进程日志落盘失败：", *report["log_errors"]] + if report.get("evidence_errors"): + lines += ["", "失败明细保存异常（原始验证错误保留）：", *report["evidence_errors"]] + if report.get("report_write_errors"): + lines += ["", "报告保存异常（原始验证错误保留）：", *report["report_write_errors"]] + concise_audit = {key: value for key, value in audit.items() if key != "aliases"} + lines += ["", "## 审计摘要", "", "```json", json.dumps(concise_audit, ensure_ascii=False, indent=2), "```", "", + "`nodes.json` 保存节点翻译记录；`bindings.json` 保存逐张量绑定和内容哈希；" + "`ir.txt` 保存 IR 文本，均不包含原始权重数组。", "", + "缺文件、解析错误、审计失败、超时或进程异常均失败；未执行的检查不能视为通过。" + "耗时和内存为观察数据，没有新增性能通过阈值。"] + markdown = "\n".join(lines) + "\n" + details = json.dumps({"graph": graph, "audit": audit, "ir_verifier": report.get("ir_verifier")}, + ensure_ascii=False, indent=2) + html = ( + "Qwen3 完整解析验收" + "" + f"

frontend:parse-qwen3 — {'PASS' if report['passed'] else 'FAIL'}

" + f"
{escape(markdown)}
结构与绑定详情" + f"
{escape(details)}
") + return {"report.md": markdown, "report.html": html} + + +def write_reports(out, report): + """Publish JSON last, so incomplete evidence cannot leave a stale PASS.""" + failures = [] + + def record_failure(name, exc): + failures.append(f"{name}: {type(exc).__name__}: {exc}") + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", + error="Could not preserve complete validation reports") + + def write_views(*, record_errors): + for name, content in _render_reports(report).items(): + try: + (out / name).write_text(content, encoding="utf-8") + except OSError as exc: + if record_errors: + record_failure(name, exc) + + write_views(record_errors=True) + if failures: + # A view written before another failed may still contain PASS. Refresh + # every writable view, preserving the original validation failure. + write_views(record_errors=False) + try: + write_json(out / "report.json", report) + except OSError as exc: + record_failure("report.json", exc) + write_views(record_errors=False) + for error in failures: + print(error, file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--mode", choices=("verify", "download"), default="verify") + cli.add_argument("--model-dir", type=Path, required=True) + cli.add_argument("--output-dir", type=Path, default=ROOT / "output/qwen3-parse") + cli.add_argument("--timeout", type=float, default=300) + cli.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + args = cli.parse_args(argv) + if not math.isfinite(args.timeout) or args.timeout <= 0: + cli.error("--timeout must be finite and positive") + if args.worker: + return worker_main(args) + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + cli.error("--output-dir must be absent or empty; preserve previous evidence") + out.mkdir(parents=True, exist_ok=True) + report = initial_report(args.mode) + report.update(stage="worker-launch", timeout_seconds=args.timeout) + started = time.perf_counter() + result = None + try: + # Keep the attempted run identifiable even if the child never starts or + # exits before it can validate dependencies and write its own report. + report["attempt"] = {"command": worker_command(args), "cwd": str(ROOT), + "model_dir": str(args.model_dir.resolve()), + "python": sys.version, "checkout": checkout_evidence(), + "source_sha256": source_fingerprints(), + "scope": "Supervisor metadata; not proof of worker initialization or validation"} + result = launch_worker(args) + report["worker_returncode"] = result.returncode + worker = json.loads((out / "worker-report.json").read_text(encoding="utf-8")) + if worker.get("gate") != GATE or worker.get("schema_version") != 1: + raise ValueError("Invalid validation worker report") + report.update(worker) + if result.returncode or report.get("passed") is not True or report.get("stage") != "complete": + report["passed"] = False + report.setdefault("error", f"Validation worker did not pass (exit {result.returncode})") + elif any(not (out / name).is_file() for name in ("nodes.json", "bindings.json", "ir.txt")): + raise ValueError("Validation worker omitted required evidence files") + except subprocess.TimeoutExpired as exc: + report.update(passed=False, stage="timeout", error=f"Validation worker exceeded {args.timeout} seconds") + save_worker_logs(out, report, exc.stdout, exc.stderr) + except Exception as exc: + report["passed"] = False + report["error"] = f"{type(exc).__name__}: {exc}" + finally: + if result is not None: + save_worker_logs(out, report, result.stdout, result.stderr) + report["seconds"] = time.perf_counter() - started + if not report["passed"] and (out / "worker-progress.json").is_file(): + try: + report["last_progress"] = json.loads((out / "worker-progress.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + pass + write_reports(out, report) + print(f"[{GATE}] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; " + f"seconds={report['seconds']:.3f}; report={out / 'report.json'}", flush=True) + if report.get("error"): + print(report["error"], file=sys.stderr) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_qwen3_parse/validation.py b/probes/w2_qwen3_parse/validation.py new file mode 100644 index 0000000..dfc3dfe --- /dev/null +++ b/probes/w2_qwen3_parse/validation.py @@ -0,0 +1,427 @@ +"""Structural acceptance of the pinned, decomposed Qwen3 ONNX graph. + +This deliberately recognizes the exported architecture, not arbitrary graphs +whose node names contain ``layers.N``. It follows actual operands from the +weights through both residuals in every decoder layer to the returned logits. +No external tensor data is read and no full-model numerical claim is made. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +import math + +import numpy as np +import onnx + + +QWEN3_06B_CONFIG = { + "num_hidden_layers": 28, + "hidden_size": 1024, + "intermediate_size": 3072, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 151936, + "sequence_length": 256, + "rms_norm_eps": 1e-6, +} + + +class Qwen3StructureError(ValueError): + """A missing or incorrectly connected part of the Qwen3 architecture.""" + + +def _require(condition, message): + if not condition: + raise Qwen3StructureError(message) + + +def _attrs(node): + return {a.name: onnx.helper.get_attribute_value(a) for a in node.attribute} + + +def _one(nodes, context): + _require(len(nodes) == 1, f"{context}: expected exactly one node, found {len(nodes)}") + return nodes[0] + + +class _Graph: + def __init__(self, graph): + self.graph = graph + self.initializers = {t.name: t for t in graph.initializer} + self.nodes = list(graph.node) + self.producers = {} + self.consumers = defaultdict(list) + self.by_op = defaultdict(list) + self.bits = {} + self.ancestors = {} + self.constants = {} + self.indices = {} + roots = list(dict.fromkeys([*(v.name for v in graph.input), *self.initializers])) + for index, name in enumerate(roots): + self.bits[name] = self.ancestors[name] = 1 << index + for index, node in enumerate(self.nodes): + label = node.name or f"node[{index}]" + self.indices[id(node)] = index + _require(node.domain in ("", "ai.onnx"), f"{label}: unsupported domain {node.domain!r}") + _require(len(node.output) == 1 and node.output[0], + f"{label}: this pinned graph requires one nonempty output per node") + output = node.output[0] + _require(output not in self.ancestors, f"{label}: duplicate definition {output!r}") + bit = 1 << (len(roots) + index) + inherited = bit + for value in node.input: + if not value: + continue + _require(value in self.ancestors, + f"{label}: input {value!r} has no preceding definition") + inherited |= self.ancestors[value] + self.consumers[value].append(node) + self.bits[output] = bit + self.ancestors[output] = inherited + self.producers[output] = node + self.by_op[node.op_type].append(node) + + def depends(self, output, value): + return bool(self.ancestors.get(output, 0) & self.bits.get(value, 0)) + + def strip(self, value, extra=()): + while value in self.producers: + node = self.producers[value] + if node.op_type not in ("Identity", "Cast", *extra): + break + value = node.input[0] + return value + + def producer(self, value, op, context): + node = self.producers.get(self.strip(value)) + _require(node is not None and node.op_type == op, + f"{context}: expected {op} producer for {value!r}") + return node + + def same(self, left, right, extra=()): + return self.strip(left, extra) == self.strip(right, extra) + + def constant(self, value): + """Resolve only tiny control constants; never open external weights.""" + if value in self.constants: + return self.constants[value] + if value in self.initializers: + tensor = self.initializers[value] + _require(not tensor.external_data and math.prod(tensor.dims) <= 1024, + f"constant {value!r}: external or large tensor is not a control constant") + result = onnx.numpy_helper.to_array(tensor) + else: + node = self.producers.get(value) + _require(node is not None, f"constant {value!r}: missing definition") + attrs = _attrs(node) + op = node.op_type + if op == "Constant": + if "value" in attrs: + tensor = attrs["value"] + _require(not tensor.external_data and math.prod(tensor.dims) <= 1024, + f"constant {value!r}: external or large tensor") + result = onnx.numpy_helper.to_array(tensor) + elif "value_int" in attrs: + result = np.asarray(attrs["value_int"], dtype=np.int64) + elif "value_ints" in attrs: + result = np.asarray(attrs["value_ints"], dtype=np.int64) + elif "value_float" in attrs: + result = np.asarray(attrs["value_float"], dtype=np.float32) + elif "value_floats" in attrs: + result = np.asarray(attrs["value_floats"], dtype=np.float32) + else: + raise Qwen3StructureError(f"constant {value!r}: unsupported Constant encoding") + elif op in ("Identity", "Cast", "Abs"): + result = self.constant(node.input[0]) + if op == "Cast": + result = result.astype(onnx.helper.tensor_dtype_to_np_dtype(attrs["to"])) + elif op == "Abs": + result = np.abs(result) + elif op == "Reshape": + data = self.constant(node.input[0]) + shape = self.constant(node.input[1]).reshape(-1).tolist() + shape = [data.shape[i] if d == 0 else d for i, d in enumerate(shape)] + result = data.reshape(shape) + elif op == "Concat": + result = np.concatenate([self.constant(v) for v in node.input], axis=attrs["axis"]) + elif op == "Unsqueeze": + axes = self.constant(node.input[1]).reshape(-1).tolist() + result = np.expand_dims(self.constant(node.input[0]), tuple(axes)) + else: + raise Qwen3StructureError(f"constant {value!r}: unexpected {op} dependency") + _require(result.size <= 1024, f"constant {value!r}: exceeds control-constant size limit") + self.constants[value] = result + return result + + def scalar(self, value, expected, context): + data = self.constant(value) + _require(data.size == 1 and np.isfinite(data).all() + and math.isclose(float(data.reshape(-1)[0]), expected, rel_tol=1e-6, abs_tol=0), + f"{context}: {value!r} must be scalar {expected}") + + def projection(self, weight, context): + candidates = [] + for node in self.by_op["MatMul"]: + if len(node.input) != 2: + continue + transposed = self.producers.get(self.strip(node.input[1])) + if (transposed is not None and transposed.op_type == "Transpose" + and self.same(transposed.input[0], weight)): + _require(tuple(_attrs(transposed).get("perm", (1, 0))) == (1, 0), + f"{context}: weight transpose must use perm=[1, 0]") + candidates.append(node) + return _one(candidates, f"{context}, weight {weight!r}") + + def norm(self, weight, epsilon, context): + node = _one([n for n in self.by_op["Mul"] + if len(n.input) == 2 and any(self.same(v, weight) for v in n.input)], + f"{context}, weight {weight!r}") + normalized = _one([v for v in node.input if not self.same(v, weight)], + f"{context}: normalized activation") + multiply = self.producer(normalized, "Mul", context) + reciprocals = [v for v in multiply.input + if self.producers.get(self.strip(v)) is not None + and self.producers[self.strip(v)].op_type == "Reciprocal"] + reciprocal_value = _one(reciprocals, f"{context}: RMS inverse square root") + reciprocal = self.producer(reciprocal_value, "Reciprocal", context) + raw = next(v for v in multiply.input if v != reciprocal_value) + sqrt = self.producer(reciprocal.input[0], "Sqrt", context) + add = self.producer(sqrt.input[0], "Add", context) + means = [v for v in add.input if self.producers.get(self.strip(v)) is not None + and self.producers[self.strip(v)].op_type == "ReduceMean"] + mean_value = _one(means, f"{context}: RMS mean") + self.scalar(next(v for v in add.input if v != mean_value), epsilon, context) + mean = self.producer(mean_value, "ReduceMean", context) + attrs = _attrs(mean) + axes = (self.constant(mean.input[1]).reshape(-1).tolist() + if len(mean.input) == 2 else list(attrs.get("axes", []))) + _require(axes == [-1] and attrs.get("keepdims", 1) == 1, + f"{context}: RMS ReduceMean must use axes=[-1], keepdims=1") + power = self.producer(mean.input[0], "Pow", context) + _require(self.same(power.input[0], raw), f"{context}: RMS numerator/denominator input mismatch") + self.scalar(power.input[1], 2.0, context) + return node, self.strip(raw) + + def residual(self, branch, bypass, context): + candidates = [n for n in self.by_op["Add"] if len(n.input) == 2 + and ((self.same(n.input[0], branch) and self.same(n.input[1], bypass)) + or (self.same(n.input[1], branch) and self.same(n.input[0], bypass)))] + return _one(candidates, context) + + +def make_control_constant_reader(model): + """Return ``reader(onnx_value_name) -> ndarray`` for small control values. + + Resolution uses the original ONNX graph, never the parser, its cached + constants, or the emitted IR. Every intermediate is limited to 1,024 + elements and external control tensors are rejected. This is the bounded + constant subset needed by the pinned export's axes, slices and shapes, + not a general ONNX interpreter or a way to load model weights. + """ + return _Graph(model.graph).constant + + +def _metadata(graph, config): + hidden, head = config["hidden_size"], config["head_dim"] + query, kv = config["num_attention_heads"] * head, config["num_key_value_heads"] * head + middle = config["intermediate_size"] + per_layer = { + "self_attn.q_proj.weight": (query, hidden), + "self_attn.k_proj.weight": (kv, hidden), + "self_attn.v_proj.weight": (kv, hidden), + "self_attn.o_proj.weight": (hidden, query), + "self_attn.q_norm.weight": (head,), + "self_attn.k_norm.weight": (head,), + "mlp.gate_proj.weight": (middle, hidden), + "mlp.up_proj.weight": (middle, hidden), + "mlp.down_proj.weight": (hidden, middle), + "input_layernorm.weight": (hidden,), + "post_attention_layernorm.weight": (hidden,), + } + expected = {f"model.model.layers.{i}.{name}": (shape, onnx.TensorProto.FLOAT) + for i in range(config["num_hidden_layers"]) for name, shape in per_layer.items()} + expected.update({ + "model.model.norm.weight": ((hidden,), onnx.TensorProto.FLOAT), + "model.lm_head.weight": ((config["vocab_size"], hidden), onnx.TensorProto.FLOAT), + "positions": ((1, config["sequence_length"]), onnx.TensorProto.INT64), + "model.model.rotary_emb.inv_freq": ((head // 2,), onnx.TensorProto.FLOAT), + }) + actual = {t.name: t for t in graph.initializer} + _require(len(actual) == len(graph.initializer), "initializers: duplicate weight names") + _require(set(actual) == set(expected), + f"initializers: missing={sorted(set(expected) - set(actual))}, " + f"unexpected={sorted(set(actual) - set(expected))}") + for name, (shape, dtype) in expected.items(): + tensor = actual[name] + _require(tuple(tensor.dims) == shape and tensor.data_type == dtype, + f"weight {name!r}: expected shape={shape}, dtype={dtype}; " + f"got shape={tuple(tensor.dims)}, dtype={tensor.data_type}") + length = config["sequence_length"] + signature = {"input_ids": ((1, length), onnx.TensorProto.INT64), + "attention_mask": ((1, 1, length, length), onnx.TensorProto.FLOAT), + "logits": ((1, length, config["vocab_size"]), onnx.TensorProto.FLOAT)} + _require([v.name for v in graph.input] == ["input_ids", "attention_mask"], + "graph inputs must be input_ids and attention_mask, in that order") + _require([v.name for v in graph.output] == ["logits"], "graph output must be logits") + for value in [*graph.input, *graph.output]: + tensor = value.type.tensor_type + shape, dtype = signature[value.name] + _require(all(d.HasField("dim_value") for d in tensor.shape.dim) + and tuple(d.dim_value for d in tensor.shape.dim) == shape + and tensor.elem_type == dtype, f"{value.name}: incorrect static shape or dtype") + + +def _attention(index, q, k, v, out, q_norm, k_norm, config, context): + qv, kv = q_norm.output[0], k_norm.output[0] + softmax = _one([n for n in index.by_op["Softmax"] + if index.depends(out.input[0], n.output[0]) + and index.depends(n.input[0], qv) and index.depends(n.input[0], kv)], + f"{context}: attention Softmax") + _require(_attrs(softmax).get("axis", -1) == -1, + f"{context}: attention Softmax must use axis=-1") + scores = _one([n for n in index.by_op["MatMul"] + if index.depends(softmax.input[0], n.output[0]) + and index.depends(n.input[0], qv) and index.depends(n.input[1], kv)], + f"{context}: QK attention MatMul") + _require(not index.depends(scores.input[0], kv) and not index.depends(scores.input[1], qv), + f"{context}: Q and K score operands must retain separate paths") + weighted = _one([n for n in index.by_op["MatMul"] + if index.depends(out.input[0], n.output[0]) + and index.same(n.input[0], softmax.output[0]) + and index.depends(n.input[1], v.output[0])], + f"{context}: attention probabilities times V") + mask_add = index.producer(softmax.input[0], "Add", context) + masks = [value for value in mask_add.input + if index.depends(value, "attention_mask") and not index.depends(value, scores.output[0])] + mask = _one(masks, f"{context}: additive attention mask") + scale = index.producer(next(value for value in mask_add.input if value != mask), "Mul", context) + scale_inputs = [value for value in scale.input if not index.same(value, scores.output[0])] + _require(len(scale_inputs) == 1, f"{context}: scores must be multiplied by a scalar scale") + index.scalar(scale_inputs[0], config["head_dim"] ** -0.5, f"{context}: attention scale") + for op in ("Cos", "Sin"): + rope = _one([n for n in index.by_op[op] + if index.depends(scores.input[0], n.output[0]) + and index.depends(scores.input[1], n.output[0])], f"{context}: shared RoPE {op}") + _require(index.depends(rope.input[0], "positions") + and index.depends(rope.input[0], "model.model.rotary_emb.inv_freq"), + f"{context}: RoPE {op} must use positions and inv_freq") + if config["num_attention_heads"] != config["num_key_value_heads"]: + expected = (1, config["num_key_value_heads"], + config["num_attention_heads"] // config["num_key_value_heads"], + config["sequence_length"], config["head_dim"]) + key_transpose = index.producer(scores.input[1], "Transpose", f"{context}: K score transpose") + _require(tuple(_attrs(key_transpose).get("perm", ())) == (0, 1, 3, 2), + f"{context}: K score transpose must use perm=[0, 1, 3, 2]") + for value, source, label in ((key_transpose.input[0], kv, "K"), + (weighted.input[1], v.output[0], "V")): + reshape = index.producer(value, "Reshape", f"{context}: GQA {label} flattened heads") + shape = tuple(index.constant(reshape.input[1]).reshape(-1).tolist()) + target = (1, config["num_attention_heads"], config["sequence_length"], config["head_dim"]) + _require(shape == target, f"{context}: GQA {label} head reshape {shape} != {target}") + expand = index.producer(reshape.input[0], "Expand", f"{context}: GQA {label} expansion") + _require(index.depends(expand.input[0], source), + f"{context}: GQA {label} expansion does not consume its projection") + shape = tuple(index.constant(expand.input[1]).reshape(-1).tolist()) + _require(shape == expected, f"{context}: GQA {label} expansion shape {shape} != {expected}") + source_value = index.strip(expand.input[0], ("Slice",)) + unsqueeze = index.producer(source_value, "Unsqueeze", f"{context}: GQA {label} group axis") + axes = tuple(index.constant(unsqueeze.input[1]).reshape(-1).tolist()) + _require(axes == (2,), f"{context}: GQA {label} repeat group must be axis 2") + return {"scores": scores.name, "softmax": softmax.name, "weighted_values": weighted.name} + + +def audit_graph_structure(model, config=None): + """Check the complete decoder dataflow without loading external weights. + + ``config`` overrides architecture dimensions for small test fixtures. This + gate recognizes the pinned export's decomposed RMSNorm/RoPE/GQA/SwiGLU; + a different exporter representation needs a reviewed new pattern. + """ + config = dict(QWEN3_06B_CONFIG) | dict(config or {}) + for name, value in config.items(): + if name == "rms_norm_eps": + _require(isinstance(value, (float, int)) and not isinstance(value, bool) + and math.isfinite(value) and value > 0, f"config: invalid {name}") + else: + _require(isinstance(value, int) and not isinstance(value, bool) and value > 0, + f"config: invalid {name}") + _require(config["head_dim"] % 2 == 0, "config: head_dim must be even for RoPE") + _require(config["num_attention_heads"] % config["num_key_value_heads"] == 0, + "config: attention heads must be divisible by KV heads") + _metadata(model.graph, config) + index = _Graph(model.graph) + shared_weight = "model.lm_head.weight" + embedding = _one([n for n in index.by_op["Gather"] + if list(n.input) == [shared_weight, "input_ids"]], + "embedding: tied LM Head weight and input_ids Gather") + _require(_attrs(embedding).get("axis", 0) == 0, "embedding: Gather axis must be zero") + previous = embedding.output[0] + rows = [] + for layer in range(config["num_hidden_layers"]): + context = f"layer {layer}" + prefix = f"model.model.layers.{layer}." + projections = {name: index.projection(prefix + name + ".weight", f"{context} {name}") + for name in ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", + "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj")} + norms = {name: index.norm(prefix + name + ".weight", config["rms_norm_eps"], f"{context} {name}") + for name in ("input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm")} + input_norm, raw = norms["input_layernorm"] + _require(index.same(raw, previous), f"{context}: input norm must consume previous layer residual") + q, k, v, out = (projections[f"self_attn.{name}_proj"] for name in ("q", "k", "v", "o")) + _require(all(index.same(n.input[0], input_norm.output[0]) for n in (q, k, v)), + f"{context}: Q/K/V projections must share this layer's input norm") + q_norm, q_raw = norms["self_attn.q_norm"] + k_norm, k_raw = norms["self_attn.k_norm"] + _require(index.same(q_raw, q.output[0], ("Reshape",)) + and index.same(k_raw, k.output[0], ("Reshape",)), + f"{context}: Q/K norm must consume the matching projection") + attention = _attention(index, q, k, v, out, q_norm, k_norm, config, context) + first_residual = index.residual(out.output[0], previous, f"{context}: attention residual") + post_norm, post_raw = norms["post_attention_layernorm"] + _require(index.same(post_raw, first_residual.output[0]), + f"{context}: post-attention norm must consume attention residual") + gate, up, down = (projections[f"mlp.{name}_proj"] for name in ("gate", "up", "down")) + _require(all(index.same(n.input[0], post_norm.output[0]) for n in (gate, up)), + f"{context}: gate/up projections must share post-attention norm") + product = index.producer(down.input[0], "Mul", f"{context}: SwiGLU") + activation_values = [value for value in product.input if not index.same(value, up.output[0])] + _require(len(activation_values) == 1, f"{context}: SwiGLU must multiply by up projection") + activation = index.producer(activation_values[0], "Mul", f"{context}: SiLU") + sigmoid_values = [value for value in activation.input if not index.same(value, gate.output[0])] + _require(len(sigmoid_values) == 1, f"{context}: SiLU must multiply gate by sigmoid(gate)") + sigmoid = index.producer(sigmoid_values[0], "Sigmoid", f"{context}: SiLU") + _require(index.same(sigmoid.input[0], gate.output[0]), f"{context}: SiLU sigmoid input must be gate") + second_residual = index.residual(down.output[0], first_residual.output[0], + f"{context}: MLP residual") + previous = second_residual.output[0] + rows.append({"layer": layer, "passed": True, "input": raw, "output": previous, + "projections": {name: node.name for name, node in projections.items()}, + "norms": {name: node.name for name, (node, _) in norms.items()}, + "attention": attention, + "residuals": [first_residual.name, second_residual.name]}) + final_norm, raw = index.norm("model.model.norm.weight", config["rms_norm_eps"], "final norm") + _require(index.same(raw, previous), "final norm must consume the last layer residual") + head = index.projection(shared_weight, "LM Head") + _require(index.same(head.input[0], final_norm.output[0], ("Slice",)), + "LM Head must consume final norm (allowing the exporter's logits slices)") + _require(index.same("logits", head.output[0]), "logits must return the tied LM Head projection") + live = [node for node in index.nodes if index.depends("logits", node.output[0])] + dead = [node.name or node.output[0] for node in index.nodes if not index.depends("logits", node.output[0])] + _require(not dead, f"graph has nodes disconnected from logits: {dead[:8]}") + for name in index.initializers: + _require(index.depends("logits", name), f"initializer {name!r} is disconnected from logits") + return {"passed": True, "config": config, + "graph": {"node_count": len(index.nodes), "initializer_count": len(index.initializers), + "operators": dict(sorted(Counter(n.op_type for n in index.nodes).items())), + "inputs": [v.name for v in model.graph.input], "outputs": ["logits"]}, + "layers": rows, "layer_count": len(rows), "live_node_count": len(live), + "embedding": embedding.name, "final_norm": final_norm.name, "lm_head": head.name, + "tied_embeddings": True, + "checks": ["exact weight inventory, shapes and dtypes", "static input/output signature", + "RMSNorm paths and constants", "Q/K/V and GQA attention dataflow", + "shared RoPE position/frequency dependencies", "SwiGLU dataflow", + "both residuals and ordered layer chain", "final norm and tied LM Head", + "every node and initializer reaches logits"], + "scope": "Structural acceptance of the pinned export; no numerical execution or external weight reads."} diff --git a/probes/w2_qwen3_small/README.md b/probes/w2_qwen3_small/README.md index 44779a7..66ccc2b 100644 --- a/probes/w2_qwen3_small/README.md +++ b/probes/w2_qwen3_small/README.md @@ -152,12 +152,19 @@ padding 隔离。每个比较仍要求 FP32、形状一致、有限值、最大 失败、输入目录混用或哈希不一致应直接失败，不能只靠当前 ORT 比较通过。该绑定 用于避免意外混用产物，不是第三方认证；参考计算仍须在指定环境重新生成。 +当前执行环境与完整源码归属的报告增强单独见 [W1 PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。 +W2 分支未包含该报告代码修复；未合并前不能把原导出环境当成本次 QEMU 对照的执行环境。 + `output/qwen3-riscv/` 包含 `report.json/md/html`、生成的 C、启动汇编与链接脚本、 ELF、编译器/QEMU 命令与日志、每次执行的输入和 UART 二进制以及 QEMU/ORT 数组。 JSON 记录源码/模型/输入/ELF 哈希、工具版本、优化统计、工作区大小、最差元素与 首次发生偏差的检查点。缺工具、编译失败、trap、超时、损坏/截断的 UART、错误 退出或任一数值不匹配都会失败，不允许跳过后返回成功。 +编译器或 QEMU 的日志文件保存失败时，仍分别尝试保存 stdout 和 stderr。 +已有的编译错误、guest 状态、协议错误或超时保持为主要错误，日志保存错误另行附加； +执行成功但必需日志无法保存时，入口仍返回失败，避免把不完整证据当作完整验收。 + ### 查看执行时长 使用上面的 `--output-dir output/qwen3-riscv` 命令即可生成时长报告，无需另开计时 diff --git a/probes/w2_runtime/README.md b/probes/w2_runtime/README.md new file mode 100644 index 0000000..462324a --- /dev/null +++ b/probes/w2_runtime/README.md @@ -0,0 +1,59 @@ +# W2 Host 运行时验收 + +`unit:runtime` 验证文本 tokenizer、固定输入准备、最后有效位置 greedy 采样及停止边界，不执行完整模型或生成循环。生产模块只依赖 NumPy 与可选的 `tokenizers`；Transformers 只用于此门禁的官方对照。 + +## 固定资产与环境 + +模型为 `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`。复用 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个 `source_files` 哈希：配置、tokenizer JSON、merges、vocab 与许可证，合计约 15.2 MiB。下载模式只获取这些文件，不下载模型权重。已有资产必须全部匹配哈希；损坏目录保留并失败，不静默覆盖。 + +使用 [两层模型指南](../w2_qwen3_small/README.md) 中的 Python 3.12 固定 CPU 环境；其中已包含 `numpy==2.2.6`、`tokenizers==0.21.4`、`transformers==4.51.3`。仅使用生产 tokenizer 时可安装 `pip install -e ".[llm]"`，无需 Torch；复现 gate 仍使用固定版本和官方参考依赖。 + +在仓库根目录运行，PowerShell： + +```powershell +$runtimePython = ".\output\qwen3-probe-venv\Scripts\python.exe" +& $runtimePython -B -X utf8 probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +$LASTEXITCODE +``` + +Linux： + +```bash +output/qwen3-probe-venv/bin/python -B -X utf8 probes/w2_runtime/run.py \ + --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +``` + +已有七个文件时使用 `--mode verify`，该模式不访问网络。输出目录必须为空或不存在，每次复现选择新的目录。CLI 退出 0 且本次 `report.json` 的 `passed=true` 才代表通过。报告写入不完整、依赖缺失、下载或哈希校验失败均为失败。 + +## 验收内容 + +- 固定 20 组语料涵盖中英混合、空白、换行、代码、数字、CJK/RTL、emoji、组合字符和特殊 token；保存固定 IDs 与解码文本作为 golden corpus。 +- 每组均对照官方 `Qwen2TokenizerFast` 和独立 Python `Qwen2Tokenizer`，覆盖默认参数、`add_special_tokens`、`skip_special_tokens` 与空格清理组合。官方 JSON 的 NFC 规范化会把 `e` 加组合重音转换为 `é`；“100%”指与官方输出完全一致，不是任意原始 Unicode 字节可逆。 +- 分开核对基础词表 151643、含 added tokens 的词表 151669、模型输出维度 151936；模型中空余行不等于可解码 ID。未知 ID 报错，不丢弃；不静默屏蔽 logits 或改变 greedy。 +- tokenizer EOS 为 151645，generation EOS 为 `(151645,151643)`，pad 为 151643。普通文本编码不注入 BOS/EOS，不渲染聊天模板。 +- 检查有效长度 1/17/255/256、右 padding、INT64 IDs、FP32 加性 mask、最后有效位置和 greedy 平局。mask 仅当 `key <= query` 且 `key < valid_length` 时为 0，其余为 FP32 最小有限值。 +- 验证生成 EOS、256 容量和 token 预算停止。空 prompt、超长输入、非法 ID、非有限 logits 等反例在单测中必须失败。 + +## 生产接口 + +```python +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer +from scratchv.runtime.llm_inputs import prepare_inputs, greedy_next_token + +tokenizer = Qwen3Tokenizer.from_directory("output/qwen3-tokenizer") +ids = tokenizer.encode("The capital of France is", add_special_tokens=False) +inputs = prepare_inputs(ids, pad_token_id=tokenizer.pad_token_id, + vocab_size=tokenizer.model_vocab_size) +# outputs = your_verified_model_runner(inputs.as_feed()) +# next_id = greedy_next_token(outputs, inputs.valid_length, +# vocab_size=tokenizer.model_vocab_size) +# tokenizer.validate_token_ids([next_id]) +``` + +`generation_stop_reason` 供后续生成循环在前向前判断 EOS/容量/预算。模型输出仍按原 ABI 完整校验，最后有效位置选择是 host 操作。这里未实现聊天模板、多轮历史、完整模型前向或端到端文本生成；也不能将官方 tokenizer 的 IDs 输入词表只有 128 的随机小模型。 + +## 报告与 CI + +`report.json/md/html` 保留逐语料结果、对照次数、词表/特殊 ID、资产与执行源码 SHA256、checkout、环境、阶段及错误。报告目录不含模型或 tokenizer 大文件；失败保留已完成的记录，不把未执行项计作通过。 + +LLM workflow 的 `unit:runtime` 在固定环境运行，下载固定的七个资产，随后全程离线调用官方参考；`w2-unit-gates-output` artifact 保存报告和单测 XML。`tests/test_llm_inputs.py`、`tests/test_qwen3_tokenizer.py` 和 `tests/test_w2_runtime_*.py` 覆盖输入、适配层和门禁的反例。具体运行结果见 [W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md)。 diff --git a/probes/w2_runtime/corpus.json b/probes/w2_runtime/corpus.json new file mode 100644 index 0000000..dc33765 --- /dev/null +++ b/probes/w2_runtime/corpus.json @@ -0,0 +1,487 @@ +{ + "schema_version": 1, + "model_id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "reference": { + "transformers": "4.51.3", + "tokenizers": "0.21.4", + "classes": [ + "Qwen2Tokenizer", + "Qwen2TokenizerFast" + ], + "add_special_tokens": false, + "skip_special_tokens": false, + "clean_up_tokenization_spaces": false + }, + "cases": [ + { + "name": "empty", + "text": "", + "ids": [], + "decoded": "" + }, + { + "name": "english", + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "name": "chinese", + "text": "你好，世界！ScratchV 编译器。", + "ids": [ + 108386, + 3837, + 99489, + 6313, + 65508, + 754, + 53, + 84238, + 244, + 102610, + 31548, + 1773 + ], + "decoded": "你好，世界！ScratchV 编译器。" + }, + { + "name": "mixed", + "text": "Qwen3 在 RISC-V 上运行 0.6B 模型。", + "ids": [ + 48, + 16948, + 18, + 73562, + 431, + 27629, + 19625, + 64118, + 104001, + 220, + 15, + 13, + 21, + 33, + 6567, + 44401, + 24300, + 1773 + ], + "decoded": "Qwen3 在 RISC-V 上运行 0.6B 模型。" + }, + { + "name": "whitespace", + "text": " leading spaces\tand tabs\n\ntrailing ", + "ids": [ + 220, + 6388, + 220, + 12621, + 52477, + 22398, + 271, + 376, + 14277, + 256 + ], + "decoded": " leading spaces\tand tabs\n\ntrailing " + }, + { + "name": "line_endings", + "text": "line1\r\nline2\rline3\n", + "ids": [ + 1056, + 16, + 319, + 1056, + 17, + 201, + 1056, + 18, + 198 + ], + "decoded": "line1\r\nline2\rline3\n" + }, + { + "name": "code", + "text": "def f(x):\n return x * 2 + 1\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 353, + 220, + 17, + 488, + 220, + 16, + 198 + ], + "decoded": "def f(x):\n return x * 2 + 1\n" + }, + { + "name": "math", + "text": "x² + y² = z²; ∑ₖ αₖ ≈ 3.1415926", + "ids": [ + 87, + 29456, + 488, + 379, + 29456, + 284, + 1147, + 29456, + 26, + 11995, + 239, + 15896, + 244, + 19043, + 15896, + 244, + 21103, + 230, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 17, + 21 + ], + "decoded": "x² + y² = z²; ∑ₖ αₖ ≈ 3.1415926" + }, + { + "name": "emoji", + "text": "👩🏽‍💻 🧪 🚀 🇨🇳 👨‍👩‍👧‍👦", + "ids": [ + 145233, + 145375, + 378, + 235, + 145851, + 11162, + 100, + 103, + 11162, + 248, + 222, + 11162, + 229, + 101, + 145754, + 61804, + 101, + 378, + 235, + 145233, + 378, + 235, + 145665, + 378, + 235, + 145988 + ], + "decoded": "👩🏽‍💻 🧪 🚀 🇨🇳 👨‍👩‍👧‍👦" + }, + { + "name": "combining", + "text": "café café Å Å", + "ids": [ + 924, + 58858, + 51950, + 79252, + 79252 + ], + "decoded": "café café Å Å" + }, + { + "name": "cjk", + "text": "日本語のテスト。한국어 테스트。繁體中文測試。", + "ids": [ + 101059, + 102819, + 15767, + 140592, + 1773, + 23573, + 124785, + 31079, + 10764, + 72509, + 53189, + 1773, + 100112, + 100480, + 104811, + 107636, + 102854, + 1773 + ], + "decoded": "日本語のテスト。한국어 테스트。繁體中文測試。" + }, + { + "name": "rtl", + "text": "مرحبا بالعالم שלום עולם", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634, + 124756, + 123881, + 123855, + 124907 + ], + "decoded": "مرحبا بالعالم שלום עולם" + }, + { + "name": "punctuation", + "text": "Hello , world ! Isn't this fun ?", + "ids": [ + 9707, + 1154, + 1879, + 753, + 220, + 52196, + 944, + 419, + 2464, + 937 + ], + "decoded": "Hello , world ! Isn't this fun ?" + }, + { + "name": "numbers", + "text": "0 1 -2 1e-5 3.14159 1234567890 00123", + "ids": [ + 15, + 220, + 16, + 481, + 17, + 220, + 16, + 68, + 12, + 20, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 220, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 15, + 220, + 15, + 15, + 16, + 17, + 18 + ], + "decoded": "0 1 -2 1e-5 3.14159 1234567890 00123" + }, + { + "name": "json", + "text": "{\"name\":\"Qwen3\",\"enabled\":true,\"items\":[1,2,3]}", + "ids": [ + 4913, + 606, + 3252, + 48, + 16948, + 18, + 2198, + 15868, + 788, + 1866, + 1335, + 3615, + 8899, + 16, + 11, + 17, + 11, + 18, + 13989 + ], + "decoded": "{\"name\":\"Qwen3\",\"enabled\":true,\"items\":[1,2,3]}" + }, + { + "name": "controls", + "text": "a\u0000b\t\u000b\f\n", + "ids": [ + 64, + 188, + 65, + 197, + 199, + 68762 + ], + "decoded": "a\u0000b\t\u000b\f\n" + }, + { + "name": "special", + "text": "<|endoftext|><|im_start|>user\nhello<|im_end|>", + "ids": [ + 151643, + 151644, + 872, + 198, + 14990, + 151645 + ], + "decoded": "<|endoftext|><|im_start|>user\nhello<|im_end|>" + }, + { + "name": "thinking", + "text": "Let me reason.\n42", + "ids": [ + 151667, + 10061, + 752, + 2874, + 13, + 151668, + 198, + 19, + 17 + ], + "decoded": "Let me reason.\n42" + }, + { + "name": "repeated", + "text": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello ", + "ids": [ + 14990, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 220 + ], + "decoded": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello " + }, + { + "name": "all_added", + "text": "<|endoftext|><|im_start|><|im_end|><|object_ref_start|><|object_ref_end|><|box_start|><|box_end|><|quad_start|><|quad_end|><|vision_start|><|vision_end|><|vision_pad|><|image_pad|><|video_pad|><|fim_prefix|><|fim_middle|><|fim_suffix|><|fim_pad|><|repo_name|><|file_sep|>", + "ids": [ + 151643, + 151644, + 151645, + 151646, + 151647, + 151648, + 151649, + 151650, + 151651, + 151652, + 151653, + 151654, + 151655, + 151656, + 151657, + 151658, + 151659, + 151660, + 151661, + 151662, + 151663, + 151664, + 151665, + 151666, + 151667, + 151668 + ], + "decoded": "<|endoftext|><|im_start|><|im_end|><|object_ref_start|><|object_ref_end|><|box_start|><|box_end|><|quad_start|><|quad_end|><|vision_start|><|vision_end|><|vision_pad|><|image_pad|><|video_pad|><|fim_prefix|><|fim_middle|><|fim_suffix|><|fim_pad|><|repo_name|><|file_sep|>" + } + ] +} diff --git a/probes/w2_runtime/run.py b/probes/w2_runtime/run.py new file mode 100644 index 0000000..7ec2e08 --- /dev/null +++ b/probes/w2_runtime/run.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +"""W2 host-runtime gate against pinned official tokenizer assets and two oracles.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import html +import importlib.metadata +import json +from pathlib import Path +import platform +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +MANIFEST_PATH = ROOT / "probes/w1_qwen3_export/manifest.json" +MANIFEST = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) +CORPUS = Path(__file__).with_name("corpus.json") +PINNED = {"numpy": "2.2.6", "tokenizers": "0.21.4", "transformers": "4.51.3"} +MAX_FILE_BYTES = 16 * 1024 * 1024 + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def verify_assets(directory, manifest=MANIFEST): + directory = Path(directory).resolve() + rows, seen = [], set() + for item in manifest["source_files"]: + name = item["name"] + if (not name or name in seen or name in (".", "..") + or any(c in name for c in "/\\:")): + raise ValueError(f"Non-flat or duplicate asset name: {name}") + seen.add(name) + path = (directory / name).resolve() + if not path.is_relative_to(directory) or not path.is_file(): + raise FileNotFoundError(f"Missing tokenizer asset: {name}") + size = path.stat().st_size + if size > MAX_FILE_BYTES or sha256(path) != item["sha256"]: + raise ValueError(f"Tokenizer asset SHA256/size mismatch: {name}") + rows.append({**item, "bytes": size}) + required = {"tokenizer.json", "tokenizer_config.json", "config.json", + "generation_config.json", "vocab.json", "merges.txt", "LICENSE"} + if seen != required: + raise ValueError("Tokenizer gate requires exactly the seven pinned source files") + return rows + + +def download_file(url, destination): + """Bound downloads; retry transient server failures without trusting a cache.""" + for attempt in range(3): + try: + request = urllib.request.Request(url, headers={"User-Agent": "ScratchV-W2-runtime"}) + with urllib.request.urlopen(request, timeout=30) as incoming, Path(destination).open("wb") as out: + count = 0 + while block := incoming.read(1024 * 1024): + count += len(block) + if count > MAX_FILE_BYTES: + raise ValueError("Tokenizer download exceeds per-file limit") + out.write(block) + return + except urllib.error.HTTPError as exc: + if exc.code not in (429, 500, 502, 503, 504) or attempt == 2: + raise + time.sleep(attempt + 1) + + +def acquire_assets(directory, manifest=MANIFEST): + directory = Path(directory).resolve() + if directory.exists(): + return {"reused": True, "files": verify_assets(directory, manifest)} + directory.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="qwen3-tokenizer-", dir=directory.parent) as tmp: + stage = Path(tmp).resolve() + for item in manifest["source_files"]: + name = item["name"] + if not name or name in (".", "..") or any(c in name for c in "/\\:"): + raise ValueError(f"Invalid tokenizer asset name: {name}") + url = f"https://huggingface.co/{manifest['model_id']}/resolve/{manifest['revision']}/{name}" + download_file(url, stage / name) + rows = verify_assets(stage, manifest) + if not stage.is_relative_to(directory.parent) or directory.exists(): + raise ValueError("Tokenizer destination changed during download") + stage.rename(directory) + return {"reused": False, "files": rows} + + +def require_environment(): + versions = {name: importlib.metadata.version(name) for name in PINNED} + if sys.version_info[:2] != (3, 12) or versions != PINNED: + raise RuntimeError(f"Use Python 3.12 and the pinned CPU probe environment: {versions}") + return {"python": platform.python_version(), "platform": platform.platform(), + "packages": versions, "reference": "Official Transformers fast and slow tokenizers, offline"} + + +def checkout_evidence(): + """Source archives remain runnable; never attribute them to a parent repo.""" + result = {"root": str(ROOT), "head": None, "status": None, "clean": None} + if not (ROOT / ".git").exists(): + result["unavailable"] = "Source directory has no .git entry; use source_sha256 for identity" + return result + try: + for key, arguments in (("head", ["rev-parse", "HEAD"]), + ("status", ["status", "--porcelain=v1", "--untracked-files=all"])): + result[key] = subprocess.check_output( + ["git", *arguments], cwd=ROOT, text=True, encoding="utf-8", errors="replace", + stderr=subprocess.PIPE, timeout=10, + ).rstrip("\r\n") + result["clean"] = not bool(result["status"]) + except (OSError, subprocess.SubprocessError) as exc: + result["unavailable"] = f"{type(exc).__name__}: {exc}" + return result + + +def tokenizer_cases(adapter, fast, slow, corpus, rows): + if not corpus["cases"]: + raise ValueError("Tokenizer corpus must not be empty") + names = set() + for case in corpus["cases"]: + if case["name"] in names: + raise ValueError("Duplicate corpus case name") + names.add(case["name"]) + row = {"name": case["name"], "passed": False, "checks": 0} + rows.append(row) + # Exercise public defaults as well as explicit flags. A default-only + # regression would otherwise leave every explicit-option check green. + default_ids = adapter.encode(case["text"]) + default_text = adapter.decode(default_ids) + for reference in (fast, slow): + if default_ids != reference.encode(case["text"]) or default_ids != case["ids"]: + raise ValueError(f"{case['name']}: default encoding differs from official/golden IDs") + row["checks"] += 1 + if default_text != reference.decode(default_ids) or default_text != case["decoded"]: + raise ValueError(f"{case['name']}: default decode differs from official/golden text") + row["checks"] += 1 + for special in (False, True): + actual = adapter.encode(case["text"], add_special_tokens=special) + for reference in (fast, slow): + expected = reference.encode(case["text"], add_special_tokens=special) + if actual != expected or actual != case["ids"]: + raise ValueError(f"{case['name']}: encoding differs from official/golden IDs") + row["checks"] += 1 + for skip in (False, True): + for cleanup in (False, True): + result = adapter.decode(actual, skip_special_tokens=skip, + clean_up_tokenization_spaces=cleanup) + for reference in (fast, slow): + expected = reference.decode(actual, skip_special_tokens=skip, + clean_up_tokenization_spaces=cleanup) + if result != expected: + raise ValueError(f"{case['name']}: decode differs (skip={skip}, cleanup={cleanup})") + row["checks"] += 1 + if not skip and not cleanup and result != case["decoded"]: + raise ValueError(f"{case['name']}: decode differs from golden text") + row.update(passed=True, token_count=len(case["ids"]), + raw_text_preserved=case["decoded"] == case["text"]) + + +def tokenizer_metadata(adapter, fast, slow, directory): + """Check metadata independently: model rows are not tokenizer membership.""" + directory = Path(directory) + model = json.loads((directory / "config.json").read_text(encoding="utf-8")) + generation = json.loads((directory / "generation_config.json").read_text(encoding="utf-8")) + expected_eos = generation["eos_token_id"] + if isinstance(expected_eos, int): + expected_eos = [expected_eos] + if adapter.model_vocab_size != model["vocab_size"]: + raise ValueError("Tokenizer model_vocab_size differs from the official model configuration") + if adapter.generation_eos_token_ids != tuple(expected_eos): + raise ValueError("Tokenizer generation EOS IDs differ from the official generation configuration") + if adapter.pad_token_id != generation["pad_token_id"]: + raise ValueError("Tokenizer pad ID differs from the official generation configuration") + for reference in (fast, slow): + if (adapter.base_vocab_size != reference.vocab_size + or adapter.tokenizer_vocab_size != len(reference) + or adapter.valid_token_ids != frozenset(reference.get_vocab().values())): + raise ValueError("Tokenizer vocabulary metadata differs from the official reference") + for name in ("pad_token_id", "eos_token_id", "bos_token_id", "unk_token_id"): + if getattr(adapter, name) != getattr(reference, name): + raise ValueError(f"Tokenizer {name} differs from the official reference") + return {"model_vocab_size": adapter.model_vocab_size, + "base_vocab_size": adapter.base_vocab_size, + "tokenizer_vocab_size": adapter.tokenizer_vocab_size, + "pad_token_id": adapter.pad_token_id, + "eos_token_id": adapter.eos_token_id, + "bos_token_id": adapter.bos_token_id, + "unk_token_id": adapter.unk_token_id, + "generation_eos_token_ids": list(adapter.generation_eos_token_ids)} + + +def input_cases(adapter, rows): + import numpy as np + from scratchv.runtime.llm_inputs import prepare_inputs, greedy_next_token, generation_stop_reason + # Explicit loop oracle, independent of the vectorized production mask. + for length in (1, 17, 255, 256): + ids = [42] * length + ids[0] = adapter.pad_token_id # A pad-like ID inside the prompt remains a valid token. + prepared = prepare_inputs(ids, pad_token_id=adapter.pad_token_id, + vocab_size=adapter.model_vocab_size) + expected = np.full((1, 1, 256, 256), np.finfo(np.float32).min, dtype=np.float32) + for query in range(256): + for key in range(min(query + 1, length)): + expected[0, 0, query, key] = 0 + if (prepared.valid_length != length or prepared.input_ids.dtype != np.int64 + or prepared.input_ids.shape != (1, 256) + or not np.array_equal(prepared.input_ids[0, :length], ids) + or not np.all(prepared.input_ids[0, length:] == adapter.pad_token_id) + or prepared.attention_mask.dtype != np.float32 + or not np.array_equal(prepared.attention_mask, expected)): + raise ValueError(f"Input/mask mismatch for valid length {length}") + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, -1, 7] = 100 + logits[0, length - 1, 7] = 0 + logits[0, length - 1, [2, 5]] = 1 + if greedy_next_token(logits, length, vocab_size=8) != 2: + raise ValueError("Greedy used a padding row or wrong tie rule") + rows.append({"name": f"mask-and-greedy-{length}", "passed": True}) + for eos in adapter.generation_eos_token_ids: + reason = generation_stop_reason(valid_length=2, generated_tokens=1, max_new_tokens=3, + last_token_id=eos, eos_token_ids=adapter.generation_eos_token_ids) + if reason != "eos": + raise ValueError(f"Generation EOS not recognized: {eos}") + if generation_stop_reason(valid_length=256, generated_tokens=0, max_new_tokens=1) != "capacity": + raise ValueError("Full context must stop before a new forward call") + if generation_stop_reason(valid_length=1, generated_tokens=0, max_new_tokens=0) != "max_new_tokens": + raise ValueError("Zero generation budget must stop") + rows.append({"name": "eos-capacity-budget", "passed": True}) + + +def run_checks(args, report): + report["stage"] = "environment" + report["environment"] = require_environment() + report["stage"] = "assets" + if args.mode == "download": + report["acquisition"] = acquire_assets(args.tokenizer_dir) + report["files"] = verify_assets(args.tokenizer_dir) + report["stage"] = "tokenizer" + from transformers import AutoTokenizer + from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + adapter = Qwen3Tokenizer.from_directory(args.tokenizer_dir) + fast = AutoTokenizer.from_pretrained(args.tokenizer_dir, use_fast=True, local_files_only=True, + trust_remote_code=False) + slow = AutoTokenizer.from_pretrained(args.tokenizer_dir, use_fast=False, local_files_only=True, + trust_remote_code=False) + corpus = json.loads(CORPUS.read_text(encoding="utf-8")) + if corpus["revision"] != MANIFEST["revision"] or corpus["model_id"] != MANIFEST["model_id"]: + raise ValueError("Corpus/model identity mismatch") + if (type(fast).__name__, type(slow).__name__) != ("Qwen2TokenizerFast", "Qwen2Tokenizer"): + raise ValueError("Expected two distinct official tokenizer reference classes") + report["tokenizer"] = tokenizer_metadata(adapter, fast, slow, args.tokenizer_dir) + tokenizer_cases(adapter, fast, slow, corpus, report["tokenizer_cases"]) + report["stage"] = "inputs-sampling" + input_cases(adapter, report["runtime_cases"]) + report.update(passed=True, stage="complete") + + +def save_report(directory, report): + """Keep the primary failure and make incomplete evidence an explicit FAIL.""" + def write(name, data): + try: + (directory / name).write_text(data, encoding="utf-8") + except OSError as exc: + return f"{name}: {type(exc).__name__}: {exc}" + return None + + lines = ["# W2 unit:runtime", "", f"Result: **{'PASS' if report['passed'] else 'FAIL'}**", + f"Stage: {report['stage']}; seconds: {report['seconds']:.3f}", "", + "Fixed official tokenizer assets; encode/decode vs Transformers fast + slow and golden corpus.", + "NFC normalization is official behavior; equivalence does not mean arbitrary raw-text byte preservation.", + "No pretrained weights, full-model inference or text generation is executed.", "", + "| Case | Checks | Passed |", "|---|---:|---|"] + for row in report["tokenizer_cases"] + report["runtime_cases"]: + lines.append(f"| {row['name']} | {row.get('checks', 1)} | {row['passed']} |") + if "error" in report: + lines.extend(["", "Error: " + report["error"]]) + markdown = "\n".join(lines) + "\n" + failures = [] + documents = { + "report.md": markdown, + "report.html": 'W2 runtime gate
'
+        + html.escape(markdown) + "
", + } + for name, content in documents.items(): + if error := write(name, content): + failures.append(error) + def record_failure(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete runtime gate evidence") + + def recover_views(): + # Preserve the primary probe error. Try both human-readable formats even + # when one destination is unwritable. + failure_text = ( + "# W2 unit:runtime\n\nResult: **FAIL**\n\n" + + f"Stage: {report['stage']}\n\nError: {report['error']}\n\n" + + "Report write errors:\n" + "\n".join(f"- {failure}" for failure in failures) + "\n" + ) + recovery = { + "report.md": failure_text, + "report.html": '
' + html.escape(failure_text) + "
", + } + for name, content in recovery.items(): + write(name, content) + + if failures: + record_failure() + recover_views() + # Publish machine-readable status only after the view outcomes are known. + # In particular, never write PASS and rely on a later corrective write: that + # second write could fail and leave a misleading successful JSON artifact. + data = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + # A write can complete its payload and still fail while flushing/closing. + # Keep even a fully written PASS private until write_text has returned; + # same-directory replacement then publishes the completed evidence. + pending = directory / ".report.json.tmp" + error = None + try: + pending.write_text(data, encoding="utf-8") + pending.replace(directory / "report.json") + except OSError as exc: + error = f"report.json: {type(exc).__name__}: {exc}" + try: + pending.unlink(missing_ok=True) + except OSError as cleanup_exc: + error += f"; temporary cleanup: {type(cleanup_exc).__name__}: {cleanup_exc}" + if error: + failures.append(error) + record_failure() + recover_views() + if failures: + print(f"[unit:runtime] {report['error']}; report write errors: {'; '.join(failures)}", file=sys.stderr) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--tokenizer-dir", type=Path, required=True) + ap.add_argument("--mode", choices=("verify", "download"), default="verify") + ap.add_argument("--output-dir", type=Path, default=Path("output/w2-runtime")) + args = ap.parse_args(argv) + args.tokenizer_dir = args.tokenizer_dir.resolve() + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + ap.error("--output-dir must be absent or empty; preserve earlier evidence") + out.mkdir(parents=True, exist_ok=True) + report = {"gate": "unit:runtime", "passed": False, "stage": "initialize", + "created_at": datetime.now(timezone.utc).isoformat(), "mode": args.mode, + "model_id": MANIFEST["model_id"], "revision": MANIFEST["revision"], + "tokenizer_dir": str(args.tokenizer_dir), "tokenizer_cases": [], "runtime_cases": []} + started = time.perf_counter() + try: + report["source_sha256"] = {p: sha256(ROOT / p) for p in ( + "probes/w2_runtime/run.py", "probes/w2_runtime/corpus.json", + "probes/w1_qwen3_export/manifest.json", "scratchv/runtime/llm_inputs.py", + "scratchv/runtime/qwen3_tokenizer.py")} + report["checkout"] = checkout_evidence() + run_checks(args, report) + except Exception as exc: + report.update(passed=False, error=f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + save_report(out, report) + print(f"[unit:runtime] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; report={out / 'report.json'}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_runtime_model/README.md b/probes/w2_runtime_model/README.md new file mode 100644 index 0000000..794fa0e --- /dev/null +++ b/probes/w2_runtime_model/README.md @@ -0,0 +1,42 @@ +# W2 Runtime → Model 单步接口探测 + +把已实现的 Tokenizer、输入准备、模型调用、最后有效位置选择及 greedy 解码真正串起来。模型使用 `transformers==4.51.3` 官方 `Qwen3ForCausalLM`，两层、hidden=32、FFN=96、Q/KV heads=4/2、head_dim=16、随机种子 0、绑定 embedding/LM head。保留**完整模型词表 151936**；官方 Token ID 直接进入模型，不取模、不重新编号、不裁剪词表。 + +这是 W2 的接口集成证据；权重随机，不代表预训练 Qwen3-0.6B 的语言能力。执行单步前向与一个 greedy token，不执行生成循环、KV cache 或 QEMU。 + +## 本地复现 + +使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 + +```bash +python probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +python probes/w2_runtime_model/run.py --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime-model +``` + +输出目录必须不存在或为空。既有证据不会被覆盖；版本不匹配、资产缺失/损坏、数值或采样不一致、报告写入失败均返回非零。 + +## 验收内容 + +- 两个固定原始文本：中文混合文本，以及包含官方特殊 token 和 `endoftext` 的输入。后者包含 pad ID，仍必须按原始 token 数计算有效长度。 +- 生产 `Qwen3Tokenizer` 的编码与官方 fast、slow Tokenizer 相同。生产 `prepare_inputs` 的 ID、dtype、shape、因果 mask 和 padding 由独立标量循环核验。 +- 只导出普通 logits ONNX：INT64 `[1,256]` input_ids、FP32 `[1,1,256,256]` attention_mask，FP32 `[1,256,151936]` logits；FP32、opset 18、固定形状、无 KV cache、eager attention。 +- Torch、ORT、共享 IR `none` 和 `all` 真正执行相同输入。ORT 对 Torch、IR 对 Torch 和 ORT 均检查所有位置，含 padding，严格要求有限 FP32、shape 完全相同、`max_abs < 1e-5`、`rtol=0`。 +- 每条路径选择 `valid_length - 1` 的 logits，greedy ID 必须与独立索引的 Torch argmax 相同；解码必须与官方 fast、slow 相同。即使精度误差小于阈值，argmax 改变仍会失败；若抽中模型词表的未定义 Tokenizer 行，明确失败，不能修改候选 mask 来掩盖。 + +## 输出及资源 + +`report.json`、`report.md`、`report.html` 包含结果、阶段、首次失败、源码 SHA256、实际 checkout、环境版本、资产 SHA256、模型参数与权重 hash、输入 token IDs/hash、ONNX hash、全/有效/padding 误差、最差位置、greedy ID/解码/top-2 margin、构建/导出/解析/各引擎执行耗时。JSON 在完整写入并关闭后原子发布；报告生成失败不能留下 PASS。 + +无 `.git` 的源码快照也可复现：checkout HEAD 明确标为未知，身份由运行前记录的源码 SHA256 确定，不借用上层仓库的 Git 信息。报告同时记录实际导入的 parser、IR interpreter、Tokenizer 和输入模块的 `__file__`；若这些模块来自当前源码树之外则失败，防止误用旧安装。 + +报告中的引擎 `seconds` 仅计该次主机前向调用；总 `seconds` 还包含环境加载、资产校验、导出、报告前的数据保存与数值比较。都不是 QEMU 时间，也不是目标硬件性能数据。 + +`model.onnx`、`config.json`、每个文本的 `*.inputs.npz`、`*.torch.npy`、`*.ort.npy` 供独立复核。一个完整输出约 148.4 MiB；参考输出保存后通过 mmap 顺序读取，ORT 会话在 IR 前释放，各次完整输出按顺序释放，比较按 8 个 token 的块进行。两种参考、两个文本的持久化输出合计约 594 MiB，另加约 20 MiB 模型；预留至少 1 GiB 磁盘和数 GiB 空闲内存。这里没有声称限制了解释器内部内存峰值。 + +负例回归： + +```bash +python -m pytest tests/test_w2_runtime_model.py -q +``` + +负例覆盖 ID 映射、padding/因果 mask、错误行选择、argmax 翻转、非法采样 ID、非有限/错误 shape/dtype、只在 padding 超差、解码差异及报告失败。单测不冒充真实模型运行；必须同时执行上面的集成 CLI 才能得到接口集成 PASS。 diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py new file mode 100644 index 0000000..05f6f29 --- /dev/null +++ b/probes/w2_runtime_model/run.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Text -> official tokenizer -> static Qwen3 -> shared IR -> one greedy ID. + +This is an interface integration probe with seeded random weights, not a +pretrained 0.6B model or a generation loop. The complete vocabulary is retained. +""" +from __future__ import annotations + +import argparse +import copy +from datetime import datetime, timezone +import gc +import html +import json +from pathlib import Path +import sys +import time + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w2_runtime.run import MANIFEST, checkout_evidence, require_environment, sha256, verify_assets +from probes.w2_qwen3_small.run import arrays_sha256, environment +from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token, prepare_inputs + +SEQ, VOCAB, ATOL = 256, 151936, 1e-5 +LEVELS = ("none", "all") +CASES = (("chinese", "你好，ScratchV！"), + ("special_pad_in_prompt", "<|im_start|>user\nTest<|endoftext|>!<|im_end|>\n")) + + +def checked_inputs(ids, prepared, pad_token_id): + """Independent ABI oracle: never infer valid length from pad-ID values.""" + length = len(ids) + if prepared.valid_length != length or not 1 <= length <= SEQ: + raise ValueError("Input preparation changed the prompt length") + actual = prepared.as_feed() + expected_ids = np.full((1, SEQ), pad_token_id, dtype=np.int64) + expected_ids[0, :length] = ids + expected_mask = np.full((1, 1, SEQ, SEQ), np.finfo(np.float32).min, dtype=np.float32) + for query in range(SEQ): + for key in range(min(query + 1, length)): + expected_mask[0, 0, query, key] = 0 + for name, expected in (("input_ids", expected_ids), ("attention_mask", expected_mask)): + value = actual[name] + if value.shape != expected.shape or value.dtype != expected.dtype or not np.array_equal(value, expected): + raise ValueError(f"Input preparation changed {name} ABI, IDs, causal mask or padding") + return actual + + +def compare_logits(actual, expected, valid_length, *, vocab_size=VOCAB): + """Compare every element in bounded FP64 chunks, including padding rows.""" + shape = (1, SEQ, vocab_size) + for name, array in (("actual", actual), ("expected", expected)): + if not isinstance(array, np.ndarray) or np.ma.isMaskedArray(array): + raise TypeError(f"{name} logits must be an ordinary ndarray") + if array.shape != shape or array.dtype != np.dtype(np.float32): + raise ValueError(f"{name} logits require FP32 {shape}; got {array.dtype} {array.shape}") + if not 1 <= valid_length <= SEQ: + raise ValueError("Invalid prompt length") + maxima = {"max_abs": 0.0, "valid_max_abs": 0.0, "padding_max_abs": 0.0} + worst = None + for start in range(0, SEQ, 8): + stop = min(start + 8, SEQ) + left, right = actual[0, start:stop], expected[0, start:stop] + if not np.isfinite(left).all() or not np.isfinite(right).all(): + raise ValueError(f"Nonfinite logits in positions {start}:{stop}") + difference = np.abs(left.astype(np.float64) - right.astype(np.float64)) + maximum = float(difference.max()) + if maximum > maxima["max_abs"]: + position, token = np.unravel_index(int(difference.argmax()), difference.shape) + worst = [0, start + int(position), int(token)] + maxima["max_abs"] = max(maxima["max_abs"], maximum) + boundary = min(max(valid_length - start, 0), stop - start) + if boundary: + maxima["valid_max_abs"] = max(maxima["valid_max_abs"], float(difference[:boundary].max())) + if boundary < stop - start: + maxima["padding_max_abs"] = max(maxima["padding_max_abs"], float(difference[boundary:].max())) + return {**maxima, "worst_index": worst, "elements": int(actual.size), + "passed": maxima["max_abs"] < ATOL, "atol": ATOL, "rtol": 0} + + +def sample_checked(logits, reference, length, adapter, references, *, vocab_size=VOCAB): + vector = last_valid_logits(logits, length, vocab_size=vocab_size) + expected = reference[0, length - 1, :] + # Independently index the reference to catch an incorrect host row selector. + if vector.shape != expected.shape or not np.array_equal(vector, logits[0, length - 1, :]): + raise ValueError("Host selected the wrong logits row") + actual_id, expected_id = greedy_token(vector), int(np.argmax(expected)) + if actual_id != expected_id: + raise ValueError(f"Greedy ID differs: actual={actual_id}, reference={expected_id}") + adapter.validate_token_ids([actual_id]) # Never hide invalid padded model rows. + text = adapter.decode([actual_id], skip_special_tokens=False, clean_up_tokenization_spaces=False) + for tokenizer in references: + if text != tokenizer.decode([actual_id], skip_special_tokens=False, clean_up_tokenization_spaces=False): + raise ValueError("Sampled token decode differs from official tokenizer") + ordered = np.partition(expected, -2)[-2:] + return {"id": actual_id, "decoded": text, "position": length - 1, + "reference_top2_margin": float(ordered[-1] - ordered[-2]), "passed": True} + + +def build_model(adapter): + import torch + from transformers import Qwen3Config, Qwen3ForCausalLM + from probes.w2_qwen3_small.model import MODEL_CONFIG + + config_values = {**MODEL_CONFIG, "vocab_size": VOCAB, + "pad_token_id": adapter.pad_token_id, + "bos_token_id": None, "eos_token_id": list(adapter.generation_eos_token_ids)} + config = Qwen3Config(**config_values) + config._attn_implementation = "eager" + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + model = Qwen3ForCausalLM(config).float().eval() + if model.lm_head.weight is not model.model.embed_tokens.weight: + raise ValueError("Expected official tied embedding and LM head") + + class NormalModel(torch.nn.Module): + def __init__(self, qwen): + super().__init__() + self.model = qwen + self.register_buffer("positions", torch.arange(SEQ).reshape(1, SEQ), persistent=False) + + def forward(self, input_ids, attention_mask): + return self.model(input_ids=input_ids, attention_mask=attention_mask, + position_ids=self.positions, use_cache=False, + output_attentions=False, output_hidden_states=False, + return_dict=False, logits_to_keep=0)[0] + + return NormalModel(model).eval(), config_values + + +def check_onnx_contract(model): + import onnx + onnx.checker.check_model(model, full_check=True) + standard_opsets = [item.version for item in model.opset_import if item.domain in ("", "ai.onnx")] + if standard_opsets != [18]: + raise ValueError(f"Expected exactly one standard ONNX opset 18: {standard_opsets}") + expected = {"input_ids": (onnx.TensorProto.INT64, [1, SEQ]), + "attention_mask": (onnx.TensorProto.FLOAT, [1, 1, SEQ, SEQ]), + "logits": (onnx.TensorProto.FLOAT, [1, SEQ, VOCAB])} + if [v.name for v in model.graph.input] != ["input_ids", "attention_mask"] or [v.name for v in model.graph.output] != ["logits"]: + raise ValueError("ONNX must have two ordinary inputs and only logits output") + for value in [*model.graph.input, *model.graph.output]: + element = value.type.tensor_type + actual = element.elem_type, [d.dim_value for d in element.shape.dim] + if actual != expected[value.name]: + raise ValueError(f"Unexpected static ONNX ABI: {value.name}: {actual}") + if any(t.data_location == onnx.TensorProto.EXTERNAL for t in model.graph.initializer): + raise ValueError("This small model must embed its weights") + + +def source_evidence(): + paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", + ROOT / "probes/w2_qwen3_small/model.py", ROOT / "probes/w2_qwen3_small/run.py", + ROOT / "probes/w1_qwen3_export/manifest.json", + ROOT / "scratchv/runtime/qwen3_tokenizer.py", ROOT / "scratchv/runtime/llm_inputs.py", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/verification/ir_interpreter.py", + ROOT / "scratchv/verification/ir_numpy_ops.py", ROOT / "scratchv/pass_manager.py", + ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/ir/types.py", ROOT / "scratchv/ir/builder.py", + ROOT / "requirements/qwen3-small-probe.txt", + *sorted((ROOT / "scratchv/optimizer").glob("*.py")), + *sorted((ROOT / "scratchv/analysis").glob("*.py"))] + return {str(p.relative_to(ROOT)).replace("\\", "/"): sha256(p) for p in paths} + + +def run_probe(directory, out, report): + report.update(stage="environment", environment=environment(), tokenizer_environment=require_environment()) + import onnx + import onnxruntime as ort + import torch + from transformers import AutoTokenizer + from scratchv.analysis.ir_verifier import verify_ir + from scratchv.frontend.onnx_parser import ONNXParser + from scratchv.pass_manager import create_optimization_pass_manager + from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + from scratchv.verification.ir_interpreter import IRInterpreter + + def require_valid(program): + valid, errors = verify_ir(program) + if not valid: + raise ValueError(f"Invalid shared IR: {errors}") + + from scratchv.runtime import llm_inputs + modules = {"onnx_parser": sys.modules[ONNXParser.__module__], + "ir_interpreter": sys.modules[IRInterpreter.__module__], + "llm_inputs": llm_inputs, + "qwen3_tokenizer": sys.modules[Qwen3Tokenizer.__module__]} + report["module_files"] = {name: str(Path(module.__file__).resolve()) for name, module in modules.items()} + for name, path in report["module_files"].items(): + if not Path(path).is_relative_to(ROOT): + raise ValueError(f"{name} imported outside the current source tree: {path}") + report.update(stage="tokenizer", assets=verify_assets(directory)) + adapter = Qwen3Tokenizer.from_directory(directory) + if adapter.model_vocab_size != VOCAB: + raise ValueError("The complete official model vocabulary is mandatory") + references = [AutoTokenizer.from_pretrained(directory, use_fast=fast, local_files_only=True, + trust_remote_code=False) for fast in (True, False)] + if [type(t).__name__ for t in references] != ["Qwen2TokenizerFast", "Qwen2Tokenizer"]: + raise ValueError("Require two distinct official tokenizer references") + report["cases"] = [] + feeds = [] + for name, text in CASES: + ids = adapter.encode(text) + if any(ids != reference.encode(text) for reference in references): + raise ValueError(f"Official prompt encoding differs: {name}") + prepared = prepare_inputs(ids, pad_token_id=adapter.pad_token_id) + feed = checked_inputs(ids, prepared, adapter.pad_token_id) + np.savez(out / f"{name}.inputs.npz", **feed) + feeds.append((prepared.valid_length, feed)) + report["cases"].append({"name": name, "text": text, "token_ids": ids, + "valid_length": prepared.valid_length, "input_sha256": arrays_sha256(feed), + "pad_id_inside_prompt": adapter.pad_token_id in ids, + "passed": False, "executions": []}) + if len(feeds) != 2 or not report["cases"][1]["pad_id_inside_prompt"]: + raise ValueError("Require two original-text cases including an in-prompt pad ID") + torch.set_num_threads(1) + torch.use_deterministic_algorithms(True) + report["stage"] = "model" + started = time.perf_counter() + wrapper, config = build_model(adapter) + report.update(config=config, model_seed=0, model_seconds=time.perf_counter() - started, + model_class=type(wrapper.model).__module__ + "." + type(wrapper.model).__name__, + weights="seed 0 random initialization; no pretrained weights", + weights_sha256=arrays_sha256({name: value.detach().numpy() + for name, value in wrapper.model.state_dict().items()})) + (out / "config.json").write_text(wrapper.model.config.to_json_string(use_diff=False), encoding="utf-8") + # Persist references and release their complete tensors before ORT/IR runs. + for case, (length, feed) in zip(report["cases"], feeds): + report.update(stage="torch", current_case=case["name"]) + started = time.perf_counter() + with torch.inference_mode(): + value = wrapper(*(torch.from_numpy(feed[n]) for n in ("input_ids", "attention_mask"))).numpy() + case["torch_seconds"] = time.perf_counter() - started + check = compare_logits(value, value, length) + case["torch"] = {"contract": check, + "sampling": sample_checked(value, value, length, adapter, references)} + np.save(out / f"{case['name']}.torch.npy", value) + del value + gc.collect() + report["stage"] = "export" + started = time.perf_counter() + with torch.inference_mode(): + exported = torch.onnx.export(wrapper, tuple(torch.from_numpy(feeds[0][1][n]) for n in ("input_ids", "attention_mask")), + dynamo=True, opset_version=18, input_names=["input_ids", "attention_mask"], + output_names=["logits"], dynamic_shapes=None, optimize=False, fallback=False) + exported.save(str(out / "model.onnx"), external_data=False) + report["export_seconds"] = time.perf_counter() - started + del exported, wrapper + gc.collect() + exported = onnx.load(out / "model.onnx") + check_onnx_contract(exported) + report["onnx"] = {"sha256": sha256(out / "model.onnx"), "bytes": (out / "model.onnx").stat().st_size, + "nodes": len(exported.graph.node), "opset": 18, + "exporter": "torch.onnx.export(dynamo=True, optimize=False, fallback=False)", + "inputs": {"input_ids": "INT64[1,256]", "attention_mask": "FP32[1,1,256,256]"}, + "output": "FP32[1,256,151936]", "output_bytes": SEQ * VOCAB * 4} + del exported + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + options.enable_cpu_mem_arena = False + report["stage"] = "ort" + started = time.perf_counter() + session = ort.InferenceSession(str(out / "model.onnx"), options, providers=["CPUExecutionProvider"]) + report["ort_load_seconds"] = time.perf_counter() - started + for case, (length, feed) in zip(report["cases"], feeds): + report["current_case"] = case["name"] + started = time.perf_counter() + actual = session.run(["logits"], feed)[0] + row = {"engine": "ort", "seconds": time.perf_counter() - started, "passed": False} + case["executions"].append(row) + expected = np.load(out / f"{case['name']}.torch.npy", mmap_mode="r") + row["vs_torch"] = compare_logits(actual, expected, length) + row["sampling"] = sample_checked(actual, expected, length, adapter, references) + row["passed"] = row["vs_torch"]["passed"] + if not row["passed"]: + raise ValueError(f"ORT numeric error for {case['name']}") + # IR is compared independently to both references without retaining a + # second live full-size output array from another execution engine. + np.save(out / f"{case['name']}.ort.npy", actual) + del actual, expected + del session + gc.collect() + report["stage"] = "parse" + started = time.perf_counter() + parser = ONNXParser() + program = parser.parse(str(out / "model.onnx")) + require_valid(program) + report["parse_seconds"] = time.perf_counter() - started + report["ir_optimization_seconds"] = {} + for level in LEVELS: + report.update(stage="ir", current_level=level) + started = time.perf_counter() + manager = create_optimization_pass_manager(level) + manager.before_pass = lambda p, value: require_valid(value) + manager.after_pass = lambda p, value: require_valid(value) + optimized = manager.run_pipeline(copy.deepcopy(program)).data + require_valid(optimized) + report["ir_optimization_seconds"][level] = time.perf_counter() - started + for case, (length, feed) in zip(report["cases"], feeds): + report["current_case"] = case["name"] + row = {"engine": f"ir-{level}", "passed": False} + case["executions"].append(row) + started = time.perf_counter() + result = IRInterpreter(optimized).run(feed, initializers=parser.initializers) + row.update(seconds=time.perf_counter() - started, executed_steps=result.executed_steps) + actual = result.return_value + expected = np.load(out / f"{case['name']}.torch.npy", mmap_mode="r") + row["vs_torch"] = compare_logits(actual, expected, length) + row["sampling"] = sample_checked(actual, expected, length, adapter, references) + del expected + expected = np.load(out / f"{case['name']}.ort.npy", mmap_mode="r") + row["vs_ort"] = compare_logits(actual, expected, length) + row["passed"] = row["vs_torch"]["passed"] and row["vs_ort"]["passed"] + del result, actual, expected + gc.collect() + if not row["passed"]: + raise ValueError(f"IR {level} numeric error for {case['name']}") + print(f"[{case['name']}/ir-{level}] PASS max_abs={row['vs_torch']['max_abs']:.9g}", flush=True) + del optimized + for case in report["cases"]: + case["passed"] = ([row["engine"] for row in case["executions"]] == ["ort", "ir-none", "ir-all"] + and all(row["passed"] for row in case["executions"])) + report.update(passed=bool(report["cases"]) and all(case["passed"] for case in report["cases"]), stage="complete") + report["completed_model_invocations"] = len(report["cases"]) + sum(len(case["executions"]) for case in report["cases"]) + report.pop("current_case", None) + report.pop("current_level", None) + + +def save_reports(out, report): + failures = [] + + def views(): + text = ("# W2 runtime/model integration\n\nResult: **" + ("PASS" if report["passed"] else "FAIL") + + "**\n\nOfficial two-layer Qwen3, random weights, full vocabulary; one forward and greedy token.\n" + + "Not pretrained inference or a generation loop. All positions use strict max_abs < 1e-5.\n\n```json\n" + + json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n```\n") + return {"report.md": text, "report.html": '
' + html.escape(text) + "
"} + + def failed(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete integration evidence") + + def write_views(): + for name, value in views().items(): + try: + (out / name).write_text(value, encoding="utf-8") + except OSError as exc: + failures.append(f"{name}: {type(exc).__name__}: {exc}") + + write_views() + if failures: + failed() + write_views() + pending = out / ".report.json.tmp" + try: + pending.write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") + pending.replace(out / "report.json") + except OSError as exc: + failures.append(f"report.json: {type(exc).__name__}: {exc}") + failed() + write_views() + finally: + try: + pending.unlink(missing_ok=True) + except OSError: + pass + if failures: + print("; ".join(failures), file=sys.stderr) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokenizer-dir", required=True, type=Path) + parser.add_argument("--output-dir", default=Path("output/w2-runtime-model"), type=Path) + args = parser.parse_args(argv) + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + parser.error("--output-dir must be absent or empty; preserve earlier evidence") + out.mkdir(parents=True, exist_ok=True) + report = {"gate": "integration:runtime-model", "passed": False, "stage": "initialize", + "created_at": datetime.now(timezone.utc).isoformat(), "model_id": MANIFEST["model_id"], + "tokenizer_revision": MANIFEST["revision"], "cases": []} + started = time.perf_counter() + try: + report["source_sha256"] = source_evidence() + report["checkout"] = checkout_evidence() + run_probe(args.tokenizer_dir.resolve(), out, report) + except Exception as exc: + report.update(passed=False, error=f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + save_reports(out, report) + print(f"[integration:runtime-model] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; report={out / 'report.json'}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index a588db5..670aa63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ riscv = ["tinyfive"] llvm = ["llvmlite"] # optional: LLVM IR JIT execution verify = ["onnxruntime"] # optional: ONNX Runtime comparison +llm = ["tokenizers>=0.21,<0.22"] # optional: local Qwen3 text tokenizer; no Torch all = ["tinyfive", "llvmlite", "onnxruntime"] topic06 = ["tinyfive", "pytest"] topic06-report = ["tinyfive", "pytest", "jinja2", "matplotlib"] diff --git a/scratchv/backend/tensor_c_codegen.py b/scratchv/backend/tensor_c_codegen.py index f333946..5b45db4 100644 --- a/scratchv/backend/tensor_c_codegen.py +++ b/scratchv/backend/tensor_c_codegen.py @@ -304,6 +304,15 @@ def _constant(self, value, data): spec = self.specs[value.name] if not isinstance(data, np.ndarray) or data.shape != spec.shape or data.dtype != spec.numpy_dtype: raise TensorCCodegenError(f"Initializer {value.name} dtype/shape disagrees with IR") + if value.is_constant: + # A scalar literal is part of the IR's meaning (and may have been + # used by optimizations). A caller binding cannot override it. + try: + expected = np.asarray(value.const_value, dtype=spec.numpy_dtype) + except (TypeError, ValueError, OverflowError) as exc: + raise TensorCCodegenError(f"Invalid scalar constant {value.name}") from exc + if data.shape != () or expected.shape != () or not np.array_equal(data, expected): + raise TensorCCodegenError(f"Initializer {value.name} disagrees with scalar constant") self.constant_bytes += spec.nbytes if self.constant_bytes > self.max_constant_bytes: raise TensorCCodegenError("Constant tensor storage exceeds configured limit") diff --git a/scratchv/runtime/llm_inputs.py b/scratchv/runtime/llm_inputs.py new file mode 100644 index 0000000..c1fadb6 --- /dev/null +++ b/scratchv/runtime/llm_inputs.py @@ -0,0 +1,179 @@ +"""Host input preparation and greedy sampling for fixed-length Qwen3 graphs. + +These helpers do not tokenize text, invoke a model, or run a generation loop. +The model ABI remains batch 1, sequence length 256, INT64 IDs and FP32 tensors. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal + +import numpy as np + + +SEQ_LENGTH = 256 +QWEN3_VOCAB_SIZE = 151936 +StopReason = Literal["eos", "capacity", "max_new_tokens"] + + +def _integer(value: object, name: str, *, minimum: int = 0, + maximum: int | None = None) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)): + raise TypeError(f"{name} must be an integer, not {type(value).__name__}") + result = int(value) + if result < minimum or (maximum is not None and result > maximum): + bound = f"[{minimum}, {maximum}]" if maximum is not None else f">= {minimum}" + raise ValueError(f"{name} must be in {bound}; got {result}") + return result + + +def _vocabulary_size(vocab_size: object) -> int: + # A vocabulary size is a count; its largest valid ID must fit INT64. + return _integer(vocab_size, "vocab_size", minimum=1, + maximum=int(np.iinfo(np.int64).max) + 1) + + +def _valid_length(valid_length: object) -> int: + return _integer(valid_length, "valid_length", minimum=1, maximum=SEQ_LENGTH) + + +def _token_sequence(token_ids: Sequence[int] | np.ndarray, *, + vocab_size: int, name: str) -> list[int]: + if isinstance(token_ids, np.ndarray): + if np.ma.isMaskedArray(token_ids): + raise TypeError(f"{name} must not be a masked array") + if token_ids.ndim != 1: + raise ValueError(f"{name} must be a one-dimensional sequence") + if token_ids.dtype.kind not in "iu": + raise TypeError(f"{name} array must have an integer dtype") + elif isinstance(token_ids, (str, bytes)) or not isinstance(token_ids, Sequence): + raise TypeError(f"{name} must be a sequence of integer token IDs") + return [_integer(value, f"{name}[{index}]", maximum=vocab_size - 1) + for index, value in enumerate(token_ids)] + + +@dataclass(frozen=True) +class PreparedInputs: + """Owned model input arrays plus the unpadded token count. + + Arrays are mutable for the model caller. ``as_feed`` returns references to + these arrays, so changing the feed also changes this instance's tensors. + Construct instances through :func:`prepare_inputs` for validated inputs. + """ + + input_ids: np.ndarray + attention_mask: np.ndarray + valid_length: int + + def as_feed(self) -> dict[str, np.ndarray]: + return {"input_ids": self.input_ids, "attention_mask": self.attention_mask} + + +def build_attention_mask(valid_length: int) -> np.ndarray: + """Return causal + key-padding additive mask ``[1, 1, 256, 256]``. + + Query row ``q`` may attend to key ``k`` exactly when ``k <= q`` and + ``k < valid_length``. Padded query rows still see earlier valid keys; fully + masking those rows is not the fixed Qwen3 export's mask contract. + """ + length = _valid_length(valid_length) + positions = np.arange(SEQ_LENGTH) + allowed = (positions[None, :] <= positions[:, None]) & (positions[None, :] < length) + mask = np.where(allowed, np.float32(0), np.finfo(np.float32).min) + return mask.reshape(1, 1, SEQ_LENGTH, SEQ_LENGTH) + + +def prepare_inputs(token_ids: Sequence[int] | np.ndarray, *, pad_token_id: int, + vocab_size: int = QWEN3_VOCAB_SIZE) -> PreparedInputs: + """Validate unpadded IDs and return right-padded, contiguous ABI inputs. + + The length comes from the supplied sequence, never from occurrences of the + pad ID: a valid prompt may itself contain that ID. Empty or over-capacity + prompts fail instead of being silently truncated. Integer arrays of other + widths are accepted only after every ID passes the INT64 range checks. + """ + vocab = _vocabulary_size(vocab_size) + pad = _integer(pad_token_id, "pad_token_id", maximum=vocab - 1) + tokens = _token_sequence(token_ids, vocab_size=vocab, name="token_ids") + length = _valid_length(len(tokens)) + ids = np.full((1, SEQ_LENGTH), pad, dtype=np.int64) + ids[0, :length] = tokens + return PreparedInputs(ids, build_attention_mask(length), length) + + +def _fp32_finite(array: object, name: str) -> np.ndarray: + if not isinstance(array, np.ndarray): + raise TypeError(f"{name} must be a NumPy ndarray") + if np.ma.isMaskedArray(array): + raise TypeError(f"{name} must not be a masked array") + if array.dtype != np.dtype(np.float32): + raise TypeError(f"{name} must have dtype float32; got {array.dtype}") + if not np.isfinite(array).all(): + raise ValueError(f"{name} must contain only finite values") + return array + + +def last_valid_logits(logits: np.ndarray, valid_length: int, *, + vocab_size: int = QWEN3_VOCAB_SIZE) -> np.ndarray: + """Validate the full model output and copy the last valid token's logits. + + All positions, including padding, must be finite. Returning an owned vector + lets a runner release its full output buffer after selecting this row. + """ + length = _valid_length(valid_length) + vocab = _vocabulary_size(vocab_size) + if not isinstance(logits, np.ndarray): + raise TypeError("logits must be a NumPy ndarray") + expected = (1, SEQ_LENGTH, vocab) + if logits.shape != expected: + raise ValueError(f"logits must have shape {expected}; got {logits.shape}") + checked = _fp32_finite(logits, "logits") + return checked[0, length - 1, :].copy() + + +def greedy_token(logits: np.ndarray) -> int: + """Argmax of one finite FP32 vocabulary vector; ties choose the lowest ID.""" + if not isinstance(logits, np.ndarray): + raise TypeError("logits must be a NumPy ndarray") + if logits.ndim != 1 or logits.size == 0: + raise ValueError("logits must be a nonempty one-dimensional vocabulary vector") + checked = _fp32_finite(logits, "logits") + return int(np.argmax(checked)) + + +def greedy_next_token(logits: np.ndarray, valid_length: int, *, + vocab_size: int = QWEN3_VOCAB_SIZE) -> int: + """Select the next ID from ``logits[0, valid_length - 1]``, without softmax.""" + return greedy_token(last_valid_logits(logits, valid_length, vocab_size=vocab_size)) + + +def generation_stop_reason(*, valid_length: int, generated_tokens: int, + max_new_tokens: int, last_token_id: int | None = None, + eos_token_ids: Sequence[int] | np.ndarray = (), + vocab_size: int = QWEN3_VOCAB_SIZE) -> StopReason | None: + """Report whether a host loop must stop before its next forward call. + + Call initially with ``generated_tokens=0`` and after appending each newly + sampled ID. ``valid_length`` includes the nonempty prompt and generated IDs. + EOS applies only to a generated token, not an EOS-like token in the prompt. + Simultaneous stop conditions resolve as EOS, capacity, then token budget. + No truncation, model execution or token append is performed here. + """ + length = _valid_length(valid_length) + generated = _integer(generated_tokens, "generated_tokens", maximum=length - 1) + budget = _integer(max_new_tokens, "max_new_tokens") + vocab = _vocabulary_size(vocab_size) + eos = _token_sequence(eos_token_ids, vocab_size=vocab, name="eos_token_ids") + if generated and last_token_id is None: + raise ValueError("last_token_id is required after generating a token") + last = None if last_token_id is None else _integer(last_token_id, "last_token_id", + maximum=vocab - 1) + if generated and last in eos: + return "eos" + if length == SEQ_LENGTH: + return "capacity" + if generated >= budget: + return "max_new_tokens" + return None diff --git a/scratchv/runtime/qwen3_tokenizer.py b/scratchv/runtime/qwen3_tokenizer.py new file mode 100644 index 0000000..70ae2d7 --- /dev/null +++ b/scratchv/runtime/qwen3_tokenizer.py @@ -0,0 +1,223 @@ +"""Local Qwen3 text tokenization, without a Torch or Transformers dependency. + +The caller supplies a verified local snapshot containing tokenizer.json, +tokenizer_config.json, config.json and generation_config.json. This adapter never +downloads files or renders chat templates. Encoding follows tokenizer.json, +including its NFC normalization; a text round trip need not preserve decomposed +Unicode byte-for-byte. Model output width and the set of decodable token IDs are +different contracts (151936 versus 151669 entries in Qwen3-0.6B). +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Set +from numbers import Integral +from pathlib import Path + +from tokenizers import Tokenizer + + +def _read_config(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path.name} must contain a JSON object") + return value + + +def _boolean(value: object, name: str) -> bool: + if not isinstance(value, bool): + raise TypeError(f"{name} must be bool") + return value + + +def _cleanup(text: str) -> str: + # Match PreTrainedTokenizerBase.clean_up_tokenization in Transformers 4.51.3. + for before, after in ( + (" .", "."), (" ?", "?"), (" !", "!"), (" ,", ","), (" ' ", "'"), + (" n't", "n't"), (" 'm", "'m"), (" 's", "'s"), (" 've", "'ve"), + (" 're", "'re"), + ): + text = text.replace(before, after) + return text + + +class Qwen3Tokenizer: + """Tokenizer for a local Qwen3 snapshot; use :meth:`from_directory`. + + ``eos_token_id`` is the tokenizer's EOS. ``generation_eos_token_ids`` comes + from generation_config.json and includes *all* generation stop IDs. The + model's BOS metadata does not cause encoding to insert a BOS token. + """ + + @classmethod + def from_directory(cls, directory: str | Path) -> Qwen3Tokenizer: + directory = Path(directory) + config = _read_config(directory / "tokenizer_config.json") + model = _read_config(directory / "config.json") + generation = _read_config(directory / "generation_config.json") + if config.get("tokenizer_class") not in ("Qwen2Tokenizer", "Qwen2TokenizerFast"): + raise ValueError("tokenizer_class must identify the Qwen2 tokenizer used by Qwen3") + if model.get("model_type") != "qwen3": + raise ValueError("config.json model_type must be qwen3") + model_vocab_size = model.get("vocab_size") + if type(model_vocab_size) is not int or model_vocab_size <= 0: + raise ValueError("model vocab_size must be a positive integer") + for name in ("add_bos_token", "add_eos_token", "split_special_tokens", "add_prefix_space"): + if _boolean(config.get(name, False), name): + raise ValueError(f"Qwen3 text adapter does not support {name}=True") + # Transformers can add/mark these tokens during initialization. Loading + # tokenizer.json alone would silently ignore those extra semantics. + for name in ("mask_token", "sep_token", "cls_token"): + if config.get(name) is not None: + raise ValueError(f"Qwen3 text adapter does not support {name}") + if config.get("extra_special_tokens") not in (None, {}): + raise ValueError("Qwen3 text adapter does not support extra_special_tokens") + + self = cls() + self._tokenizer = Tokenizer.from_file(str(directory / "tokenizer.json")) + if self._tokenizer.padding is not None or self._tokenizer.truncation is not None: + raise ValueError("tokenizer.json must not enable automatic padding or truncation") + self._validate_prefix_space() + self.model_vocab_size = model_vocab_size + self.base_vocab_size = self._tokenizer.get_vocab_size(with_added_tokens=False) + self._valid_token_ids = frozenset(self._tokenizer.get_vocab(with_added_tokens=True).values()) + self.tokenizer_vocab_size = len(self._valid_token_ids) + if not self._valid_token_ids or any( + token_id < 0 or token_id >= model_vocab_size for token_id in self._valid_token_ids + ): + raise ValueError("tokenizer vocabulary does not fit the model vocab_size") + self.clean_up_tokenization_spaces = _boolean( + config.get("clean_up_tokenization_spaces", False), "clean_up_tokenization_spaces" + ) + self._validate_added_tokens(config) + self.pad_token_id = self._special_id(config, "pad_token", required=True) + self.eos_token_id = self._special_id(config, "eos_token", required=True) + self.bos_token_id = self._special_id(config, "bos_token", required=False) + # The official Qwen2 fast class defaults a missing unk_token to + # endoftext; an explicit JSON null intentionally disables it instead. + self.unk_token_id = self._special_id( + {"unk_token": config.get("unk_token", "<|endoftext|>")}, "unk_token", required=False + ) + eos_ids = generation.get("eos_token_id") + if type(eos_ids) is int: + eos_ids = [eos_ids] + if not isinstance(eos_ids, list) or not eos_ids: + raise ValueError("generation eos_token_id must be an integer or nonempty list") + self.generation_eos_token_ids = tuple(self.validate_token_ids(eos_ids)) + if self.eos_token_id not in self.generation_eos_token_ids: + raise ValueError("generation EOS IDs must include the tokenizer EOS") + if ( + generation.get("pad_token_id") != self.pad_token_id + or type(generation.get("pad_token_id")) is not int + ): + raise ValueError("generation pad_token_id must match the tokenizer pad token") + # Qwen3's JSON post-processor is ByteLevel and inserts no token IDs. + if self._tokenizer.num_special_tokens_to_add(is_pair=False) != 0: + raise ValueError("Qwen3 text encoding must not automatically insert BOS/EOS tokens") + return self + + def _validate_prefix_space(self) -> None: + # The fixed snapshot uses a Sequence containing ByteLevel(False). + # Transformers may override a top-level ByteLevel setting from config; + # refusing prefix insertion keeps this narrow adapter unambiguous for + # both that layout and the official nested layout. + pre_tokenizer = self._tokenizer.pre_tokenizer + pending = [] if pre_tokenizer is None else [json.loads(pre_tokenizer.__getstate__())] + while pending: + state = pending.pop() + if state.get("type") == "ByteLevel" and state.get("add_prefix_space"): + raise ValueError("tokenizer.json must not enable add_prefix_space") + if state.get("type") == "Sequence": + pending.extend(state["pretokenizers"]) + + @property + def valid_token_ids(self) -> frozenset[int]: + """Actual vocabulary members, including added tokens; not model rows.""" + return self._valid_token_ids + + def is_valid_token_id(self, token_id: object) -> bool: + return ( + isinstance(token_id, Integral) + and not isinstance(token_id, bool) + and int(token_id) in self._valid_token_ids + ) + + def validate_token_ids(self, token_ids: Iterable[int]) -> list[int]: + """Copy an ordered iterable to ints, refusing unknown IDs or bools.""" + if ( + isinstance(token_ids, (str, bytes, bytearray, Mapping, Set)) + or not isinstance(token_ids, Iterable) + ): + raise TypeError("token_ids must be an ordered iterable of integer IDs") + result = [] + for index, token_id in enumerate(token_ids): + if not isinstance(token_id, Integral) or isinstance(token_id, bool): + raise TypeError(f"token_ids[{index}] must be an integer (not bool)") + if not self.is_valid_token_id(token_id): + raise ValueError(f"token_ids[{index}]={token_id} is not in the tokenizer vocabulary") + result.append(int(token_id)) + return result + + def encode(self, text: str, *, add_special_tokens: bool = True) -> list[int]: + """Encode one string, with no padding, truncation, BOS/EOS, or chat template.""" + if not isinstance(text, str): + raise TypeError("text must be str") + text.encode("utf-8", errors="strict") # Reject lone surrogates explicitly. + _boolean(add_special_tokens, "add_special_tokens") + return self._tokenizer.encode(text, add_special_tokens=add_special_tokens).ids + + def decode( + self, + token_ids: Iterable[int] | int, + *, + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool | None = None, + ) -> str: + """Decode valid IDs; defaults match the official fast tokenizer API. + + Unlike raw tokenizers.decode, unknown IDs are errors, including unused + rows below model_vocab_size. A sampled unused row must not disappear. + """ + _boolean(skip_special_tokens, "skip_special_tokens") + cleanup = self.clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None: + cleanup = _boolean(clean_up_tokenization_spaces, "clean_up_tokenization_spaces") + if isinstance(token_ids, Integral): + token_ids = [token_ids] + ids = self.validate_token_ids(token_ids) + text = self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return _cleanup(text) if cleanup else text + + def _validate_added_tokens(self, config: dict) -> None: + decoder = config.get("added_tokens_decoder") + if not isinstance(decoder, dict): + raise ValueError("added_tokens_decoder must be an object") + actual = self._tokenizer.get_added_tokens_decoder() + if set(decoder) != {str(token_id) for token_id in actual}: + raise ValueError("tokenizer.json and tokenizer_config.json added token IDs differ") + for token_id, expected in decoder.items(): + if not isinstance(expected, dict): + raise ValueError(f"added token {token_id} must be an object") + token = actual[int(token_id)] + for name in ("content", "lstrip", "rstrip", "single_word", "normalized", "special"): + value = expected.get(name) + if type(value) is not type(getattr(token, name)) or value != getattr(token, name): + raise ValueError(f"added token {token_id} has inconsistent {name}") + additional = config.get("additional_special_tokens", []) + if not isinstance(additional, list): + raise ValueError("additional_special_tokens must be a list") + for content in additional: + self._special_id({"token": content}, "token", required=True) + + def _special_id(self, config: dict, name: str, *, required: bool) -> int | None: + content = config.get(name) + if content is None and not required: + return None + if not isinstance(content, str) or not content: + raise ValueError(f"{name} must be a nonempty token string") + token_id = self._tokenizer.token_to_id(content) + token = self._tokenizer.get_added_tokens_decoder().get(token_id) + if token_id is None or token is None or not token.special: + raise ValueError(f"{name}={content!r} is not a configured special token") + return token_id diff --git a/scratchv/runtime/riscv_tensor.py b/scratchv/runtime/riscv_tensor.py index 82d56aa..d6a401f 100644 --- a/scratchv/runtime/riscv_tensor.py +++ b/scratchv/runtime/riscv_tensor.py @@ -17,6 +17,7 @@ import signal import struct import subprocess +import threading import time from typing import Any, Mapping, Sequence @@ -165,6 +166,13 @@ def _run_process(command, *, cwd, timeout, env=None) -> subprocess.CompletedProc with subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) as process: job = _windows_job(process) + # The W2 supervisor first sends SIGTERM to the probe. This invocation + # owns a different session, so killing the probe group alone would + # orphan its compiler/QEMU. Unwind through cleanup before exiting. + previous_term = None + if os.name != "nt" and threading.current_thread() is threading.main_thread(): + previous_term = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, _terminate_invocation) try: stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired as exc: @@ -182,12 +190,44 @@ def _run_process(command, *, cwd, timeout, env=None) -> subprocess.CompletedProc process.wait(timeout=5) stdout, stderr = drain.output or b"", drain.stderr or b"" raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from exc + except BaseException: + # Cancellation and Ctrl-C must also close the owned invocation. + # Do not replace the primary interruption with a cleanup failure. + try: + _stop_process_tree(process, job) + process.communicate(timeout=5) + except (OSError, subprocess.SubprocessError): + process.kill() + for stream in (process.stdout, process.stderr): + if stream: + stream.close() + raise finally: + if previous_term is not None: + signal.signal(signal.SIGTERM, previous_term) if job is not None: job[0].CloseHandle(job[1]) return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) +def _terminate_invocation(signum, frame): + raise SystemExit(128 + signum) + + +def _save_process_logs(directory: Path, prefix: str, stdout, stderr): + """Attempt both logs without allowing a secondary I/O error to hide execution.""" + errors = [] + for stream, content in (("stdout", stdout), ("stderr", stderr)): + path = directory / f"{prefix}.{stream}" + try: + path.write_bytes(content or b"") + except OSError as exc: + errors.append((path.name, exc)) + detail = "; ".join(f"{name}: {error}" for name, error in errors) + suffix = f"; failed to preserve {prefix} logs: {detail}" if errors else "" + return suffix, errors[0][1] if errors else None + + def toolchain_versions(toolchain: RiscVToolchain) -> dict[str, str]: compiler = ([toolchain.cc[0], "version"] if toolchain.is_zig else [*toolchain.cc, "--version"]) @@ -516,15 +556,15 @@ def build_riscv_tensor( try: process = _run_process(command, cwd=build_dir, timeout=timeout, env=environment) except subprocess.TimeoutExpired as exc: - (build_dir / "compiler.stdout").write_bytes(exc.stdout or b"") - (build_dir / "compiler.stderr").write_bytes(exc.stderr or b"") - raise TimeoutError(f"RISC-V compilation exceeded {timeout} seconds") from exc + log_message, _ = _save_process_logs(build_dir, "compiler", exc.stdout, exc.stderr) + raise TimeoutError(f"RISC-V compilation exceeded {timeout} seconds{log_message}") from exc elapsed = time.perf_counter() - started - (build_dir / "compiler.stdout").write_bytes(process.stdout) - (build_dir / "compiler.stderr").write_bytes(process.stderr) + log_message, log_error = _save_process_logs(build_dir, "compiler", process.stdout, process.stderr) if process.returncode: message = process.stderr.decode("utf-8", errors="replace")[-12000:] - raise RuntimeError(f"RISC-V tensor compilation failed ({process.returncode}):\n{message}") + raise RuntimeError(f"RISC-V tensor compilation failed ({process.returncode}):\n{message}{log_message}") + if log_error is not None: + raise RuntimeError(log_message.lstrip("; ")) from log_error validate_riscv_elf(elf) return RiscVTensorExecutable(elf, tuple(artifact.inputs), artifact.output, toolchain, tuple(command), elapsed, int(artifact.workspace_bytes), @@ -562,17 +602,12 @@ def run_riscv_tensor( except subprocess.TimeoutExpired as exc: elapsed = time.perf_counter() - started message = f"RISC-V QEMU execution exceeded {timeout} seconds" - try: - stdout_path.write_bytes(exc.stdout or b"") - stderr_path.write_bytes(exc.stderr or b"") - except OSError as log_error: - message += f"; failed to preserve QEMU logs: {log_error}" - raise RiscVTensorTimeoutError(message, elapsed_s=elapsed, + log_message, _ = _save_process_logs(run_dir, "qemu", exc.stdout, exc.stderr) + raise RiscVTensorTimeoutError(message + log_message, elapsed_s=elapsed, command=command, timeout_s=timeout) from exc elapsed = time.perf_counter() - started + log_message, log_error = _save_process_logs(run_dir, "qemu", process.stdout, process.stderr) try: - stdout_path.write_bytes(process.stdout) - stderr_path.write_bytes(process.stderr) if process.returncode: diagnostic = process.stderr.decode("utf-8", errors="replace")[-4000:] if uart_path.exists(): @@ -584,5 +619,8 @@ def run_riscv_tensor( raise RuntimeError("QEMU exited without producing UART tensor output") result = decode_tensor_frame(uart_path.read_bytes(), executable.output) except Exception as exc: - raise RiscVTensorExecutionError(str(exc), elapsed_s=elapsed, command=command) from exc + raise RiscVTensorExecutionError(str(exc) + log_message, elapsed_s=elapsed, command=command) from exc + if log_error is not None: + raise RiscVTensorExecutionError(log_message.lstrip("; "), elapsed_s=elapsed, + command=command) from log_error return RiscVTensorRun(result, elapsed, tuple(command), uart_path, stdout_path, stderr_path) diff --git a/scripts/check_w2_ci_results.py b/scripts/check_w2_ci_results.py new file mode 100644 index 0000000..935fa55 --- /dev/null +++ b/scripts/check_w2_ci_results.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Fail W2 CI acceptance if a required job failed, was skipped or is absent.""" +from __future__ import annotations + +import json +import os +import sys + +REQUIRED_JOBS = ("llm-deploy", "full-qwen3-frontend") + + +def validate_results(results): + if not isinstance(results, dict): + raise ValueError("W2_JOB_RESULTS must contain a JSON object") + failures = [] + for name in REQUIRED_JOBS: + job = results.get(name) + outcome = job.get("result") if isinstance(job, dict) else None + if outcome != "success": + failures.append(f"{name}: {outcome or 'missing result'}") + if failures: + raise ValueError("Required W2 jobs did not succeed: " + "; ".join(failures)) + + +def main(): + print("## W2 required acceptance\n") + try: + validate_results(json.loads(os.environ.get("W2_JOB_RESULTS", "{}"))) + except (ValueError, TypeError) as exc: + print(f"FAIL: {exc}") + return 1 + print("PASS: small numerical/runtime gates and complete 28-layer frontend audit succeeded.") + print("\nThis does not certify second-person reproduction or full-model IR/QEMU inference.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_w2_acceptance.py b/scripts/run_w2_acceptance.py new file mode 100644 index 0000000..60439dd --- /dev/null +++ b/scripts/run_w2_acceptance.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Run fresh W2 acceptance gates, without installing dependencies or publishing. + +Use an existing pinned Python environment and explicit local asset/tool paths. +All seven gates are required for PASS. A successful --gates subset is PARTIAL +(exit 2); failures exit 1. Reports never certify independent-person signoff, +full-model numerical execution, or a complete text-generation loop. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +from html import escape +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[1] +GATES = ("frontend", "backend", "runtime", "runtime-model", "small-ir", "small-qemu", "full-parse") +SCRIPTS = { + "backend": "probes/w2_backend_ops/run.py", "runtime": "probes/w2_runtime/run.py", + "runtime-model": "probes/w2_runtime_model/run.py", "small-ir": "probes/w2_qwen3_small/run.py", + "small-qemu": "probes/w2_qwen3_small/riscv.py", "full-parse": "probes/w2_qwen3_parse/run.py", +} +SMALL_CASES = ("full_seed_0", "full_seed_42", "one_token", "short_17", "short_255", + "changed_future", "changed_padding") +FAMILIES = {"matmul", "elementwise", "softmax", "rmsnorm", "rope", "swiglu", "gqa"} + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def source_fingerprints(root=ROOT): + files = {root / "pyproject.toml", root / "scripts/run_w2_acceptance.py", + root / "scripts/check_w2_ci_results.py"} + for directory in ("scratchv", "probes", "requirements"): + files.update(p for p in (root / directory).rglob("*") + if p.is_file() and p.suffix in {".py", ".json", ".txt"} + and not any(part in {"out", "output", "__pycache__"} for part in p.relative_to(root).parts)) + files.update((root / "tests").glob("test_qwen3*.py")) + files.update((root / "tests").glob("test_w2*.py")) + files.add(root / "tests/test_llm_inputs.py") + return {p.relative_to(root).as_posix(): sha256(p) for p in sorted(files) if p.is_file()} + + +def checkout_evidence(root=ROOT): + """A source archive is valid evidence too; Git absence is not a failure.""" + if not (root / ".git").exists(): + # A copied source tree may sit under another checkout's ignored output/. + # Git's parent-directory discovery must not attribute that archive to it. + return {"head": None, "status": "unknown", "reason": "Source directory has no .git metadata"} + try: + def git(*args): + return subprocess.run(["git", *args], cwd=root, capture_output=True, + check=True, timeout=15).stdout + return {"head": git("rev-parse", "HEAD").decode().strip(), + "status": git("status", "--porcelain=v1", "--untracked-files=all").decode("utf-8", "replace"), + "tracked_diff_sha256": hashlib.sha256(git("diff", "--binary", "HEAD", "--")).hexdigest(), + "diff_scope": "Tracked staged and unstaged changes; source_sha256 also covers new source files"} + except (OSError, subprocess.SubprocessError) as exc: + return {"head": None, "status": "unknown", "error": f"{type(exc).__name__}: {exc}"} + + +@dataclass(frozen=True) +class Gate: + name: str + command: tuple[str, ...] + output: Path + dependencies: tuple[str, ...] = () + + +def build_plan(args, root=ROOT): + selected = set(args.gates) + if "small-qemu" in selected: + selected.add("small-ir") + if "runtime-model" in selected: + selected.add("runtime") + plans = [] + for name in GATES: + if name not in selected: + continue + out = args.output_dir / name + command = [args.python, "-B", "-X", "utf8"] + if name == "frontend": + command += ["-m", "pytest", "tests/test_qwen3_frontend_patterns.py", "-q", "-p", "no:cacheprovider", + "--junit-xml", str(out / "tests.xml"), "--basetemp", str(args.output_dir / "frontend-pytest-temp")] + else: + command += [str(root / SCRIPTS[name]), "--output-dir", str(out)] + if name in {"backend", "small-qemu"}: + command += ["--timeout", str(args.qemu_timeout)] + for option in ("cc", "qemu"): + if getattr(args, option): + command += ["--" + option, getattr(args, option)] + if name in {"runtime", "runtime-model"}: + command += ["--tokenizer-dir", str(args.tokenizer_dir)] + if name == "runtime": + command += ["--mode", args.tokenizer_mode] + if name == "small-qemu": + command += ["--model-dir", str(args.output_dir / "small-ir")] + if name == "full-parse": + command += ["--model-dir", str(args.model_dir), "--mode", args.model_mode, + "--timeout", str(args.parse_timeout)] + dependencies = (("small-ir",) if name == "small-qemu" else + ("runtime",) if name == "runtime-model" else ()) + plans.append(Gate(name, tuple(command), out, dependencies)) + return plans + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def check_numeric_tree(value): + """Reject contradictory nested failures and NaN/Infinity numeric evidence.""" + if isinstance(value, dict): + if "passed" in value: + require(value["passed"] is True, "Nested validation did not pass") + if "max_abs" in value: + number = value["max_abs"] + require(type(number) in (int, float) and math.isfinite(number) and 0 <= number < 1e-5, + "Numeric error is missing, nonfinite or outside strict 1e-5 threshold") + for child in value.values(): + check_numeric_tree(child) + elif isinstance(value, list): + for child in value: + check_numeric_tree(child) + elif isinstance(value, float): + require(math.isfinite(value), "Nonfinite report value cannot be accepted as evidence") + + +def checked_rows(rows, count, names=None): + require(isinstance(rows, list) and len(rows) == count, f"Expected {count} complete cases") + require(all(isinstance(row, dict) and row.get("passed") is True for row in rows), "Incomplete case results") + if names is not None: + require([row.get("name") for row in rows] == list(names), "Missing, duplicated or reordered cases") + return rows + + +def validate_gate_report(gate, sources, root=ROOT): + if gate.name == "frontend": + path = gate.output / "tests.xml" + cases = ET.parse(path).getroot().findall(".//testcase") + require(len(cases) == 22, "Frontend must execute all 22 pattern tests") + require(len({(case.get("classname"), case.get("name")) for case in cases}) == 22, + "Duplicated frontend test results") + require(not any(case.find(tag) is not None for case in cases for tag in ("failure", "error", "skipped")), + "Frontend failures, errors and skips are not passing acceptance") + return {"tests": len(cases), "report_sha256": sha256(path)} + path = gate.output / "report.json" + data = json.loads(path.read_text(encoding="utf-8")) + require(data.get("passed") is True and data.get("stage") == "complete", + str(data.get("error", "Child gate did not complete successfully"))) + for name in ("report.md", "report.html"): + require((gate.output / name).is_file(), f"Missing child report {name}") + hashes = data.get("source_sha256", data.get("provenance", {}).get("source_sha256")) + require(isinstance(hashes, dict) and bool(hashes), "Missing child source fingerprints") + require(hashes.get(SCRIPTS[gate.name]) == sources.get(SCRIPTS[gate.name]), "Gate script fingerprint mismatch") + require(all(name in sources and sources[name] == value for name, value in hashes.items()), + "Child source fingerprints do not match this acceptance run") + check_numeric_tree(data) + summary = {"report_sha256": sha256(path), "child_seconds": data.get("seconds")} + if gate.name == "backend": + require(data.get("backend") == "ScratchV tensor-c -> RV64GC -> QEMU", "Unexpected backend gate identity") + rows = checked_rows(data.get("cases"), 16) + require(len({row.get("name") for row in rows}) == 16 and {row.get("family") for row in rows} == FAMILIES, + "Missing backend families or duplicated cases") + for row in rows: + runs = checked_rows(row.get("executions"), 2) + require([run.get("optimization") for run in runs] == ["none", "all"], "Incomplete optimization levels") + for run in runs: + require(all(run.get(key, {}).get("passed") is True for key in + ("ir_vs_ort", "ir_vs_numpy", "qemu_vs_ort", "qemu_vs_numpy")), "Missing backend comparisons") + require(bool(run.get("command")) and bool(run.get("elf_sha256")), "Missing real QEMU evidence") + require((gate.output / row["name"] / run["optimization"] / "qemu.npy").is_file(), "Missing QEMU output") + require(data.get("completed_executions") == data.get("passed_executions") == 32, "Incomplete backend executions") + summary.update(executions=32, qemu_seconds=data.get("qemu_process_wall_seconds", {}).get("total")) + elif gate.name == "runtime": + require(data.get("gate") == "unit:runtime", "Unexpected runtime gate identity") + corpus = json.loads((root / "probes/w2_runtime/corpus.json").read_text(encoding="utf-8")) + rows = checked_rows(data.get("tokenizer_cases"), 20, [row["name"] for row in corpus["cases"]]) + require(all(row.get("checks") == 24 for row in rows), "Incomplete tokenizer comparisons") + checked_rows(data.get("runtime_cases"), 5) + summary.update(corpus_cases=20, tokenizer_comparisons=480, runtime_cases=5) + elif gate.name == "runtime-model": + require(data.get("gate") == "integration:runtime-model", "Unexpected runtime-model gate identity") + require(data.get("config", {}).get("vocab_size") == 151936 + and data.get("config", {}).get("num_hidden_layers") == 2, "Missing real tokenizer/model vocabulary contract") + require(data.get("completed_model_invocations") == 8, "Incomplete runtime-model forward invocations") + rows = checked_rows(data.get("cases"), 2, ("chinese", "special_pad_in_prompt")) + require(rows[1].get("pad_id_inside_prompt") is True, "Missing in-prompt PAD test") + for row in rows: + require(row.get("torch", {}).get("contract", {}).get("passed") is True + and row.get("torch", {}).get("sampling", {}).get("passed") is True, + "Missing independent model and sampling reference") + executions = checked_rows(row.get("executions"), 3) + require([run.get("engine") for run in executions] == ["ort", "ir-none", "ir-all"], + "Missing runtime-model execution engines") + for run in executions: + require(run.get("vs_torch", {}).get("passed") is True + and run.get("sampling", {}).get("passed") is True, "Missing runtime-model comparisons") + if run["engine"].startswith("ir-"): + require(run.get("vs_ort", {}).get("passed") is True, "Missing IR/ORT comparison") + summary.update(cases=2, executions=6) + elif gate.name == "small-ir": + require(data.get("config", {}).get("num_hidden_layers") == 2, "Expected real two-layer configuration") + checked_rows(data.get("cases"), 7, SMALL_CASES) + checked_rows(data.get("invariants"), 6) + for filename, field in (("model.onnx", "model_sha256"), ("diagnostics.onnx", "diagnostics_sha256")): + require(sha256(gate.output / filename) == data.get("onnx", {}).get(field), "Export artifact hash mismatch") + require((gate.output / "checkpoints.json").is_file(), "Missing diagnostic checkpoint schema") + summary["cases"] = 7 + elif gate.name == "small-qemu": + rows = checked_rows(data.get("cases"), 7, SMALL_CASES) + groups = [(graph, level) for graph in ("normal", "diagnostic") for level in ("none", "all")] + for row in rows: + runs = checked_rows(row.get("qemu"), 4) + require([(run.get("graph"), run.get("optimization")) for run in runs] == groups, "Missing QEMU graph/level") + require(all(run.get("status") == "success" and run.get("command") for run in runs), "Missing QEMU execution") + for run in runs: + require((gate.output / "runs" / row["name"] / f"{run['graph']}-{run['optimization']}" / "qemu.npy").is_file(), + "Missing two-layer QEMU output") + timing = data.get("timing", {}) + require(timing.get("completed_count") == timing.get("status_counts", {}).get("success") == 28, + "Incomplete two-layer QEMU executions") + require(data.get("export_evidence", {}).get("report_sha256") == sha256(gate.output.parent / "small-ir/report.json"), + "QEMU did not use the fresh export from this run") + summary.update(executions=28, qemu_seconds=timing.get("qemu_process_wall_seconds_total")) + elif gate.name == "full-parse": + require(data.get("gate") == "frontend:parse-qwen3" and data.get("schema_version") == 1, + "Unexpected full parse gate identity") + require(data.get("graph", {}).get("config", {}).get("num_hidden_layers") == 28, "Missing complete model configuration") + layers = checked_rows(data.get("graph", {}).get("layers"), 28) + require([row.get("layer") for row in layers] == list(range(28)), "Incomplete 28-layer chain") + require(data.get("audit", {}).get("passed") is True and data.get("ir_verifier", {}).get("passed") is True, + "Missing binding audit or IR verification") + for name in ("nodes.json", "bindings.json", "ir.txt"): + require(sha256(gate.output / name) == data.get("evidence", {}).get(name, {}).get("sha256"), + f"Missing or altered parse evidence: {name}") + summary["layers"] = 28 + return summary + + +def terminate_process_tree(process): + """The gates spawn compilers/QEMU/workers; stop owned descendants on timeout.""" + try: + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True, timeout=20, check=True) + else: + try: + # Let the probe unwind _run_process and stop its separate + # compiler/QEMU/worker session before the final group kill. + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=20) + + +def child_failure(row, gate, log): + try: + child = json.loads((gate.output / "report.json").read_text(encoding="utf-8")) + row.update(child_stage=child.get("stage"), child_error=child.get("error")) + except (OSError, ValueError): + pass + return row.get("child_error") or f"Child exited {row['returncode']}; inspect {log.name}" + + +def run_gate(gate, sources, *, timeout, root=ROOT): + started = time.perf_counter() + row = {"name": gate.name, "status": "FAIL", "passed": False, "command": list(gate.command), + "output": str(gate.output), "timeout_seconds": timeout} + log = gate.output.parent / "logs" / (gate.name + ".log") + row["log"] = str(log) + try: + require(not gate.output.exists(), "Gate output already exists; cannot reuse previous evidence") + log.parent.mkdir(exist_ok=True) + env = os.environ.copy() + env.update(PYTHONHASHSEED="0", OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1") + options = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == "nt" else {"start_new_session": True} + with log.open("xb") as stream: + process = subprocess.Popen(gate.command, cwd=root, stdout=stream, stderr=subprocess.STDOUT, + env=env, **options) + try: + row["returncode"] = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + row["error"] = f"Gate exceeded {timeout} seconds" + try: + terminate_process_tree(process) + except (OSError, subprocess.SubprocessError) as exc: + row["cleanup_error"] = f"{type(exc).__name__}: {exc}" + raise + if row["returncode"] != 0: + # Save the primary failure before closing the log; close itself + # can fail and must not replace a child model/tool error. + row["error"] = child_failure(row, gate, log) + raise ValueError(row["error"]) + row["validation"] = validate_gate_report(gate, sources, root) + row.update(status="PASS", passed=True) + except Exception as exc: + row.setdefault("error", f"{type(exc).__name__}: {exc}") + row["seconds"] = time.perf_counter() - started + return row + + +def finalize_status(report): + if report.get("error") or any(row["status"] in {"FAIL", "BLOCKED"} for row in report["gates"]): + report.update(status="FAIL", passed=False) + elif all(row["status"] == "PASS" for row in report["gates"]) and len(report["gates"]) == len(GATES): + report.update(status="PASS", passed=True) + else: + report.update(status="PARTIAL", passed=False) + + +def write_reports(out, report): + errors = [] + + def fail(name, exc): + errors.append(f"{name}: {type(exc).__name__}: {exc}") + report.setdefault("error", "Could not preserve complete W2 acceptance evidence") + report.update(passed=False, status="FAIL", report_write_errors=list(errors)) + + def write(name, text): + temporary = out / ("." + name + ".tmp") + try: + temporary.write_text(text, encoding="utf-8") + temporary.replace(out / name) + return True + except OSError as exc: + fail(name, exc) + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + return False + + def views(): + lines = ["# W2 local acceptance", "", f"Result: **{report['status']}**", "", + "Fresh executions only. Subsets and blocked gates cannot certify all W2 technical gates.", "", + "| Gate | Result | Wall seconds | Reason |", "|---|---|---:|---|"] + for row in report["gates"]: + reason = str(row.get("error", "")).replace("|", "/").replace("\n", " ") + lines.append(f"| {row['name']} | {row['status']} | {row.get('seconds', 0):.3f} | {reason} |") + if report.get("error"): + lines += ["", "Error: " + report["error"]] + lines += ["", "Source hashes and checkout metadata: report.json. Per-gate logs: logs/.", + "This does not establish team signoff, full-model IR/QEMU correctness or text generation."] + markdown = "\n".join(lines) + "\n" + return {"report.md": markdown, "report.html": 'W2 acceptance' + '
'
+                + escape(markdown) + "
"} + + for name, value in views().items(): + write(name, value) + if errors: + for name, value in views().items(): + write(name, value) + if not write("report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n"): + for name, value in views().items(): + write(name, value) + write("report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + if errors: + print("; ".join(errors), file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--python", default=sys.executable, help="Existing pinned Python interpreter; no installation") + cli.add_argument("--output-dir", type=Path, required=True, help="Must not exist, even if empty") + cli.add_argument("--tokenizer-dir", type=Path, help="Verified local official tokenizer assets") + cli.add_argument("--tokenizer-mode", choices=("verify", "download"), default="verify") + cli.add_argument("--model-dir", type=Path, help="Pinned full Qwen3 ONNX release directory") + cli.add_argument("--model-mode", choices=("verify", "download"), default="verify") + cli.add_argument("--cc", help="RISC-V compiler / Zig executable") + cli.add_argument("--qemu", help="qemu-system-riscv64 executable") + cli.add_argument("--gate-timeout", type=float, default=1800, help="Wall seconds per gate including child workers") + cli.add_argument("--qemu-timeout", type=float, default=180, help="Seconds per QEMU execution") + cli.add_argument("--parse-timeout", type=float, default=300, help="Full frontend worker timeout") + cli.add_argument("--gates", nargs="+", choices=GATES, default=list(GATES), + help="Optional subset: dependencies auto-added; successful subset exits 2 (PARTIAL)") + args = cli.parse_args(argv) + for field in ("gate_timeout", "qemu_timeout", "parse_timeout"): + if not math.isfinite(getattr(args, field)) or getattr(args, field) <= 0: + cli.error("Timeouts must be finite and positive") + if len(set(args.gates)) != len(args.gates): + cli.error("Duplicate gates are not allowed") + args.output_dir = args.output_dir.resolve() + for field in ("tokenizer_dir", "model_dir"): + if getattr(args, field) is not None: + setattr(args, field, getattr(args, field).resolve()) + if args.output_dir.exists(): + cli.error("--output-dir must not exist; preserve previous evidence") + args.output_dir.mkdir(parents=True) + report = {"schema_version": 1, "gate": "acceptance:w2", "status": "FAIL", "passed": False, + "created_at": datetime.now(timezone.utc).isoformat(), "gates": [], "requested_gates": args.gates, + "python": args.python, "checkout": checkout_evidence()} + started = time.perf_counter() + try: + report["source_sha256"] = source_fingerprints() + plan = {gate.name: gate for gate in build_plan(args)} + states = {} + for name in GATES: + gate = plan.get(name) + missing = ("--tokenizer-dir" if name in {"runtime", "runtime-model"} and args.tokenizer_dir is None else + "--model-dir" if name == "full-parse" and args.model_dir is None else None) + if gate is None: + row = {"name": name, "status": "NOT_RUN", "passed": False, "error": "Not selected"} + elif missing or any(states.get(dependency) != "PASS" for dependency in gate.dependencies): + row = {"name": name, "status": "BLOCKED", "passed": False, + "error": "Missing " + missing if missing else "Required preceding gate did not pass"} + else: + print(f"[W2] Running {name}", flush=True) + row = run_gate(gate, report["source_sha256"], timeout=args.gate_timeout) + report["gates"].append(row) + states[name] = row["status"] + print(f"[W2] {name}: {row['status']}", flush=True) + final_sources = source_fingerprints() + if final_sources != report["source_sha256"]: + report["error"] = "Source files changed while acceptance was running; rerun after edits finish" + report["final_source_sha256"] = final_sources + except Exception as exc: + report.setdefault("error", f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + finalize_status(report) + write_reports(args.output_dir, report) + print(f"[W2] {report['status']}: {args.output_dir / 'report.json'}", flush=True) + return 0 if report["passed"] else 2 if report["status"] == "PARTIAL" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_llm_inputs.py b/tests/test_llm_inputs.py new file mode 100644 index 0000000..8aebbc5 --- /dev/null +++ b/tests/test_llm_inputs.py @@ -0,0 +1,228 @@ +"""Fixed Qwen3 host ABI, padding, finite greedy selection and stop boundaries.""" + +import numpy as np +import pytest + +from scratchv.runtime.llm_inputs import ( + QWEN3_VOCAB_SIZE, SEQ_LENGTH, build_attention_mask, generation_stop_reason, + greedy_next_token, greedy_token, last_valid_logits, prepare_inputs, +) + + +def test_prepare_inputs_preserves_prompt_and_does_not_infer_length_from_pad_id(): + original = np.array([7, 0, 9], dtype=np.int32) + prepared = prepare_inputs(original, pad_token_id=0, vocab_size=10) + original[:] = 1 + assert prepared.valid_length == 3 + assert prepared.input_ids.shape == (1, 256) + assert prepared.input_ids.dtype == np.int64 + np.testing.assert_array_equal(prepared.input_ids[0, :3], [7, 0, 9]) + assert np.all(prepared.input_ids[0, 3:] == 0) + assert prepared.input_ids.flags.c_contiguous + assert prepared.attention_mask.flags.c_contiguous + assert prepared.as_feed() == { + "input_ids": prepared.input_ids, "attention_mask": prepared.attention_mask, + } + assert set(prepared.as_feed()) == {"input_ids", "attention_mask"} + + +def test_prepare_inputs_accepts_maximum_int64_without_rounding_through_float(): + maximum = int(np.iinfo(np.int64).max) + prepared = prepare_inputs([maximum - 1, maximum], pad_token_id=maximum, + vocab_size=maximum + 1) + assert int(prepared.input_ids[0, 0]) == maximum - 1 + assert int(prepared.input_ids[0, 1]) == maximum + assert int(prepared.input_ids[0, -1]) == maximum + + +def test_prepare_inputs_defaults_to_full_qwen_vocabulary(): + prepared = prepare_inputs([QWEN3_VOCAB_SIZE - 1], pad_token_id=151643) + assert prepared.input_ids[0, 0] == 151935 + assert prepared.input_ids[0, 1] == 151643 + + +def test_prepare_inputs_accepts_full_capacity_without_truncation(): + prepared = prepare_inputs(tuple(range(256)), pad_token_id=0, vocab_size=256) + assert prepared.valid_length == 256 + np.testing.assert_array_equal(prepared.input_ids[0], np.arange(256)) + + +@pytest.mark.parametrize("tokens", [[], [1] * 257]) +def test_prepare_inputs_rejects_empty_and_oversize_prompts(tokens): + with pytest.raises(ValueError, match="valid_length"): + prepare_inputs(tokens, pad_token_id=0, vocab_size=2) + + +@pytest.mark.parametrize("tokens,exception", [ + ([-1], ValueError), ([4], ValueError), ([2**64], ValueError), + ([True], TypeError), ([np.bool_(False)], TypeError), ([1.0], TypeError), + (["1"], TypeError), ([None], TypeError), ("123", TypeError), + (b"123", TypeError), (iter([1]), TypeError), (1, TypeError), + (np.array([[1]], dtype=np.int64), ValueError), + (np.array([1.0], dtype=np.float32), TypeError), + (np.array([1], dtype=object), TypeError), + (np.array([2**64 - 1], dtype=np.uint64), ValueError), +]) +def test_prepare_inputs_rejects_invalid_ids_without_casting_them(tokens, exception): + with pytest.raises(exception): + prepare_inputs(tokens, pad_token_id=0, vocab_size=4) + + +@pytest.mark.parametrize("kwargs,exception", [ + ({"pad_token_id": -1}, ValueError), ({"pad_token_id": 4}, ValueError), + ({"pad_token_id": True}, TypeError), ({"pad_token_id": 0.0}, TypeError), + ({"vocab_size": 0}, ValueError), ({"vocab_size": -1}, ValueError), + ({"vocab_size": True}, TypeError), ({"vocab_size": 4.0}, TypeError), + ({"vocab_size": 2**63 + 1}, ValueError), +]) +def test_prepare_inputs_rejects_invalid_metadata(kwargs, exception): + options = {"pad_token_id": 0, "vocab_size": 4} + options.update(kwargs) + with pytest.raises(exception): + prepare_inputs([1], **options) + + +@pytest.mark.parametrize("length", [1, 17, 255, 256]) +def test_attention_mask_blocks_future_and_padding_keys_but_not_all_padded_queries(length): + mask = build_attention_mask(length) + blocked = np.finfo(np.float32).min + assert mask.shape == (1, 1, 256, 256) + assert mask.dtype == np.float32 + assert np.isfinite(mask).all() + assert set(np.unique(mask)) == {np.float32(0), blocked} + # Counts independently characterize every row, including padded queries. + counts = np.count_nonzero(mask[0, 0] == 0, axis=1) + np.testing.assert_array_equal(counts, [min(q + 1, length) for q in range(256)]) + assert mask[0, 0, 0, 0] == 0 + assert np.all(mask[0, 0, 0, 1:] == blocked) + assert np.all(mask[0, 0, 255, :length] == 0) + if length < SEQ_LENGTH: + assert np.all(mask[0, 0, :, length:] == blocked) + assert mask[0, 0, length, length - 1] == 0 + + +def test_mask_suppresses_large_padding_scores_and_never_hides_an_earlier_valid_key(): + mask = build_attention_mask(3)[0, 0] + scores = np.full((256, 256), 1000, dtype=np.float32) + scores[:, :3] = 0 + masked = scores + mask + weights = np.exp(masked - masked.max(axis=-1, keepdims=True)) + weights /= weights.sum(axis=-1, keepdims=True) + np.testing.assert_array_equal(weights[:, 3:], 0) + np.testing.assert_array_equal(weights[0, :3], [1, 0, 0]) + np.testing.assert_array_equal(weights[1, :3], [0.5, 0.5, 0]) + np.testing.assert_allclose(weights[2:, :3], np.float32(1 / 3), rtol=0, atol=0) + + +@pytest.mark.parametrize("length,exception", [ + (0, ValueError), (-1, ValueError), (257, ValueError), + (True, TypeError), (np.bool_(True), TypeError), (1.0, TypeError), (None, TypeError), +]) +def test_mask_rejects_invalid_lengths(length, exception): + with pytest.raises(exception, match="valid_length"): + build_attention_mask(length) + + +@pytest.mark.parametrize("length", [1, 17, 255, 256]) +def test_greedy_uses_last_valid_position_and_lowest_id_tie(length): + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, :, 7] = 100 + logits[0, length - 1, :] = [-9, 2, -1, 2, -5, 0, -3, 1] + selected = last_valid_logits(logits, length, vocab_size=8) + assert selected.shape == (8,) + assert selected.dtype == np.float32 + assert not np.shares_memory(selected, logits) + assert greedy_next_token(logits, length, vocab_size=8) == 1 + selected[:] = 0 + assert logits[0, length - 1, 1] == 2 + + +def test_greedy_supports_noncontiguous_fp32_view_and_negative_logits(): + values = np.array([-9, 0, -1, 0, -3, 0, -1, 0], dtype=np.float32)[::2] + assert not values.flags.c_contiguous + assert greedy_token(values) == 1 + assert greedy_token(np.array([-0.0, 0.0], dtype=np.float32)) == 0 + + +@pytest.mark.parametrize("logits,exception", [ + ([0, 1], TypeError), (np.zeros(3, dtype=np.float64), TypeError), + (np.zeros(3, dtype=np.float16), TypeError), (np.zeros(3, dtype=np.int64), TypeError), + (np.zeros(0, dtype=np.float32), ValueError), (np.zeros((), dtype=np.float32), ValueError), + (np.zeros((1, 3), dtype=np.float32), ValueError), + (np.array([np.nan, 0], dtype=np.float32), ValueError), + (np.array([0, np.inf], dtype=np.float32), ValueError), + (np.array([-np.inf, 0], dtype=np.float32), ValueError), +]) +def test_greedy_rejects_malformed_or_nonfinite_vocabulary(logits, exception): + with pytest.raises(exception): + greedy_token(logits) + + +def test_masked_arrays_cannot_hide_invalid_ids_or_nan_logits(): + ids = np.ma.array([1, -1], mask=[False, True], dtype=np.int64) + with pytest.raises(TypeError, match="masked array"): + prepare_inputs(ids, pad_token_id=0, vocab_size=8) + scores = np.ma.array([0, np.nan], mask=[False, True], dtype=np.float32) + with pytest.raises(TypeError, match="masked array"): + greedy_token(scores) + + +@pytest.mark.parametrize("shape", [(256, 8), (1, 255, 8), (2, 256, 8), (1, 256, 9)]) +def test_last_valid_logits_rejects_invalid_model_output_shape(shape): + with pytest.raises(ValueError, match="shape"): + last_valid_logits(np.zeros(shape, dtype=np.float32), 3, vocab_size=8) + + +def test_last_valid_logits_rejects_implicit_list_and_dtype_conversion(): + logits = np.zeros((1, 256, 8), dtype=np.float64) + with pytest.raises(TypeError, match="float32"): + last_valid_logits(logits, 3, vocab_size=8) + with pytest.raises(TypeError, match="ndarray"): + last_valid_logits(logits.tolist(), 3, vocab_size=8) + + +@pytest.mark.parametrize("position", [0, 2, 255]) +@pytest.mark.parametrize("invalid", [np.nan, np.inf, -np.inf]) +def test_full_output_must_be_finite_including_unused_padding(position, invalid): + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, position, 7] = invalid + with pytest.raises(ValueError, match="finite"): + greedy_next_token(logits, 3, vocab_size=8) + + +def stop_reason(**kwargs): + options = {"valid_length": 3, "generated_tokens": 0, "max_new_tokens": 10, + "vocab_size": 8, "eos_token_ids": [6, 7]} + options.update(kwargs) + return generation_stop_reason(**options) + + +def test_stop_reason_keeps_prompt_eos_and_stops_on_either_generated_eos(): + assert stop_reason(last_token_id=7) is None + assert stop_reason(generated_tokens=1, last_token_id=6) == "eos" + assert stop_reason(generated_tokens=1, last_token_id=7) == "eos" + assert stop_reason(generated_tokens=1, last_token_id=5) is None + + +def test_stop_reason_respects_zero_budget_exact_budget_and_capacity(): + assert stop_reason(max_new_tokens=0) == "max_new_tokens" + assert stop_reason(generated_tokens=2, max_new_tokens=2, last_token_id=1) == "max_new_tokens" + assert stop_reason(valid_length=255) is None + assert stop_reason(valid_length=256) == "capacity" + assert stop_reason(valid_length=256, generated_tokens=1, last_token_id=7, + max_new_tokens=1) == "eos" + assert stop_reason(valid_length=256, generated_tokens=1, last_token_id=1, + max_new_tokens=1) == "capacity" + + +@pytest.mark.parametrize("kwargs,exception", [ + ({"valid_length": 257}, ValueError), ({"generated_tokens": -1}, ValueError), + ({"generated_tokens": 3}, ValueError), ({"generated_tokens": True}, TypeError), + ({"generated_tokens": 1}, ValueError), ({"max_new_tokens": -1}, ValueError), + ({"max_new_tokens": 1.5}, TypeError), ({"last_token_id": 8}, ValueError), + ({"eos_token_ids": [8]}, ValueError), ({"eos_token_ids": [False]}, TypeError), + ({"eos_token_ids": 7}, TypeError), +]) +def test_stop_reason_rejects_inconsistent_state_or_invalid_configuration(kwargs, exception): + with pytest.raises(exception): + stop_reason(**kwargs) diff --git a/tests/test_qwen3_frontend_patterns.py b/tests/test_qwen3_frontend_patterns.py new file mode 100644 index 0000000..5f11a6d --- /dev/null +++ b/tests/test_qwen3_frontend_patterns.py @@ -0,0 +1,56 @@ +"""unit:frontend-ops: four Qwen3 decomposed patterns, before/after optimization.""" +import copy + +import numpy as np +import onnx +from onnx import helper, numpy_helper +import pytest + +from probes.w2_backend_ops.cases import build_cases +from probes.w2_backend_ops.run import ATOL, FRONTEND_FAMILIES, interpret, reference_case +from probes.w2_qwen3_small.diagnostics import tensor_diff + +CASES = [case for case in build_cases() if case.family in FRONTEND_FAMILIES] + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) +@pytest.mark.parametrize("level", ["none", "all"]) +def test_four_qwen3_patterns_match_independent_formula_and_ort(tmp_path, case, level): + expected, parser, program = reference_case(case, tmp_path / "pattern.onnx") + assert tensor_diff(expected, case.expected, ATOL)["passed"] + actual = interpret(program, parser.initializers, case.feed, level) + assert tensor_diff(actual, expected, ATOL)["passed"] + assert tensor_diff(actual, case.expected, ATOL)["passed"] + + +@pytest.mark.parametrize("family", FRONTEND_FAMILIES) +def test_pattern_cases_detect_semantic_mutations(tmp_path, family): + case = next(copy.deepcopy(case) for case in CASES if case.family == family) + graph = case.model.graph + if family == "rmsnorm": + epsilon = next(value for value in graph.initializer if value.name == "epsilon") + epsilon.CopyFrom(numpy_helper.from_array(np.array(1e-3, np.float32), "epsilon")) + elif family == "rope": + # Wrong rotate_half sign; rotation still preserves shape/dtype. + next(node for node in graph.node if node.op_type == "Neg").op_type = "Identity" + elif family == "swiglu": + next(node for node in graph.node if node.op_type == "Sigmoid").input[0] = "up" + else: + # Wrong interleaved KV repetition still has the expected head count. + graph.initializer.append(numpy_helper.from_array(np.array([0, 2, 1, 3], np.int64), "wrong_heads")) + for name in ("k", "v"): + nodes = list(graph.node) + position = next(index for index, node in enumerate(nodes) if f"{name}_repeated" in node.output) + nodes.insert(position + 1, helper.make_node("Gather", [f"{name}_repeated", "wrong_heads"], + [f"{name}_wrong"], axis=1)) + for node in nodes[position + 2:]: + for index, value in enumerate(node.input): + if value == f"{name}_repeated": + node.input[index] = f"{name}_wrong" + del graph.node[:] + graph.node.extend(nodes) + onnx.checker.check_model(case.model) + actual, parser, program = reference_case(case, tmp_path / "mutated.onnx") + ir = interpret(program, parser.initializers, case.feed, "all") + assert tensor_diff(actual, ir, ATOL)["passed"] + assert not tensor_diff(actual, case.expected, ATOL)["passed"], family diff --git a/tests/test_qwen3_full_audit.py b/tests/test_qwen3_full_audit.py new file mode 100644 index 0000000..1674c1e --- /dev/null +++ b/tests/test_qwen3_full_audit.py @@ -0,0 +1,375 @@ +"""The full-model acceptance audit must reject plausible but incomplete parses.""" + +import json + +import numpy as np +import onnx +import pytest +from onnx import TensorProto as T, helper, numpy_helper + +from probes.w2_qwen3_parse.audit import ( + AuditError, AuditedONNXParser, audit_parsed_model, +) +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.ir.types import DataType, OpCode, Value + + +def save(tmp_path, nodes, arrays=None, outputs=None, external=False): + graph = helper.make_graph( + nodes, "audit_fixture", [helper.make_tensor_value_info("x", T.FLOAT, [2, 2])], + outputs or [helper.make_tensor_value_info("logits", T.FLOAT, [2, 2])], + [numpy_helper.from_array(array, name) for name, array in (arrays or {}).items()], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)], ir_version=9) + onnx.checker.check_model(model) + path = tmp_path / "model.onnx" + if external: + onnx.save_model(model, path, save_as_external_data=True, + all_tensors_to_one_file=True, location="weights.data", size_threshold=0) + else: + onnx.save_model(model, path) + return path + + +@pytest.fixture +def parsed(tmp_path): + nodes = [helper.make_node("MatMul", ["x", "w1"], ["a"], name="project"), + helper.make_node("Add", ["a", "w2"], ["b"], name="add_weight"), + helper.make_node("Identity", ["b"], ["alias"], name="alias"), + helper.make_node("Constant", [], ["bias"], name="bias", + value=numpy_helper.from_array(np.full((2, 2), .5, np.float32))), + helper.make_node("Add", ["alias", "bias"], ["logits"], name="final")] + path = save(tmp_path, nodes, {"w1": np.arange(4, dtype=np.float32).reshape(2, 2), + "w2": np.full((2, 2), 8, dtype=np.float32)}, external=True) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + return onnx.load(path, load_external_data=False), parser, program, tmp_path + + +def test_complete_parse_has_serializable_mapping_and_binding_evidence(parsed): + result = audit_parsed_model(*parsed) + json.dumps(result, allow_nan=False) + assert result["passed"] and result["node_count"] == 5 + assert result["value_count"] == 8 + assert result["initializer_count"] == 2 + assert result["constant_count"] == 1 + assert result["global_count"] == result["binding_count"] == 3 + assert result["binding_bytes"] == 48 + assert result["alias_count"] == 1 + assert result["return"]["onnx_name"] == "logits" + assert all(len(row["sha256"]) == 64 for row in result["tensor_bindings"]) + assert parsed[0].graph.initializer[0].raw_data == b"" + + +def test_parser_reuse_resets_observations_and_progress(parsed): + _, parser, _, directory = parsed + events = [] + parser.progress = events.append + parser.parse(str(directory / "model.onnx")) + assert len(parser.node_records) == 5 + assert events[0]["stage"] == "parse" and events[-1]["stage"] == "parsed" + assert [row["node_index"] for row in events if row["stage"] == "translate_node"] == list(range(5)) + + +@pytest.mark.parametrize("fault", ["missing_record", "missing_map", "missing_instruction", + "shape", "dtype", "alias", "return", "global", + "duplicate_global", "swapped_weights", "constant_content", + "operand", "opcode", "attrs", "parameter"]) +def test_incomplete_or_corrupt_translation_fails_closed(parsed, fault): + _, parser, program, _ = parsed + instructions = program.functions[0].blocks[0].instructions + if fault == "missing_record": + parser.node_records.pop() + elif fault == "missing_map": + del parser._value_map["a"] + elif fault == "missing_instruction": + instructions.pop(0) + elif fault == "shape": + parser._value_map["a"].shape = (4,) + elif fault == "dtype": + parser._value_map["a"].dtype = DataType.FLOAT64 + elif fault == "alias": + parser._value_map["alias"] = parser._value_map["a"] + elif fault == "return": + instructions[-1].operands = [parser._value_map["a"]] + elif fault == "global": + program.global_values.pop() + elif fault == "duplicate_global": + program.global_values.append(program.global_values[0]) + elif fault == "swapped_weights": + parser.initializers["w1"], parser.initializers["w2"] = ( + parser.initializers["w2"], parser.initializers["w1"]) + elif fault == "constant_content": + parser.initializers["bias"] = np.zeros((2, 2), dtype=np.float32) + elif fault == "operand": + instructions[0].operands[1] = parser._value_map["w2"] + elif fault == "opcode": + instructions[0].opcode = OpCode.ADD + elif fault == "attrs": + instructions[0].attrs["k"] = 1 + elif fault == "parameter": + program.functions[0].params = [] + with pytest.raises(AuditError): + audit_parsed_model(*parsed) + + +def test_skip_handler_is_not_counted_as_success(parsed, monkeypatch): + model, _, _, directory = parsed + parser = AuditedONNXParser() + monkeypatch.setattr(parser, "_handle_add", lambda node, inputs, outputs: + parser._define_outputs(outputs, inputs[0])) + program = parser.parse(str(directory / "model.onnx")) + with pytest.raises(AuditError, match="exactly one instruction"): + audit_parsed_model(model, parser, program, directory) + + +def test_equal_shape_value_copy_does_not_prove_identity_alias(parsed): + _, parser, _, _ = parsed + source = parser._value_map["alias"] + parser._value_map["alias"] = Value(source.name, source.dtype, shape=source.shape) + with pytest.raises(AuditError, match="alias binding"): + audit_parsed_model(*parsed) + + +@pytest.mark.parametrize("fault", ["missing", "truncated", "escape", "negative_offset", + "bad_length", "duplicate_key", "changed_content"]) +def test_external_data_errors_are_rejected_without_loading_second_weights(parsed, fault): + model, _, _, directory = parsed + tensor = model.graph.initializer[0] + fields = {row.key: row for row in tensor.external_data} + path = directory / "weights.data" + if fault == "missing": + path.unlink() + elif fault == "truncated": + path.write_bytes(b"\0" * 3) + elif fault == "escape": + fields["location"].value = "../outside.data" + elif fault == "negative_offset": + fields["offset"].value = "-1" + elif fault == "bad_length": + fields["length"].value = "15" + elif fault == "duplicate_key": + tensor.external_data.add(key="location", value="weights.data") + elif fault == "changed_content": + with path.open("r+b") as stream: + stream.write(b"\xFF" * 4) + with pytest.raises(AuditError): + audit_parsed_model(*parsed) + + +def test_noncontiguous_equal_binding_hashes_logical_c_order(parsed): + _, parser, _, _ = parsed + parser.initializers["w1"] = np.asfortranarray(parser.initializers["w1"]) + assert not parser.initializers["w1"].flags.c_contiguous + assert audit_parsed_model(*parsed)["passed"] + + +def test_multiple_graph_outputs_are_explicitly_rejected(tmp_path): + outputs = [helper.make_tensor_value_info(name, T.FLOAT, [2, 2]) for name in ("x", "logits")] + path = save(tmp_path, [helper.make_node("Identity", ["x"], ["logits"])], outputs=outputs) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + with pytest.raises(AuditError, match="exactly one graph output"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +def test_scalar_constants_and_static_shape_cast_chain(tmp_path): + nodes = [helper.make_node("Constant", [], ["dim"], value_int=2), + helper.make_node("Cast", ["dim"], ["cast"], to=T.INT64), + helper.make_node("Constant", [], ["axis"], value_ints=[0]), + helper.make_node("Unsqueeze", ["cast", "axis"], ["vec"]), + helper.make_node("Concat", ["vec", "vec"], ["shape"], axis=0), + helper.make_node("Reshape", ["x", "shape"], ["reshaped"]), + helper.make_node("Add", ["reshaped", "scalar"], ["logits"])] + path = save(tmp_path, nodes, {"scalar": np.array(1, dtype=np.float32)}) + model = onnx.load(path) + model.graph.value_info.append(helper.make_tensor_value_info("reshaped", T.FLOAT, [2, 2])) + onnx.save(model, path) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + result = audit_parsed_model(onnx.load(path), parser, program, tmp_path) + assert result["binding_count"] == 3 + assert result["node_count"] == 7 + + +def test_failed_translation_preserves_node_context(tmp_path, monkeypatch): + path = save(tmp_path, [helper.make_node("Identity", ["x"], ["logits"], name="failed_node")]) + parser = AuditedONNXParser() + + def broken(*args): + raise RuntimeError("injected") + + monkeypatch.setattr(parser, "_handle_identity", broken) + with pytest.raises(RuntimeError, match="injected"): + parser.parse(str(path)) + assert parser.current_context["node_name"] == "failed_node" + assert parser.node_records[-1]["status"] == "failed" + + +def test_scalar_constant_signed_zero_is_preserved(tmp_path): + path = save(tmp_path, [helper.make_node("Constant", [], ["zero"], value_float=-0.0), + helper.make_node("Add", ["x", "zero"], ["logits"])]) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + model = onnx.load(path) + assert audit_parsed_model(model, parser, program, tmp_path)["passed"] + program.functions[0].blocks[0].instructions[0].attrs["value"] = 0.0 + with pytest.raises(AuditError, match="Constant.*scalar instruction"): + audit_parsed_model(model, parser, program, tmp_path) + + +def test_complete_decoder_fixture_uses_production_parser_and_audit(tmp_path): + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + path = tmp_path / "decoder.onnx" + onnx.save_model(model, path, save_as_external_data=True, + all_tensors_to_one_file=True, location="decoder.data", size_threshold=0) + original = onnx.load(path, load_external_data=False) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + result = audit_parsed_model(original, parser, program, tmp_path) + assert result["passed"] and result["initializer_count"] == 26 + assert result["node_count"] == len(original.graph.node) + assert result["return"]["shape"] == [1, 3, 9] + shared = parser._value_map["model.lm_head.weight"] + consumers = [inst for inst in program.functions[0].blocks[0].instructions + if any(operand is shared for operand in inst.operands)] + assert sorted(inst.opcode.value for inst in consumers) == ["gather", "transpose"] + + +@pytest.mark.parametrize("operation", ["Reshape", "Slice"]) +def test_parser_control_evaluation_cannot_certify_its_own_wrong_results(tmp_path, operation): + from scratchv.analysis.ir_verifier import verify_ir + + if operation == "Reshape": + nodes = [helper.make_node("Reshape", ["x", "shape"], ["logits"])] + arrays = {"shape": np.array([2, 2], np.int64)} + shape = [2, 2] + wrong = {"shape": (1, 4)} + else: + nodes = [helper.make_node("Slice", ["x", "starts", "ends", "axes"], ["logits"])] + arrays = {name: np.array(value, np.int64) for name, value in + {"starts": [0], "ends": [1], "axes": [0]}.items()} + shape = [1, 2] + # The wrong row has the SAME output shape: metadata cannot catch this. + wrong = {"starts": (1,), "ends": (2,)} + path = save(tmp_path, nodes, arrays, + outputs=[helper.make_tensor_value_info("logits", T.FLOAT, shape)]) + parser = AuditedONNXParser() + correct = parser._constant_ints + parser._constant_ints = lambda value: wrong[value.name] if value.name in wrong else correct(value) + program = parser.parse(str(path)) + assert verify_ir(program) == (True, []) + with pytest.raises(AuditError, match="IR attributes mismatch"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +@pytest.fixture +def qwen_parsed(tmp_path): + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + path = tmp_path / "qwen.onnx" + onnx.save_model(model, path) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + return model, parser, program, tmp_path + + +@pytest.mark.parametrize("fault", ["transpose", "epsilon"]) +def test_parser_attribute_reader_cannot_certify_its_own_wrong_results(tmp_path, monkeypatch, fault): + from scratchv.analysis.ir_verifier import verify_ir + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + original_bytes = model.SerializeToString() + epsilon_name = next(node.input[1] for node in model.graph.node + if list(node.output) == ["layer_0_input_norm_eps"]) + path = tmp_path / "qwen.onnx" + onnx.save(model, path) + correct = ONNXParser._attributes + + def wrong_reader(node): + attrs = correct(node) + if fault == "transpose" and list(node.output) == ["layer_0_q_weight"]: + attrs["perm"] = [0, 1] # Square Q projection: output shape stays unchanged. + if fault == "epsilon" and list(node.output) == [epsilon_name]: + attrs["value"] = numpy_helper.from_array(np.array(1e-3, dtype=np.float32)) + return attrs + + monkeypatch.setattr(ONNXParser, "_attributes", staticmethod(wrong_reader)) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + assert verify_ir(program) == (True, []) + expected = "IR attributes mismatch" if fault == "transpose" else "content SHA256 mismatch" + with pytest.raises(AuditError, match=expected): + audit_parsed_model(model, parser, program, tmp_path) + assert model.SerializeToString() == original_bytes + + +@pytest.mark.parametrize("parameter", [0, 1]) +@pytest.mark.parametrize("fault", ["name", "constant"]) +def test_qwen_parameter_binding_contract_is_checked(qwen_parsed, parameter, fault): + from scratchv.analysis.ir_verifier import verify_ir + + _, _, program, _ = qwen_parsed + value = program.functions[0].params[parameter] + if fault == "name": + value.name = "renamed_" + value.name + else: + value.is_constant = True + value.const_value = 0 + assert verify_ir(program) == (True, []) + expected = "parameter names" if fault == "name" else "must not be constants" + with pytest.raises(AuditError, match=expected): + audit_parsed_model(*qwen_parsed) + + +@pytest.mark.parametrize("source", ["initializer", "Constant"]) +@pytest.mark.parametrize("fault", ["shape", "constant_flag", "constant_value"]) +def test_auxiliary_scalar_load_result_metadata_is_checked(tmp_path, source, fault): + nodes = [helper.make_node("Add", ["x", "scalar"], ["logits"])] + arrays = {} + if source == "Constant": + nodes.insert(0, helper.make_node("Constant", [], ["scalar"], value_float=1.0)) + else: + arrays["scalar"] = np.array(1.0, dtype=np.float32) + path = save(tmp_path, nodes, arrays) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + scalar_load = program.functions[0].blocks[0].instructions[0] + assert scalar_load.opcode is OpCode.LOAD_CONST + assert scalar_load.dest is not parser._value_map["scalar"] + if fault == "shape": + scalar_load.dest.shape = (2,) + elif fault == "constant_flag": + scalar_load.dest.is_constant = False + else: + scalar_load.dest.const_value = 2.0 + with pytest.raises(AuditError, match="scalar instruction metadata mismatch"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +@pytest.mark.parametrize("binding", ["initializer", "Constant"]) +def test_qwen_bindings_are_not_silently_coerced_to_numpy(qwen_parsed, binding): + _, parser, _, _ = qwen_parsed + name = "positions" if binding == "initializer" else next( + name for name, value in parser.initializers.items() + if name.startswith("constant_") and value.dtype == np.dtype("int64") and value.ndim == 1) + original = parser.initializers[name] + parser.initializers[name] = original.tolist() + # Coercion would give the exact same bytes, shape and dtype, but execution + # requires the original binding object to already be a NumPy ndarray. + converted = np.asarray(parser.initializers[name]) + assert converted.dtype == original.dtype and converted.shape == original.shape + np.testing.assert_array_equal(converted, original) + with pytest.raises(AuditError, match="binding must be a NumPy array"): + audit_parsed_model(*qwen_parsed) + + +def test_qwen_read_only_ndarray_bindings_remain_valid(qwen_parsed): + _, parser, _, _ = qwen_parsed + for array in parser.initializers.values(): + array.setflags(write=False) + assert audit_parsed_model(*qwen_parsed)["passed"] diff --git a/tests/test_qwen3_full_parse.py b/tests/test_qwen3_full_parse.py new file mode 100644 index 0000000..970abec --- /dev/null +++ b/tests/test_qwen3_full_parse.py @@ -0,0 +1,422 @@ +"""Full-model gate orchestration and failure evidence; no large checkpoint needed.""" +import json +from pathlib import Path +import subprocess +from types import SimpleNamespace + +import pytest + +from probes.w2_qwen3_parse import run as gate + + +def completed_report(out, *, passed=True, stage="complete"): + report = gate.initial_report("verify") + report.update(passed=passed, stage=stage, validation_seconds=1.0, + graph={"passed": True, "layers": [{"layer": index} for index in range(28)]}, + stages_seconds={"scratchv-parse": 0.25}, + ir={"instructions": 12, "globals": 3}, ir_verifier={"passed": True}, + audit={"passed": True, "binding_bytes": 16}, + peak_memory={"bytes": 1024, "method": "test"}) + for name, content in (("nodes.json", "[]"), ("bindings.json", "[]"), ("ir.txt", "fixture IR")): + (out / name).write_text(content, encoding="utf-8") + gate.write_json(out / "worker-report.json", report) + return report + + +def invoke(tmp_path): + return ["--model-dir", str(tmp_path / "model"), "--output-dir", str(tmp_path / "report")] + + +def test_success_keeps_separate_parse_time_and_scope(tmp_path, monkeypatch): + def launch(args): + completed_report(args.output_dir) + return subprocess.CompletedProcess([], 0, b"parsed", b"") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 0 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["passed"] and report["stage"] == "complete" + assert report["stages_seconds"]["scratchv-parse"] == 0.25 + assert report["seconds"] >= 0 and report["validation_seconds"] == 1 + assert report["timeout_seconds"] == 300 + for name in ("report.md", "report.html"): + content = (out / name).read_text(encoding="utf-8") + assert "PASS" in content and "28 层" in content + assert "不证明完整 IR 数值或 QEMU 前向通过" in content + assert (out / "worker.stdout").read_bytes() == b"parsed" + + +@pytest.mark.parametrize("failure", ["nonzero", "failed", "unfinished", "missing_report", + "bad_json", "wrong_gate", "missing_evidence", "spawn", "killed_missing_report"]) +def test_worker_failure_cannot_be_success(tmp_path, monkeypatch, failure): + def launch(args): + if failure == "spawn": + raise OSError("cannot start worker ") + report = completed_report(args.output_dir) + path = args.output_dir / "worker-report.json" + if failure == "failed": + report.update(passed=False, stage="ir-verifier", error="invalid IR") + elif failure == "unfinished": + report["stage"] = "scratchv-parse" + elif failure == "wrong_gate": + report["gate"] = "ORT-only" + gate.write_json(path, report) + if failure in ("missing_report", "killed_missing_report"): + path.unlink() + elif failure == "bad_json": + path.write_text("{broken", encoding="utf-8") + elif failure == "missing_evidence": + (args.output_dir / "bindings.json").unlink() + code = -9 if failure == "killed_missing_report" else (4 if failure == "nonzero" else 0) + return subprocess.CompletedProcess([], code, b"partial", b"failure") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["passed"] is False and report["error"] + assert report["attempt"]["command"][0] == gate.sys.executable + assert report["attempt"]["model_dir"] == str((tmp_path / "model").resolve()) + assert "probes/w2_qwen3_parse/run.py" in report["attempt"]["source_sha256"] + if failure == "killed_missing_report": + assert report["worker_returncode"] == -9 + assert "FAIL" in (out / "report.md").read_text(encoding="utf-8") + html = (out / "report.html").read_text(encoding="utf-8") + assert "" not in html + + +def test_timeout_keeps_partial_progress_and_actual_elapsed_time(tmp_path, monkeypatch): + clock = iter((10.0, 13.75)) + monkeypatch.setattr(gate.time, "perf_counter", lambda: next(clock)) + def launch(args): + gate.write_json(args.output_dir / "worker-progress.json", + {"context": {"node_index": 17, "op": "MatMul"}, "elapsed_seconds": 2.0}) + raise subprocess.TimeoutExpired(["python"], 2, output=b"partial stdout", stderr=b"partial stderr") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main([*invoke(tmp_path), "--timeout", "2"]) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["stage"] == "timeout" and report["timeout_seconds"] == 2 + assert report["attempt"]["source_sha256"] + assert report["seconds"] == 3.75 + assert report["last_progress"]["context"]["node_index"] == 17 + assert (out / "worker.stderr").read_bytes() == b"partial stderr" + + +def test_timeout_log_write_failure_still_produces_failure_report(tmp_path, monkeypatch): + original = Path.write_bytes + def locked_stderr(path, content): + if path.name == "worker.stderr": + raise PermissionError("log locked") + return original(path, content) + def timeout(args): + raise subprocess.TimeoutExpired(["python"], 1, output=b"partial") + monkeypatch.setattr(Path, "write_bytes", locked_stderr) + monkeypatch.setattr(gate, "launch_worker", timeout) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["stage"] == "timeout" and not report["passed"] + assert "log locked" in report["log_errors"][0] + assert "log locked" in (out / "report.md").read_text(encoding="utf-8") + assert (out / "worker.stdout").read_bytes() == b"partial" + + +@pytest.mark.parametrize("worker_passed", [False, True]) +@pytest.mark.parametrize("locked", ["worker.stdout", "worker.stderr"]) +def test_normal_log_failure_preserves_worker_result(tmp_path, monkeypatch, worker_passed, locked): + original = Path.write_bytes + def write(path, content): + if path.name == locked: + raise PermissionError("log locked") + return original(path, content) + def launch(args): + report = completed_report(args.output_dir, passed=worker_passed, + stage="complete" if worker_passed else "ir-and-bindings") + report.update(checkout={"head": "review-sha"}, failure_context={"tensor_name": "positions"}) + if not worker_passed: + report["error"] = "positions: bound tensor content SHA256 mismatch" + gate.write_json(args.output_dir / "worker-report.json", report) + return subprocess.CompletedProcess([], 0 if worker_passed else 1, b"out", b"err") + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["checkout"]["head"] == "review-sha" + assert report["failure_context"] == {"tensor_name": "positions"} + assert len(report["log_errors"]) == 1 and locked in report["log_errors"][0] + assert report["worker_returncode"] == (0 if worker_passed else 1) + if worker_passed: + assert report["stage"] == "log-write" and "logs" in report["error"] + else: + assert report["stage"] == "ir-and-bindings" and "SHA256 mismatch" in report["error"] + assert "SHA256 mismatch" in (out / "report.md").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("worker_passed", [False, True]) +@pytest.mark.parametrize("locked", ["report.md", "report.html", "report.json"]) +def test_report_write_failure_cannot_publish_pass(tmp_path, monkeypatch, worker_passed, locked, capsys): + write_text, replace = Path.write_text, Path.replace + json_publications = [] + + def write(path, content, **kwargs): + if path.name == locked: + raise PermissionError("injected report lock") + return write_text(path, content, **kwargs) + + def publish(path, target): + if Path(target).name == "report.json": + if locked == "report.json": + raise PermissionError("injected report lock") + json_publications.append(json.loads(path.read_text(encoding="utf-8"))) + return replace(path, target) + + def launch(args): + report = completed_report(args.output_dir, passed=worker_passed, + stage="complete" if worker_passed else "ir-and-bindings") + if not worker_passed: + report["error"] = "positions: bound tensor content SHA256 mismatch" + report["failure_context"] = {"tensor_name": "positions"} + gate.write_json(args.output_dir / "worker-report.json", report) + return subprocess.CompletedProcess([], 0 if worker_passed else 1, b"out", b"err") + + monkeypatch.setattr(Path, "write_text", write) + monkeypatch.setattr(Path, "replace", publish) + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + assert "injected report lock" in capsys.readouterr().err + if locked == "report.json": + assert not (out / "report.json").exists() + assert not json_publications + else: + # JSON must never have been published as PASS, even temporarily. + assert len(json_publications) == 1 + report = json_publications[0] + assert report["passed"] is False + assert locked in report["report_write_errors"][0] + if worker_passed: + assert report["stage"] == "report-write" + else: + assert report["stage"] == "ir-and-bindings" + assert "SHA256 mismatch" in report["error"] + assert report["failure_context"] == {"tensor_name": "positions"} + for name in ("report.md", "report.html"): + if name != locked: + content = (out / name).read_text(encoding="utf-8") + assert "FAIL" in content and "PASS" not in content + assert "injected report lock" in content + if not worker_passed: + assert "SHA256 mismatch" in content + + +def test_w2_docs_links_resolve_without_local_output(monkeypatch): + from scripts import check_docs_links + exists = Path.exists + def clean_checkout(path): + if path.resolve().is_relative_to(gate.ROOT / "output"): + return False + return exists(path) + monkeypatch.setattr(Path, "exists", clean_checkout) + _, broken = check_docs_links.check_links([ + "docs/llm-deploy-v1.0/W2/README.md", "probes/w2_qwen3_parse/README.md"]) + assert not broken, broken + + +def test_checkout_evidence_preserves_porcelain_columns(monkeypatch, tmp_path): + # A linked worktree stores .git as a file, not a directory. + (tmp_path / ".git").write_text("gitdir: unused-test-path\n", encoding="utf-8") + monkeypatch.setattr(gate, "ROOT", tmp_path) + def run(command, **kwargs): + assert kwargs["timeout"] == 10 and kwargs["cwd"] == tmp_path + return SimpleNamespace(stdout="abcdef\n" if command[1] == "rev-parse" else " M file.py\n") + monkeypatch.setattr(gate.subprocess, "run", run) + report = gate.checkout_evidence() + assert report["head"] == "abcdef" and report["status"] == " M file.py" + assert report["clean"] is False + + +def test_source_archive_does_not_query_parent_checkout(monkeypatch, tmp_path): + (tmp_path / ".git").mkdir() + archive = tmp_path / "source-archive" + archive.mkdir() + monkeypatch.setattr(gate, "ROOT", archive) + monkeypatch.setattr(gate.subprocess, "run", lambda *a, **kw: pytest.fail("must not search parent Git")) + report = gate.checkout_evidence() + assert report["head"] is None and report["status"] == "unknown" and report["clean"] is None + assert report["reason"] == "Source directory has no .git metadata" + + +@pytest.mark.parametrize("failure_at", ["rev-parse", "status"]) +def test_git_timeout_is_unknown_and_does_not_leave_partial_identity(monkeypatch, tmp_path, failure_at): + (tmp_path / ".git").mkdir() + monkeypatch.setattr(gate, "ROOT", tmp_path) + def query(command, **kwargs): + assert kwargs["timeout"] == 10 + if command[1] == failure_at: + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + return SimpleNamespace(stdout="partial-sha\n") + monkeypatch.setattr(gate.subprocess, "run", query) + report = gate.checkout_evidence() + assert report["head"] is None and report["status"] == "unknown" and report["clean"] is None + assert report["reason"] == "Checkout metadata unavailable" + assert "TimeoutExpired" in report["error"] + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +def test_invalid_timeout_does_not_start_worker(tmp_path, monkeypatch, timeout): + monkeypatch.setattr(gate, "launch_worker", lambda args: pytest.fail("must not launch")) + with pytest.raises(SystemExit, match="2"): + gate.main([*invoke(tmp_path), "--timeout", timeout]) + assert not (tmp_path / "report").exists() + + +def test_existing_evidence_is_preserved(tmp_path): + out = tmp_path / "report" + out.mkdir() + evidence = out / "report.json" + evidence.write_text("prior evidence", encoding="utf-8") + with pytest.raises(SystemExit, match="2"): + gate.main(invoke(tmp_path)) + assert evidence.read_text(encoding="utf-8") == "prior evidence" + + +def test_worker_uses_same_python_explicit_paths_and_timeout(tmp_path, monkeypatch): + import scratchv.runtime.riscv_tensor as runtime + captured = {} + def process(command, **kwargs): + captured.update(command=command, **kwargs) + return subprocess.CompletedProcess(command, 0, b"", b"") + monkeypatch.setattr(runtime, "_run_process", process) + args = SimpleNamespace(mode="verify", model_dir=tmp_path / "model with spaces", + output_dir=tmp_path / "output with spaces", timeout=12.5) + gate.launch_worker(args) + assert captured["command"][0] == gate.sys.executable + assert str(args.model_dir.resolve()) in captured["command"] + assert "--worker" in captured["command"] and captured["timeout"] == 12.5 + + +def test_worker_dependency_failure_writes_report_before_parsing(tmp_path, monkeypatch): + def bad_environment(): + raise RuntimeError("pinned environment mismatch") + monkeypatch.setattr(gate.artifacts, "environment", bad_environment) + monkeypatch.setattr(gate, "run_validation", lambda *args, **kwargs: pytest.fail("must not parse")) + args = SimpleNamespace(mode="verify", model_dir=tmp_path / "model", output_dir=tmp_path) + assert gate.worker_main(args) == 1 + result = json.loads((tmp_path / "worker-report.json").read_text(encoding="utf-8")) + assert result["stage"] == "environment" and not result["passed"] + assert "mismatch" in result["error"] and result["validation_seconds"] >= 0 + + +def test_corrupt_hash_cannot_reach_scratchv_parser(tmp_path, monkeypatch): + from probes.w2_qwen3_parse import audit + def bad_files(*args): + raise ValueError("Artifact size/SHA-256 mismatch: weights.data") + monkeypatch.setattr(gate.artifacts, "verify_files", bad_files) + monkeypatch.setattr(audit.AuditedONNXParser, "parse", lambda *args: pytest.fail("must not parse")) + report = gate.initial_report("verify") + with pytest.raises(ValueError, match="SHA-256"): + gate.run_validation(tmp_path / "model", tmp_path, report) + assert report["stage"] == "hashes" and not report["passed"] + assert "hashes" in report["stages_seconds"] + + +@pytest.mark.parametrize("fault,evidence_locked", [ + (None, False), ("parse", False), ("verifier", False), ("binding", False), + ("parse", True), ("binding", True), (None, True), +]) +def test_real_decoder_pipeline_and_failure_context(tmp_path, monkeypatch, fault, evidence_locked): + """Exercise the stage wiring with real files, parser and audits, cheaply.""" + import onnx + from scratchv.analysis import ir_verifier + from probes.w2_qwen3_parse import audit, validation + from tests.test_qwen3_full_structure import qwen3_fixture + + model, config = qwen3_fixture() + model_dir, out = tmp_path / "model", tmp_path / "out" + model_dir.mkdir() + out.mkdir() + onnx.save_model(model, model_dir / "model.onnx", save_as_external_data=True, + all_tensors_to_one_file=True, location="weights.data", size_threshold=0) + manifest = {"files": [{"name": path.name, "bytes": path.stat().st_size, + "sha256": gate.artifacts.sha256(path)} for path in model_dir.iterdir()]} + # Only the full-model dimensions are adapted; loading, file integrity, + # lowering, verifier and content audits still run against real fixtures. + structure = validation.audit_graph_structure + monkeypatch.setattr(validation, "audit_graph_structure", lambda graph: structure(graph, config)) + def inspect(directory, files): + onnx.checker.check_model(str(directory / "model.onnx"), full_check=True) + return {"operators": {"fixture": len(model.graph.node)}} + monkeypatch.setattr(gate.artifacts, "inspect_model", inspect) + if fault == "parse": + def broken_gather(*args): + raise RuntimeError("injected Gather failure") + monkeypatch.setattr(audit.AuditedONNXParser, "_handle_gather", broken_gather) + elif fault == "verifier": + monkeypatch.setattr(ir_verifier, "verify_ir", lambda *args, **kwargs: (False, ["injected verifier failure"])) + elif fault == "binding": + original = audit.AuditedONNXParser.parse + def misbind(self, path): + program = original(self, path) + self.initializers["model.lm_head.weight"] = self.initializers["model.lm_head.weight"].copy() + self.initializers["model.lm_head.weight"][0, 0] += 1 + return program + monkeypatch.setattr(audit.AuditedONNXParser, "parse", misbind) + report = gate.initial_report("verify") + if evidence_locked: + original_write = gate.write_json + def write(path, value): + # The binding failure occurs after the first, successful nodes.json + # save; lock only its diagnostic retry, not the earlier operation. + locked = ((fault == "parse" and path.name == "nodes.partial.json") + or (fault == "binding" and path.name == "nodes.json" + and "failure_context" in report) + or (fault is None and path.name == "bindings.json")) + if locked: + raise PermissionError("injected evidence lock") + return original_write(path, value) + monkeypatch.setattr(gate, "write_json", write) + if fault or evidence_locked: + expected_error = ("injected Gather failure" if fault == "parse" else + "content SHA256 mismatch" if fault == "binding" else + "injected evidence lock" if evidence_locked else "IR verifier rejected") + with pytest.raises((RuntimeError, ValueError, PermissionError), match=expected_error) as failure: + gate.run_validation(model_dir, out, report, manifest=manifest) + assert report["passed"] is False + assert report["stage"] == {"parse": "scratchv-parse", "verifier": "ir-verifier", + "binding": "ir-and-bindings", None: "ir-and-bindings"}[fault] + if fault == "parse": + assert report["failure_context"]["op_type"] == "Gather" + if evidence_locked: + assert not (out / "nodes.partial.json").exists() + assert report["nodes_completed"] == 0 + else: + partial = json.loads((out / "nodes.partial.json").read_text(encoding="utf-8")) + assert partial[-1]["status"] == "failed" + assert report["nodes_completed"] == len(partial) - 1 + elif fault == "verifier": + assert report["ir_verifier"]["issues"] == ["injected verifier failure"] + assert (out / "nodes.json").is_file() + elif fault == "binding": + assert "model.lm_head.weight" in json.dumps(report["failure_context"]) + assert (out / "nodes.json").is_file() + if evidence_locked and fault: + assert len(report["evidence_errors"]) == 1 + assert "injected evidence lock" in report["evidence_errors"][0] + report["error"] = f"{type(failure.value).__name__}: {failure.value}" + gate.write_reports(out, report) + markdown = (out / "report.md").read_text(encoding="utf-8") + assert expected_error in markdown and "injected evidence lock" in markdown + elif evidence_locked: + # Successful validation cannot pass if required evidence is lost. + assert isinstance(failure.value, PermissionError) + assert "evidence_errors" not in report + assert not (out / "bindings.json").exists() + else: + gate.run_validation(model_dir, out, report, manifest=manifest) + assert report["passed"] and report["stage"] == "complete" + assert report["graph"]["layer_count"] == 2 + assert report["audit"]["node_count"] == len(model.graph.node) + assert report["audit"]["return"]["shape"] == [1, 3, 9] + for name, evidence in report["evidence"].items(): + assert evidence["sha256"] == gate.artifacts.sha256(out / name) diff --git a/tests/test_qwen3_full_structure.py b/tests/test_qwen3_full_structure.py new file mode 100644 index 0000000..e199933 --- /dev/null +++ b/tests/test_qwen3_full_structure.py @@ -0,0 +1,327 @@ +"""Small complete decoder fixtures and deliberately corrupted structure gates.""" + +from copy import deepcopy + +import numpy as np +import pytest + +onnx = pytest.importorskip("onnx") + +from probes.w2_qwen3_parse.validation import ( + Qwen3StructureError, audit_graph_structure, make_control_constant_reader, +) + + +def qwen3_fixture(layers=2): + """A tiny, valid ONNX decoder using every required architectural path.""" + config = {"num_hidden_layers": layers, "hidden_size": 4, "intermediate_size": 6, + "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 2, + "vocab_size": 9, "sequence_length": 3, "rms_norm_eps": 1e-6} + nodes, weights, constants = [], [], {} + + def node(op, inputs, output, **attrs): + nodes.append(onnx.helper.make_node(op, inputs, [output], name=f"opaque_{len(nodes)}", **attrs)) + return output + + def constant(value, dtype=np.int64): + array = np.asarray(value, dtype=dtype) + key = (array.dtype.str, array.shape, array.tobytes()) + if key not in constants: + constants[key] = node("Constant", [], f"constant_{len(constants)}", + value=onnx.numpy_helper.from_array(array)) + return constants[key] + + def weight(name, shape): + data = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) / 100 + len(weights) / 1000 + weights.append(onnx.numpy_helper.from_array(data, name)) + + shapes = {"self_attn.q_proj.weight": (4, 4), "self_attn.k_proj.weight": (2, 4), + "self_attn.v_proj.weight": (2, 4), "self_attn.o_proj.weight": (4, 4), + "self_attn.q_norm.weight": (2,), "self_attn.k_norm.weight": (2,), + "mlp.gate_proj.weight": (6, 4), "mlp.up_proj.weight": (6, 4), + "mlp.down_proj.weight": (4, 6), "input_layernorm.weight": (4,), + "post_attention_layernorm.weight": (4,)} + for layer in range(layers): + for name, shape in shapes.items(): + weight(f"model.model.layers.{layer}.{name}", shape) + weight("model.model.norm.weight", (4,)) + weight("model.lm_head.weight", (9, 4)) + weights.append(onnx.numpy_helper.from_array(np.arange(3, dtype=np.int64).reshape(1, 3), "positions")) + weights.append(onnx.numpy_helper.from_array(np.array([1.0], dtype=np.float32), "model.model.rotary_emb.inv_freq")) + + def norm(raw, name, weight_name): + square = node("Pow", [raw, constant(2, np.float32)], name + "_square") + mean = node("ReduceMean", [square, constant([-1])], name + "_mean", keepdims=1) + plus = node("Add", [mean, constant(config["rms_norm_eps"], np.float32)], name + "_eps") + sqrt = node("Sqrt", [plus], name + "_sqrt") + inv = node("Reciprocal", [sqrt], name + "_inv") + scaled = node("Mul", [raw, inv], name + "_scaled") + return node("Mul", [weight_name, scaled], name) + + def projection(raw, name, weight_name): + transpose = node("Transpose", [weight_name], name + "_weight", perm=[1, 0]) + return node("MatMul", [raw, transpose], name) + + def rope(raw, name, cosine, sine): + first = node("Slice", [raw, constant([0]), constant([1]), constant([-1])], name + "_first") + second = node("Slice", [raw, constant([1]), constant([2]), constant([-1])], name + "_second") + negative = node("Neg", [second], name + "_negative") + rotated = node("Concat", [negative, first], name + "_rotated", axis=-1) + original = node("Mul", [raw, cosine], name + "_cos") + rotated = node("Mul", [rotated, sine], name + "_sin") + return node("Add", [original, rotated], name) + + def repeat(raw, name): + expanded = node("Unsqueeze", [raw, constant([2])], name + "_unsqueezed") + expanded = node("Expand", [expanded, constant([1, 1, 2, 3, 2])], name + "_expanded") + return node("Reshape", [expanded, constant([1, 2, 3, 2])], name) + + previous = node("Gather", ["model.lm_head.weight", "input_ids"], "embedding", axis=0) + positions = node("Cast", ["positions"], "float_positions", to=onnx.TensorProto.FLOAT) + positions = node("Unsqueeze", [positions, constant([1])], "positions_3d") + freq = node("Unsqueeze", ["model.model.rotary_emb.inv_freq", constant([0, 2])], "freq_3d") + freq = node("MatMul", [freq, positions], "freq_matmul") + freq = node("Transpose", [freq], "freq_transpose", perm=[0, 2, 1]) + freq = node("Concat", [freq, freq], "freq_full_head", axis=-1) + cosine = node("Cos", [freq], "cosine") + sine = node("Sin", [freq], "sine") + cosine = node("Unsqueeze", [cosine, constant([1])], "cosine_4d") + sine = node("Unsqueeze", [sine, constant([1])], "sine_4d") + for layer in range(layers): + prefix = f"layer_{layer}" + path = f"model.model.layers.{layer}." + normalized = norm(previous, prefix + "_input_norm", path + "input_layernorm.weight") + q = projection(normalized, prefix + "_q", path + "self_attn.q_proj.weight") + k = projection(normalized, prefix + "_k", path + "self_attn.k_proj.weight") + v = projection(normalized, prefix + "_v", path + "self_attn.v_proj.weight") + q = node("Reshape", [q, constant([1, 3, 2, 2])], prefix + "_q_view") + k = node("Reshape", [k, constant([1, 3, 1, 2])], prefix + "_k_view") + v = node("Reshape", [v, constant([1, 3, 1, 2])], prefix + "_v_view") + q = norm(q, prefix + "_q_norm", path + "self_attn.q_norm.weight") + k = norm(k, prefix + "_k_norm", path + "self_attn.k_norm.weight") + q = node("Transpose", [q], prefix + "_q_transpose", perm=[0, 2, 1, 3]) + k = node("Transpose", [k], prefix + "_k_transpose", perm=[0, 2, 1, 3]) + v = node("Transpose", [v], prefix + "_v_transpose", perm=[0, 2, 1, 3]) + q = rope(q, prefix + "_q_rope", cosine, sine) + k = rope(k, prefix + "_k_rope", cosine, sine) + k = repeat(k, prefix + "_k_repeat") + v = repeat(v, prefix + "_v_repeat") + k = node("Transpose", [k], prefix + "_k_scores", perm=[0, 1, 3, 2]) + scores = node("MatMul", [q, k], prefix + "_scores") + scores = node("Mul", [scores, constant(2 ** -0.5, np.float32)], prefix + "_scaled_scores") + scores = node("Add", [scores, "attention_mask"], prefix + "_masked_scores") + probabilities = node("Softmax", [scores], prefix + "_probabilities", axis=-1) + values = node("MatMul", [probabilities, v], prefix + "_weighted_values") + values = node("Transpose", [values], prefix + "_attention_transpose", perm=[0, 2, 1, 3]) + values = node("Reshape", [values, constant([1, 3, 4])], prefix + "_attention_view") + output = projection(values, prefix + "_o", path + "self_attn.o_proj.weight") + residual = node("Add", [previous, output], prefix + "_attention_residual") + normalized = norm(residual, prefix + "_post_norm", path + "post_attention_layernorm.weight") + gate = projection(normalized, prefix + "_gate", path + "mlp.gate_proj.weight") + up = projection(normalized, prefix + "_up", path + "mlp.up_proj.weight") + sigmoid = node("Sigmoid", [gate], prefix + "_sigmoid") + activated = node("Mul", [gate, sigmoid], prefix + "_silu") + swiglu = node("Mul", [activated, up], prefix + "_swiglu") + down = projection(swiglu, prefix + "_down", path + "mlp.down_proj.weight") + previous = node("Add", [residual, down], prefix + "_mlp_residual") + normalized = norm(previous, "final_norm", "model.model.norm.weight") + projection(normalized, "logits", "model.lm_head.weight") + helper = onnx.helper.make_tensor_value_info + graph = onnx.helper.make_graph(nodes, "small_full_structure", + [helper("input_ids", onnx.TensorProto.INT64, [1, 3]), + helper("attention_mask", onnx.TensorProto.FLOAT, [1, 1, 3, 3])], + [helper("logits", onnx.TensorProto.FLOAT, [1, 3, 9])], weights) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 18)], ir_version=10) + return model, config + + +def output_node(model, name): + return next(node for node in model.graph.node if name in node.output) + + +def replace_control(model, output, input_index, value, dtype=np.int64): + target_index = next(i for i, node in enumerate(model.graph.node) if output in node.output) + constant_name = output + "_corrupt_control" + constant = onnx.helper.make_node("Constant", [], [constant_name], + value=onnx.numpy_helper.from_array(np.asarray(value, dtype=dtype))) + model.graph.node.insert(target_index, constant) + output_node(model, output).input[input_index] = constant_name + + +def test_complete_small_graph_validates_actual_dataflow(): + model, config = qwen3_fixture() + onnx.checker.check_model(model, full_check=True) + result = audit_graph_structure(model, config) + assert result["passed"] and result["tied_embeddings"] + assert result["layer_count"] == 2 + assert result["graph"]["initializer_count"] == 26 + assert result["live_node_count"] == len(model.graph.node) + assert result["layers"][1]["input"] == "layer_0_mlp_residual" + + +def test_layer_detection_does_not_depend_on_node_names_or_exporter_metadata(): + model, config = qwen3_fixture() + for node in model.graph.node: + node.name = "" + del node.metadata_props[:] + assert audit_graph_structure(model, config)["layer_count"] == 2 + + +def test_external_weight_bytes_are_not_loaded(monkeypatch): + model, config = qwen3_fixture() + for tensor in model.graph.initializer: + onnx.external_data_helper.set_external_data(tensor, "does-not-exist.data", offset=0, + length=len(tensor.raw_data)) + tensor.ClearField("raw_data") + tensor.data_location = onnx.TensorProto.EXTERNAL + def forbidden(*args, **kwargs): + raise AssertionError("structure validation must not read external weight data") + monkeypatch.setattr(onnx.external_data_helper, "load_external_data_for_tensor", forbidden) + assert audit_graph_structure(model, config)["passed"] + + +def test_control_reader_uses_original_onnx_graph_without_parser_state(): + model, _ = qwen3_fixture() + reader = make_control_constant_reader(model) + shape_name = output_node(model, "layer_0_k_repeat_expanded").input[1] + np.testing.assert_array_equal(reader(shape_name), [1, 1, 2, 3, 2]) + assert reader(shape_name).dtype == np.int64 + + +def test_control_reader_rejects_external_and_large_tensors_without_loading(): + model, _ = qwen3_fixture() + tensor = model.graph.initializer[0] + onnx.external_data_helper.set_external_data(tensor, "missing.data", offset=0, length=len(tensor.raw_data)) + tensor.ClearField("raw_data") + tensor.data_location = onnx.TensorProto.EXTERNAL + reader = make_control_constant_reader(model) + with pytest.raises(Qwen3StructureError, match="external or large tensor"): + reader(tensor.name) + big = onnx.numpy_helper.from_array(np.zeros(1025, dtype=np.int64), "big_control") + model.graph.initializer.append(big) + with pytest.raises(Qwen3StructureError, match="external or large tensor"): + make_control_constant_reader(model)("big_control") + + +@pytest.mark.parametrize("change,expected", [("shape", "expected shape"), ("dtype", "dtype"), + ("missing", "missing="), ("extra", "unexpected="), + ("duplicate", "duplicate weight")]) +def test_weight_inventory_rejects_incorrect_metadata(change, expected): + model, config = qwen3_fixture() + tensor = model.graph.initializer[0] + if change == "shape": + tensor.dims[0] += 1 + elif change == "dtype": + tensor.data_type = onnx.TensorProto.FLOAT16 + elif change == "missing": + del model.graph.initializer[0] + elif change == "extra": + extra = deepcopy(tensor) + extra.name = "model.model.layers.99.self_attn.q_proj.weight" + model.graph.initializer.append(extra) + else: + model.graph.initializer.append(deepcopy(tensor)) + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("target", ["layer_1_input_norm_square", "layer_1_input_norm_scaled"]) +def test_cross_layer_norm_must_use_same_previous_residual(target): + model, config = qwen3_fixture() + output_node(model, target).input[0] = "embedding" + with pytest.raises(Qwen3StructureError, match="layer 1.*input mismatch"): + audit_graph_structure(model, config) + + +def test_bypassed_layer_is_rejected_even_when_all_weights_remain(): + model, config = qwen3_fixture() + for name in ("final_norm_square", "final_norm_scaled"): + output_node(model, name).input[0] = "layer_0_mlp_residual" + with pytest.raises(Qwen3StructureError, match="final norm must consume the last layer"): + audit_graph_structure(model, config) + + +def test_wrong_residual_bypass_is_rejected(): + model, config = qwen3_fixture() + output_node(model, "layer_1_attention_residual").input[0] = "embedding" + with pytest.raises(Qwen3StructureError, match="layer 1: attention residual"): + audit_graph_structure(model, config) + + +def test_dead_final_norm_is_rejected(): + model, config = qwen3_fixture() + output_node(model, "logits").input[0] = "layer_1_mlp_residual" + with pytest.raises(Qwen3StructureError, match="LM Head must consume final norm"): + audit_graph_structure(model, config) + + +def test_shared_embedding_weight_cannot_be_replaced_by_another_matrix(): + model, config = qwen3_fixture() + output_node(model, "embedding").input[0] = "model.model.layers.0.self_attn.q_proj.weight" + with pytest.raises(Qwen3StructureError, match="embedding: tied LM Head"): + audit_graph_structure(model, config) + + +def test_same_shape_layer_weights_cannot_be_substituted(): + model, config = qwen3_fixture() + output_node(model, "layer_0_q_weight").input[0] = "model.model.layers.1.self_attn.q_proj.weight" + with pytest.raises(Qwen3StructureError, match="layer 0 self_attn.q_proj"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("node_name,new_input,expected", [ + ("layer_0_q_norm_square", "layer_0_k_view", "input mismatch"), + ("layer_0_sigmoid", "layer_0_up", "sigmoid input must be gate"), + ("layer_0_masked_scores", "layer_0_scores", "additive attention mask"), +]) +def test_key_computations_cannot_be_rewired(node_name, new_input, expected): + model, config = qwen3_fixture() + node = output_node(model, node_name) + node.input[1 if node_name == "layer_0_masked_scores" else 0] = new_input + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +def test_disconnected_extra_compute_node_cannot_hide_in_a_complete_graph(): + model, config = qwen3_fixture() + model.graph.node.append(onnx.helper.make_node("Identity", ["logits"], ["dead_output"], name="dead_node")) + with pytest.raises(Qwen3StructureError, match="disconnected from logits.*dead_node"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("output,input_index,value,dtype,expected", [ + ("layer_0_input_norm_square", 1, 3, np.float32, "must be scalar 2"), + ("layer_0_input_norm_mean", 1, [0], np.int64, "axes=\\[-1\\]"), + ("layer_0_input_norm_eps", 1, 0.001, np.float32, "must be scalar 1e-06"), + ("layer_0_scaled_scores", 1, 1, np.float32, "attention scale"), + ("layer_0_k_repeat_expanded", 1, [1, 2, 1, 3, 2], np.int64, "GQA K expansion shape"), + ("layer_0_v_repeat", 1, [1, 1, 3, 4], np.int64, "GQA V head reshape"), + ("layer_0_v_repeat_unsqueezed", 1, [1], np.int64, "GQA V repeat group must be axis 2"), +]) +def test_numerical_structure_constants_and_group_layout_are_checked(output, input_index, value, dtype, expected): + model, config = qwen3_fixture() + replace_control(model, output, input_index, value, dtype) + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +def test_norm_weight_used_as_both_operands_has_a_clear_validation_error(): + model, config = qwen3_fixture() + node = output_node(model, "layer_0_input_norm") + node.input[1] = node.input[0] + with pytest.raises(Qwen3StructureError, match="normalized activation"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("change,expected", [("output", "graph output"), ("shape", "incorrect static shape"), + ("dtype", "incorrect static shape")]) +def test_static_interface_is_required(change, expected): + model, config = qwen3_fixture() + if change == "output": + model.graph.output[0].name = "layer_1_mlp_residual" + elif change == "shape": + model.graph.output[0].type.tensor_type.shape.dim[1].dim_value = 2 + else: + model.graph.input[0].type.tensor_type.elem_type = onnx.TensorProto.INT32 + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) diff --git a/tests/test_qwen3_tokenizer.py b/tests/test_qwen3_tokenizer.py new file mode 100644 index 0000000..81eb29d --- /dev/null +++ b/tests/test_qwen3_tokenizer.py @@ -0,0 +1,221 @@ +"""Offline contract tests; official assets are checked by the runtime probe.""" + +import json +from pathlib import Path + +import numpy as np +import pytest + +pytest.importorskip("tokenizers") +from tokenizers import AddedToken, Tokenizer, decoders, models, normalizers, pre_tokenizers, processors + +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + + +@pytest.fixture +def source(tmp_path): + alphabet = sorted(pre_tokenizers.ByteLevel.alphabet()) + engine = Tokenizer(models.BPE(vocab={s: i for i, s in enumerate(alphabet)}, merges=[])) + engine.normalizer = normalizers.NFC() + engine.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + engine.decoder = decoders.ByteLevel() + engine.post_processor = processors.ByteLevel(trim_offsets=False) + engine.add_special_tokens([ + AddedToken("<|endoftext|>", special=True, normalized=False), + AddedToken("<|im_start|>", special=True, normalized=False), + AddedToken("<|im_end|>", special=True, normalized=False), + ]) + engine.add_tokens([AddedToken("", special=False, normalized=False)]) + engine.save(str(tmp_path / "tokenizer.json")) + added = { + str(token_id): {name: getattr(token, name) for name in ( + "content", "lstrip", "rstrip", "single_word", "normalized", "special" + )} + for token_id, token in engine.get_added_tokens_decoder().items() + } + _write(tmp_path, "tokenizer_config.json", { + "tokenizer_class": "Qwen2Tokenizer", "add_bos_token": False, + "bos_token": None, "unk_token": None, "eos_token": "<|im_end|>", + "pad_token": "<|endoftext|>", "clean_up_tokenization_spaces": False, + "added_tokens_decoder": added, "additional_special_tokens": ["<|im_start|>", "<|im_end|>"], + }) + _write(tmp_path, "config.json", {"model_type": "qwen3", "vocab_size": 288, "bos_token_id": 256}) + _write(tmp_path, "generation_config.json", {"eos_token_id": [258, 256], "pad_token_id": 256}) + return tmp_path + + +def _write(directory: Path, filename: str, value: dict): + (directory / filename).write_text(json.dumps(value), encoding="utf-8") + + +def _change(directory, filename, **changes): + data = json.loads((directory / filename).read_text(encoding="utf-8")) + data.update(changes) + _write(directory, filename, data) + + +def test_metadata_distinguishes_vocab_and_generation_eos(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + assert tokenizer.base_vocab_size == 256 + assert tokenizer.tokenizer_vocab_size == 260 + assert tokenizer.model_vocab_size == 288 + assert tokenizer.pad_token_id == 256 + assert tokenizer.eos_token_id == 258 + assert tokenizer.generation_eos_token_ids == (258, 256) + assert tokenizer.bos_token_id is None + assert tokenizer.unk_token_id is None + assert tokenizer.valid_token_ids == frozenset(range(260)) + assert tokenizer.is_valid_token_id(np.int64(259)) + assert not tokenizer.is_valid_token_id(260) + assert not tokenizer.is_valid_token_id(True) + + +def test_missing_unk_uses_official_default_but_explicit_null_disables_it(source): + config = json.loads((source / "tokenizer_config.json").read_text(encoding="utf-8")) + assert Qwen3Tokenizer.from_directory(source).unk_token_id is None + del config["unk_token"] + _write(source, "tokenizer_config.json", config) + assert Qwen3Tokenizer.from_directory(source).unk_token_id == 256 + + +@pytest.mark.parametrize("text", ["", "Hello, world!", " a\t\r\nb ", "中文日本語", "😀👩‍💻", "a\x00b"]) +def test_text_round_trip_and_no_implicit_special_tokens(source, text): + tokenizer = Qwen3Tokenizer.from_directory(source) + ids = tokenizer.encode(text) + assert ids == tokenizer.encode(text, add_special_tokens=False) + assert not set(ids) & {256, 257, 258} + assert tokenizer.decode(ids) == text + + +def test_normalization_is_official_json_behavior(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + assert tokenizer.encode("e\u0301") == tokenizer.encode("é") + assert tokenizer.decode(tokenizer.encode("e\u0301")) == "é" + + +def test_special_tokens_and_ordinary_added_tokens(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + text = "a<|im_start|><|im_end|><|endoftext|>b" + ids = tokenizer.encode(text) + assert all(token_id in ids for token_id in (256, 257, 258, 259)) + assert tokenizer.decode(ids) == text + assert tokenizer.decode(ids, skip_special_tokens=True) == "ab" + assert tokenizer.decode(np.int64(258)) == "<|im_end|>" + assert tokenizer.decode(np.array([257, 258], dtype=np.int64)) == "<|im_start|><|im_end|>" + + +def test_cleanup_is_opt_in_and_matches_transformers_rules(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + text = "I 'm sure , you 're here . We 've seen it ! He 's fine ? I do n't know ." + ids = tokenizer.encode(text) + assert tokenizer.decode(ids) == text + assert tokenizer.decode(ids, clean_up_tokenization_spaces=True) == "I'm sure, you're here. We've seen it! He's fine? I don't know." + _change(source, "tokenizer_config.json", clean_up_tokenization_spaces=True) + configured = Qwen3Tokenizer.from_directory(source) + assert configured.decode(ids) == tokenizer.decode(ids, clean_up_tokenization_spaces=True) + assert configured.decode(ids, clean_up_tokenization_spaces=False) == text + + +@pytest.mark.parametrize("text", [None, 12, b"text", ["text"]]) +def test_encode_rejects_non_text(source, text): + with pytest.raises(TypeError, match="text must be str"): + Qwen3Tokenizer.from_directory(source).encode(text) + + +def test_encode_rejects_invalid_unicode(source): + with pytest.raises(UnicodeEncodeError): + Qwen3Tokenizer.from_directory(source).encode("\ud800") + + +@pytest.mark.parametrize("ids", [ + [True], [1.0], ["1"], [[1]], [None], "123", b"123", bytearray(b"123"), + None, True, {1, 2}, {1: "one"}, +]) +def test_decode_rejects_noninteger_ids(source, ids): + with pytest.raises(TypeError): + Qwen3Tokenizer.from_directory(source).decode(ids) + + +@pytest.mark.parametrize("ids", [[-1], [260], [287], [288], [2**80]]) +def test_decode_refuses_unknown_or_unused_model_rows(source, ids): + with pytest.raises(ValueError, match="not in the tokenizer vocabulary"): + Qwen3Tokenizer.from_directory(source).decode(ids) + + +@pytest.mark.parametrize("option", ["add_special_tokens", "skip_special_tokens", "clean_up_tokenization_spaces"]) +def test_flags_are_boolean(source, option): + tokenizer = Qwen3Tokenizer.from_directory(source) + with pytest.raises(TypeError, match="must be bool"): + if option == "add_special_tokens": + tokenizer.encode("text", **{option: 1}) + else: + tokenizer.decode([], **{option: 1}) + + +@pytest.mark.parametrize(("filename", "changes", "message"), [ + ("config.json", {"model_type": "other"}, "model_type"), + ("config.json", {"vocab_size": True}, "vocab_size"), + ("config.json", {"vocab_size": 259}, "vocab_size"), + ("tokenizer_config.json", {"tokenizer_class": "OtherTokenizer"}, "tokenizer_class"), + ("tokenizer_config.json", {"add_bos_token": True}, "add_bos_token"), + ("tokenizer_config.json", {"add_prefix_space": True}, "add_prefix_space"), + ("tokenizer_config.json", {"split_special_tokens": True}, "split_special_tokens"), + ("tokenizer_config.json", {"mask_token": ""}, "mask_token"), + ("tokenizer_config.json", {"sep_token": ""}, "sep_token"), + ("tokenizer_config.json", {"cls_token": ""}, "cls_token"), + ("tokenizer_config.json", {"extra_special_tokens": {"image_token": ""}}, "extra_special_tokens"), + ("tokenizer_config.json", {"pad_token": "absent"}, "pad_token"), + ("tokenizer_config.json", {"additional_special_tokens": [""]}, "special token"), + ("tokenizer_config.json", {"added_tokens_decoder": {}}, "added token IDs differ"), + ("generation_config.json", {"eos_token_id": []}, "eos_token_id"), + ("generation_config.json", {"eos_token_id": [256]}, "include the tokenizer EOS"), + ("generation_config.json", {"eos_token_id": [258, 280]}, "not in the tokenizer vocabulary"), + ("generation_config.json", {"pad_token_id": 257}, "pad_token_id"), +]) +def test_configuration_mismatch_is_not_silently_ignored(source, filename, changes, message): + _change(source, filename, **changes) + with pytest.raises(ValueError, match=message): + Qwen3Tokenizer.from_directory(source) + + +def test_inconsistent_added_token_flags_rejected(source): + data = json.loads((source / "tokenizer_config.json").read_text(encoding="utf-8")) + data["added_tokens_decoder"]["259"]["special"] = True + _write(source, "tokenizer_config.json", data) + with pytest.raises(ValueError, match="inconsistent special"): + Qwen3Tokenizer.from_directory(source) + + +@pytest.mark.parametrize("setting", ["padding", "truncation", "post_processor"]) +def test_no_implicit_padding_truncation_or_bos(source, setting): + engine = Tokenizer.from_file(str(source / "tokenizer.json")) + if setting == "padding": + engine.enable_padding(length=10, pad_id=256, pad_token="<|endoftext|>") + elif setting == "truncation": + engine.enable_truncation(max_length=2) + else: + engine.post_processor = processors.TemplateProcessing(single="<|im_start|> $A", special_tokens=[("<|im_start|>", 257)]) + engine.save(str(source / "tokenizer.json")) + with pytest.raises(ValueError, match="automatic"): + Qwen3Tokenizer.from_directory(source) + + +@pytest.mark.parametrize("nested", [False, True]) +def test_json_cannot_insert_prefix_space_even_when_config_disables_it(source, nested): + engine = Tokenizer.from_file(str(source / "tokenizer.json")) + prefix = pre_tokenizers.ByteLevel(add_prefix_space=True) + engine.pre_tokenizer = pre_tokenizers.Sequence([prefix]) if nested else prefix + engine.save(str(source / "tokenizer.json")) + with pytest.raises(ValueError, match="add_prefix_space"): + Qwen3Tokenizer.from_directory(source) + + +def test_generation_single_eos_supported(source): + _change(source, "generation_config.json", eos_token_id=258) + assert Qwen3Tokenizer.from_directory(source).generation_eos_token_ids == (258,) + + +def test_files_are_local_and_required(source): + (source / "generation_config.json").unlink() + with pytest.raises(FileNotFoundError): + Qwen3Tokenizer.from_directory(source) diff --git a/tests/test_riscv_tensor_runtime.py b/tests/test_riscv_tensor_runtime.py index de1c8de..517a9c3 100644 --- a/tests/test_riscv_tensor_runtime.py +++ b/tests/test_riscv_tensor_runtime.py @@ -217,6 +217,32 @@ def _assert_timed_child_terminated(pid, started): _assert_posix_process_terminated(pid, starttime=started) +@pytest.mark.parametrize("interruption", [KeyboardInterrupt, SystemExit]) +def test_interruption_stops_actual_owned_process(monkeypatch, tmp_path, interruption): + communicate = subprocess.Popen.communicate + interrupted = [] + + def interrupt_once(process, *args, **kwargs): + if not interrupted: + interrupted.append(process) + raise interruption(143) + return communicate(process, *args, **kwargs) + + monkeypatch.setattr(subprocess.Popen, "communicate", interrupt_once) + with pytest.raises(interruption): + _run_process([sys.executable, "-B", "-c", "import time;time.sleep(60)"], + cwd=tmp_path, timeout=30) + assert len(interrupted) == 1 + assert interrupted[0].poll() is not None + + +def test_termination_signal_requests_normal_cleanup(): + from scratchv.runtime.riscv_tensor import _terminate_invocation + with pytest.raises(SystemExit) as caught: + _terminate_invocation(15, None) + assert caught.value.code == 143 + + @pytest.mark.parametrize("field,value", [ ("class", 1), ("endian", 2), ("machine", 62), ("entry", INPUT_BASE), ("abi", 0), ]) @@ -347,6 +373,68 @@ def process(command, *, cwd, timeout): assert error.__cause__ is not None +@pytest.mark.parametrize("outcome,message", [ + ("guest", "failure status 3"), ("nonzero", r"QEMU failed \(7\): rejected"), + ("bad_frame", "truncated or missing"), +]) +def test_qemu_log_failure_keeps_primary_error_and_other_log( + tmp_path, monkeypatch, timed_guest, outcome, message): + runtime, executable, feed, clock = timed_guest + original_write = Path.write_bytes + + def write(path, data): + if path.name == "qemu.stdout": + raise PermissionError("stdout is locked") + return original_write(path, data) + + def process(command, *, cwd, timeout): + clock.now += 2.5 + if outcome == "guest": + (cwd / "uart.bin").write_bytes(frame(b"", status=3)) + elif outcome == "bad_frame": + (cwd / "uart.bin").write_bytes(b"broken") + return subprocess.CompletedProcess(command, 7 if outcome != "bad_frame" else 0, + b"partial stdout", b"rejected") + + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(runtime, "_run_process", process) + run_dir = tmp_path / "run" + with pytest.raises(RiscVTensorExecutionError, match=message) as captured: + run_riscv_tensor(executable, feed, run_dir) + assert "stdout is locked" in str(captured.value) + assert captured.value.elapsed_s == 2.5 and clock.reads == 2 + assert (run_dir / "qemu.stderr").read_bytes() == b"rejected" + + +@pytest.mark.parametrize("timeout", [False, True]) +def test_compiler_log_failure_keeps_primary_error_and_other_log(tmp_path, monkeypatch, timeout): + from scratchv.runtime import riscv_tensor as runtime + + original_write = Path.write_bytes + + def write(path, data): + if path.name == "compiler.stdout": + raise PermissionError("compiler stdout is locked") + return original_write(path, data) + + def process(command, **kwargs): + if timeout: + raise subprocess.TimeoutExpired(command, kwargs["timeout"], output=b"partial", stderr=b"rejected") + return subprocess.CompletedProcess(command, 11, b"partial", b"rejected") + + artifact = SimpleNamespace(source="int scratchv_run(void){return 0;}", inputs=(), + output=spec("result", (1,), "float32"), function_name="scratchv_run") + tools = SimpleNamespace(cc=("fixture-clang",), qemu="fixture-qemu", is_zig=False) + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(runtime, "_run_process", process) + error = TimeoutError if timeout else RuntimeError + message = "compilation exceeded 2" if timeout else r"compilation failed \(11\)" + with pytest.raises(error, match=message) as captured: + build_riscv_tensor(artifact, tmp_path / "build", tools, timeout=2) + assert "compiler stdout is locked" in str(captured.value) + assert (tmp_path / "build/compiler.stderr").read_bytes() == b"rejected" + + @pytest.mark.parametrize("log_error", [False, True]) def test_timeout_reports_actual_time_including_cleanup(tmp_path, monkeypatch, timed_guest, log_error): runtime, executable, feed, clock = timed_guest @@ -380,7 +468,7 @@ def write(path, data): assert "cannot preserve timeout log" in str(error) else: assert (tmp_path / "run/qemu.stdout").read_bytes() == b"partial" - assert (tmp_path / "run/qemu.stderr").read_bytes() == b"timed out" + assert (tmp_path / "run/qemu.stderr").read_bytes() == b"timed out" @pytest.mark.parametrize("failure", ["elf", "input", "stale", "write_input", "spawn"]) diff --git a/tests/test_tensor_c_codegen.py b/tests/test_tensor_c_codegen.py index 6fccf71..23dd265 100644 --- a/tests/test_tensor_c_codegen.py +++ b/tests/test_tensor_c_codegen.py @@ -330,3 +330,45 @@ def test_same_generator_is_reusable_and_never_mutates_ir(): first, second = generator.generate(), generator.generate() assert first == second assert b.program.dump() == before + + +@pytest.mark.parametrize("dtype,numpy_dtype", [(D.FLOAT32, np.float32), (D.INT32, np.int32), + (D.INT64, np.int64)]) +def test_scalar_initializer_cannot_override_ir_literal(dtype, numpy_dtype): + from scratchv.verification.ir_interpreter import IRExecutionError, IRInterpreter + + scale = Value("scale", dtype, is_constant=True, const_value=1) + b = builder() + b.program.global_values.append(scale) + b.ret(scale) + bindings = {"scale": np.array(2, dtype=numpy_dtype)} + with pytest.raises(IRExecutionError, match="initializer disagrees with scalar constant"): + IRInterpreter(b.program).run({}, initializers=bindings) + with pytest.raises(TensorCCodegenError, match="scale disagrees with scalar constant"): + TensorCCodegen(b.program, bindings).generate() + + +def test_scalar_constant_binding_cannot_be_a_vector(): + scale = Value("scale", D.FLOAT32, is_constant=True, const_value=1.0, shape=(1,)) + b = builder() + b.program.global_values.append(scale) + b.ret(scale) + with pytest.raises(TensorCCodegenError, match="scale disagrees with scalar constant"): + TensorCCodegen(b.program, {"scale": np.array([1], np.float32)}).generate() + + +def test_matching_scalar_initializer_preserves_compiled_result(tmp_path, compiler): + from scratchv.verification.ir_interpreter import IRInterpreter + + x = Value("x", shape=(2,)) + scale = Value("scale", D.FLOAT32, is_constant=True, const_value=1.25) + b = builder(x) + b.program.global_values.append(scale) + b.ret(b.mul(x, scale)) + bindings = {"scale": np.array(1.25, np.float32)} + feed = {"x": np.array([2, -3], np.float32)} + expected = IRInterpreter(b.program).run(feed, initializers=bindings).return_value + with compiled(tmp_path, TensorCCodegen(b.program, bindings).generate(), compiler) as execute: + status, actual = execute(feed) + assert status == 0 + np.testing.assert_array_equal(actual, expected) diff --git a/tests/test_w2_acceptance.py b/tests/test_w2_acceptance.py new file mode 100644 index 0000000..44a8796 --- /dev/null +++ b/tests/test_w2_acceptance.py @@ -0,0 +1,296 @@ +"""Acceptance orchestration must not turn partial, stale or failed evidence into PASS.""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +import pytest + +SPEC = importlib.util.spec_from_file_location("w2_acceptance", Path(__file__).parents[1] / "scripts/run_w2_acceptance.py") +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +def arguments(tmp_path, gates=None): + return argparse.Namespace(output_dir=tmp_path / "fresh", gates=list(runner.GATES) if gates is None else gates, + python=sys.executable, tokenizer_dir=tmp_path / "assets", tokenizer_mode="verify", + model_dir=tmp_path / "model", model_mode="verify", cc="compiler", qemu="emulator", + qemu_timeout=180, parse_timeout=300) + + +def runtime_fixture(tmp_path): + root = tmp_path / "source" + corpus = root / "probes/w2_runtime/corpus.json" + corpus.parent.mkdir(parents=True) + names = [f"case_{i}" for i in range(20)] + corpus.write_text(json.dumps({"cases": [{"name": name} for name in names]}), encoding="utf-8") + folder = tmp_path / "run/runtime" + folder.mkdir(parents=True) + for name in ("report.md", "report.html"): + (folder / name).write_text("PASS", encoding="utf-8") + sources = {runner.SCRIPTS["runtime"]: "a" * 64} + data = {"gate": "unit:runtime", "passed": True, "stage": "complete", "source_sha256": sources, + "tokenizer_cases": [{"name": name, "passed": True, "checks": 24} for name in names], + "runtime_cases": [{"name": str(i), "passed": True} for i in range(5)]} + return root, runner.Gate("runtime", (), folder), sources, data + + +def validate_fixture(fixture, mutate=lambda data: None): + root, gate, sources, data = fixture + mutate(data) + (gate.output / "report.json").write_text(json.dumps(data), encoding="utf-8") + return runner.validate_gate_report(gate, sources, root) + + +def test_plan_is_fresh_serial_and_explicit(tmp_path): + args = arguments(tmp_path) + plan = runner.build_plan(args) + assert [gate.name for gate in plan] == list(runner.GATES) + for gate in plan: + assert gate.command[:4] == (sys.executable, "-B", "-X", "utf8") + assert str(gate.output) in " ".join(gate.command) + assert plan[5].dependencies == ("small-ir",) + assert str(args.output_dir / "small-ir") in plan[5].command + assert "--mode" in plan[-1].command and "verify" in plan[-1].command + assert "compiler" in plan[1].command and "emulator" in plan[1].command + + +def test_frontend_real_pytest_tmp_path_in_fresh_gate_directory(tmp_path): + """Pytest's basetemp.mkdir has no parents=True; its parent must already exist.""" + source = tmp_path / "source" + tests = source / "tests" + tests.mkdir(parents=True) + (tests / "test_qwen3_frontend_patterns.py").write_text( + "import pytest\n@pytest.mark.parametrize('number', range(22))\n" + "def test_real_tmp_path(tmp_path, number):\n" + " path = tmp_path / str(number)\n" + " path.write_text('fresh')\n" + " assert path.read_text() == 'fresh'\n", encoding="utf-8") + args = arguments(tmp_path, ["frontend"]) + args.output_dir.mkdir() + gate = runner.build_plan(args, source)[0] + assert not gate.output.exists() + result = runner.run_gate(gate, {}, timeout=30, root=source) + assert result["status"] == "PASS", Path(result["log"]).read_text(encoding="utf-8") + assert result["validation"]["tests"] == 22 + + +@pytest.mark.parametrize("chosen,expected", [(["small-qemu"], ["small-ir", "small-qemu"]), + (["runtime-model"], ["runtime", "runtime-model"]), + (["frontend"], ["frontend"])]) +def test_subset_auto_adds_only_required_dependencies(tmp_path, chosen, expected): + assert [gate.name for gate in runner.build_plan(arguments(tmp_path, chosen))] == expected + + +@pytest.mark.parametrize("status,expected", [("PASS", "PASS"), ("NOT_RUN", "PARTIAL"), ("BLOCKED", "FAIL"), ("FAIL", "FAIL")]) +def test_overall_requires_every_gate(status, expected): + report = {"gates": [{"name": name, "status": "PASS"} for name in runner.GATES]} + report["gates"][-1]["status"] = status + runner.finalize_status(report) + assert report["status"] == expected + assert report["passed"] is (expected == "PASS") + + +def test_top_level_source_error_blocks_pass(): + report = {"error": "source changed", "gates": [{"status": "PASS"} for _ in runner.GATES]} + runner.finalize_status(report) + assert report["status"] == "FAIL" + + +def test_git_unavailable_archive_is_not_failed(monkeypatch, tmp_path): + monkeypatch.setattr(runner.subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("git"))) + result = runner.checkout_evidence(tmp_path) + assert result["head"] is None and result["status"] == "unknown" + + +def test_new_source_files_have_fingerprints(tmp_path): + path = tmp_path / "scratchv/runtime/new.py" + path.parent.mkdir(parents=True) + path.write_text("x = 1\n") + before = runner.source_fingerprints(tmp_path) + path.write_text("x = 2\n") + assert before != runner.source_fingerprints(tmp_path) + + +def test_complete_runtime_evidence(tmp_path): + assert validate_fixture(runtime_fixture(tmp_path))["tokenizer_comparisons"] == 480 + + +@pytest.mark.parametrize("mutation", [ + lambda d: d.update(gate="different"), lambda d: d.update(passed=False), + lambda d: d.update(stage="tokenizer"), lambda d: d["tokenizer_cases"].pop(), + lambda d: d["runtime_cases"].pop(), lambda d: d["tokenizer_cases"][0].update(checks=23), + lambda d: d["tokenizer_cases"][1].update(name="case_0"), + lambda d: d["runtime_cases"][0].update(passed=False), + lambda d: d.update(source_sha256={}), + lambda d: d.update(source_sha256={runner.SCRIPTS["runtime"]: "wrong"}), +]) +def test_runtime_missing_or_inconsistent_evidence_fails(tmp_path, mutation): + with pytest.raises(ValueError): + validate_fixture(runtime_fixture(tmp_path), mutation) + + +@pytest.mark.parametrize("number", [None, float("nan"), float("inf"), -1, 1e-5, 0.1, True]) +def test_invalid_numeric_evidence_fails(number): + with pytest.raises(ValueError): + runner.check_numeric_tree({"passed": True, "nested": [{"passed": True, "max_abs": number}]}) + + +@pytest.mark.parametrize("field", ["seconds", "qemu_process_wall_seconds"]) +def test_nonfinite_timing_cannot_break_machine_report(field): + with pytest.raises(ValueError): + runner.check_numeric_tree({"passed": True, field: float("nan")}) + + +@pytest.mark.parametrize("tag", ["skipped", "failure", "error"]) +def test_frontend_skips_are_failures(tmp_path, tag): + cases = [f'{"<" + tag + "/>" if i == 0 else ""}' for i in range(22)] + (tmp_path / "tests.xml").write_text("" + "".join(cases) + "") + with pytest.raises(ValueError): + runner.validate_gate_report(runner.Gate("frontend", (), tmp_path), {}) + + +def test_existing_output_is_not_read_as_pass(tmp_path): + folder = tmp_path / "old" + folder.mkdir() + (folder / "report.json").write_text('{"passed":true}') + result = runner.run_gate(runner.Gate("runtime", (), folder), {}, timeout=1) + assert result["status"] == "FAIL" and "already exists" in result["error"] + + +def test_zero_exit_without_report_is_failure(tmp_path): + gate = runner.Gate("runtime", (sys.executable, "-c", "print('PASS')"), tmp_path / "gate") + result = runner.run_gate(gate, {}, timeout=10) + assert result["returncode"] == 0 and result["status"] == "FAIL" + assert "PASS" in Path(result["log"]).read_text() + + +def test_child_error_is_preserved(tmp_path): + folder = tmp_path / "gate" + code = "import pathlib,json; p=pathlib.Path(" + repr(str(folder)) + ");p.mkdir();(p/'report.json').write_text(json.dumps({'passed':False,'stage':'assets','error':'Checksum mismatch'}));raise SystemExit(1)" + result = runner.run_gate(runner.Gate("runtime", (sys.executable, "-c", code), folder), {}, timeout=10) + assert result["status"] == "FAIL" and "Checksum mismatch" in result["error"] + assert result["child_stage"] == "assets" + + +def test_timeout_is_failure(tmp_path): + result = runner.run_gate(runner.Gate("runtime", (sys.executable, "-c", "import time;time.sleep(30)"), tmp_path / "gate"), {}, timeout=0.2) + assert result["status"] == "FAIL" and "exceeded" in result["error"] + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux nested-session SIGTERM integration") +def test_supervisor_cancels_nested_worker_session(tmp_path): + """Real probe -> _run_process -> different session, not a mocked killpg.""" + pidfile = tmp_path / "ready.json" + worker = ("import os,time,json;from pathlib import Path;" + "pid=os.getpid();stat=Path(f'/proc/{pid}/stat').read_text();" + f"Path({str(pidfile)!r}).write_text(json.dumps([pid,stat.rsplit(')',1)[1].split()[19]]));" + "time.sleep(60)") + probe = ("import sys;from scratchv.runtime.riscv_tensor import _run_process;" + f"_run_process([sys.executable,'-B','-c',{worker!r}],cwd=None,timeout=50)") + unrelated = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(60)"], + start_new_session=True) + process = subprocess.Popen([sys.executable, "-B", "-c", probe], cwd=runner.ROOT, + start_new_session=True) + pid = None + try: + deadline = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, "Probe exited before worker readiness" + assert time.monotonic() < deadline, "Worker did not become ready" + time.sleep(0.01) + # Existence can precede completion of the tiny JSON write. + while True: + try: + pid, started = json.loads(pidfile.read_text()) + break + except ValueError: + assert time.monotonic() < deadline + time.sleep(0.01) + assert os.getpgid(pid) != process.pid + runner.terminate_process_tree(process) + assert process.returncode == 143 + assert unrelated.poll() is None + deadline = time.monotonic() + 5 + while True: + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): + break + if fields[0] in {"Z", "X"} or fields[19] != started: + break + assert time.monotonic() < deadline, "Detached worker survived supervisor cleanup" + time.sleep(0.01) + finally: + for child in (process, unrelated): + if child.poll() is None: + child.kill() + child.wait(timeout=5) + if pid is not None: + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + if fields[19] == started and fields[0] not in {"Z", "X"}: + os.kill(pid, 9) + except (FileNotFoundError, ProcessLookupError): + pass + + +@pytest.mark.parametrize("filename", ["report.md", "report.html", "report.json"]) +def test_report_write_failure_cannot_publish_pass(tmp_path, monkeypatch, filename): + original = Path.replace + def replace(path, target): + if Path(target).name == filename: + raise OSError("disk failed") + return original(path, target) + monkeypatch.setattr(Path, "replace", replace) + report = {"status": "PASS", "passed": True, "gates": []} + runner.write_reports(tmp_path, report) + assert report["status"] == "FAIL" and not report["passed"] + if (tmp_path / "report.json").exists(): + assert json.loads((tmp_path / "report.json").read_text())["passed"] is False + + +def test_report_secondary_error_preserves_primary(tmp_path, monkeypatch): + monkeypatch.setattr(Path, "replace", lambda *a: (_ for _ in ()).throw(OSError("disk failed"))) + report = {"status": "FAIL", "passed": False, "gates": [], "error": "original numerical divergence"} + runner.write_reports(tmp_path, report) + assert report["error"] == "original numerical divergence" + + +def test_subset_main_is_partial_and_exit_two(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + monkeypatch.setattr(runner, "source_fingerprints", lambda: {"source": "hash"}) + monkeypatch.setattr(runner, "run_gate", lambda gate, *a, **k: {"name": gate.name, "status": "PASS", "passed": True}) + out = tmp_path / "new" + assert runner.main(["--output-dir", str(out), "--gates", "frontend"]) == 2 + report = json.loads((out / "report.json").read_text()) + assert report["status"] == "PARTIAL" and not report["passed"] + + +def test_source_mutation_during_run_fails(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + values = iter([{"source": "old"}, {"source": "new"}]) + monkeypatch.setattr(runner, "source_fingerprints", lambda: next(values)) + monkeypatch.setattr(runner, "run_gate", lambda gate, *a, **k: {"name": gate.name, "status": "PASS", "passed": True}) + assert runner.main(["--output-dir", str(tmp_path / "new"), "--gates", "frontend"]) == 1 + + +def test_missing_assets_block_and_do_not_launch_dependent_gates(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + monkeypatch.setattr(runner, "source_fingerprints", lambda: {}) + monkeypatch.setattr(runner, "run_gate", lambda *a, **k: pytest.fail("must not run without required assets")) + out = tmp_path / "new" + assert runner.main(["--output-dir", str(out), "--gates", "runtime-model"]) == 1 + data = json.loads((out / "report.json").read_text()) + assert [row["status"] for row in data["gates"] if row["name"] in {"runtime", "runtime-model"}] == ["BLOCKED", "BLOCKED"] + + +def test_help_has_no_heavy_imports(tmp_path): + code = "import runpy,sys;sys.argv=['run_w2_acceptance.py','--help'];\ntry: runpy.run_path(" + repr(str(Path(runner.__file__))) + ",run_name='__main__')\nexcept SystemExit: pass\nassert not ({'torch','numpy','onnx','transformers'} & sys.modules.keys())" + result = subprocess.run([sys.executable, "-S", "-c", code], capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr diff --git a/tests/test_w2_backend_ops_gate.py b/tests/test_w2_backend_ops_gate.py new file mode 100644 index 0000000..cf295fb --- /dev/null +++ b/tests/test_w2_backend_ops_gate.py @@ -0,0 +1,220 @@ +"""Mandatory QEMU gate failure/report handling; target execution is a separate job.""" +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from probes.w2_backend_ops import run as probe +from scratchv.runtime.riscv_tensor import RiscVTensorTimeoutError + + +@pytest.fixture +def fake_target(monkeypatch): + case = probe.build_cases()[0] + monkeypatch.setattr(probe, "build_cases", lambda: [case]) + monkeypatch.setattr(probe, "FAMILIES", (case.family,)) + monkeypatch.setattr(probe, "discover_toolchain", lambda **kwargs: SimpleNamespace(cc=("fixture-cc",), qemu="fixture-qemu")) + monkeypatch.setattr(probe, "build_riscv_tensor", lambda *args: SimpleNamespace( + elf_sha256="fixture", compile_command=("fixture-cc",), compile_seconds=0.1, + workspace_bytes=16, tool_versions={"fixture": "unit-test-only"})) + monkeypatch.setattr(probe, "run_riscv_tensor", lambda *args, **kwargs: SimpleNamespace( + output=case.expected.copy(), elapsed_s=0.25, command=("fixture-qemu",))) + return case + + +def result(tmp_path): + directory = tmp_path / "report" + code = probe.main(["--output-dir", str(directory)]) + report = json.loads((directory / "report.json").read_text(encoding="utf-8")) + assert (directory / "report.md").is_file() and (directory / "report.html").is_file() + return code, report + + +def test_missing_tools_fail_with_all_report_formats(tmp_path, monkeypatch): + def unavailable(**kwargs): + raise FileNotFoundError("QEMU unavailable") + monkeypatch.setattr(probe, "discover_toolchain", unavailable) + code, report = result(tmp_path) + assert code == 1 and report["passed"] is False and report["stage"] == "toolchain" + assert "QEMU unavailable" in report["error"] + assert report["planned_executions"] == 32 + + +def test_success_requires_both_optimization_levels(tmp_path, monkeypatch, fake_target): + code, report = result(tmp_path) + assert code == 0 and report["passed"] is True + assert report["completed_executions"] == report["planned_executions"] == 2 + assert report["passed_executions"] == 2 + assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.5} + assert report["atol"] == 1e-5 and report["rtol"] == 0 + + +@pytest.mark.parametrize("fault", ["numeric", "nan", "shape", "dtype", "second_level"]) +def test_incorrect_qemu_output_is_not_a_pass(tmp_path, monkeypatch, fake_target, fault): + calls = [] + def execute(*args, **kwargs): + actual = fake_target.expected.copy() + calls.append(1) + if fault == "numeric" or (fault == "second_level" and len(calls) == 2): + actual.flat[0] += np.float32(0.01) + elif fault == "nan": + actual.flat[0] = np.nan + elif fault == "shape": + actual = actual.flatten() + elif fault == "dtype": + actual = actual.astype(np.float64) + return SimpleNamespace(output=actual, elapsed_s=0.25, command=("fixture-qemu",)) + monkeypatch.setattr(probe, "run_riscv_tensor", execute) + code, report = result(tmp_path) + assert code == 1 and report["passed"] is False + assert report["completed_executions"] == 2 + assert report["passed_executions"] == (1 if fault == "second_level" else 0) + + +def test_qemu_timeout_preserves_measured_time_and_failures(tmp_path, monkeypatch, fake_target): + def timeout(*args, **kwargs): + raise RiscVTensorTimeoutError("QEMU test timeout", elapsed_s=0.125, + command=["fixture-qemu"], timeout_s=0.1) + monkeypatch.setattr(probe, "run_riscv_tensor", timeout) + code, report = result(tmp_path) + assert code == 1 + assert report["completed_executions"] == 0 + assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.25} + assert all(item["status"] == "timeout" for item in report["cases"][0]["executions"]) + + +def test_formula_disagreement_cannot_enter_qemu(tmp_path, monkeypatch, fake_target): + fake_target.expected.flat[0] += np.float32(1) + def unexpected(*args, **kwargs): + pytest.fail("Invalid reference must stop before QEMU") + monkeypatch.setattr(probe, "run_riscv_tensor", unexpected) + code, report = result(tmp_path) + assert code == 1 and report["cases"][0]["executions"] == [] + assert "NumPy formula" in report["cases"][0]["error"] + + +def test_compiler_failure_cannot_become_qemu_success(tmp_path, monkeypatch, fake_target): + monkeypatch.setattr(probe.CompilerDriver, "compile", lambda *args: SimpleNamespace( + success=False, errors=["fixture unsupported operator"])) + def unexpected(*args, **kwargs): + pytest.fail("Compiler failure must stop before QEMU") + monkeypatch.setattr(probe, "run_riscv_tensor", unexpected) + code, report = result(tmp_path) + assert code == 1 and report["completed_executions"] == 0 + assert all(item["stage"] == "compile" and "unsupported operator" in item["error"] + for item in report["cases"][0]["executions"]) + + +@pytest.mark.parametrize("fault", ["empty", "missing_family", "duplicate_name"]) +def test_invalid_coverage_fails_before_tools(tmp_path, monkeypatch, fault): + cases = probe.build_cases() + selected = [] if fault == "empty" else (cases[:-2] if fault == "missing_family" else cases + [cases[0]]) + monkeypatch.setattr(probe, "build_cases", lambda: selected) + def unexpected(**kwargs): + pytest.fail("Invalid coverage must stop before toolchain discovery") + monkeypatch.setattr(probe, "discover_toolchain", unexpected) + code, report = result(tmp_path) + assert code == 1 and "cover every required operator family" in report["error"] + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +def test_invalid_timeout_is_rejected(tmp_path, timeout): + with pytest.raises(SystemExit, match="2"): + probe.main(["--output-dir", str(tmp_path / "report"), "--timeout", timeout]) + + +def test_existing_report_is_preserved(tmp_path): + sentinel = tmp_path / "existing.txt" + sentinel.write_text("keep", encoding="utf-8") + with pytest.raises(SystemExit, match="2"): + probe.main(["--output-dir", str(tmp_path)]) + assert sentinel.read_text(encoding="utf-8") == "keep" + + +@pytest.mark.parametrize("filename", ["report.json", "report.md", "report.html"]) +def test_report_failure_invalidates_cli_success(tmp_path, monkeypatch, capsys, filename): + def successful_probe(out, report, **kwargs): + report.update(passed=True, stage="complete", cases=[]) + + original = Path.write_text + + def blocked(path, data, *args, **kwargs): + if path.name.removesuffix(".tmp") == filename: + raise PermissionError("blocked evidence") + return original(path, data, *args, **kwargs) + + monkeypatch.setattr(probe, "run_probe", successful_probe) + monkeypatch.setattr(Path, "write_text", blocked) + directory = tmp_path / "report" + assert probe.main(["--output-dir", str(directory)]) == 1 + for name in ("report.json", "report.md", "report.html"): + if name == filename: + continue + content = (directory / name).read_text(encoding="utf-8") + if name == "report.json": + saved = json.loads(content) + assert saved["passed"] is False and saved["stage"] == "report-write" + assert "blocked evidence" in saved["report_write_errors"][0] + else: + assert "Result: FAIL" in content + assert filename in capsys.readouterr().err + + +def test_report_failure_does_not_replace_primary_error(tmp_path, monkeypatch): + def failed_probe(out, report, **kwargs): + report.update(stage="toolchain", cases=[]) + raise RuntimeError("original missing compiler") + + original = Path.write_text + + def blocked(path, data, *args, **kwargs): + if path.name == "report.md": + raise OSError("disk full") + return original(path, data, *args, **kwargs) + + monkeypatch.setattr(probe, "run_probe", failed_probe) + monkeypatch.setattr(Path, "write_text", blocked) + directory = tmp_path / "report" + assert probe.main(["--output-dir", str(directory)]) == 1 + saved = json.loads((directory / "report.json").read_text(encoding="utf-8")) + assert saved["error"] == "RuntimeError: original missing compiler" + assert saved["stage"] == "toolchain" + assert "disk full" in saved["report_write_errors"][0] + + +def test_json_write_error_after_bytes_written_cannot_publish_pass(tmp_path, monkeypatch): + original = Path.write_text + + def written_then_failed(path, data, *args, **kwargs): + result = original(path, data, *args, **kwargs) + if path.name.removesuffix(".tmp") == "report.json": + raise OSError("late flush failure") + return result + + monkeypatch.setattr(Path, "write_text", written_then_failed) + report = {"passed": True, "stage": "complete", "cases": [], "seconds": 0.1} + probe.write_reports(tmp_path, report) + assert report["passed"] is False + assert not (tmp_path / "report.json").exists() + assert "Result: FAIL" in (tmp_path / "report.md").read_text(encoding="utf-8") + assert "late flush failure" in report["report_write_errors"][0] + + +def test_json_publish_failure_is_not_success(tmp_path, monkeypatch): + original = Path.replace + + def fail_publish(path, target): + if Path(target).name == "report.json": + raise PermissionError("JSON destination locked") + return original(path, target) + + monkeypatch.setattr(Path, "replace", fail_publish) + report = {"passed": True, "stage": "complete", "cases": [], "seconds": 0.1} + probe.write_reports(tmp_path, report) + assert report["passed"] is False + assert report["stage"] == "report-write" + assert not (tmp_path / "report.json").exists() + assert not (tmp_path / "report.json.tmp").exists() + assert "Result: FAIL" in (tmp_path / "report.md").read_text(encoding="utf-8") diff --git a/tests/test_w2_ci_results.py b/tests/test_w2_ci_results.py new file mode 100644 index 0000000..b411e65 --- /dev/null +++ b/tests/test_w2_ci_results.py @@ -0,0 +1,49 @@ +"""A skipped heavy frontend job must never yield green W2 acceptance.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/check_w2_ci_results.py" +SPEC = importlib.util.spec_from_file_location("w2_ci_results", SCRIPT) +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +def successful_jobs(): + return {name: {"result": "success"} for name in gate.REQUIRED_JOBS} + + +@pytest.mark.parametrize("name", gate.REQUIRED_JOBS) +@pytest.mark.parametrize("outcome", ["skipped", "failure", "cancelled", "pending", None, True]) +def test_required_job_must_actually_succeed(name, outcome): + results = successful_jobs() + results[name]["result"] = outcome + with pytest.raises(ValueError, match=name): + gate.validate_results(results) + + +@pytest.mark.parametrize("payload", [{}, [], None, {"llm-deploy": {"result": "success"}}, + {"llm-deploy": "success", "full-qwen3-frontend": {"result": "success"}}]) +def test_absent_or_malformed_job_results_fail(payload): + with pytest.raises(ValueError): + gate.validate_results(payload) + + +@pytest.mark.parametrize("payload,expected", [("invalid-json", 1), ("{}", 1), + (json.dumps(successful_jobs()), 0)]) +def test_cli_exit_matches_gate_status(monkeypatch, payload, expected): + monkeypatch.setenv("W2_JOB_RESULTS", payload) + result = subprocess.run([sys.executable, "-B", str(SCRIPT)], capture_output=True, + text=True, timeout=10) + assert result.returncode == expected + assert ("PASS:" if expected == 0 else "FAIL:") in result.stdout + + +def test_optional_full_ort_skip_does_not_replace_required_frontend(): + results = successful_jobs() + results["full-qwen3-onnx"] = {"result": "skipped"} + gate.validate_results(results) diff --git a/tests/test_w2_runtime_gate.py b/tests/test_w2_runtime_gate.py new file mode 100644 index 0000000..b33f478 --- /dev/null +++ b/tests/test_w2_runtime_gate.py @@ -0,0 +1,192 @@ +"""Runtime acceptance must reject missing/corrupt assets and reference drift.""" +import copy +import hashlib +import io +import json +from pathlib import Path +import urllib.error + +import pytest + +from probes.w2_runtime import run as gate + + +@pytest.fixture +def assets(tmp_path): + data = {item["name"]: ("fixture:" + item["name"]).encode() for item in gate.MANIFEST["source_files"]} + manifest = copy.deepcopy(gate.MANIFEST) + for item in manifest["source_files"]: + item["sha256"] = hashlib.sha256(data[item["name"]]).hexdigest() + source = tmp_path / "source" + source.mkdir() + for name, payload in data.items(): + (source / name).write_bytes(payload) + return source, data, manifest + + +@pytest.mark.parametrize("problem", ["missing", "corrupt", "lfs", "duplicate", "traversal", "missing_manifest_entry"]) +def test_asset_verification_rejects_incomplete_or_altered_source(assets, problem): + source, _, manifest = assets + file = source / "tokenizer.json" + if problem == "missing": + file.unlink() + elif problem == "corrupt": + file.write_bytes(b"corrupted") + elif problem == "lfs": + file.write_bytes(b"version https://git-lfs.github.com/spec/v1\n") + elif problem == "duplicate": + manifest["source_files"].append(manifest["source_files"][0]) + elif problem == "traversal": + manifest["source_files"][0]["name"] = "../config.json" + else: + manifest["source_files"].pop() + with pytest.raises((ValueError, FileNotFoundError)): + gate.verify_assets(source, manifest) + + +def test_download_only_pinned_tokenizer_files_and_reuses_verified_assets(assets, tmp_path, monkeypatch): + _, data, manifest = assets + urls = [] + def download(url, path): + urls.append(url) + Path(path).write_bytes(data[Path(path).name]) + monkeypatch.setattr(gate, "download_file", download) + out = tmp_path / "download" + result = gate.acquire_assets(out, manifest) + assert not result["reused"] and len(result["files"]) == 7 + assert len(urls) == 7 and all(manifest["revision"] in url for url in urls) + assert not any("safetensors" in url or ".onnx" in url for url in urls) + assert gate.acquire_assets(out, manifest)["reused"] + assert len(urls) == 7 + + +def test_download_hash_failure_never_installs_partial_directory(assets, tmp_path, monkeypatch): + _, _, manifest = assets + monkeypatch.setattr(gate, "download_file", lambda url, path: Path(path).write_bytes(b"wrong")) + out = tmp_path / "download" + with pytest.raises(ValueError, match="SHA256"): + gate.acquire_assets(out, manifest) + assert not out.exists() + assert not list(tmp_path.glob("qwen3-tokenizer-*")) + + +def test_corrupt_existing_cache_is_preserved_not_silently_replaced(assets, monkeypatch): + source, _, manifest = assets + (source / "tokenizer.json").write_bytes(b"keep this evidence") + monkeypatch.setattr(gate, "download_file", lambda *a: pytest.fail("must not redownload")) + with pytest.raises(ValueError, match="SHA256"): + gate.acquire_assets(source, manifest) + assert (source / "tokenizer.json").read_bytes() == b"keep this evidence" + + +def test_download_retries_only_transient_http_errors(tmp_path, monkeypatch): + attempts = [] + def open_url(*args, **kwargs): + attempts.append(1) + if len(attempts) == 1: + raise urllib.error.HTTPError("https://example.invalid/", 503, "busy", {}, None) + return io.BytesIO(b"payload") + monkeypatch.setattr(gate.urllib.request, "urlopen", open_url) + monkeypatch.setattr(gate.time, "sleep", lambda _: None) + gate.download_file("https://example.invalid/test", tmp_path / "asset") + assert len(attempts) == 2 and (tmp_path / "asset").read_bytes() == b"payload" + + +def test_download_rejects_oversize_data(tmp_path, monkeypatch): + monkeypatch.setattr(gate, "MAX_FILE_BYTES", 4) + monkeypatch.setattr(gate.urllib.request, "urlopen", lambda *a, **k: io.BytesIO(b"12345")) + with pytest.raises(ValueError, match="exceeds"): + gate.download_file("https://example.invalid/test", tmp_path / "asset") + + +def test_missing_assets_produces_failed_report_with_source_identity(tmp_path, monkeypatch): + monkeypatch.setattr(gate, "require_environment", lambda: {"fixture": True}) + out = tmp_path / "report" + assert gate.main(["--tokenizer-dir", str(tmp_path / "absent"), "--output-dir", str(out)]) == 1 + r = json.loads((out / "report.json").read_text()) + assert not r["passed"] and r["stage"] == "assets" + assert "Missing tokenizer asset" in r["error"] + assert r["source_sha256"]["scratchv/runtime/llm_inputs.py"] + assert "FAIL" in (out / "report.md").read_text() + + +def test_validation_failure_preserves_completed_cases_and_escapes_html(tmp_path, monkeypatch): + def fail(args, report): + report["stage"] = "tokenizer" + report["tokenizer_cases"].append({"name": "first", "passed": True}) + raise ValueError("bad ") + monkeypatch.setattr(gate, "run_checks", fail) + out = tmp_path / "report" + assert gate.main(["--tokenizer-dir", str(tmp_path / "fixture"), "--output-dir", str(out)]) == 1 + r = json.loads((out / "report.json").read_text()) + assert r["stage"] == "tokenizer" and len(r["tokenizer_cases"]) == 1 and not r["passed"] + assert " + +ONNX 算子 benchmark Linux 复跑入口 +

ONNX 算子 benchmark：Linux 复跑入口

+

正式环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。本页是入口，不是新测量报告。

+

请按环境与单测指南准备依赖、固定 Git 基线和 LFS 模型，再按Linux benchmark 命令生成自己的 JSON、Markdown 和 HTML。

+

当前同名 JSON 为 historical_benchmark_index，不能作为 benchmark 结果。历史个人测量不改名为 Linux 数据，也不能与不同环境的耗时直接计算加速比。

+

查看固定提交中的原始 HTML 测量

+ diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" index 9b68647..1ce68a3 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" @@ -1,2388 +1,14 @@ { + "document_kind": "historical_benchmark_index", "schema_version": 1, - "baseline": { - "commit": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "source_sha256": "a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c" + "measurement_available": false, + "reproduction_target": "Ubuntu 24.04 x86_64 / Bash / Python 3.12", + "guide": "算子补齐与单测-benchmark.md", + "original_commit": "2d07ccc42936dcfde19d552bca02a324cd33821c", + "historical_reports": { + "md": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md", + "json": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json", + "html": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html" }, - "current": { - "head": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "dirty": true, - "source_sha256": "57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf" - }, - "environment": { - "python": "3.12.13", - "platform": "Windows-11-10.0.26200-SP0", - "numpy": "2.5.3", - "onnx": "1.23.0", - "onnxruntime": "1.29.0", - "threads": { - "OMP_NUM_THREADS": "1", - "OPENBLAS_NUM_THREADS": "1", - "MKL_NUM_THREADS": "1", - "NUMEXPR_NUM_THREADS": "1", - "VECLIB_MAXIMUM_THREADS": "1" - }, - "blas": { - "Compilers": { - "c": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - }, - "cython": { - "name": "cython", - "linker": "cython", - "version": "3.3.0", - "commands": "cython" - }, - "c++": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - } - }, - "Machine Information": { - "host": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - }, - "build": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - } - }, - "Build Dependencies": { - "blas": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - }, - "lapack": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - } - }, - "Python Information": { - "version": "3.12" - }, - "SIMD Extensions": { - "baseline": [ - "X86_V2" - ], - "found": [ - "X86_V3" - ] - } - }, - "ort": "CPUExecutionProvider, sequential, one thread, optimizations disabled" - }, - "warmup": 1, - "repeats": 10, - "repeats_per_round": 5, - "rounds_per_variant": 2, - "execution_order": [ - "before", - "after", - "after", - "before" - ], - "timing_scope": "parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU.", - "binding_policy": "Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change.", - "results": [ - { - "name": "add_float32", - "group": "control", - "description": "FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs.", - "operators": [ - "Add" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba", - "inputs_sha256": "8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036", - "expected_sha256": "9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012459990102797747, - 0.00012260000221431255, - 0.00012610002886503935, - 0.00012240000069141388, - 0.00011799996718764305, - 0.00017060001846402884, - 0.00017589994240552187, - 0.0001556000206619501, - 0.00013669999316334724, - 0.0001281999284401536 - ], - "run_samples_s": [ - 9.820004925131798e-05, - 9.109999518841505e-05, - 9.620003402233124e-05, - 9.129999671131372e-05, - 8.999998681247234e-05, - 0.0001316999550908804, - 0.0001227000029757619, - 0.00012240000069141388, - 9.77999297901988e-05, - 9.400001727044582e-05 - ], - "parse_median_s": 0.00012714997865259647, - "run_median_s": 9.699998190626502e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00011799996718764305, - "parse_max_s": 0.00017589994240552187, - "run_min_s": 8.999998681247234e-05, - "run_max_s": 0.0001316999550908804 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004192000487819314, - 0.00039460009429603815, - 0.00038620003033429384, - 0.0003771999618038535, - 0.00038330000825226307, - 0.00037699996028095484, - 0.00035340001340955496, - 0.00035879993811249733, - 0.00034790008794516325, - 0.0003451999509707093 - ], - "run_samples_s": [ - 0.00011930009350180626, - 0.00010579999070614576, - 0.00010310008656233549, - 0.00010159995872527361, - 0.0001004999503493309, - 0.00010600010864436626, - 0.00010289996862411499, - 0.0001020999625325203, - 0.00010160007514059544, - 0.00010059995111078024 - ], - "parse_median_s": 0.0003770999610424042, - "run_median_s": 0.00010249996557831764, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003451999509707093, - "parse_max_s": 0.0004192000487819314, - "run_min_s": 0.0001004999503493309, - "run_max_s": 0.00011930009350180626 - }, - "fixture_files": { - "model": "add_float32.onnx", - "inputs": "add_float32.inputs.npz", - "expected": "add_float32.expected.npy", - "initializers": "add_float32.initializers.npz" - } - }, - { - "name": "matmul_32", - "group": "control", - "description": "FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs.", - "operators": [ - "MatMul" - ], - "atol": 1e-05, - "rtol": 1e-05, - "model_sha256": "c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203", - "inputs_sha256": "3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0", - "expected_sha256": "d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012470001820474863, - 0.00012370001059025526, - 0.0001233000075444579, - 0.0001382000045850873, - 0.00012390001211315393, - 0.00015219999477267265, - 0.00013090006541460752, - 0.0001327999634668231, - 0.00012919993605464697, - 0.00012370001059025526 - ], - "run_samples_s": [ - 0.00010169995948672295, - 9.799993131309748e-05, - 9.8800053820014e-05, - 9.50000248849392e-05, - 9.69000393524766e-05, - 0.00011190003715455532, - 0.00010029994882643223, - 0.00010699999984353781, - 0.00010060006752610207, - 9.679992217570543e-05 - ], - "parse_median_s": 0.0001269499771296978, - "run_median_s": 9.955000132322311e-05, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0001233000075444579, - "parse_max_s": 0.00015219999477267265, - "run_min_s": 9.50000248849392e-05, - "run_max_s": 0.00011190003715455532 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000394199974834919, - 0.00036610011011362076, - 0.00036529998760670424, - 0.0003619999624788761, - 0.00037210003938525915, - 0.00037340004928410053, - 0.00036669999826699495, - 0.0003709999145939946, - 0.00040339992847293615, - 0.000367900007404387 - ], - "run_samples_s": [ - 0.00011040002573281527, - 0.00010629999451339245, - 0.00010939990170300007, - 0.00010389997623860836, - 0.00011880008969455957, - 0.00010979990474879742, - 0.00010659999679774046, - 0.00010810000821948051, - 0.00010469998233020306, - 0.0001050999853760004 - ], - "parse_median_s": 0.0003694499609991908, - "run_median_s": 0.00010735000250861049, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003619999624788761, - "parse_max_s": 0.00040339992847293615, - "run_min_s": 0.00010389997623860836, - "run_max_s": 0.00011880008969455957 - }, - "fixture_files": { - "model": "matmul_32.onnx", - "inputs": "matmul_32.inputs.npz", - "expected": "matmul_32.expected.npy", - "initializers": "matmul_32.initializers.npz" - } - }, - { - "name": "gather", - "group": "operators", - "description": "FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4].", - "operators": [ - "Gather" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e", - "inputs_sha256": "dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf", - "expected_sha256": "4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000382099999114871, - 0.0003637999761849642, - 0.00036070006899535656, - 0.00037329993210732937, - 0.0003640999784693122, - 0.0003535000141710043, - 0.0003445000620558858, - 0.00034570007119327784, - 0.00036100007127970457, - 0.000354600022546947 - ], - "run_samples_s": [ - 0.00010419997852295637, - 0.00010269996710121632, - 0.00010080006904900074, - 0.0001020999625325203, - 0.00010219996329396963, - 0.00010059995111078024, - 0.00010079995263367891, - 0.00010109995491802692, - 0.00010179996024817228, - 0.00010019994806498289 - ], - "parse_median_s": 0.00036085007013753057, - "run_median_s": 0.0001014499575830996, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003445000620558858, - "parse_max_s": 0.000382099999114871, - "run_min_s": 0.00010019994806498289, - "run_max_s": 0.00010419997852295637 - }, - "fixture_files": { - "model": "gather.onnx", - "inputs": "gather.inputs.npz", - "expected": "gather.expected.npy", - "initializers": "gather.initializers.npz" - } - }, - { - "name": "sqrt", - "group": "operators", - "description": "Positive FP32 x[2,4] -> Sqrt [2,4].", - "operators": [ - "Sqrt" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97", - "inputs_sha256": "dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6", - "expected_sha256": "2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sqrt", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003466999623924494, - 0.0003605000674724579, - 0.0003635999746620655, - 0.00034799997229129076, - 0.00034380005672574043, - 0.00040040002204477787, - 0.0003725000424310565, - 0.0003456000704318285, - 0.0003480000887066126, - 0.0003408000338822603 - ], - "run_samples_s": [ - 0.00010339997243136168, - 9.490002412348986e-05, - 9.290000889450312e-05, - 9.370001498609781e-05, - 9.099999442696571e-05, - 9.990006219595671e-05, - 9.940005838871002e-05, - 9.69000393524766e-05, - 9.69000393524766e-05, - 9.26000066101551e-05 - ], - "parse_median_s": 0.0003480000304989517, - "run_median_s": 9.590003173798323e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003408000338822603, - "parse_max_s": 0.00040040002204477787, - "run_min_s": 9.099999442696571e-05, - "run_max_s": 0.00010339997243136168 - }, - "fixture_files": { - "model": "sqrt.onnx", - "inputs": "sqrt.inputs.npz", - "expected": "sqrt.expected.npy", - "initializers": "sqrt.initializers.npz" - } - }, - { - "name": "reduce_mean", - "group": "operators", - "description": "FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1].", - "operators": [ - "ReduceMean" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132", - "inputs_sha256": "037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17", - "expected_sha256": "43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7", - "initializers_sha256": "31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: ReduceMean", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 9.933077874436559e-08, - "parse_samples_s": [ - 0.0003934999695047736, - 0.00037949997931718826, - 0.00037489994429051876, - 0.000372299924492836, - 0.00037619995418936014, - 0.0003952999832108617, - 0.00040259992238134146, - 0.0003915999550372362, - 0.0003774999640882015, - 0.0003698000218719244 - ], - "run_samples_s": [ - 0.00011030002497136593, - 0.00010720000136643648, - 0.0001078000059351325, - 0.00010830000974237919, - 0.00010569998994469643, - 0.00010750000365078449, - 0.00010809989180415869, - 0.00010909989941865206, - 0.00010730000212788582, - 0.00010609999299049377 - ], - "parse_median_s": 0.0003784999717026949, - "run_median_s": 0.0001076500047929585, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003698000218719244, - "parse_max_s": 0.00040259992238134146, - "run_min_s": 0.00010569998994469643, - "run_max_s": 0.00011030002497136593 - }, - "fixture_files": { - "model": "reduce_mean.onnx", - "inputs": "reduce_mean.inputs.npz", - "expected": "reduce_mean.expected.npy", - "initializers": "reduce_mean.initializers.npz" - } - }, - { - "name": "transpose", - "group": "operators", - "description": "FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4].", - "operators": [ - "Transpose" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6", - "inputs_sha256": "c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a", - "expected_sha256": "b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Transpose", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003711999161168933, - 0.00037619995418936014, - 0.0003662999952211976, - 0.00041259999852627516, - 0.0003730000462383032, - 0.00035260000731796026, - 0.0003722000401467085, - 0.0003524000057950616, - 0.0003497001016512513, - 0.0003529998939484358 - ], - "run_samples_s": [ - 0.00010429997928440571, - 0.00010159995872527361, - 9.989994578063488e-05, - 0.00010140007361769676, - 0.00010170007590204477, - 0.00010349997319281101, - 9.93999419733882e-05, - 9.840005077421665e-05, - 9.790004696696997e-05, - 9.710004087537527e-05 - ], - "parse_median_s": 0.00036874995566904545, - "run_median_s": 0.00010065000969916582, - "executed_steps": 2, - "actual_shape": [ - 1, - 3, - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003497001016512513, - "parse_max_s": 0.00041259999852627516, - "run_min_s": 9.710004087537527e-05, - "run_max_s": 0.00010429997928440571 - }, - "fixture_files": { - "model": "transpose.onnx", - "inputs": "transpose.inputs.npz", - "expected": "transpose.expected.npy", - "initializers": "transpose.initializers.npz" - } - }, - { - "name": "concat", - "group": "operators", - "description": "FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4].", - "operators": [ - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977", - "inputs_sha256": "09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9", - "expected_sha256": "c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Concat", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003693000180646777, - 0.00038620003033429384, - 0.0003693000180646777, - 0.0003645999822765589, - 0.00036700000055134296, - 0.00041009997949004173, - 0.0003698000218719244, - 0.00037619995418936014, - 0.00035749992821365595, - 0.00035550002939999104 - ], - "run_samples_s": [ - 9.770004544407129e-05, - 0.00010369997471570969, - 9.69000393524766e-05, - 9.790004696696997e-05, - 9.599991608411074e-05, - 0.00010299996938556433, - 9.889993816614151e-05, - 9.699992369860411e-05, - 9.910005610436201e-05, - 9.550002869218588e-05 - ], - "parse_median_s": 0.0003693000180646777, - "run_median_s": 9.780004620552063e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00035550002939999104, - "parse_max_s": 0.00041009997949004173, - "run_min_s": 9.550002869218588e-05, - "run_max_s": 0.00010369997471570969 - }, - "fixture_files": { - "model": "concat.onnx", - "inputs": "concat.inputs.npz", - "expected": "concat.expected.npy", - "initializers": "concat.initializers.npz" - } - }, - { - "name": "slice", - "group": "operators", - "description": "FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4].", - "operators": [ - "Slice" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30", - "inputs_sha256": "04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa", - "expected_sha256": "15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df", - "initializers_sha256": "6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Slice", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00040770007763057947, - 0.0004149000160396099, - 0.00039679999463260174, - 0.000395399983972311, - 0.000394199974834919, - 0.0004005999071523547, - 0.0003872000379487872, - 0.0003947000950574875, - 0.00039820000529289246, - 0.0003963999915868044 - ], - "run_samples_s": [ - 0.00010409997776150703, - 0.00010469998233020306, - 0.00010289996862411499, - 0.00010169995948672295, - 0.00010259996633976698, - 0.00010469998233020306, - 0.00010290008503943682, - 0.0001039999770000577, - 0.00010349997319281101, - 0.00010409997776150703 - ], - "parse_median_s": 0.00039659999310970306, - "run_median_s": 0.00010374997509643435, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003872000379487872, - "parse_max_s": 0.0004149000160396099, - "run_min_s": 0.00010169995948672295, - "run_max_s": 0.00010469998233020306 - }, - "fixture_files": { - "model": "slice.onnx", - "inputs": "slice.inputs.npz", - "expected": "slice.expected.npy", - "initializers": "slice.initializers.npz" - } - }, - { - "name": "unsqueeze", - "group": "operators", - "description": "INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1].", - "operators": [ - "Unsqueeze" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd", - "inputs_sha256": "4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50", - "expected_sha256": "d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193", - "initializers_sha256": "a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Unsqueeze", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004278999986127019, - 0.00037889997474849224, - 0.0003894000547006726, - 0.0004752000095322728, - 0.00046479993034154177, - 0.00037020002491772175, - 0.00035470002330839634, - 0.0003568999236449599, - 0.0003622000804170966, - 0.0003958999877795577 - ], - "run_samples_s": [ - 8.939998224377632e-05, - 8.649996016174555e-05, - 8.329993579536676e-05, - 8.98001017048955e-05, - 8.560006972402334e-05, - 8.510006591677666e-05, - 8.279993198812008e-05, - 8.359993807971478e-05, - 8.44000605866313e-05, - 8.329993579536676e-05 - ], - "parse_median_s": 0.00038415001472458243, - "run_median_s": 8.475006325170398e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 2, - 3, - 1 - ], - "actual_dtype": "int64", - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00035470002330839634, - "parse_max_s": 0.0004752000095322728, - "run_min_s": 8.279993198812008e-05, - "run_max_s": 8.98001017048955e-05 - }, - "fixture_files": { - "model": "unsqueeze.onnx", - "inputs": "unsqueeze.inputs.npz", - "expected": "unsqueeze.expected.npy", - "initializers": "unsqueeze.initializers.npz" - } - }, - { - "name": "expand", - "group": "operators", - "description": "FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4].", - "operators": [ - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab", - "inputs_sha256": "4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6", - "expected_sha256": "eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6", - "initializers_sha256": "7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Expand", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004062999505549669, - 0.0003779999678954482, - 0.0004357000580057502, - 0.00038139999378472567, - 0.0004078999627381563, - 0.0003739000530913472, - 0.0003749999450519681, - 0.0003727000439539552, - 0.00036589999217540026, - 0.000359599944204092 - ], - "run_samples_s": [ - 0.00011890009045600891, - 0.00010760000441223383, - 0.00010830000974237919, - 0.00010639999527484179, - 0.0001023000804707408, - 0.00010429997928440571, - 0.00010469998233020306, - 0.00010419997852295637, - 0.00010289996862411499, - 0.00010410009417682886 - ], - "parse_median_s": 0.00037649995647370815, - "run_median_s": 0.00010449998080730438, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000359599944204092, - "parse_max_s": 0.0004357000580057502, - "run_min_s": 0.0001023000804707408, - "run_max_s": 0.00011890009045600891 - }, - "fixture_files": { - "model": "expand.onnx", - "inputs": "expand.inputs.npz", - "expected": "expand.expected.npy", - "initializers": "expand.initializers.npz" - } - }, - { - "name": "abs", - "group": "operators", - "description": "Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype.", - "operators": [ - "Abs" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf", - "inputs_sha256": "898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a", - "expected_sha256": "7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Abs", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00036099995486438274, - 0.0003568999236449599, - 0.0003541000187397003, - 0.0003519000019878149, - 0.0003469999646767974, - 0.00033569999504834414, - 0.00035450002178549767, - 0.00034719996619969606, - 0.0003350999904796481, - 0.000327100045979023 - ], - "run_samples_s": [ - 8.889997843652964e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.420005906373262e-05, - 8.320005144923925e-05, - 8.759996853768826e-05, - 8.549995254725218e-05, - 8.499994874000549e-05, - 8.68999632075429e-05, - 8.470006287097931e-05 - ], - "parse_median_s": 0.0003495499840937555, - "run_median_s": 8.585001341998577e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.000327100045979023, - "parse_max_s": 0.00036099995486438274, - "run_min_s": 8.320005144923925e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "abs.onnx", - "inputs": "abs.inputs.npz", - "expected": "abs.expected.npy", - "initializers": "abs.initializers.npz" - } - }, - { - "name": "cast", - "group": "operators", - "description": "INT64 position IDs[1,8] -> FP32 [1,8].", - "operators": [ - "Cast" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5", - "inputs_sha256": "93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f", - "expected_sha256": "5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.00036179996095597744, - 0.0004943999228999019, - 0.0005131000652909279, - 0.00037509994581341743, - 0.00034440006129443645, - 0.00033980002626776695, - 0.00034899997990578413, - 0.00033760000951588154, - 0.00035889993887394667 - ], - "run_samples_s": [ - 7.98999099060893e-05, - 0.00017800007481127977, - 0.00011080002877861261, - 8.82000895217061e-05, - 8.230004459619522e-05, - 7.829989772289991e-05, - 7.860001642256975e-05, - 7.840001489967108e-05, - 8.060003165155649e-05, - 7.970002479851246e-05 - ], - "parse_median_s": 0.00036034994991496205, - "run_median_s": 8.02499707788229e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033760000951588154, - "parse_max_s": 0.0005131000652909279, - "run_min_s": 7.829989772289991e-05, - "run_max_s": 0.00017800007481127977 - }, - "fixture_files": { - "model": "cast.onnx", - "inputs": "cast.inputs.npz", - "expected": "cast.expected.npy", - "initializers": "cast.initializers.npz" - } - }, - { - "name": "constant", - "group": "operators", - "description": "Constant tensor attribute FP32 [2,3]; no runtime inputs.", - "operators": [ - "Constant" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.000381799996830523, - 0.00039549998473376036, - 0.000381799996830523, - 0.0003598000621423125, - 0.00036259996704757214, - 0.0003591999411582947, - 0.0003554000286385417, - 0.0003502999898046255, - 0.0003415999235585332 - ], - "run_samples_s": [ - 6.250001024454832e-05, - 6.170000415295362e-05, - 6.10999995842576e-05, - 6.200000643730164e-05, - 6.0399994254112244e-05, - 6.150000263005495e-05, - 6.219989154487848e-05, - 6.0399994254112244e-05, - 6.049999501556158e-05, - 5.909998435527086e-05 - ], - "parse_median_s": 0.00036120001459494233, - "run_median_s": 6.130000110715628e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003415999235585332, - "parse_max_s": 0.00039549998473376036, - "run_min_s": 5.909998435527086e-05, - "run_max_s": 6.250001024454832e-05 - }, - "fixture_files": { - "model": "constant.onnx", - "inputs": "constant.inputs.npz", - "expected": "constant.expected.npy", - "initializers": "constant.initializers.npz" - } - }, - { - "name": "cos", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4].", - "operators": [ - "Cos" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298", - "inputs_sha256": "8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998", - "expected_sha256": "7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cos", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 5.960464477539063e-08, - "max_rel_error": 8.023689783612319e-08, - "parse_samples_s": [ - 0.0003530000103637576, - 0.00034549995325505733, - 0.00033980002626776695, - 0.000340400030836463, - 0.0003451999509707093, - 0.00034739996772259474, - 0.000332699972204864, - 0.0003412999212741852, - 0.00033599999733269215, - 0.00033199996687471867 - ], - "run_samples_s": [ - 8.690007962286472e-05, - 8.670007809996605e-05, - 8.889997843652964e-05, - 8.739996701478958e-05, - 8.559995330870152e-05, - 8.60999571159482e-05, - 8.629995863884687e-05, - 8.690007962286472e-05, - 8.579995483160019e-05, - 8.400005754083395e-05 - ], - "parse_median_s": 0.0003408499760553241, - "run_median_s": 8.650001836940646e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033199996687471867, - "parse_max_s": 0.0003530000103637576, - "run_min_s": 8.400005754083395e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "cos.onnx", - "inputs": "cos.inputs.npz", - "expected": "cos.expected.npy", - "initializers": "cos.initializers.npz" - } - }, - { - "name": "identity", - "group": "operators", - "description": "INT64 indices[2,3] -> Identity [2,3]; frontend alias.", - "operators": [ - "Identity" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b", - "inputs_sha256": "19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413", - "expected_sha256": "a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Identity", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000341000035405159, - 0.00034380005672574043, - 0.0003355999942868948, - 0.0003361001145094633, - 0.0003378000110387802, - 0.00033820001408457756, - 0.0003301999531686306, - 0.00032760004978626966, - 0.0003744999412447214, - 0.0003630999708548188 - ], - "run_samples_s": [ - 5.170004442334175e-05, - 5.280005279928446e-05, - 5.219993181526661e-05, - 5.2200048230588436e-05, - 5.080003757029772e-05, - 5.179992876946926e-05, - 5.079992115497589e-05, - 5.139992572367191e-05, - 6.809993647038937e-05, - 5.4600066505372524e-05 - ], - "parse_median_s": 0.0003380000125616789, - "run_median_s": 5.1999930292367935e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00032760004978626966, - "parse_max_s": 0.0003744999412447214, - "run_min_s": 5.079992115497589e-05, - "run_max_s": 6.809993647038937e-05 - }, - "fixture_files": { - "model": "identity.onnx", - "inputs": "identity.inputs.npz", - "expected": "identity.expected.npy", - "initializers": "identity.initializers.npz" - } - }, - { - "name": "pow", - "group": "operators", - "description": "Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3].", - "operators": [ - "Pow" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771", - "inputs_sha256": "b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1", - "expected_sha256": "dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004491000436246395, - 0.00043999997433274984, - 0.00040019990410655737, - 0.00037569995038211346, - 0.00038789992686361074, - 0.0003453000681474805, - 0.00048369995784014463, - 0.00044920004438608885, - 0.00047619990073144436, - 0.0004214000655338168 - ], - "run_samples_s": [ - 0.00014999997802078724, - 0.00014180003199726343, - 0.00011959997937083244, - 0.00011729996185749769, - 0.00011809996794909239, - 0.00011589995119720697, - 0.00015820004045963287, - 0.00015029998030513525, - 0.00015840004198253155, - 0.0001241000136360526 - ], - "parse_median_s": 0.00043070001993328333, - "run_median_s": 0.00013295002281665802, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003453000681474805, - "parse_max_s": 0.00048369995784014463, - "run_min_s": 0.00011589995119720697, - "run_max_s": 0.00015840004198253155 - }, - "fixture_files": { - "model": "pow.onnx", - "inputs": "pow.inputs.npz", - "expected": "pow.expected.npy", - "initializers": "pow.initializers.npz" - } - }, - { - "name": "reciprocal", - "group": "operators", - "description": "Nonzero signed FP32 x[2,4] -> Reciprocal [2,4].", - "operators": [ - "Reciprocal" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3", - "inputs_sha256": "fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f", - "expected_sha256": "c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Reciprocal", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00035170000046491623, - 0.0003496999852359295, - 0.00034989998675882816, - 0.00034769997000694275, - 0.00033870001789182425, - 0.00037759996484965086, - 0.00035119999665766954, - 0.0003469000803306699, - 0.00033660000190138817, - 0.00033329997677356005 - ], - "run_samples_s": [ - 9.190000128000975e-05, - 9.019998833537102e-05, - 9.029998909682035e-05, - 9.14999982342124e-05, - 8.899997919797897e-05, - 9.609991684556007e-05, - 9.510002564638853e-05, - 9.06999921426177e-05, - 9.109999518841505e-05, - 9.059999138116837e-05 - ], - "parse_median_s": 0.0003486999776214361, - "run_median_s": 9.089999366551638e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033329997677356005, - "parse_max_s": 0.00037759996484965086, - "run_min_s": 8.899997919797897e-05, - "run_max_s": 9.609991684556007e-05 - }, - "fixture_files": { - "model": "reciprocal.onnx", - "inputs": "reciprocal.inputs.npz", - "expected": "reciprocal.expected.npy", - "initializers": "reciprocal.initializers.npz" - } - }, - { - "name": "sin", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4].", - "operators": [ - "Sin" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a", - "inputs_sha256": "efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e", - "expected_sha256": "e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sin", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00034879997838288546, - 0.0003532000118866563, - 0.00035470002330839634, - 0.0003448999486863613, - 0.00033730000723153353, - 0.00033710000570863485, - 0.0003510999958962202, - 0.0003322999691590667, - 0.00032819993793964386, - 0.000327700050547719 - ], - "run_samples_s": [ - 8.570007048547268e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.480006363242865e-05, - 8.730008266866207e-05, - 8.71000811457634e-05, - 8.659996092319489e-05, - 8.569995407015085e-05, - 8.390005677938461e-05, - 8.750008419156075e-05 - ], - "parse_median_s": 0.0003410999779589474, - "run_median_s": 8.640001760795712e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000327700050547719, - "parse_max_s": 0.00035470002330839634, - "run_min_s": 8.390005677938461e-05, - "run_max_s": 8.750008419156075e-05 - }, - "fixture_files": { - "model": "sin.onnx", - "inputs": "sin.inputs.npz", - "expected": "sin.expected.npy", - "initializers": "sin.initializers.npz" - } - }, - { - "name": "scalar_fp32_add_chain", - "group": "regressions", - "description": "FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output.", - "operators": [ - "Add" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a", - "initializers_sha256": "32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a", - "before": { - "status": "NUMERIC_ERROR", - "phase": "compare", - "error": "NumericMismatch: output differs from ORT: max_abs_error=2", - "max_abs_error": 2.0, - "max_rel_error": 1.1920928955078125e-07, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0005631999811157584, - 0.0004637000383809209, - 0.0004089000867679715, - 0.00039579998701810837, - 0.0003945999778807163, - 0.000381799996830523, - 0.00041089998558163643, - 0.00040959997568279505, - 0.0003945999778807163, - 0.0003902000607922673 - ], - "run_samples_s": [ - 0.00019599997904151678, - 0.00015169999096542597, - 0.00014780007768422365, - 0.0001476999605074525, - 0.0001452999422326684, - 0.00015129998791962862, - 0.00014889996964484453, - 0.0001441999338567257, - 0.00014589994680136442, - 0.0001444999361410737 - ], - "parse_median_s": 0.00040235003689303994, - "run_median_s": 0.00014775001909583807, - "executed_steps": 5, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": 0.000381799996830523, - "parse_max_s": 0.0005631999811157584, - "run_min_s": 0.0001441999338567257, - "run_max_s": 0.00019599997904151678 - }, - "fixture_files": { - "model": "scalar_fp32_add_chain.onnx", - "inputs": "scalar_fp32_add_chain.inputs.npz", - "expected": "scalar_fp32_add_chain.expected.npy", - "initializers": "scalar_fp32_add_chain.initializers.npz" - } - }, - { - "name": "singleton_initializer", - "group": "regressions", - "description": "FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar.", - "operators": [], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8", - "initializers_sha256": "7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df", - "before": { - "status": "EXECUTION_ERROR", - "phase": "execute", - "error": "IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003521000035107136, - 0.00034039991442114115, - 0.00033650000113993883, - 0.0003445999464020133, - 0.0003336999798193574, - 0.0003494999837130308, - 0.0003339999821037054, - 0.00035889993887394667, - 0.00032849994022399187, - 0.00033849989995360374 - ], - "run_samples_s": [ - 6.290001329034567e-05, - 6.280001252889633e-05, - 6.140000186860561e-05, - 6.009999196976423e-05, - 6.079999729990959e-05, - 6.370001938194036e-05, - 6.200000643730164e-05, - 6.330001633614302e-05, - 6.260001100599766e-05, - 6.0399994254112244e-05 - ], - "parse_median_s": 0.00033944990718737245, - "run_median_s": 6.230000872164965e-05, - "executed_steps": 1, - "actual_shape": [ - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00032849994022399187, - "parse_max_s": 0.00035889993887394667, - "run_min_s": 6.009999196976423e-05, - "run_max_s": 6.370001938194036e-05 - }, - "fixture_files": { - "model": "singleton_initializer.onnx", - "inputs": "singleton_initializer.inputs.npz", - "expected": "singleton_initializer.expected.npy", - "initializers": "singleton_initializer.initializers.npz" - } - }, - { - "name": "constant_cast_abs_expand", - "group": "composite", - "description": "Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3].", - "operators": [ - "Constant", - "Cast", - "Abs", - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3", - "inputs_sha256": "9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822", - "expected_sha256": "eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00043970008846372366, - 0.00044329999946057796, - 0.0004450000124052167, - 0.0004384000785648823, - 0.0006199999479576945, - 0.00047199998516589403, - 0.00045169994700700045, - 0.00044700002763420343, - 0.00044139998499304056, - 0.00043539993930608034 - ], - "run_samples_s": [ - 0.00013560010120272636, - 0.0001338999718427658, - 0.00013189995661377907, - 0.0002367999404668808, - 0.00017560005653649569, - 0.00013229995965957642, - 0.0001343999756500125, - 0.000130399945192039, - 0.00013109995052218437, - 0.00013479997869580984 - ], - "parse_median_s": 0.00044415000593289733, - "run_median_s": 0.00013414997374638915, - "executed_steps": 4, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00043539993930608034, - "parse_max_s": 0.0006199999479576945, - "run_min_s": 0.000130399945192039, - "run_max_s": 0.0002367999404668808 - }, - "fixture_files": { - "model": "constant_cast_abs_expand.onnx", - "inputs": "constant_cast_abs_expand.inputs.npz", - "expected": "constant_cast_abs_expand.expected.npy", - "initializers": "constant_cast_abs_expand.initializers.npz" - } - }, - { - "name": "qwen_rmsnorm", - "group": "composite", - "description": "FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8].", - "operators": [ - "Pow", - "ReduceMean", - "Add", - "Sqrt", - "Reciprocal", - "Mul" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237", - "inputs_sha256": "f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48", - "expected_sha256": "5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec", - "initializers_sha256": "6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000506900018081069, - 0.0005015999777242541, - 0.0005096999229863286, - 0.0005014999769628048, - 0.0004960000514984131, - 0.0005388999124988914, - 0.0005028999876230955, - 0.000512199942022562, - 0.000493900035507977, - 0.0004851999692618847 - ], - "run_samples_s": [ - 0.0002813000464811921, - 0.0002788000274449587, - 0.0002808000426739454, - 0.00028170004952698946, - 0.00027820002287626266, - 0.0002930000191554427, - 0.00028490007389336824, - 0.00028249993920326233, - 0.0002837000647559762, - 0.0002848999574780464 - ], - "parse_median_s": 0.0005022499826736748, - "run_median_s": 0.0002820999943651259, - "executed_steps": 10, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0004851999692618847, - "parse_max_s": 0.0005388999124988914, - "run_min_s": 0.00027820002287626266, - "run_max_s": 0.0002930000191554427 - }, - "fixture_files": { - "model": "qwen_rmsnorm.onnx", - "inputs": "qwen_rmsnorm.inputs.npz", - "expected": "qwen_rmsnorm.expected.npy", - "initializers": "qwen_rmsnorm.initializers.npz" - } - }, - { - "name": "rope_numeric", - "group": "composite", - "description": "INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8].", - "operators": [ - "Cast", - "Mul", - "Cos", - "Sin", - "Sub", - "Add", - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa", - "inputs_sha256": "823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a", - "expected_sha256": "eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 1.698927025647682e-07, - "parse_samples_s": [ - 0.0005266999360173941, - 0.000520200002938509, - 0.0005088000325486064, - 0.0006041000597178936, - 0.0006553001003339887, - 0.0005080000264570117, - 0.0005031999899074435, - 0.0005010999739170074, - 0.0005029999883845448, - 0.00049919995944947 - ], - "run_samples_s": [ - 0.0002498000394552946, - 0.00024560000747442245, - 0.0002465000143274665, - 0.0003208999987691641, - 0.00031370006036013365, - 0.00025370006915181875, - 0.0002449000021442771, - 0.00024610001128166914, - 0.0002482000272721052, - 0.0002460000105202198 - ], - "parse_median_s": 0.000508400029502809, - "run_median_s": 0.00024735002079978585, - "executed_steps": 12, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00049919995944947, - "parse_max_s": 0.0006553001003339887, - "run_min_s": 0.0002449000021442771, - "run_max_s": 0.0003208999987691641 - }, - "fixture_files": { - "model": "rope_numeric.onnx", - "inputs": "rope_numeric.inputs.npz", - "expected": "rope_numeric.expected.npy", - "initializers": "rope_numeric.initializers.npz" - } - }, - { - "name": "pr89_transformer_2l", - "group": "composite", - "operators": [ - "Add", - "Concat", - "Div", - "Expand", - "Gather", - "MatMul", - "Mul", - "Neg", - "ReduceMean", - "Reshape", - "Sigmoid", - "Slice", - "Softmax", - "Sqrt", - "Transpose", - "Unsqueeze" - ], - "description": "PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3", - "atol": 1e-05, - "rtol": 0, - "model_sha256": "17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1", - "inputs_sha256": "bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457", - "expected_sha256": "413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c", - "initializers_sha256": "955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 7.152557373046875e-07, - "max_rel_error": 0.0005722416551695097, - "parse_samples_s": [ - 0.004760700045153499, - 0.004594899946823716, - 0.004717700066976249, - 0.004990800051018596, - 0.004586199996992946, - 0.004221900017000735, - 0.004203699994832277, - 0.004572000005282462, - 0.004674999974668026, - 0.004224899923428893 - ], - "run_samples_s": [ - 0.010629900032654405, - 0.011460799956694245, - 0.009812200092710555, - 0.009985200013034046, - 0.010765800019726157, - 0.009604100021533668, - 0.010238799965009093, - 0.009660700103268027, - 0.010715999989770353, - 0.009805700043216348 - ], - "parse_median_s": 0.004590549971908331, - "run_median_s": 0.01011199998902157, - "executed_steps": 159, - "actual_shape": [ - 1, - 256, - 128 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": 0.004203699994832277, - "parse_max_s": 0.004990800051018596, - "run_min_s": 0.009604100021533668, - "run_max_s": 0.011460799956694245 - }, - "fixture_files": { - "model": "pr89_transformer_2l.onnx", - "inputs": "pr89_transformer_2l.inputs.npz", - "expected": "pr89_transformer_2l.expected.npy", - "initializers": "pr89_transformer_2l.initializers.npz" - } - } - ], - "passed": true, - "archive_note": "归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。" + "note": "Index only; not a benchmark measurement or a Linux execution result. Generate a new report with benchmarks.bench_onnx_operators." } diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" index 87d5a06..2428b17 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" @@ -1,1228 +1,25 @@ -# ONNX 算子支持与正确性：修改前后对照 +# ONNX 算子补齐 benchmark：Linux 复跑入口 -> 归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。 +正式交付与复现使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。本页是入口，**不是新测量报告**。环境、Git 历史和 LFS 准备见 [算子补齐与单测报告](算子补齐与单测报告.txt)，通用交付约定见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。 -总验收：通过。修改后 24/24 个用例通过；原先不支持、现通过的用例 20 个，原先错误、现通过的用例 2 个，出现回归的用例 0 个。用例数不等于新增算子种类数。 +在仓库根目录激活 `.venv-operators` 后执行： -本次改动扩展算子支持并修正计算语义；计时用于本地观测，不代表性能优化结论。只有修改前后均 PASS 的用例才计算执行耗时比。不支持或缺少有效测量时显示“—”，不能解释为零耗时或无限加速。 - -## 基线兼容适配与输入绑定 - -输入绑定兼容适配不属于基线编译器实现；以下为 runner 的实际策略，需与数值结果一起审阅。 - -> Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change. - -## 按组通过情况 - -| 分组 | 用例总数 | 修改前 PASS | 修改后 PASS | -| --- | --- | --- | --- | -| 独立算子 | 16 | 0/16 | 16/16 | -| 已有支持对照 | 2 | 2/2 | 2/2 | -| 错误回归 | 2 | 0/2 | 2/2 | -| 组合链路 | 4 | 0/4 | 4/4 | - -## 版本与测量条件 - -| 项目 | 值 | -| --- | --- | -| 报告 schema | 1 | -| 基线提交 | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 基线源码 SHA-256 | a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c | -| 当前 HEAD | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 当前包含未提交改动 | 是 | -| 当前源码 SHA-256 | 57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf | -| 执行顺序 | ABBA：修改前 → 修改后 → 修改后 → 修改前 | -| 每版本轮数 | 2 | -| 每轮预热次数 | 1 | -| 每轮正式计时次数 | 5 | -| 每版本正式样本总数 | 10 | -| 计时统计 | 合并各轮正式样本后计算中位数（ms），预热不计入；原始秒数见用例详情 | -| runner 计时说明 | parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU. | - -执行计时覆盖完整 IRInterpreter.run，包括 IR 校验与输入/权重复制；不含 ONNX 解析和 ONNX Runtime 参考计算。ONNX → IR 解析单独计时。 - -环境： - -| 项目 | 值 | -| --- | --- | -| blas | {"Build Dependencies": {"blas": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}, "lapack": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}}, "Compilers": {"c": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "c++": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "cython": {"commands": "cython", "linker": "cython", "name": "cython", "version": "3.3.0"}}, "Machine Information": {"build": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}, "host": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}}, "Python Information": {"version": "3.12"}, "SIMD Extensions": {"baseline": ["X86_V2"], "found": ["X86_V3"]}} | -| numpy | 2.5.3 | -| onnx | 1.23.0 | -| onnxruntime | 1.29.0 | -| ort | CPUExecutionProvider, sequential, one thread, optimizations disabled | -| platform | Windows-11-10.0.26200-SP0 | -| python | 3.12.13 | -| threads | {"MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1"} | - -## 全部用例 - -耗时比 = 修改前 / 修改后；大于 1 表示该次观测中修改后较快，小于 1 表示较慢。 - -| 用例 | 分组 | 修改前 | 修改后 | 前最大绝对误差 | 后最大绝对误差 | 前解析 ms | 后解析 ms | 前执行 ms | 后执行 ms | 执行耗时比 前/后 | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| add_float32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1271 | 0.3771 | 0.0970 | 0.1025 | 0.946× | -| matmul_32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1269 | 0.3694 | 0.0996 | 0.1074 | 0.927× | -| gather | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3609 | — | 0.1014 | — | -| sqrt | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3480 | — | 0.0959 | — | -| reduce_mean | 独立算子 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.3785 | — | 0.1077 | — | -| transpose | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3687 | — | 0.1007 | — | -| concat | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3693 | — | 0.0978 | — | -| slice | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3966 | — | 0.1037 | — | -| unsqueeze | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3842 | — | 0.0848 | — | -| expand | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3765 | — | 0.1045 | — | -| abs | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3495 | — | 0.0859 | — | -| cast | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3603 | — | 0.0802 | — | -| constant | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3612 | — | 0.0613 | — | -| cos | 独立算子 | UNSUPPORTED | PASS | — | 5.960e-08 | — | 0.3408 | — | 0.0865 | — | -| identity | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3380 | — | 0.0520 | — | -| pow | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4307 | — | 0.1330 | — | -| reciprocal | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3487 | — | 0.0909 | — | -| sin | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3411 | — | 0.0864 | — | -| scalar_fp32_add_chain | 错误回归 | NUMERIC_ERROR | PASS | 2.000e+00 | 0.000e+00 | — | 0.4024 | — | 0.1478 | — | -| singleton_initializer | 错误回归 | EXECUTION_ERROR | PASS | — | 0.000e+00 | — | 0.3394 | — | 0.0623 | — | -| constant_cast_abs_expand | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4442 | — | 0.1341 | — | -| qwen_rmsnorm | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.5022 | — | 0.2821 | — | -| rope_numeric | 组合链路 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.5084 | — | 0.2474 | — | -| pr89_transformer_2l | 组合链路 | UNSUPPORTED | PASS | — | 7.153e-07 | — | 4.5905 | — | 10.1120 | — | - -## 用例与错误详情 - -
add_float32 · 前后均通过 - -FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba | -| 输入 SHA-256 | 8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036 | -| 参考输出 SHA-256 (expected_sha256) | 9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012459990102797747, 0.00012260000221431255, 0.00012610002886503935, 0.00012240000069141388, 0.00011799996718764305, 0.00017060001846402884, 0.00017589994240552187, 0.0001556000206619501, 0.00013669999316334724, 0.0001281999284401536] | -| 执行原始样本 s | [9.820004925131798e-05, 9.109999518841505e-05, 9.620003402233124e-05, 9.129999671131372e-05, 8.999998681247234e-05, 0.0001316999550908804, 0.0001227000029757619, 0.00012240000069141388, 9.77999297901988e-05, 9.400001727044582e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004192000487819314, 0.00039460009429603815, 0.00038620003033429384, 0.0003771999618038535, 0.00038330000825226307, 0.00037699996028095484, 0.00035340001340955496, 0.00035879993811249733, 0.00034790008794516325, 0.0003451999509707093] | -| 执行原始样本 s | [0.00011930009350180626, 0.00010579999070614576, 0.00010310008656233549, 0.00010159995872527361, 0.0001004999503493309, 0.00010600010864436626, 0.00010289996862411499, 0.0001020999625325203, 0.00010160007514059544, 0.00010059995111078024] | - -
- -
matmul_32 · 前后均通过 - -FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["MatMul"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 1e-05 | -| 模型 SHA-256 | c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203 | -| 输入 SHA-256 | 3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0 | -| 参考输出 SHA-256 (expected_sha256) | d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012470001820474863, 0.00012370001059025526, 0.0001233000075444579, 0.0001382000045850873, 0.00012390001211315393, 0.00015219999477267265, 0.00013090006541460752, 0.0001327999634668231, 0.00012919993605464697, 0.00012370001059025526] | -| 执行原始样本 s | [0.00010169995948672295, 9.799993131309748e-05, 9.8800053820014e-05, 9.50000248849392e-05, 9.69000393524766e-05, 0.00011190003715455532, 0.00010029994882643223, 0.00010699999984353781, 0.00010060006752610207, 9.679992217570543e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000394199974834919, 0.00036610011011362076, 0.00036529998760670424, 0.0003619999624788761, 0.00037210003938525915, 0.00037340004928410053, 0.00036669999826699495, 0.0003709999145939946, 0.00040339992847293615, 0.000367900007404387] | -| 执行原始样本 s | [0.00011040002573281527, 0.00010629999451339245, 0.00010939990170300007, 0.00010389997623860836, 0.00011880008969455957, 0.00010979990474879742, 0.00010659999679774046, 0.00010810000821948051, 0.00010469998233020306, 0.0001050999853760004] | - -
- -
gather · 原先不支持 → 现通过 - -FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Gather"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e | -| 输入 SHA-256 | dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf | -| 参考输出 SHA-256 (expected_sha256) | 4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000382099999114871, 0.0003637999761849642, 0.00036070006899535656, 0.00037329993210732937, 0.0003640999784693122, 0.0003535000141710043, 0.0003445000620558858, 0.00034570007119327784, 0.00036100007127970457, 0.000354600022546947] | -| 执行原始样本 s | [0.00010419997852295637, 0.00010269996710121632, 0.00010080006904900074, 0.0001020999625325203, 0.00010219996329396963, 0.00010059995111078024, 0.00010079995263367891, 0.00010109995491802692, 0.00010179996024817228, 0.00010019994806498289] | - -
- -
sqrt · 原先不支持 → 现通过 - -Positive FP32 x[2,4] -> Sqrt [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sqrt"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97 | -| 输入 SHA-256 | dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6 | -| 参考输出 SHA-256 (expected_sha256) | 2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sqrt -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003466999623924494, 0.0003605000674724579, 0.0003635999746620655, 0.00034799997229129076, 0.00034380005672574043, 0.00040040002204477787, 0.0003725000424310565, 0.0003456000704318285, 0.0003480000887066126, 0.0003408000338822603] | -| 执行原始样本 s | [0.00010339997243136168, 9.490002412348986e-05, 9.290000889450312e-05, 9.370001498609781e-05, 9.099999442696571e-05, 9.990006219595671e-05, 9.940005838871002e-05, 9.69000393524766e-05, 9.69000393524766e-05, 9.26000066101551e-05] | - -
- -
reduce_mean · 原先不支持 → 现通过 - -FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["ReduceMean"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132 | -| 输入 SHA-256 | 037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17 | -| 参考输出 SHA-256 (expected_sha256) | 43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7 | -| 初始化器 SHA-256 (initializers_sha256) | 31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: ReduceMean -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 1] / float32 | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 9.933e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003934999695047736, 0.00037949997931718826, 0.00037489994429051876, 0.000372299924492836, 0.00037619995418936014, 0.0003952999832108617, 0.00040259992238134146, 0.0003915999550372362, 0.0003774999640882015, 0.0003698000218719244] | -| 执行原始样本 s | [0.00011030002497136593, 0.00010720000136643648, 0.0001078000059351325, 0.00010830000974237919, 0.00010569998994469643, 0.00010750000365078449, 0.00010809989180415869, 0.00010909989941865206, 0.00010730000212788582, 0.00010609999299049377] | - -
- -
transpose · 原先不支持 → 现通过 - -FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Transpose"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6 | -| 输入 SHA-256 | c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a | -| 参考输出 SHA-256 (expected_sha256) | b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Transpose -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 3, 2, 4] / float32 | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003711999161168933, 0.00037619995418936014, 0.0003662999952211976, 0.00041259999852627516, 0.0003730000462383032, 0.00035260000731796026, 0.0003722000401467085, 0.0003524000057950616, 0.0003497001016512513, 0.0003529998939484358] | -| 执行原始样本 s | [0.00010429997928440571, 0.00010159995872527361, 9.989994578063488e-05, 0.00010140007361769676, 0.00010170007590204477, 0.00010349997319281101, 9.93999419733882e-05, 9.840005077421665e-05, 9.790004696696997e-05, 9.710004087537527e-05] | - -
- -
concat · 原先不支持 → 现通过 - -FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977 | -| 输入 SHA-256 | 09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9 | -| 参考输出 SHA-256 (expected_sha256) | c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Concat -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003693000180646777, 0.00038620003033429384, 0.0003693000180646777, 0.0003645999822765589, 0.00036700000055134296, 0.00041009997949004173, 0.0003698000218719244, 0.00037619995418936014, 0.00035749992821365595, 0.00035550002939999104] | -| 执行原始样本 s | [9.770004544407129e-05, 0.00010369997471570969, 9.69000393524766e-05, 9.790004696696997e-05, 9.599991608411074e-05, 0.00010299996938556433, 9.889993816614151e-05, 9.699992369860411e-05, 9.910005610436201e-05, 9.550002869218588e-05] | - -
- -
slice · 原先不支持 → 现通过 - -FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Slice"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30 | -| 输入 SHA-256 | 04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa | -| 参考输出 SHA-256 (expected_sha256) | 15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df | -| 初始化器 SHA-256 (initializers_sha256) | 6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Slice +```bash +set -euo pipefail +source .venv-operators/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 ``` -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00040770007763057947, 0.0004149000160396099, 0.00039679999463260174, 0.000395399983972311, 0.000394199974834919, 0.0004005999071523547, 0.0003872000379487872, 0.0003947000950574875, 0.00039820000529289246, 0.0003963999915868044] | -| 执行原始样本 s | [0.00010409997776150703, 0.00010469998233020306, 0.00010289996862411499, 0.00010169995948672295, 0.00010259996633976698, 0.00010469998233020306, 0.00010290008503943682, 0.0001039999770000577, 0.00010349997319281101, 0.00010409997776150703] | - -
- -
unsqueeze · 原先不支持 → 现通过 - -INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Unsqueeze"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd | -| 输入 SHA-256 | 4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50 | -| 参考输出 SHA-256 (expected_sha256) | d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193 | -| 初始化器 SHA-256 (initializers_sha256) | a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Unsqueeze -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 2, 3, 1] / int64 | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004278999986127019, 0.00037889997474849224, 0.0003894000547006726, 0.0004752000095322728, 0.00046479993034154177, 0.00037020002491772175, 0.00035470002330839634, 0.0003568999236449599, 0.0003622000804170966, 0.0003958999877795577] | -| 执行原始样本 s | [8.939998224377632e-05, 8.649996016174555e-05, 8.329993579536676e-05, 8.98001017048955e-05, 8.560006972402334e-05, 8.510006591677666e-05, 8.279993198812008e-05, 8.359993807971478e-05, 8.44000605866313e-05, 8.329993579536676e-05] | - -
- -
expand · 原先不支持 → 现通过 - -FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab | -| 输入 SHA-256 | 4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6 | -| 参考输出 SHA-256 (expected_sha256) | eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6 | -| 初始化器 SHA-256 (initializers_sha256) | 7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Expand -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004062999505549669, 0.0003779999678954482, 0.0004357000580057502, 0.00038139999378472567, 0.0004078999627381563, 0.0003739000530913472, 0.0003749999450519681, 0.0003727000439539552, 0.00036589999217540026, 0.000359599944204092] | -| 执行原始样本 s | [0.00011890009045600891, 0.00010760000441223383, 0.00010830000974237919, 0.00010639999527484179, 0.0001023000804707408, 0.00010429997928440571, 0.00010469998233020306, 0.00010419997852295637, 0.00010289996862411499, 0.00010410009417682886] | - -
- -
abs · 原先不支持 → 现通过 - -Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Abs"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf | -| 输入 SHA-256 | 898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a | -| 参考输出 SHA-256 (expected_sha256) | 7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Abs -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00036099995486438274, 0.0003568999236449599, 0.0003541000187397003, 0.0003519000019878149, 0.0003469999646767974, 0.00033569999504834414, 0.00035450002178549767, 0.00034719996619969606, 0.0003350999904796481, 0.000327100045979023] | -| 执行原始样本 s | [8.889997843652964e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.420005906373262e-05, 8.320005144923925e-05, 8.759996853768826e-05, 8.549995254725218e-05, 8.499994874000549e-05, 8.68999632075429e-05, 8.470006287097931e-05] | - -
- -
cast · 原先不支持 → 现通过 - -INT64 position IDs[1,8] -> FP32 [1,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5 | -| 输入 SHA-256 | 93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f | -| 参考输出 SHA-256 (expected_sha256) | 5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 8] / float32 | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003632999723777175, 0.00036179996095597744, 0.0004943999228999019, 0.0005131000652909279, 0.00037509994581341743, 0.00034440006129443645, 0.00033980002626776695, 0.00034899997990578413, 0.00033760000951588154, 0.00035889993887394667] | -| 执行原始样本 s | [7.98999099060893e-05, 0.00017800007481127977, 0.00011080002877861261, 8.82000895217061e-05, 8.230004459619522e-05, 7.829989772289991e-05, 7.860001642256975e-05, 7.840001489967108e-05, 8.060003165155649e-05, 7.970002479851246e-05] | - -
- -
constant · 原先不支持 → 现通过 - -Constant tensor attribute FP32 [2,3]; no runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003632999723777175, 0.000381799996830523, 0.00039549998473376036, 0.000381799996830523, 0.0003598000621423125, 0.00036259996704757214, 0.0003591999411582947, 0.0003554000286385417, 0.0003502999898046255, 0.0003415999235585332] | -| 执行原始样本 s | [6.250001024454832e-05, 6.170000415295362e-05, 6.10999995842576e-05, 6.200000643730164e-05, 6.0399994254112244e-05, 6.150000263005495e-05, 6.219989154487848e-05, 6.0399994254112244e-05, 6.049999501556158e-05, 5.909998435527086e-05] | - -
- -
cos · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cos"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298 | -| 输入 SHA-256 | 8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998 | -| 参考输出 SHA-256 (expected_sha256) | 7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cos -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 5.960e-08 | -| 最大相对误差 | 8.024e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003530000103637576, 0.00034549995325505733, 0.00033980002626776695, 0.000340400030836463, 0.0003451999509707093, 0.00034739996772259474, 0.000332699972204864, 0.0003412999212741852, 0.00033599999733269215, 0.00033199996687471867] | -| 执行原始样本 s | [8.690007962286472e-05, 8.670007809996605e-05, 8.889997843652964e-05, 8.739996701478958e-05, 8.559995330870152e-05, 8.60999571159482e-05, 8.629995863884687e-05, 8.690007962286472e-05, 8.579995483160019e-05, 8.400005754083395e-05] | - -
- -
identity · 原先不支持 → 现通过 - -INT64 indices[2,3] -> Identity [2,3]; frontend alias. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Identity"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b | -| 输入 SHA-256 | 19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413 | -| 参考输出 SHA-256 (expected_sha256) | a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Identity -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.000341000035405159, 0.00034380005672574043, 0.0003355999942868948, 0.0003361001145094633, 0.0003378000110387802, 0.00033820001408457756, 0.0003301999531686306, 0.00032760004978626966, 0.0003744999412447214, 0.0003630999708548188] | -| 执行原始样本 s | [5.170004442334175e-05, 5.280005279928446e-05, 5.219993181526661e-05, 5.2200048230588436e-05, 5.080003757029772e-05, 5.179992876946926e-05, 5.079992115497589e-05, 5.139992572367191e-05, 6.809993647038937e-05, 5.4600066505372524e-05] | - -
- -
pow · 原先不支持 → 现通过 - -Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771 | -| 输入 SHA-256 | b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1 | -| 参考输出 SHA-256 (expected_sha256) | dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004491000436246395, 0.00043999997433274984, 0.00040019990410655737, 0.00037569995038211346, 0.00038789992686361074, 0.0003453000681474805, 0.00048369995784014463, 0.00044920004438608885, 0.00047619990073144436, 0.0004214000655338168] | -| 执行原始样本 s | [0.00014999997802078724, 0.00014180003199726343, 0.00011959997937083244, 0.00011729996185749769, 0.00011809996794909239, 0.00011589995119720697, 0.00015820004045963287, 0.00015029998030513525, 0.00015840004198253155, 0.0001241000136360526] | - -
- -
reciprocal · 原先不支持 → 现通过 - -Nonzero signed FP32 x[2,4] -> Reciprocal [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Reciprocal"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3 | -| 输入 SHA-256 | fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f | -| 参考输出 SHA-256 (expected_sha256) | c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Reciprocal -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00035170000046491623, 0.0003496999852359295, 0.00034989998675882816, 0.00034769997000694275, 0.00033870001789182425, 0.00037759996484965086, 0.00035119999665766954, 0.0003469000803306699, 0.00033660000190138817, 0.00033329997677356005] | -| 执行原始样本 s | [9.190000128000975e-05, 9.019998833537102e-05, 9.029998909682035e-05, 9.14999982342124e-05, 8.899997919797897e-05, 9.609991684556007e-05, 9.510002564638853e-05, 9.06999921426177e-05, 9.109999518841505e-05, 9.059999138116837e-05] | - -
- -
sin · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sin"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a | -| 输入 SHA-256 | efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e | -| 参考输出 SHA-256 (expected_sha256) | e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sin -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00034879997838288546, 0.0003532000118866563, 0.00035470002330839634, 0.0003448999486863613, 0.00033730000723153353, 0.00033710000570863485, 0.0003510999958962202, 0.0003322999691590667, 0.00032819993793964386, 0.000327700050547719] | -| 执行原始样本 s | [8.570007048547268e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.480006363242865e-05, 8.730008266866207e-05, 8.71000811457634e-05, 8.659996092319489e-05, 8.569995407015085e-05, 8.390005677938461e-05, 8.750008419156075e-05] | - -
- -
scalar_fp32_add_chain · 原先错误 → 现正确 - -FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a | -| 初始化器 SHA-256 (initializers_sha256) | 32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | NUMERIC_ERROR / compare | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 2.000e+00 | -| 最大相对误差 | 1.192e-07 | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -NumericMismatch: output differs from ORT: max_abs_error=2 -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 5 | -| 解析原始样本 s | [0.0005631999811157584, 0.0004637000383809209, 0.0004089000867679715, 0.00039579998701810837, 0.0003945999778807163, 0.000381799996830523, 0.00041089998558163643, 0.00040959997568279505, 0.0003945999778807163, 0.0003902000607922673] | -| 执行原始样本 s | [0.00019599997904151678, 0.00015169999096542597, 0.00014780007768422365, 0.0001476999605074525, 0.0001452999422326684, 0.00015129998791962862, 0.00014889996964484453, 0.0001441999338567257, 0.00014589994680136442, 0.0001444999361410737] | - -
- -
singleton_initializer · 原先错误 → 现正确 - -FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar. - -| 项目 | 值 | -| --- | --- | -| 算子 | [] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8 | -| 初始化器 SHA-256 (initializers_sha256) | 7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | EXECUTION_ERROR / execute | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1] / float32 | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003521000035107136, 0.00034039991442114115, 0.00033650000113993883, 0.0003445999464020133, 0.0003336999798193574, 0.0003494999837130308, 0.0003339999821037054, 0.00035889993887394667, 0.00032849994022399187, 0.00033849989995360374] | -| 执行原始样本 s | [6.290001329034567e-05, 6.280001252889633e-05, 6.140000186860561e-05, 6.009999196976423e-05, 6.079999729990959e-05, 6.370001938194036e-05, 6.200000643730164e-05, 6.330001633614302e-05, 6.260001100599766e-05, 6.0399994254112244e-05] | - -
- -
constant_cast_abs_expand · 原先不支持 → 现通过 - -Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant", "Cast", "Abs", "Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3 | -| 输入 SHA-256 | 9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822 | -| 参考输出 SHA-256 (expected_sha256) | eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 4 | -| 解析原始样本 s | [0.00043970008846372366, 0.00044329999946057796, 0.0004450000124052167, 0.0004384000785648823, 0.0006199999479576945, 0.00047199998516589403, 0.00045169994700700045, 0.00044700002763420343, 0.00044139998499304056, 0.00043539993930608034] | -| 执行原始样本 s | [0.00013560010120272636, 0.0001338999718427658, 0.00013189995661377907, 0.0002367999404668808, 0.00017560005653649569, 0.00013229995965957642, 0.0001343999756500125, 0.000130399945192039, 0.00013109995052218437, 0.00013479997869580984] | - -
- -
qwen_rmsnorm · 原先不支持 → 现通过 - -FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow", "ReduceMean", "Add", "Sqrt", "Reciprocal", "Mul"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237 | -| 输入 SHA-256 | f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48 | -| 参考输出 SHA-256 (expected_sha256) | 5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec | -| 初始化器 SHA-256 (initializers_sha256) | 6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 10 | -| 解析原始样本 s | [0.000506900018081069, 0.0005015999777242541, 0.0005096999229863286, 0.0005014999769628048, 0.0004960000514984131, 0.0005388999124988914, 0.0005028999876230955, 0.000512199942022562, 0.000493900035507977, 0.0004851999692618847] | -| 执行原始样本 s | [0.0002813000464811921, 0.0002788000274449587, 0.0002808000426739454, 0.00028170004952698946, 0.00027820002287626266, 0.0002930000191554427, 0.00028490007389336824, 0.00028249993920326233, 0.0002837000647559762, 0.0002848999574780464] | - -
- -
rope_numeric · 原先不支持 → 现通过 - -INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast", "Mul", "Cos", "Sin", "Sub", "Add", "Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa | -| 输入 SHA-256 | 823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a | -| 参考输出 SHA-256 (expected_sha256) | eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 1.699e-07 | -| 执行 IR 步数 | 12 | -| 解析原始样本 s | [0.0005266999360173941, 0.000520200002938509, 0.0005088000325486064, 0.0006041000597178936, 0.0006553001003339887, 0.0005080000264570117, 0.0005031999899074435, 0.0005010999739170074, 0.0005029999883845448, 0.00049919995944947] | -| 执行原始样本 s | [0.0002498000394552946, 0.00024560000747442245, 0.0002465000143274665, 0.0003208999987691641, 0.00031370006036013365, 0.00025370006915181875, 0.0002449000021442771, 0.00024610001128166914, 0.0002482000272721052, 0.0002460000105202198] | - -
- -
pr89_transformer_2l · 原先不支持 → 现通过 - -PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3 - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add", "Concat", "Div", "Expand", "Gather", "MatMul", "Mul", "Neg", "ReduceMean", "Reshape", "Sigmoid", "Slice", "Softmax", "Sqrt", "Transpose", "Unsqueeze"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 0 | -| 模型 SHA-256 | 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 | -| 输入 SHA-256 | bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457 | -| 参考输出 SHA-256 (expected_sha256) | 413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c | -| 初始化器 SHA-256 (initializers_sha256) | 955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` +输出目录每次换新名字，生成本次 JSON、Markdown、HTML 及原始样本。仅前后均 PASS 的用例比较耗时；缺失、不支持或失败不等于零耗时。结果用于支持/正确性对比，IR 解释器计时不等于 QEMU 或目标硬件性能。 -**修改后** +原始个人测量保持原平台身份和数据，分别保存于固定提交： -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 256, 128] / float32 | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | 7.153e-07 | -| 最大相对误差 | 5.722e-04 | -| 执行 IR 步数 | 159 | -| 解析原始样本 s | [0.004760700045153499, 0.004594899946823716, 0.004717700066976249, 0.004990800051018596, 0.004586199996992946, 0.004221900017000735, 0.004203699994832277, 0.004572000005282462, 0.004674999974668026, 0.004224899923428893] | -| 执行原始样本 s | [0.010629900032654405, 0.011460799956694245, 0.009812200092710555, 0.009985200013034046, 0.010765800019726157, 0.009604100021533668, 0.010238799965009093, 0.009660700103268027, 0.010715999989770353, 0.009805700043216348] | +- [原始 MD](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md) +- [原始 JSON](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json) +- [原始 HTML](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html) -
+当前同名 JSON 的 `document_kind=historical_benchmark_index`，只保存归档链接，不含 PASS、环境测量或计时字段，不能作为 benchmark 结果输入。新 Linux 测量必须从 CLI 输出目录读取。 diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" index 13b7df2..4d516da 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" @@ -1,148 +1,92 @@ -Qwen3 ONNX 算子补齐与单测报告 -日期：2026-10-01 - -工作目录：仓库根目录；本文使用仓库相对路径。 -主干基线：20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 -历史测量状态：原始算子探测和 benchmark 在实现尚未提交时执行；归档报告保留测量时的 HEAD、dirty 和源码指纹。 -本文用于随算子实现及测试 PR 归档，不以历史 dirty 标记描述当前提交或远程仓库状态。 -本报告更新上一阶段“真实 Qwen3 尚缺八类前端算子”的临时结论。 - -一、完成范围 - -完成 ONNX 前端 → 共享 IR → NumPy 解释器路径所需的两组算子： -1. PR #89 探测图八类：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 -2. 真实 Qwen3 导出图额外八类：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 -第一组复用已有 IR 算子与内核，补齐前端和边界测试；第二组新增六个 IR opcode，Constant 绑定为全局常量，Identity 复用输入值。 - -二、本轮代码改动 - -scratchv/frontend/onnx_parser.py -- 补齐第二组前端映射；支持 Constant 的 tensor、value_int/value_ints、value_float/value_floats。 -- 保持零维标量、单元素向量与 INT64 值的区别，支持外部权重及 Constant 外部数据。 -- 求值 Constant → Cast/Abs/Reshape 等静态 shape/axis 链，适配真实 Qwen3 导出形式。 -- 不将中间结果冒充 initializer；限制静态参数求值规模，避免超大 Expand 与尺寸乘积溢出。 -- Add/Mul 统一发出 IR，修复 Python 标量折叠造成的 FP32 舍入与整数宽度错误。 -- 预留全图 v_N 名字，修复 ONNX 原始名字和临时结果名碰撞。 -- 明确拒绝当前 IR 不支持的数据类型。 - -scratchv/ir/types.py、scratchv/ir/builder.py -- 新增 ABS、CAST、COS、SIN、POW、RECIPROCAL 及 builder 方法。 -- Cast 的结果 dtype 表示目标类型；Pow 允许底数与指数类型不同，结果保持底数类型。 - -scratchv/analysis/ir_verifier.py -- 增加新算子签名、浮点算子类型约束和 Pow 输出类型校验。 - -scratchv/verification/ir_numpy_ops.py -- 增加新算子的实际 NumPy 求值和数值错误诊断。 -- Cast 整数缩窄保留低位；浮点转整数按截断后范围检查，避免 INT64 边界错误。 -- Pow 支持广播、混合类型以及有界的精确整数计算。 -- Slice 使用 ONNX 的正反向钳制规则，修复过小负 start 的反向切片，并处理空维和 INT64 哨兵。 - -三、测试 - -本轮新增四个测试文件，共 282 个用例： -- tests/test_onnx_shape_ops.py：116 个，第一组八类算子与 ORT 对照、边界和非法参数。 -- tests/test_onnx_qwen_numeric_ops.py：78 个，第二组算子、外部常量、实际静态参数链及 RMSNorm 组合。 -- tests/test_ir_qwen_numeric_ops.py：74 个，IR builder、类型签名、广播、整数精度、数值边界与错误定位。 -- tests/test_onnx_constant_regressions.py：14 个，FP32 舍入、整数宽度、名字碰撞和静态求值规模限制。 -更新 tests/test_ir_verifier.py 的穷举签名表，覆盖六个新增 opcode。 -延续上一阶段的 tests/test_onnx_transformer_execution.py 和 initializer 测试修复。 - -算子及原有相关回归共 715 项，加上 22 项 benchmark 自检；提交前合并重跑结果为 737 passed，无失败。 -该次合并运行有 7 条 NumPy 配置提示 warnings，未影响测试通过。 -其中 715 项算子及原有相关回归的分组记录如下： -- IR、解释器、校验器、优化器、PassManager 相关：439 passed。 -- 前端、ONNX 算子、Transformer 组合、诊断和相关 benchmark：262 passed。 -- Constant 专项回归：14 passed（另与已计入的 85 项用例联合运行，结果 99 passed）。 -这是相关回归集合，未声称运行整个项目测试全集。 -单测测量环境：Python 3.12.13，NumPy 2.5.3、ONNX 1.23.0、ORT 1.29.0，Python -X utf8 -B。 - -四、模型验证 - -1. PR #89 原始 LFS 模型未修改： - probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx - SHA-256：17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 - 本轮重新运行 seed=1：ORT vs 独立 NumPy 参考最大绝对误差 1.09718e-6；IR vs ORT 为 7.1525574e-7。 - 均通过最大绝对误差 <1e-5；IR 执行 159 步。 - 另有 seed=0/1/42 的组合回归通过。 - 原始探测 JSON 保留在本机，未随本 PR 归档；此处保留模型指纹和数值结论。 - -2. 本地真实 Qwen3-0.6B 导出模型： - models/qwen3-0.6b-onnx/model.onnx（额外交付模型，不包含在本次算子 PR 中） - 7847 个 ONNX 节点、23 类算子；未识别算子为 0。 - 完整载入外部权重，得到 2148 个全局定义、5910 条 IR 指令。 - verify_ir 通过，问题数为 0。 - 原始结构审计 JSON 保留在本机，未随本 PR 归档。 - 此项是完整图解析及结构/类型校验，不是全模型 IR 推理的数值对照。 -模型探测使用独立导出依赖环境（NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1）。 - -五、支持边界与下一步 - -本次范围是部署方案所需的静态图及 FP32 路径，同时覆盖 IR 已有的 FLOAT64、INT32、INT64 类型。 -没有实现 ONNX 的全部 dtype/opset 扩展；动态 shape/axis 参数及 Reshape allowzero=1 仍明确拒绝。 -解释器沿用严格数值策略：非法定义域、非有限结果以及新增 Abs/整数 Pow 的溢出会报错。 -整数 Pow 对大整数采用精确计算，ORT CPU 在超 2**53 的整数幂上可能存在浮点精度差异；实际 Qwen 使用 FP32 底数和 INT64 指数 2。 - -下一步应运行真实 Qwen3 结构的小模型，包含 Q/K RMSNorm、完整 RoPE、GQA、SwiGLU，逐层对照 PyTorch/ORT/IR。 -之后再验证优化流水线、完整 0.6B 推理的数值/内存，以及 RISC-V 后端和 QEMU。 -本轮没有补 RISC-V 降低和执行支持，不能据此前端/IR 结果判断 QEMU 已可运行。 - -六、修改前后 benchmark 对比 - -归档文件（同目录）： -- 算子补齐与单测-benchmark.json：移除本机路径后的测量数据，保留全部样本、误差及指纹。 -- 算子补齐与单测-benchmark.md：GitHub 可直接阅读的结果报告。 -- 算子补齐与单测-benchmark.html：独立 HTML 报告，下载后用浏览器打开，可筛选并展开错误详情。 - -共 24 个固定用例，修改前 2/24 PASS，修改后 24/24 PASS： -- 16 个独立算子用例：0/16 → 16/16。 -- 2 个已有支持的对照用例：2/2 → 2/2。 -- 2 个错误回归用例：0/2 → 2/2。 -- 4 个组合链路用例：0/4 → 4/4，包含 PR #89 的原始两层 Transformer。 -因此，“原先不支持、现通过”是 20 个用例，不是新增 20 种算子。 - -使用同一批模型、输入、ORT 参考及初始化器，并校验指纹；基线通过解释器已有 initializers 参数接受原始 ONNX 数组，属于基准适配，不改基线实现，也不重塑权重以掩盖旧错误。 -ABBA 顺序为修改前、修改后、修改后、修改前；每版本 2 轮，每轮预热 1 次、正式计时 5 次，合计每版本 10 个正式样本。 -IR.run 完整计时包括校验、执行规划、绑定复制、NumPy 计算及返回复制；ONNX 解析单独计时，ORT 参考计算不计入。 -只在前后均 PASS 时显示执行耗时比；本报告证明支持和正确性改善，不宣称性能优化或 QEMU 性能。 -新增 tests/test_onnx_operator_benchmark.py 的 22 项自检与原 715 项回归已合并运行，共 737 passed。 -CI 的测试 checkout 获取完整历史，以便历史基线集成测试实际运行。 - -七、复跑准备与命令（在仓库根目录 PowerShell 执行） - -建议 Python 3.12，并使用独立虚拟环境。依赖安装： -python -m pip install -e ".[all]" pytest -其中项目 all 依赖包含 onnxruntime；基准和测试直接依赖 NumPy、ONNX、onnxruntime、pytest。 -本次单测/benchmark 版本为 NumPy 2.5.3、ONNX 1.23.0、onnxruntime 1.29.0；解释器路径不是复跑前提。 - -确保 Git 历史中存在固定基线提交，浅克隆需补充该历史；origin 应指向包含该提交的项目仓库或 fork： +Qwen3 ONNX 算子补齐与单测：Linux 复现说明 +更新：2026-10-06 + +正式交付与他人复现环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。 +所有命令在仓库根目录执行。通用环境约定见 docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md。 +个人本机测量的原始版本保留在固定提交： +https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测报告.txt +原始平台、计数、计时和数值不改名为 Linux 数据；新结果必须由下面的 Linux 命令重新生成。 + +一、实现范围 + +ONNX 前端 → 共享 IR → NumPy 解释器覆盖两组算子： +1. PR #89 探测图：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 +2. 真实 Qwen3 导出额外需要：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 +第一组复用已有 IR 与内核，补前端/边界；第二组新增 ABS、CAST、COS、SIN、POW、RECIPROCAL，Constant 使用绑定，Identity 复用输入值。 + +语义修复包括零维标量与向量区别、INT64 精度、静态 shape/axis 参数链、外部权重和 Constant、FP32 常量舍入、临时值命名冲突、静态求值规模限制、Cast 截断/溢出、Pow 广播与整数精度，以及 Slice 的反向边界和 INT64 哨兵。前端不将中间值伪装为 initializer。 + +二、验收用例 + +tests/test_onnx_shape_ops.py：八类形状/索引算子，ORT 对照、边界和非法参数。 +tests/test_onnx_qwen_numeric_ops.py：数值算子、常量、静态参数链及 RMSNorm 组合。 +tests/test_ir_qwen_numeric_ops.py：IR 类型、广播、整数精度与错误定位。 +tests/test_onnx_constant_regressions.py：FP32 舍入、整数宽度、名字碰撞、求值规模。 +tests/test_onnx_operator_benchmark.py：基准自身的身份、正确性及失败传播。 +实际 passed/skipped/failed 以本次 Linux 输出为准，不沿用历史计数。 + +三、Linux 环境与资产准备 + +以下为 Bash，已存在环境时可复用，但必须核对依赖版本。benchmark 使用单独环境，以保持历史前后对照依赖；它不替代完整模型门禁的固定导出环境。 + +set -euo pipefail +sudo apt-get update +sudo apt-get install --no-install-recommends python3.12-venv git-lfs +python3.12 -m venv .venv-operators +source .venv-operators/bin/activate +python -m pip install "numpy==2.5.3" "onnx==1.23.0" "onnxruntime==1.29.0" "pytest==9.1.1" +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 + +git rev-parse HEAD +git status --short git fetch origin 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 - -安装 Git LFS 后，下载已由 PR #89 提交的模型实体，不能把 LFS 指针作为 ONNX 运行： -git lfs install +git lfs install --local git lfs pull --include="probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx" -以下 24 用例基准不需要完整 Qwen3-0.6B 权重；完整模型结构审计需要另外准备本文第四节所述模型。 -生成新的基准报告及临时 fixtures，不覆盖 docs 中的历史归档： -python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5 - -复跑相关测试： - -$operatorTests = @( - 'tests/test_ir.py', 'tests/test_ir_qwen_numeric_ops.py', - 'tests/test_ir_interpreter.py', 'tests/test_ir_interpreter_ops.py', - 'tests/test_ir_interpreter_control_flow.py', 'tests/test_ir_verifier.py', - 'tests/test_ir_verifier_integration.py', 'tests/test_optimizer.py', - 'tests/test_optimizer_advanced.py', 'tests/test_pass_manager.py', - 'tests/test_pass_registry.py', 'tests/test_parser.py', - 'tests/test_onnx_initializers.py', 'tests/test_onnx_shape_ops.py', - 'tests/test_onnx_qwen_numeric_ops.py', 'tests/test_onnx_transformer_execution.py', - 'tests/test_ir_interpreter_benchmark.py', 'tests/test_ir_verifier_benchmark.py', - 'tests/test_ir_diagnostics.py', 'tests/test_onnx_constant_regressions.py', - 'tests/test_onnx_operator_benchmark.py' +基线为固定提交 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6，不能用不同分支 HEAD 代替。浅克隆必须补齐历史。原模型 SHA256 应为 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1，不可把 LFS pointer 当 ONNX。 + +四、生成修改前后 benchmark + +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 + +每轮选择新的输出目录。CLI 生成自己的 fixtures、参考和 JSON/Markdown/HTML 报告，不读取 docs 目录中的历史索引作为测量。源码、模型、输入、环境身份与错误详情由新报告保存。 + +24 个固定用例包含 16 独立算子、2 已支持的对照、2 错误回归和 4 组合链路，含原 PR #89 两层 Transformer。ABBA 顺序比较修改前/后，每版本两轮；前后都通过的用例才计算耗时比。不支持、失败或缺测不能解释为零耗时或无限加速。 + +IR.run 计时包含验证、执行规划、绑定复制、NumPy 计算与返回复制；ONNX 解析单独计时，ORT 参考不计入。此基准验证支持与正确性变化，不证明 QEMU 性能改善。重新生成的 Linux 样本不可与不同平台的历史耗时直接算加速比。 + +五、运行相关回归与合成图 + +operator_tests=( + tests/test_ir.py tests/test_ir_qwen_numeric_ops.py + tests/test_ir_interpreter.py tests/test_ir_interpreter_ops.py + tests/test_ir_interpreter_control_flow.py tests/test_ir_verifier.py + tests/test_ir_verifier_integration.py tests/test_optimizer.py + tests/test_optimizer_advanced.py tests/test_pass_manager.py + tests/test_pass_registry.py tests/test_parser.py + tests/test_onnx_initializers.py tests/test_onnx_shape_ops.py + tests/test_onnx_qwen_numeric_ops.py tests/test_onnx_transformer_execution.py + tests/test_ir_interpreter_benchmark.py tests/test_ir_verifier_benchmark.py + tests/test_ir_diagnostics.py tests/test_onnx_constant_regressions.py + tests/test_onnx_operator_benchmark.py ) -python -X utf8 -B -m pytest @operatorTests -q --basetemp=output/pytest-operator-completion +mkdir -p output +python -X utf8 -B -m pytest "${operator_tests[@]}" -q \ + --junit-xml=output/operator-tests-linux-01.xml +python -X utf8 -B probes/w1_tiny_transformer/run.py \ + --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx \ + --output-dir output/operator-completion-linux-01 --seed 1 + +六、范围与后续门禁 + +这是静态图和 FP32 所需前端/IR 能力，同时覆盖现有 FLOAT64、INT32、INT64；不宣称全部 ONNX dtype/opset 或动态 shape/axis。Reshape allowzero=1 等不支持语义明确拒绝；非法定义域、非有限值及部分整数溢出明确报错。当前固定 Qwen 图使用 FP32 底数与整数指数 2；通用 Pow 范围需要分别验证。 -python -X utf8 -B probes/w1_tiny_transformer/run.py --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx --output-dir output/operator-completion-probe --seed 1 +完整 28 层解析、真实两层 Qwen3 数值与 RV64/QEMU 分别从 W1/README.md、W2/README.md 和 probes/w2_qwen3_small/README.md 进入；基础算子 benchmark 通过不能代替这些门禁。 -上述探测命令默认使用已下载的 ONNX，不要求重新导出；切换导出依赖环境时请遵循模型探测的已记录版本。 +同目录 benchmark.md 为 Linux 复跑入口，benchmark.html 为可打开的指引，benchmark.json 为明确标识的历史索引，不包含新测量。原始完整测量保留在固定提交历史中。 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index e233014..0db79de 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -1,262 +1,85 @@ -# W2：完整前端解析、基础后端与 Host 运行时 +# W2 Linux 独立复现与验收 -本目录记录开发计划中的 W2 验收：完整 28 层 Qwen3-0.6B 的前端解析与审计、四类模型模式回归、基础算子 QEMU，以及 Tokenizer/输入/mask/greedy。已有真实两层模型的 IR/ORT/QEMU 门禁继续保留。 +正式交付、CI 和第二人复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。完整环境准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。Linux 为宿主环境；QEMU 内的 RV64GC/LP64D 程序仍使用当前裸机 ABI。 -完整图的范围仍止于前端验收。完整 IR 与 ORT 的数值对照、真实维度预训练子图、权重独立装载、完整 QEMU 前向和生成循环仍是后续工作。本轮也不自动关闭 W1 的第二人复现、接口确认和出口评审。 +W2 的范围是完整前端解析、小模型 IR 数值、基础后端和 Host 单步接口。完整 28 层模型的 IR 数值前向属于 W3；完整模型 QEMU 与生成循环属于后续阶段。W1 的人工接口决议继续见 [W1 记录](../W1/README.md) 和 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92)。 -**当前交付状态（2026-10-03）**：本次按 W1/W2 拆分为独立分支。W2 直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含另一 W1 分支的两层报告/参考测试修复。当前分支的范围、重新执行的证据及 PR 归属见 [分支验收与发布说明](W2-分支验收与发布说明.md)。下方原本地整合记录及历史 CI 按各自源码归属，不代表本分支已通过远端 CI。 +## 独立复现版本 -## W2 独立技术验收入口 +[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 的既有 Linux 记录对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。后续文档或源码更新不自动继承该提交的执行结果。开始前记录约定 SHA、`git rev-parse HEAD` 和 `git status --short`；有未提交增量则保留完整 diff 和新增文件，不能只报告 HEAD。 -使用已有的 Python 3.12 固定 CPU 环境（[依赖](../../../requirements/qwen3-small-probe.txt)）、Zig 0.14.1、QEMU RV64，以及固定完整 ONNX 和官方 Tokenizer 资产。在仓库或完整源码副本根目录执行： +发布前个人工作树的详细测量按原始身份保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/README.md)。当前页面只提供 Linux 交付入口与可关联到具体提交的 Linux CI 记录，不把历史个人测量改名为 Linux 结果。 -```powershell -$w2Python = 'D:/path/to/qwen3-venv/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-new ` - --tokenizer-dir D:/path/to/qwen3-tokenizer ` - --model-dir D:/path/to/qwen3-0.6b-onnx ` - --cc D:/path/to/zig.exe ` - --qemu D:/path/to/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -Linux 使用相同参数，将解释器和工具替换为本机路径即可。程序不安装依赖；默认两个资产目录均使用 `verify` 离线校验。无资产时可显式传 `--tokenizer-mode download` 或 `--model-mode download`，后者需要下载约 1.24 GB 的完整模型发布包。所有结果使用本次全新的输出目录，不能复用已有目录，即使为空。 - -| 执行顺序 | 独立门禁 | 要求 | -|---|---|---| -| 1 | `frontend` | 22 项 RMSNorm/RoPE/SwiGLU/GQA 模式单测，不接受 skip | -| 2 | `backend` | 16 个小图 × none/all，32 次实际 RV64 QEMU | -| 3 | `runtime` | 官方 Tokenizer 20 组/480 对照，以及 5 组输入、mask、采样与停止条件 | -| 4 | `runtime-model` | 官方文本 ID 直入完整词表的真实两层小配置，Torch/ORT/IR none/all 单步接口集成 | -| 5 | `small-ir` | 从固定种子重新导出原两层普通/诊断图，7 组输入、29 检查点与不变性验证 | -| 6 | `small-qemu` | 使用本轮新导出的模型与报告，28 次两层实际 QEMU | -| 7 | `full-parse` | 固定 28 层完整图解析、逐节点/权重绑定审计与 IR verifier | - -只有七项全部实际执行并通过才返回 **PASS / 退出 0**。`--gates frontend backend` 等子集支持开发期间定位，但最多返回 **PARTIAL / 退出 2**；失败或阻断返回 **FAIL / 退出 1**。small-qemu 自动包含 small-ir，runtime-model 自动包含 runtime。默认每门禁截止 1800 秒、每次 QEMU 截止 180 秒、完整解析 worker 截止 300 秒，可用 `--gate-timeout`、`--qemu-timeout`、`--parse-timeout` 明确调整。 - -总报告在 `output/w2-acceptance-new/report.json/md/html`，每项有独立目录和 `logs/.log`。总入口核对退出码、报告身份、必要计数、实际输出和源码指纹；运行期间源文件变化也会失败。报告记录 HEAD/dirty diff（可用时）及本次源码 SHA256；没有 `.git` 的源码包允许运行，但必须明确标为未知 Git 身份，不能借用父目录仓库的提交。 - -新 [单步模型集成探测](../../../probes/w2_runtime_model/README.md) 保留完整词表 151936，hidden32/FFN96/两层随机权重。原文本 ID 不裁剪、不取模、不重新映射。逐元素检查完整 logits（含 padding），校验最后有效位置、原始 greedy ID 和官方解码；不要求随机权重生成有意义的文本。本地会保存较大的参考数组（约 614 MiB），CI 仅上传摘要报告，不上传这些参考数组或权重。该探测不执行完整 0.6B、完整词表 QEMU 或多步生成。 - -当前本地 workflow 已加入单步集成、完整 28 层解析必跑任务和汇总检查。`W2 / required acceptance` 要求轻量/数值任务与完整解析任务均为 success；失败、取消、跳过、缺失结果均不通过。完整 ONNX 的 ORT 前向仍是独立的手动/夜间 W1 任务。配置随本 W2 分支发布，远端 CI 以对应提交的实际执行为准；分支保护规则未变更。源码隔离复现共用本机 Python/工具链时，只能证明不依赖旧源码目录与旧产物，不能代替第二人或另一台机器复现。 - -统一验收的 Linux 超时路径会先发 SIGTERM，让运行器清理独立会话内的编译器、QEMU 或解析 worker，再执行进程组兜底清理。Windows 继续使用进程树/Job 清理。Linux 专属进程集成回归在 Windows 上明确跳过，发布前须在 Linux 实跑；不能用 Windows 结果替代它。 - -## 本轮继续交付与最终实测（2026-10-02，本地未提交） - -本轮新增完整词表的单步模型接口集成、七项统一验收入口及回归，并补齐原两层 IR 门禁的报告原子写入：MD/HTML 或 JSON 发布失败时必须返回失败，保留原数值错误，不遗留成功 JSON。此前四条报告路径的修复继续保留。 - -从当前工作树复制 **553 个源码文件**到全新的 `output/w2-independent-source-v2`，包含未提交改动，不复制 `.git`、历史 output、Python 环境或模型产物；从该目录真实运行统一入口，最终 **7/7 PASS，退出 0**。固定 ONNX/Tokenizer 资产重新验证哈希，Python 环境与 Zig/QEMU 工具仍共用本机现有安装。导入路径确认来自新副本，模型重新导出、ELF 重新编译、所有结果重新执行；总报告的 **134 项源码指纹与最终工作树一致**。 - -总证据：`output/w2-independent-source-v2/output/w2-acceptance/report.json/md/html`；每项明细、误差位置、日志和实际命令位于同一目录下对应子目录。快照来源及逐文件哈希在 `output/w2-independent-source-v2-manifest.json`，外层日志在 `output/w2-independent-acceptance-v2.log`。这些本机产物不进入版本库。 - -| 门禁 | 最终实测 | -|---|---| -| 前端四模式 | **22/22** 通过，未跳过 | -| 基础后端 | **32/32 QEMU**；最大绝对误差 **2.384185791015625e-7** | -| 运行时 | **20/20** 官方语料、**480/480** 编码/解码对照、**5/5** 输入与采样场景 | -| 单步模型集成 | **2 个原始文本、8 次模型调用**（2 Torch + 2 ORT + 4 IR）；完整输出 `[1,256,151936]`；最大绝对误差 **2.8312206268310547e-7**，采样 ID/解码一致 | -| 两层 IR | **7 组输入、29 检查点、6 项不变性**全部通过；所有数值比较最大绝对误差 **1.9073486328125e-6** | -| 两层 QEMU | **28/28**，普通/诊断图及 none/all 全通过；普通 logits 最大误差 **3.129243850708008e-7**，诊断检查点最大误差 **1.6689300537109375e-6** | -| 完整前端 | **28/28 层、7,847 节点、8,161 value、2,148 绑定**全部通过；5,910 条 IR 指令、verifier 无错误；绑定数据 **2,384,216,772 bytes** | - -统一流程 **264.340 秒**；两层 QEMU 进程累计 **157.642 秒**，其门禁全流程 **172.264 秒**；完整解析门禁 **9.538 秒**，其中 parser **2.031 秒**，worker 峰值 RSS **5,064,372,224 bytes（约 4.72 GiB）**。QEMU 计时包含进程启动、guest 运算和输出传输，诊断图包含检查点工作；这些是单次主机观察，不是纯推理性能，也不能据此声称提速。 - -首次源码副本实跑保存在 `output/w2-independent-source`，总结果正确为 FAIL：前端 pytest 临时目录的父目录未创建，导致 22 个 fixture setup 错误，其余六项通过。修复为使用统一输出根下的临时目录，新增真实 pytest 子进程回归后从新副本完整重跑，没有复用第一次的 PASS。另修复源码包向上查找 Git 的身份错误：运行时、两层 IR 和完整解析入口检查自身 `.git`；缺失时保留源码指纹并明确未知提交。最终实跑不依赖 `GIT_CEILING_DIRECTORIES` 等外部补救设置。 - -专项回归先执行 **631 passed / 0 skipped**（`output/w2-closeout-combined-tests.xml`）。最终修正上述入口后，对受影响两组分别复验 **72 passed / 0 skipped**（`output/w2-source-identity-all-tests.xml`）与 **47 passed / 0 skipped**（`output/w2-acceptance-tests-d.xml`）；批次重叠，不相加。工作流 YAML、25 段 Bash 语法、文档链接和 diff 格式检查通过。本轮未执行全仓测试或远端 CI，未提交、未推送、未更新 PR #91。 - -本轮补齐的是 W2 的独立技术验收和单步接口集成。第二人独立复现、团队接口确认以及发布后对应源码的远端 CI 仍需完成；完整预训练模型的 IR 数值、真实维度后端、独立权重装载和多步生成仍属于后续阶段。 - -## 前一轮：拆分后的本地 Review（2026-10-02） - -本轮审查完整解析、四模式前端、基础后端、Tokenizer、输入/mask/greedy，以及本地分支携带的两层 QEMU 修复。发现一类 P2 验收漏洞并修复四条报告路径：必需的报告写入失败时，可能留下可解析的 `passed=true` JSON。它会使仅消费 artifact 的脚本误判，不能以数值计算正确为理由忽略。 - -| 入口 | 复现与修复 | -|---|---| -| `probes/w2_backend_ops/run.py` | 原来先写 PASS JSON，随后 MD/HTML 失败；改为保存视图结果后发布 JSON，保存失败使门禁非零退出，保留原错误 | -| `probes/w2_qwen3_parse/run.py` | 同类先发布成功状态问题；保留验证阶段、failure_context 和主错误，报告保存异常单列，JSON 最后原子发布 | -| `probes/w2_runtime/run.py` | 虽然原来已最后写 JSON，完整内容写入后关闭/flush 抛异常仍可留下 PASS；改为同目录临时文件成功关闭后原子替换，失败不发布成功 JSON | -| `probes/w2_qwen3_small/riscv.py` | 既有两层 QEMU 门禁存在同类问题；视图与 JSON 原子写入，JSON 最后发布，保留原执行错误与时间证据。此修复只在本地，未同步 PR #91 | - -新增 **30 项**失败注入回归，覆盖 MD/HTML/JSON 无法保存、完整写入后报错、JSON 发布失败、原探测已失败及 CLI 状态。没有修改数值算法、精度阈值或模型配置。在本轮实际核对的路径和样本中，未发现新的前端/MatMul/Tokenizer 计算错误；这不构成完整模型数值正确性的证明。 - -| 本轮验证 | 结果与证据 | -|---|---| -| W2 与两层报告相关专项 | **396 passed / 0 skipped**；`output/w2-review-combined-tests.xml` | -| 前端模式、C 后端与 RISC-V 运行时相关回归 | **138 passed / 0 skipped**；`output/w2-review-compiler-regression.xml`。与上一行有重叠，不相加 | -| 基础后端实际 RV64 QEMU | **32/32**；对 ORT/独立 NumPy 最大绝对误差均 **2.384185791015625e-7**；流程 **23.588 秒**，QEMU 累计 **5.800 秒**；`output/w2-review-backend-real/report.json` | -| 官方 Tokenizer 离线探测 | **20/20** 语料、**480/480** 对照、**5/5** 输入与采样场景；流程 **3.739 秒**；`output/w2-review-runtime-offline-fixed/report.json` | - -已核对上述后端 30 项、运行时 5 项源码 SHA256 与修复后代码一致。额外 500 组随机 Unicode/特殊 token 文本编码和 4,000 次解码对照也通过；它们是本轮探索性检查，不算新增 CI 用例。本轮未重跑完整 28 层模型、整两层 QEMU 或全仓测试，也没有远端 CI；历史结果仍按原 SHA 归属。计时为单次观察，不用于声称性能改善。 - -### 可以继续的工作，按优先级 - -1. **W2 交付收尾**：第二人用自己的环境执行统一验收入口；未来按用户指示整理独立 W2 PR，明确哪些两层门禁修复应单独回补 W1。当前 PR #91 的绿色状态不是本地 W2 的验收；同机源码副本复现也不能代替第二人验收。 -2. **真实维度与预训练子图**：先验证投影 MatMul、RMSNorm、RoPE、Attention、SwiGLU，保留 Torch/ORT/IR 和适当小子图的 QEMU 误差。固定源配置为 hidden=1024、head_dim=128、16 个 Q head/8 个 KV head、FFN=3072。现有两层缩小配置和 3-token GQA case 不能证明这些长累加维度满足误差要求。优先建立基线，再决定 MatMul 分块/布局优化。 -3. **参数化中等模型与逐层定位（W3）**：将两层配置和固定 29 检查点推广成配置驱动，跑计划中的六层中等模型，记录首个偏差、最大绝对误差、耗时与峰值内存。Attention 可先沿用现有分解 IR，融合不是正确性前置。 -4. **完整 IR 内存准备（W3）**：当前解释器会复制 initializer，并在 `values` 中保留中间结果，没有最后使用后回收。先测权重、活跃张量与 RSS，设计只读权重复用和按生命周期回收；再执行完整 IR/ORT，沿用计划中的 `<1e-4` 门槛。 -5. **完整后端与生成集成（W4/W5）**：当前常量/工作区默认各 256 MiB，QEMU 为 512 MiB，权重展开为 C 字面量；仅真实 embedding 就约 593.5 MiB。需要代码与权重分离、装载 ABI、地址布局和容量验证，再接完整前向与生成循环。Host helper 与完整词表两层随机模型的单步调用现已集成；真实预训练模型调用、逐 token 循环、chat template 与流式解码仍未完成。 - -建议下一轮从第 2 项的真实维度数值基线开始，同时准备第 3 项的可配置探测，避免直接以完整模型试跑替代可定位的小范围验证。 - -## W2 基础能力补齐（2026-10-02，历史提交 ff193c7） - -该次在已通过完整解析和两层数值探测的 `dfc9437` 上补齐三个独立入口，曾更新 PR #91，现已按 W1/W2 拆分要求撤出该 PR 并保留在本地。表中“通过”必须读取对应提交的报告或 CI；本页不把旧提交或跳过项计作当前本地修改通过。 - -| W2 门禁 | 当前入口与范围 | -|---|---| -| `unit:frontend-ops` | `tests/test_qwen3_frontend_patterns.py`：RMSNorm、RoPE、SwiGLU、GQA 四类模式，含错误 epsilon、旋转符号、门控分支与 KV 重复顺序反例 | -| `unit:backend-ops` | `probes/w2_backend_ops/run.py`：16 小图，MatMul/逐元素/Softmax/RMSNorm/RoPE/SwiGLU/GQA 七类；none/all 共 32 次实际 RV64 QEMU；独立 NumPy 公式和 ORT 双参考 | -| `unit:runtime` | `probes/w2_runtime/run.py`：固定官方七份 tokenizer/配置文件，20 组语料、fast/slow 双参考与 golden IDs/解码文本；mask/最后有效位置/greedy/EOS/容量/预算 | -| `numeric:ir-small` | 既有 `probes/w2_qwen3_small/run.py`：真实两层缩小配置随机权重，继续验收 IR 对照；另保留两层 QEMU 门禁 | -| `frontend:parse-qwen3` | 既有 `probes/w2_qwen3_parse/run.py`：完整 28 层固定产物解析与绑定审计；手动/定时完整模型 CI 实际执行 | - -具体命令见 [基础后端门禁](../../../probes/w2_backend_ops/README.md) 和 [运行时门禁](../../../probes/w2_runtime/README.md)。生产新增 `scratchv/runtime/llm_inputs.py` 与 `scratchv/runtime/qwen3_tokenizer.py`；Tokenizer 只依赖可选 `tokenizers`，不在生产模块引入 Torch/Transformers。纯文本编码、完整模型输入准备与单步采样为可复用函数，尚未组合成端到端生成循环。 +## 七项统一验收 -交叉 review 修复了非默认 prefix/special-token 配置被静默接受而与官方语义不同、缺省 unk token 偏离参考的问题；门禁也增加词表/特殊 ID 元数据与默认参数对照，并修正报告写入失败遗留 PASS 的路径。官方 NFC 规范化是保留行为，不把组合 Unicode 归一化误判成编译器错误。未知或空余 logits 行解码时失败，不静默丢弃或改变 greedy 候选。 - -本次 CI 增加明确的三个门禁、固定环境单测、独立 Summary 与 `w2-unit-gates-output` artifact。运行时下载约 15.2 MiB 的 tokenizer/config/许可证，按固定 revision 和既有 manifest SHA256 校验，不下载模型权重。源码和测试文件进入 Git，模型、下载缓存、ELF 与本地 output 不进入 Git。 - -### 本轮本地验收 - -| 验证 | 结果 | 本地证据 | -|---|---|---| -| 新增专项测试（固定 CPU 环境） | **228 passed / 0 skipped**，含四类前端模式 22 项、后端门禁 18 项、运行时 188 项 | `output/w2-final-unit-tests.xml` | -| 全仓回归（通用开发环境） | **2,732 passed / 41 skipped / 0 failed** | `output/w2-final-full-ready-tests.xml` | -| 基础后端真实 RV64 QEMU | **32 / 32** 通过；对 ORT 与独立 NumPy 参考的最大绝对误差均为 **2.384185791015625e-7** | `output/w2-backend-ops-final/report.json/md/html` | -| 官方 Tokenizer 下载与运行时门禁 | **20 / 20** 语料、**480 / 480** 比较、**5 / 5** 输入与采样场景通过 | `output/w2-runtime-final-download/report.json/md/html` | -| 同一执行身份离线复验 | 禁用 Hugging Face 在线访问，复用经过哈希校验的本地资源；上述 **20 / 480 / 5** 全部通过 | `output/w2-runtime-final-offline-same-user/report.json/md/html` | - -41 个跳过项包括本机缺少 `qemu-riscv32` 的 38 项，以及通用环境没有 Torch/Tokenizers 的 3 个模块；新 Tokenizer 模块已在上述固定 CPU 环境执行。两个既有 Torch 模块在本轮未本地重跑，PR CI 的固定环境继续执行原两层数值探测。各批次测试有重叠，不能相加成通过总数。 - -首次全仓运行因子进程未获得 LLVM 工具路径失败，修正测试进程的 PATH 后得到上表结果；首次跨执行身份的离线复验因 Windows 目录读取权限失败，同一身份复验通过。失败报告仍保留，未通过修改数值门槛或跳过检查消除失败。 - -后端探测全流程 **22.194 秒**，32 次 QEMU 进程墙钟累计 **5.660 秒**；Tokenizer 下载门禁 **14.824 秒**，同一身份离线复验 **4.198 秒**。均为本机单次观察，不是性能改善结论。已核对后端 30 项、运行时 5 项源码指纹与本次源码一致。仓库指定的本地 L2 harness 文件缺失，未宣称执行；本轮以实际专项、全仓和真实探测结果交付。 - -## 完整前端解析的本地复现 - -详细环境、Windows/Linux 命令、下载模式和报告说明见 [探测入口 README](../../../probes/w2_qwen3_parse/README.md)。所有命令在仓库根目录执行；使用 Python 3.12 和固定 NumPy/ONNX/ORT/protobuf 环境，无需 torch、Zig 或 QEMU。 - -已有完整模型时： +先完成环境准备，激活 `.venv-linux`，并设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU`。模型及官方 Tokenizer 目录必须包含固定 manifest 对应的真实资产。在仓库根目录执行： ```bash -python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +set -euo pipefail +source .venv-linux/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export PYTHONIOENCODING=utf-8 +export SCRATCHV_PYTHON="$(pwd)/.venv-linux/bin/python" +export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" +export SCRATCHV_QEMU="$(command -v qemu-system-riscv64)" +W2_TOKENIZER=/absolute/path/to/fixed-qwen3-tokenizer +W2_MODEL=/absolute/path/to/fixed-qwen3-onnx + +git rev-parse HEAD +git status --short +"$SCRATCHV_PYTHON" -B -X utf8 scripts/run_w2_acceptance.py \ + --python "$SCRATCHV_PYTHON" \ + --output-dir output/w2-acceptance-linux-01 \ + --tokenizer-dir "$W2_TOKENIZER" \ + --model-dir "$W2_MODEL" \ + --cc "$SCRATCHV_CC" \ + --qemu "$SCRATCHV_QEMU" ``` -PowerShell 可使用实际虚拟环境解释器： +把两项资产路径改成 Linux 实际目录。默认使用 `verify` 离线校验；无资产时可显式增加 `--tokenizer-mode download` 与 `--model-mode download`。后者下载约 1.24 GB 的固定完整发布包，解包及解析需要额外磁盘与数 GiB 空闲内存。入口不安装依赖。`--output-dir` 必须是本次全新的路径，连已存在的空目录也会被拒绝。 -```powershell -& .\output\qwen3-parse-venv\Scripts\python.exe -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE -``` - -没有完整模型时，将 `verify` 改成 `download`，仍使用同一固定 W1 manifest。模型必须包含真实 external-data 分片。每次使用新的报告目录，失败时保留已有文件与日志。 - -## 如何判断本阶段完成 - -| 项目 | 应保留的证据 | +| 门禁 | 实际执行与判断 | |---|---| -| 源码身份 | checkout SHA、是否有本地改动、实际 diff 与新增文件 | -| 固定模型身份 | 发布产物和 external-data 文件校验结果 | -| 完整结构与层覆盖 | 本次结构审计摘要及 28 层检查结果 | -| 实际节点转换 | `nodes.json` 与审计摘要 | -| initializer 绑定 | `bindings.json` 与审计摘要 | -| 共享 IR | `ir.txt` 与 IR 验证器结果 | -| 本次执行 | 命令、退出码、`report.json/md/html`、实测时间及内存（如平台可测） | -| CI | 对应提交和实际 run URL；未执行时明确写未执行 | +| `frontend` | RMSNorm、RoPE、SwiGLU、GQA 四种基础算子分解图；独立公式与反例 | +| `backend` | 七类基础算子的 16 个小图 × none/all，共 32 次 QEMU；误差 `<1e-5` | +| `runtime` | 官方 Tokenizer 的 20 组语料、480 次对照，以及输入、掩码、greedy、停止边界 | +| `runtime-model` | 完整词表两层随机小配置；2 文本、8 次 Torch/ORT/IR 调用及单步 greedy/解码 | +| `small-ir` | 官方 Qwen3 类两层小配置，7 输入、29 检查点、none/basic/all，误差 `<1e-5` | +| `small-qemu` | 同一本轮普通/诊断图 × none/all × 7 输入，共 28 次真实 QEMU，误差 `<1e-5` | +| `full-parse` | 固定完整 28 层 ONNX 的结构、逐节点转换、真实权重绑定与 IR verifier 审计 | -退出 0 和 `passed=true` 必须来自本次执行。轻量 fixture 测试、历史 ORT 通过或一个可打开的报告文件都不足以代替完整图审计。失败发生较早时，某些明细可能尚未产生；按实际阶段和错误定位，不将缺失记录补成成功。 +所有数值门禁同时检查 shape、dtype、有限值；上述阈值使用最大绝对误差、`rtol=0`，包括 padding 输出。Attention 使用基础算子组合，不要求新增融合 opcode。Tokenizer 的“往返 100%”指与官方 NFC/特殊 Token 规则一致，不是任意字节可逆。 -本地 W2 workflow 现在让 PR、手动和定时任务均执行独立的 `full-qwen3-frontend` 完整解析任务，无须等待完整 ONNX 的 ORT 前向。该任务下载约 1.24 GB 的固定发布包，解析时需约 5 GB RSS；手动/夜间同时运行 ORT 任务时，两台 runner 会各自下载。汇总任务拒绝失败或跳过的依赖任务。PR #91 已合并；未来 W2 远端结果须关联其独立提交和实际 run，不能继承旧提交的 PASS。 +仅七项全部执行并通过才会退出 **0 / PASS**；`--gates` 选择部分门禁即使成功，也只会退出 **2 / PARTIAL**。失败退出 1，中断按实际退出状态记录。`runtime-model` 自动带上 `runtime`，`small-qemu` 自动带上 `small-ir`。缺失、跳过、非有限误差、复用旧输出或执行中源码变化均不能冒充完整通过。 -## 完整解析 Review 与实测（2026-10-02，历史提交 dfc9437） +## 报告与独立复现记录 -该次同时复核本地完整解析门禁和 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91)。核对时 PR head 为 `df18b02f152878b1ea838bbf6073def01d367456`，base 为 `d1c2b5f04cad817acf0b2f14d3bfa4ff56dbfa11`，当时差异涉及 52 个文件。审查覆盖编译入口、IR 优化、tensor-c 后端、RISC-V 运行时、数值探测、CI 和复现说明。下述完整解析与 Review 修复曾在 `dfc9437` 纳入 PR #91，现已撤出该 PR 并保留于本地；当时实际执行源码由报告中的 SHA256 确定。 +总报告位于本次 `--output-dir` 下的 `report.json`、`report.md`、`report.html`；子目录保存各门禁原始报告、命令、日志、数组和模型/ELF 身份。源码指纹核对的是实际执行文件，不能把别处安装的模块误当当前 checkout。 -### 完整解析门禁修复 +- 保存 checkout SHA、dirty/diff、Linux 发行版、CPU、Python/NumPy/ORT/Torch、Zig/QEMU 版本。 +- 保存每项实际退出码、PASS/FAIL/PARTIAL/未执行、原始误差、首次失败阶段及报告位置。 +- QEMU 时长是进程墙钟，包含启动、guest 执行、UART 和退出；不是目标硬件性能或纯 forward 时间。 +- 重新生成自己的模型、参考与输出，不能下载第一人的数组后称为独立执行。 +- 独立复现和团队接口/出口确认分开记录，作者再次执行或 CI 自动运行不替代第二人。 -| 问题 | 修复与验证 | -|---|---| -| 属性审计复用生产 parser 的属性解码，可能让相同错误在转换和审计两端同时通过 | 直接从原 ONNX protobuf 独立读取属性，拒绝重复属性；增加错误 Transpose 轴和错误 epsilon 的反例 | -| 未检查函数参数名称与常量标记 | 对照原 ONNX 输入校验名称、顺序、shape/dtype，并拒绝被标记为常量的输入 | -| 标量辅助 `LOAD_CONST` 校验不完整 | 对 initializer 和 Constant 两条路径核对 opcode、目标 Value、空操作数、shape/dtype、常量标记、常量值及指令属性 | -| 审计时隐式 `np.asarray` 可能掩盖错误的绑定类型 | 要求实际绑定是 `numpy.ndarray`；list 等错误类型直接失败，合法只读 ndarray 仍接受 | -| 日志写入失败可能覆盖 worker 原始错误 | stdout/stderr 独立尝试保存；保留原错误、阶段及退出码，附加 `log_errors`；成功路径缺少日志也不能报 PASS | -| 文档链接指向被忽略的本地输出，干净 checkout 中无法访问 | 改为明确的本地输出路径说明，并增加模拟缺少 output 目录的文档链接回归测试 | - -进一步修正失败明细写入的同类问题：保存 `nodes.partial.json` 等失败证据时，写入异常记录到 `evidence_errors`，不覆盖原始解析或绑定错误；成功路径缺少必需证据仍失败。supervisor 在启动 worker 前保存 `attempt`，记录命令、模型路径、解释器、checkout 和源码指纹，便于定位过早超时、worker 未能启动等情况；该字段不代表验证已经执行。 +## 已有 Linux CI 证据 -### PR #91 修复 +以下全部归属历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c`，不作为这次文档更新的新执行结果： -| 问题 | 修复与验证 | +| 工作流 | 已记录结果 | |---|---| -| tensor-c 接受与 IR 标量常量值不同的外部绑定，可能与解释器行为不一致 | 校验绑定与常量声明一致；增加 FP32/INT32/INT64 错值、错误向量形状和合法标量执行的回归测试 | -| 编译或 QEMU 失败时，保存日志的异常可能掩盖编译错误、超时或 guest 错误 | 独立保存两路日志；保留主错误及原耗时，并附加日志错误。覆盖编译失败/成功与 QEMU 超时/失败/成功等组合 | -| RISC-V 报告不足以确定本次 ORT/IR 对照的实际执行环境和完整源码身份 | 顶层 `environment` 记录当前进程，`export_evidence.environment` 保留导出时环境；源码指纹覆盖 parser、IR、优化、后端和运行时，共 30 项 | - -未发现本次审查范围内需要额外修改的 MatMul 实现问题；下列结果只证明已测试输入和路径。未调整精度门槛、模型权重或 D8 等团队待确认决策。 - -### 修复后的实测 - -| 验证 | 当前结果 | 本地证据 | -|---|---|---| -| 全仓回归 | 2,563 passed / 40 skipped / 0 failed | `output/review-fix-full-tests.xml`、同名 `.log` | -| 固定 CPU 环境最终相关测试 | 286 passed / 0 skipped，包含全部 114 项完整解析测试 | `output/review-fix-pinned-tests.xml`、同名 `.log` | -| 真实 Qwen3 结构两层小模型重新导出与 PyTorch/ORT/IR 对照 | 7 / 7 组输入通过；全部数值比较最大绝对误差 `1.9073486328125e-6` | `output/review-fix-qwen3-small/report.json` | -| RISC-V QEMU 两层探测 | 7 组输入 × 普通/诊断图 × none/all 优化，共 28 / 28 执行通过；QEMU 对照最大绝对误差 `1.6689300537109375e-6`，门槛 `atol=1e-5, rtol=0` | `output/review-fix-qwen3-riscv/report.json`、`report.md` | -| QEMU MatMul | 7 / 7 通过，最大绝对误差 `2.384185791015625e-7` | `output/review-fix-matmul/report.json` | -| 完整 Qwen3 前端解析 | 28 / 28 层；7,847 节点、23 类算子；5,910 条 IR；2,148 项绑定；IR verifier 0 问题 | `output/review-fix-full-qwen3-final/report.json`、`report.md`、`report.html` | - -全仓回归中的 38 个 RV32 执行测试因缺少 `qemu-riscv32` 跳过，另有 2 个依赖 torch 的模块因通用开发环境不含 torch 而跳过；最终固定 CPU 环境相关测试已覆盖这两个模块。全仓测试收集后补充了 3 项失败证据回归测试，均包含在最后的 286 项测试内；没有把不同批次或重叠测试相加为一个通过总数。RV64 tensor 真实运行时集成测试和上述实际 QEMU 探测已执行。 - -本次 QEMU 进程墙钟时间累计 **160.450 秒**，探测全流程 **174.634 秒**，4 个成功构建累计 **6.586 秒**。普通图每次平均约 **1.143 / 1.147 秒**（none/all），诊断图约 **10.138 / 10.494 秒**。QEMU 时间包含进程启动、guest 计算、输出打包/UART 和退出；诊断图还包含中间张量输出，不能当作纯前向耗时或真实硬件性能。 - -完整模型本次纯解析 **3.636 秒**，入口至验证及日志保存结束 **12.772 秒**，worker 峰值 RSS **5,064,331,264 bytes，约 4.72 GiB**。完整模型输出静态契约为 FP32 `[1,256,151936]`，绑定共 **2,384,216,772 bytes**；这些是解析与绑定审计结果，本次未执行完整模型数值前向。以上耗时均为本机单次运行，期间存在并行测试，不用于声称性能改善。 - -已核对完整解析报告的 20 项、RISC-V 报告的 30 项源码 SHA256 与当前源码一致；完整解析的 `nodes.json`、`bindings.json`、`ir.txt` 的文件大小和 SHA256 均与报告一致。测试后仅补充说明文档，执行源码未再修改。输出目录被 Git 忽略，仅在本机保存；他人复现需运行入口生成自己的报告。 - -最终还实跑了 0.1 秒截止和模型缺失两个失败路径：均正确退出 1，分别记录 `timeout`、`hashes` 阶段，且保留启动前的 20 项源码指纹；报告在 `output/review-fix-final-expected-timeout/` 和 `output/review-fix-final-expected-missing/`。文档检查包含尚未跟踪的新文档，共检查 97 个文件、196 个相对链接，0 个失效；LLM workflow YAML 与 19 段 shell 命令语法检查通过，`git diff --check` 通过。 - -当前没有已确认且尚未修复的本轮阻断性问题。完整 IR/ORT 数值与容量验收、完整模型 QEMU 前向、权重独立装载、W1 第二人复现和接口签字仍未由本次工作完成。本页实测均为本地证据，远端 CI 状态及 run 链接以 PR 描述和 Checks 为准。 - -## 首次交付结果（2026-10-02，历史记录） - -首次交付工作在独立本地分支 `codex/w2-full-qwen3-parse`，基于 PR #91 的 `df18b02f152878b1ea838bbf6073def01d367456`。实际工作目录为 `D:/cyq/code/ScratchV/output/w1-pr89-review`；当时修改未暂存、未提交、未推送，尚未更新 PR #91。上层原工作目录中的已有改动未纳入本轮。 - -首次交付的完整实物执行退出 0，结果 **PASS**。本地证据保存在 `output/w2-full-qwen3-parse-complete/`：`report.md`、`report.json`、`nodes.json`、`bindings.json` 和 `ir.txt`。这些是该次 checkout 的本地输出，`output/` 不进入版本库；其他机器需执行入口重新生成，不能依赖这些历史路径存在。 - -| 验收项 | 首次交付实测 | -|---|---| -| 固定模型 | Qwen3-0.6B，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 / L=256 / 无 KV Cache | -| 完整层链 | 28 / 28 层，含最终归一化与共享 embedding / LM Head | -| ONNX 节点 | 7,847 / 7,847；23 类基础算子；全部可达 logits | -| ONNX value | 8,161 项静态 shape / dtype 审计通过 | -| 实际 IR | 5,910 条指令；IR verifier 0 个问题 | -| 真实绑定 | 312 个 initializer + 1,836 个 Constant，共 2,148 项；逐项内容哈希通过 | -| 绑定字节数 | 2,384,216,772 bytes | -| 合法别名 | 112 个 Identity，均核对实际对象映射 | -| 返回值 | `logits`，FP32 `[1,256,151936]` | -| 纯解析耗时 | 2.493 秒 | -| 全入口耗时 | 10.533 秒，包含检查、解析、审计与子进程开销 | -| 验证进程峰值 RSS | 5,063,430,144 bytes，约 4.72 GiB，Windows PeakWorkingSetSize | - -耗时来自单次本地运行，不是平均值或性能承诺；本轮没有做解析性能优化。报告记录运行环境、实际源码 SHA256 和产物哈希。工作树有本地改动，不能将此结果仅归属于基准 HEAD。 - -首次交付的轻量测试 **91 / 91 通过**，在固定 CPU 环境和现有开发环境各执行一次，分别记录于 `output/w2-gates-pinned-delivery.xml` 和 `output/w2-gates-general-delivery.xml`。测试含真实两层 fixture 经文件哈希、结构检查、生产 parser、IR verifier 和绑定审计的贯通，以及漏节点、错权重、错 shape/dtype、外部数据损坏、解析异常、缺报告与超时等反例。当时全仓回归 **2,521 passed / 40 skipped**，记录于 `output/w2-regression-final.xml`；首次交付最后修正后重跑上述全部 91 项相关测试。当前修复后的验证以本页上方结果为准；跳过项不计为通过。 - -实际进程反例也已执行：模型缺失在 `hashes` 阶段退出 1；0.1 秒截止测试在 `timeout` 阶段退出 1，实测总耗时约 0.184 秒，均保存 FAIL 报告。记录分别位于 `output/w2-expected-missing-model/` 和 `output/w2-expected-timeout/`。 - -首次交付新增三个探测模块、三组测试与复现文档，并更新本地 LLM CI 配置。没有发现这份完整固定图需要新增前端算子，当时未修改通用 ONNX parser 或编译后端。首次互审修正了验收实现中的两个重点问题：控制参数必须从原 ONNX 独立求值，避免解析器错误自证通过；超时日志写入失败、进程异常退出及 verifier 失败时应尽量保留原始错误和节点证据。后续 Review 的更多修复见本页上方。 +| [通用 CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915411) | 成功；2927 passed / 7 skipped。Linux `test_w2_acceptance.py` 55/55、无跳过 | +| [LLM Deploy](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915420) | 成功；32/32 基础 QEMU、28/28 两层 QEMU、3/3 显式产物、20/20 超时清理；两层诊断最大误差 `1.7285346984863281e-6` | +| [Topic 06](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915428) | 成功；范围以该 run 的实际步骤与 artifact 为准 | -本阶段已具备完整前端解析的本地验收证据；远端 CI、第二人复现、完整模型 IR 数值及完整模型 QEMU 前向尚未由本轮验证。 +PR 中按条件未运行的完整 ONNX ORT 重型任务仍是未执行，不因上述绿色记录而算作完整 ORT 重验。新的提交需要新的 Checks；已有 Linux CI 不替代本次七项统一入口的第二人独立复现。 -## 后续衔接 +## 分项入口与后续边界 -1. 前端审计通过后，依据实际图和权重规模建立完整 IR 数值与峰值内存基线，逐步扩大预训练模型验证范围。 -2. 设计代码与权重分离、权重绑定 ABI 和内存布局；当前 C 常量展开及小模型 guest 配置不能直接承担完整 0.6B 模型。 -3. 完整 IR 数值验收通过后，再验证完整 QEMU 单次前向；随后将已有 Tokenizer、输入准备、最后有效 token logits、采样和停止条件组合成生成循环，并验证真实文本输出。 +- [完整前端解析](../../../probes/w2_qwen3_parse/README.md)：无须 Torch 或 QEMU，可单独准备固定解析环境。报告含 `nodes.json`、`bindings.json`、`ir.txt`。 +- [前端模式与基础后端](../../../probes/w2_backend_ops/README.md)：NumPy 独立公式、ORT、IR 和 QEMU 四路检查。 +- [Tokenizer 与输入](../../../probes/w2_runtime/README.md) 和 [单步模型接口](../../../probes/w2_runtime_model/README.md)：模型词表 151936 与实际可解码 ID 集合分开处理，不取模重映射。 +- [两层数值探测](../../../probes/w2_qwen3_small/README.md)：普通图、诊断图、产物绑定、仿真计时与精度边界。 +- [发布范围与修复](W2-分支验收与发布说明.md)：共享后端、日志和 Linux 取消清理的交付归属。 -前端解析通过不能跳过这些阶段，也不能据此宣布完整 Qwen3 已在 ScratchV 上运行。开发范围见 [总开发计划](../开发计划.md)，W1 人工验收状态见 [W1 独立复现记录](../W1/README.md)。 +完整解析通过不能替代完整 IR 数值、完整权重独立装载、完整模型 QEMU 前向或真实文本生成。阶段安排见 [开发计划](../开发计划.md)。 diff --git "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" index e08da37..1cde2ae 100644 --- "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -1,125 +1,9 @@ -# W1 修复与 W2 本地验收报告 +# W1 修复与 W2 历史验收记录索引 -日期：2026-10-03。**历史整合工作树报告**：记录的是发布前 W1+W2 合并在本地的验证；其中“未提交”等为当时状态。后来按 W1/W2 拆分独立发布，当前 W2 分支结果见 [分支验收与发布说明](W2-分支验收与发布说明.md)，不能将下述组合源码哈希当作当前分支哈希。 +本文件名保留以兼容已有链接。当前正式交付、CI 和独立复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，从 [Linux 复现约定](../LINUX_REPRODUCTION.md) 和 [W2 七项统一入口](README.md) 开始。 -## 1. 结论与源码位置 +原报告记录 2026-10-03 发布前的个人混合工作树，包含 W1 修复与 W2 增量，既不是独立 W2 分支的精确源码，也不是 Linux 实测。原始测量和当时状态完整保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W1修复与W2本地验收报告.md)。没有修改原始平台标签、计数、误差或耗时来生成新的通过结论。 -W1 的报告可靠性与源码归属修复已补入；W2 已在 PR #91 合并基线上整合，并重新执行七项技术验收，结果 **7/7 PASS**。完整图的通过范围是前端解析、结构和权重绑定审计；完整 0.6B 的 IR 数值、QEMU 前向与生成循环仍未完成。本报告不替代团队冻结接口、第二人复现或远端 CI。 +已交付的修复包括报告写入失败与源码归属、标量常量绑定一致性、Linux 嵌套会话退出清理、七项统一验收及 CI 缺失/跳过检测。具体代码归属与 Linux run 见 [W2 分支验收与发布说明](W2-分支验收与发布说明.md)；W1 报告修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。 -本轮源码在 `D:/cyq/code/ScratchV/output/w1-w2-local-closeout`，本地分支 `codex/w1-w2-local-closeout`。基线是 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，所有增量均未提交。原 `output/w1-pr89-review` 的 W2 修改、`output/w1-pr91-repro-fixes` 和最外层工作目录的代码保持原样。 - -下述 `output/...` 均相对于本轮源码目录。它们是本机证据，不是仓库发布资产；复现者应使用新的目录生成自己的报告。 - -## 2. W1 补完了什么 - -| 修复 | 原问题 | 当前行为与主要文件 | -|---|---|---| -| 两层报告完整发布 | MD/HTML 写入失败或 JSON 发布失败可能留下成功证据 | 视图先写、JSON 最后原子发布；失败返回非零，尽力保存 FAIL，保留原始数值错误。`probes/w2_qwen3_small/run.py`、`riscv.py` | -| 源码与环境归属 | 无 `.git` 的源码副本可能向父目录查找 Git，误认其他 checkout；QEMU 对照环境可能与导出环境混淆 | 无本目录元数据时明确未知 Git 身份，保留源码 SHA256；分别记录当前执行与导出环境，并补齐 parser/IR/优化/运行时指纹 | -| 回归覆盖 | 成功路径测试不足以验证失败时的证据 | 故障注入覆盖写入、关闭、替换失败与原始错误保留；补源码副本及当前环境归属回归 | -| 原有算子用例与固定 ORT 环境兼容 | ORT 1.22.1 无 DOUBLE Cos 内核，ReduceMean 的 omitted axes + noop=1 路径报内核错误 | 三项用例显式使用 ONNX 官方 ReferenceEvaluator，继续实际运行生产 parser/IR，ReduceMean 再检查与输入完全相等；其余用例继续 ORT 对照，不捕获或忽略数值错误 | -| 共享后端与运行时防错 | 外部绑定可能覆盖标量 IR 常量；日志 I/O 失败可能遮蔽原编译/QEMU 错误 | `tensor_c_codegen.py` 校验常量绑定一致；`riscv_tensor.py` 独立保存两路日志，保留主错误和耗时；一并整合之前 W2 分支暂存的修复 | -| 文档状态 | 仍使用 PR 未合并、旧提交或“所有第二人复现待做”的描述 | 固定已合并基线，保留 `df18b02f` 第二人两层复现事实，区分完整模型复现与接口签字待办 | - -在整合 W2 前，W1 四个源码/测试文件的专项测试已执行 **120 passed / 0 skipped**，证据为 `output/w1-fix-tests-v2.xml`。W1 独立检查点在 `output/w1-fix-checkpoint/`：包含补丁、四文件快照与哈希，便于后续按 W1/W2 分拆提交。这次没有重新打开或更新已合并的 PR #91。 - -最终共享修复的建议拆分补丁为 `output/w1-final-review/w1-fixes.patch`，含上述报告/归属、标量绑定、运行时清理/日志及 ORT 用例兼容的 10 个源码/测试文件，配有 `manifest.json`；它是供审阅的本地补丁，不是已暂存或已提交的改动。 - -最后还单独实跑原 W1 入口：MatMul **7/7**（4×4 小数最大误差 `1.4901161193847656e-8`，七例最大 `2.384185791015625e-7`）；PR #89 合成两层图的 IR vs ORT 最大误差约 `7.153e-7`，ORT vs 独立 NumPy 约 `1.097e-6`。证据在 `output/w1-matmul-final/`、`output/w1-transformer-final/`。完整 0.6B ONNX 的 ORT 前向本轮没有重跑，其历史验收仍按原记录归属。 - -W1 人工出口继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪：团队 D1–D8 接口决议、风险处置责任与 Plan B、完整模型第二人复现和最终出口评审。本轮没有代签或关闭这些事项。 - -## 3. W2 工作与开发计划逐项对照 - -| 开发计划要求 | 当前实现与本轮证据 | 状态边界 | -|---|---|---| -| `unit:frontend-ops`：RMSNorm/RoPE/SwiGLU/GQA | 22 项真实 ONNX 模式测试全部通过 | 通过基础算子组合解析，未新增同名融合 IR 指令 | -| `unit:backend-ops`：基础算子 QEMU | MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA，7 类/16 小图 × none/all，共 32 次通过 | 超过表中六类最低覆盖；仅证明这些 shape 与输入 | -| `numeric:ir-small`：两层 IR vs ORT | 真实 Qwen3 类的两层缩小配置，7 输入、29 检查点、6 项不变性通过；none/basic/all 优化对照 | 随机固定权重，不是预训练权重切片 | -| Attention IR 定义 | 候选接口文档明确基础算子组合、Q/K 独立 RMSNorm、全维 RoPE、GQA 分组与加性 mask 语义 | 当前不强制新增融合 ATTENTION opcode；这项接口决议仍需团队确认 | -| `unit:runtime`：Tokenizer、greedy、mask | 固定官方 Tokenizer 20 组语料/480 对照，5 组输入、mask、采样与停止条件通过 | 往返要求按官方 tokenizer 的特殊 token/规范化语义校验，不声称任意文本字节级恒等 | -| `frontend:parse-qwen3`：完整 28 层 | 固定 FP32/L=256 产物完整解析；7,847 节点、8,161 value、2,148 绑定、5,910 IR 指令通过审计，verifier 无问题 | 不代表完整 IR 数值或 QEMU 已运行 | -| 单步运行时与模型接口集成 | 2 个文本、完整词表 151936；2 Torch + 2 ORT + 4 IR，共 8 次调用；原始 token ID、最后有效位置、greedy ID 与解码一致 | hidden32/FFN96 两层随机小配置；未执行完整词表 QEMU 或多步生成 | -| W1 数值链路不退化 | 两层普通/诊断图 × none/all × 7 输入，28 次真实 QEMU 全通过 | 原 W1 的 20 次 Linux 超时回归与 3 项显式产物集成 CI 配置继续保留 | -| CI 全绿 | 本地配置增加完整 28 层解析 PR 必跑任务与汇总检查，拒绝跳过/失败/缺失 | **尚未提交，所以没有本轮远端 CI 通过结论** | - -本轮整合原先仅本地保存的完整解析、后端和运行时工作，新增修复 CI 验收缺口与嵌套进程超时清理。生产后端还保留审查所得的标量常量绑定一致性检查，拒绝调用方用不同绑定覆盖 IR 常量；日志写入失败保留原编译/QEMU 错误与耗时。这些此前 W2 分支的修复均已在本轮基线上重新验证。 - -## 4. 本轮重新执行的验收数据 - -总入口：`scripts/run_w2_acceptance.py`。总报告：`output/w2-acceptance-20261003/report.json`，另有同名 Markdown/HTML。每项包含自己的原始报告、日志、命令、输出数组、模型或 ELF 身份。总入口核对 **136 项源码指纹**，拒绝旧目录、缺失门禁、skip、非有限误差及运行中代码变化。 - -| 门禁 | 结果 | 最大绝对误差 | 顶层观测耗时 | -|---|---|---:|---:| -| frontend | 22/22 | 不适用 | 1.157 秒 | -| backend | 32/32 QEMU | `2.384185791015625e-7` | 23.769 秒 | -| runtime | 20/20、480/480、5/5 | 离散结果精确对照 | 4.905 秒 | -| runtime-model | 2 文本、8 次调用 | `2.8312206268310547e-7` | 14.889 秒 | -| small-ir | 7 输入/29 检查点/6 不变性 | `1.9073486328125e-6` | 6.971 秒 | -| small-qemu | 28/28 QEMU | 普通图 `3.129243850708008e-7`；诊断图 `1.6689300537109375e-6` | 153.658 秒 | -| full-parse | 28/28 层、逐节点/绑定审计通过 | 本项不执行数值前向 | 11.111 秒 | - -数值门槛保持 `max_abs < 1e-5`、`rtol=0`，没有放宽精度、裁剪 token ID 或用历史产物替代本次执行。small-qemu 报告也包含主机 IR 参考，其全报告最大误差 `1.9073486328125e-6` 不应误写成 QEMU 自身误差。 - -整轮 **216.597 秒（约 3 分 37 秒）**。基础后端 32 次 QEMU 进程墙钟累计 **5.545 秒**；两层 28 次累计 **140.312 秒**，4 次 C→RV64 编译累计 **6.189 秒**。两层普通图平均约 1.06/1.07 秒（none/all），诊断图约 8.94/8.97 秒；诊断输出多，不能当作纯前向速度。 - -完整前端 worker 的纯解析 **2.579 秒**，验证流程 **10.818 秒**，峰值 RSS **5,064,474,624 bytes，约 4.72 GiB**；绑定数据 **2,384,216,772 bytes**。这些是本机单次观察，不是优化前后基准对比，也不等同真实硬件性能。 - -## 5. Review、回归及 CI 检查 - -本轮 Review 先修复两个新发现的问题： - -1. 完整解析原来只在手动/夜间任务运行，PR 绿色无法证明 W2 全部门禁。现在 `full-qwen3-frontend` 在匹配路径的 PR 上必跑；`w2-acceptance` 对所有依赖要求 success。对应 21 项结果校验测试覆盖失败、跳过、取消、缺失和畸形 JSON。分支保护设置未修改。 -2. 顶层探测的 Linux 超时直接杀进程组，可能遗留 `_run_process` 创建的独立编译器/QEMU 会话。现在先 SIGTERM 触发运行器清理，再强制清理原组；增加实际进程中断回归和 Linux 嵌套会话回归。Windows 路径沿用 Job/进程树处理。 - -扩大回归时还修复了上文三项 ORT 参考内核兼容问题；相关四个测试文件最终 **251 passed / 0 skipped**，证据 `output/review-regression-fixes.xml`。没有修改编译器计算算法或数值门槛来适应参考执行器。 - -超时与 CI 专项结果 **128 passed / 1 skipped**，证据 `output/timeout-ci-tests.xml`；跳过的是 Linux 专属嵌套会话集成用例，本机 Windows 无法执行，不能计作通过。W2 集成/报告/CI 结果校验前一轮为 **111 passed / 0 skipped**，证据 `output/w2-integration-tests.xml`。各批次有重叠，不能相加。 - -最终 `tests/` 全量回归 **3020 passed / 39 skipped / 0 failed，177.66 秒**，证据为 `output/final-all-tests-v4.xml` 和同名 `.log`。39 个 skip 明确分为：本机无 `qemu-riscv32` 的 38 个旧 RV32 用户态执行测试，以及 1 个 Linux 专属嵌套会话清理测试。真实 RV64 tensor 运行时测试、三项显式产物优化集成及本轮所有可在 Windows 执行的 W1/W2 测试均已执行；不能把 39 个 skip 写成通过。 - -最终工作流 YAML、29 段 Bash 语法、文档相对链接、W1 四类接口/21 条风险及 Plan B 的结构检查、`git diff --check` 通过；详见 `output/final-static-review.json`。七项验收记录的 136 项源码指纹仍与最终执行代码一致。报告及状态文档在数值执行结束后补写，不改变该数值结果对应的执行代码。项目约定的 `.claude/harness/verify/run.py` 不存在，未宣称运行 L2 harness。 - -失败尝试也保留：W1 首次测试因自定义 pytest 临时目录的父目录尚未创建而产生 82 个 setup error，创建目录后 120 项通过；全仓第一次在收集阶段发现固定模型环境缺少可选 `llvmlite`。后者使用已有开发环境的 llvmlite 0.49.0 与 tinyfive 1.0.0 的本地副本补入仅本轮测试的路径，未改固定模型环境或全局安装。 - -中间全仓尝试 `final-all-tests-v2` 为 228 failed / 2791 passed / 40 skipped：224 项是 LLVM 工具未发现，3 项是上述 ORT 参考限制，1 项文档链接失败发生在报告文件尚未写完时。`v3` 显式指定 llvm-mca 后为 5 failed / 3014 passed / 40 skipped，剩余 5 项均因 clang/ld.lld 未从 PATH 找到。最终通过 `output/run_local_regressions.py` 在 Python 进程内配置路径，并确认三项 LLVM 工具实际可运行；不把这些中间失败隐藏或计算成通过。 - -## 6. 尚待完成与后续工作 - -- 本轮 Linux 嵌套会话清理集成回归、修改后的 GitHub workflow 和完整解析 runner 的实际容量，需要发布后取得对应源码的 Linux CI 证据。源码/配置本地通过不等于远端已通过。 -- W2 需要第二人用自己的环境和全新输出目录运行七项统一验收；W1 已确认的是旧 SHA 的两层链路，不能自动覆盖这次新增修复或 W2。 -- W1 接口冻结与风险责任决议保持候选，继续按 Issue #92 讨论；不要因技术门禁通过代替团队确认。 -- 后续按 W3 先做真实维度、预训练权重子图和更深层 IR 数值/内存基线，再做权重与代码分离、权重加载 ABI、内存复用和完整模型 QEMU 前向；最后接多步生成。原 `backend=riscv` 标量选择器并未因此支持大模型，本路线使用 `tensor-c`。 - -## 7. 本机复现命令 - -在本轮源码目录执行，输出目录必须是全新的。下面使用现有工具与固定模型资产，不自动安装或下载。 - -```powershell -$w2Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-review-new ` - --tokenizer-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` - --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` - --cc D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/zig-python/ziglang/zig.exe ` - --qemu D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/qemu/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -人工首先检查本报告、七项总报告和文件清单，再决定后续拆分提交方式。本轮没有创建、修改或关闭 GitHub PR/Issue。 - -## 8. 修改文件审阅顺序 - -| 范围 | 主要文件 | -|---|---| -| W1 两层证据可靠性 | `probes/w2_qwen3_small/run.py`、`riscv.py` 及对应 gate 单测 | -| 编译/运行时基础修复 | `scratchv/backend/tensor_c_codegen.py`、`scratchv/runtime/riscv_tensor.py` 及对应单测 | -| 固定 ORT 测试兼容 | `tests/test_onnx_qwen_numeric_ops.py`、`tests/test_onnx_shape_ops.py` | -| 完整图审计 | `probes/w2_qwen3_parse/{run,validation,audit}.py`、三组 `test_qwen3_full_*.py` | -| 基础后端与前端模式 | `probes/w2_backend_ops/`、`tests/test_qwen3_frontend_patterns.py`、后端 gate 测试 | -| Host 运行时与单步模型集成 | `scratchv/runtime/{qwen3_tokenizer,llm_inputs}.py`、`probes/w2_runtime/`、`probes/w2_runtime_model/` 及对应测试 | -| 验收入口与 CI | `scripts/run_w2_acceptance.py`、`scripts/check_w2_ci_results.py`、`.github/workflows/llm-deploy.yml` 及对应测试 | -| 文档与依赖声明 | W1/W2 指南、执行/探测计划、接口/风险候选、总开发计划、各 probe README、`pyproject.toml` 的可选 tokenizer 依赖 | - -最终全文件清单与哈希保存在 `output/final-review-manifest.json`；完整已跟踪文件差异在 `output/final-tracked.patch`，新增文件保留在上表目录。模型、参考数组、ELF、可选测试依赖副本均在被忽略目录，未纳入源码差异。 +新复现必须在指定 SHA 上自行生成报告，记录 Linux 环境、实际退出码、源码与模型身份以及误差。旧原始数据可用于调查历史问题，不能代替新提交执行或第二人确认。 diff --git "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" index 4e95d1e..fc5b57d 100644 --- "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" +++ "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" @@ -1,27 +1,27 @@ -# W2 独立分支验收与发布说明 +# W2 Linux 分支验收与发布说明 -W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。本分支直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含独立 W1 PR 中的六个两层报告/参考测试文件。两部分可分别审阅和合并。共享后端的标量绑定、运行时日志和取消清理加固随 W2 交付。 +正式复现环境为 Ubuntu 24.04 x86_64 / Bash / Python 3.12。环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)，七项统一入口和实际 CI 链接见 [W2 README](README.md)。 -## 范围 +W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。W2 初次独立交付基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`；[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 既有 Linux 执行对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。新修改按新 SHA 验收，不能继承旧 CI PASS。 -- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和权重绑定审计。 -- RMSNorm/RoPE/SwiGLU/GQA 前端模式，七类基础后端的 16 小图/32 次 QEMU。 -- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层小配置的单步模型集成。 -- 七项统一验收入口，完整前端 PR 必跑任务，以及拒绝失败、取消、跳过、缺失的 CI 汇总。 -- 标量 IR 常量外部绑定一致性；日志写入失败保留主错误；取消时清理独立编译器/QEMU 会话。 +## 交付范围 -本分支 Windows 固定 CPU 环境重新执行：**629 passed / 1 skipped**（仅 Linux 嵌套会话清理用例跳过）；**七项统一验收 7/7 PASS**，实际 32 次基础后端 + 28 次两层 QEMU 全通过。完整图为 28 层、7,847 节点、2,148 绑定、5,910 条 IR，verifier 与逐项审计通过。运行时为 20 组语料/480 对照/5 场景；完整词表单步集成为 2 文本、8 次调用。136 项源码指纹与发布前代码一致。 +- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和真实权重绑定审计。 +- RMSNorm/RoPE/SwiGLU/GQA 前端模式；七类基础后端的 16 小图、32 次 QEMU。 +- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层随机小配置的单步模型集成。 +- 七项统一验收、完整前端 PR 必跑任务，以及拒绝失败、取消、跳过和缺失的 CI 汇总。 +- 标量 IR 常量外部绑定一致性；保存日志失败时保留主错误；Linux 取消时清理独立编译器/QEMU 会话。 -总入口耗时 251.494 秒，两层 QEMU 进程累计 154.125 秒，基础后端 QEMU 累计 6.194 秒；并行验收时的本机单次观察，不用作性能对比。精度最大误差：基础后端 `2.384185791015625e-7`，单步集成 `2.8312206268310547e-7`，两层 IR `1.9073486328125e-6`，两层目标诊断 `1.6689300537109375e-6`，均小于 `1e-5`。 +## Linux 验证与源码归属 -另外，以本分支新生成的模型显式执行 none/basic/all 三项产物优化集成，**3/3 通过、0 跳过**，证据为 `output/w2-publish-artifact.xml`。 +历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c` 的通用 CI、LLM Deploy、Topic 06 均成功。通用 CI 为 2927 passed / 7 skipped；Linux 统一验收单测 55/55、无跳过；LLM 记录基础 QEMU 32/32、两层 QEMU 28/28、显式优化产物 3/3、退出清理 20/20。两层 QEMU 诊断最大绝对误差 `1.7285346984863281e-6`，小于 `1e-5`。原始 run 链接在 [W2 README](README.md)。 -证据在 `output/w2-publish-tests.xml`、`output/w2-publish-acceptance/report.json/md/html` 及各子目录。工作流 YAML、29 段 Bash、文档链接及 diff 格式检查通过。本分支没有重跑全仓 tests；原整合工作树 3020 项通过属于下述历史记录，不能用其代替本分支验证。固定 ORT 环境下三个原有算子参考用例的兼容修复归属 PR #93。项目的本地 L2 harness 文件缺失，未声称运行。此前 [W1 修复与 W2 本地验收报告](W1修复与W2本地验收报告.md) 记录的是两部分合在一起的历史工作树，不应冒充本分支的精确源码证据。 +跳过项不计为通过；按 PR 条件未运行的完整 ONNX ORT 重型任务不计为完整 ORT 重验。上述工作流集合与作者运行统一七项 CLI、第二人完整独立执行，是不同证据，分别记录。 -## 复现与限制 +发布前个人工作树的计数、数值和耗时保留于 [固定提交中的原报告](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W2-分支验收与发布说明.md)，不迁移成 Linux 测量值。早期 W1/W2 混合工作树记录见 [历史归档说明](W1修复与W2本地验收报告.md)，不代表独立 W2 分支的精确源码。 -运行 [统一入口](README.md) 的七项门禁；每次使用新的输出目录。源文件哈希、模型/ELF 身份、原始命令、误差和仿真墙钟时间均保存在生成的 JSON/Markdown/HTML 中。 +## 复现与限制 -完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合，无新增融合 opcode。第二人复现、接口冻结、完整模型权重独立装载和生成循环仍需后续完成。 +运行统一入口时每次使用不存在的新输出目录。保留 Linux 版本、CPU、依赖和工具版本、源文件哈希、模型/ELF 身份、原始命令、误差及仿真墙钟；七项完整通过才能记为 PASS。新的提交需要新的实际运行证据。 -Linux 嵌套会话清理用例在 Windows 上明确跳过；CI 配置已经提供入口，但只有对应提交的实际远端执行才能证明 Linux 通过。完整 ONNX ORT 前向继续由独立手动/夜间任务验证。 +完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合；完整词表单步模型使用随机小配置，不具备预训练模型语言能力。第二人独立复现和团队接口确认仍由参与者完成；完整权重装载、生成循环等后续阶段不能由本分支的小模型结果代替。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 030275d..8c6182e 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -11,6 +11,8 @@ --- +**交付环境约定（2026-10-06）**：W1 及后续阶段的公开指南、第二人复现和 CI 统一使用 Linux（Ubuntu 24.04 / Bash / Python 3.12）。作者个人环境不构成 Linux 验收；保留历史记录的真实平台和提交身份。环境安装、资产和后续提交规则见 [Linux 复现约定](LINUX_REPRODUCTION.md)。这只规定 Host 环境，目标仍为 RISC-V 裸机程序。 + ## 1. 项目概述 ### 1.1 背景 diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" index b39dab2..b1395f8 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -17,14 +17,22 @@ DSL -> DSLInterpreter -> 解释器结果 -> 期望值对比 ## 运行依赖 -测试命令需要在 ScratchV 仓库根目录运行。`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +复现环境统一为 Linux / Bash，使用 Python 3.12。测试命令需要在 ScratchV 仓库根目录运行。 +首次使用时创建并激活虚拟环境： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 调用真实 TinyFive 所需的配套接口位于 `scratchv/simulator/tinyfive.py`、 `scratchv/backend/register_alloc.py`、`scratchv/compiler.py` 和 `scratchv/main.py`。 安装基础测试依赖： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 基础模式需要以下依赖： @@ -34,8 +42,8 @@ pip install -e ".[topic06]" 如需生成 HTML 和 PNG 完整报告，再安装可选依赖： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` 其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 @@ -67,13 +75,13 @@ build/topic06/ # 运行时生成，不提交 在仓库根目录执行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` 默认同时执行解释器与 TinyFive，并分别记录状态。也可以显式选择： -```powershell +```bash python scripts/run_topic06_benchmarks.py --verification-backend both python scripts/run_topic06_benchmarks.py --verification-backend interpreter python scripts/run_topic06_benchmarks.py --verification-backend tinyfive @@ -102,13 +110,13 @@ python scripts/run_topic06_benchmarks.py --verification-backend tinyfive 更新性能基线： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` 性能退化阈值默认是 5%，可以按场景调整： -```powershell +```bash # 超过 2% 即判定为退化 python scripts/run_topic06_benchmarks.py --regression-threshold 2 @@ -122,19 +130,19 @@ python scripts/run_topic06_benchmarks.py --regression-threshold 10 按类别运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按用例名称进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 组合筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -160,7 +168,7 @@ benchmarks/topic06/baseline.json 测试和报告生成采用两阶段流程。先运行测试并生成与展示格式无关的元数据： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -172,7 +180,7 @@ benchmark_reports/topic06/metadata.json 随后由独立报告工具读取元数据并生成轻量报告： -```powershell +```bash python scripts/generate_topic06_report.py ``` @@ -187,7 +195,7 @@ benchmark_reports/topic06/cases/*.json 需要 HTML 和 PNG 时运行： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" index b5a687c..a95e6b0 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -366,7 +366,7 @@ Markdown 和 HTML 报告固定展示相同核心指标。CI 推荐读取 `benchm 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -374,7 +374,7 @@ python scripts/run_topic06_benchmarks.py 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` @@ -386,7 +386,7 @@ benchmarks/topic06/baseline.json 后续再次运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -398,7 +398,7 @@ python scripts/run_topic06_benchmarks.py 如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： -```powershell +```bash python scripts/run_topic06_benchmarks.py --regression-threshold 2 ``` @@ -478,48 +478,57 @@ benchmark_reports/topic06/course_report_instructions.png 测试运行时仍使用绝对路径调用编译器、读取用例和写入文件，避免工作目录变化影响执行。写入报告前，`_report_path()` 会将仓库内路径转换为相对于 ScratchV 仓库根目录的 POSIX 格式。 -例如，原报告中的： +例如，Linux 工作区中的绝对路径： ```text -D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl +/home/reviewer/ScratchV/tests/topic06/cases/activation/relu_only.dsl ``` -现在写为： +在报告中写为： ```text tests/topic06/cases/activation/relu_only.dsl ``` -该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖本机盘符与工作区位置，可以在 Windows、Linux、macOS 和 CI artifact 中查看。 +该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖作者机器的工作区位置，可在其他 Linux 工作区和 CI artifact 中查看。 --- ## 六、运行方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行，使用 Python 3.12： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +各节保留的历史测试结果不因命令格式调整而变成新的 Linux 执行证据；新结果以本次生成的报告为准。 + ### 6.1 安装依赖 基础测试： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 完整报告： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` ### 6.2 普通测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py python scripts/generate_topic06_report.py ``` 第一条命令生成 `metadata.json`，第二条命令读取元数据并生成 `report.md`、`report.json` 和单用例报告。生成 HTML 和 PNG 完整报告： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` @@ -527,19 +536,19 @@ python scripts/generate_topic06_report.py --full-report 按测试目录类别筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按 DSL 文件名进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 两个条件可以组合，组合时必须同时满足： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -549,7 +558,7 @@ python scripts/run_topic06_benchmarks.py --category tensor --filter relu ### 6.4 性能回归测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -557,7 +566,7 @@ python scripts/run_topic06_benchmarks.py ### 6.5 更新性能基线 -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" index 9cd0e8c..bf93541 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" @@ -8,7 +8,11 @@ push 使用 self-hosted）、Python 3.12 ## 专项测试 -```powershell +以下复现命令统一使用 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境： + +```bash +python3.12 -m venv .venv-dsl-diagnostics +source .venv-dsl-diagnostics/bin/activate python -m pip install -e . "pytest>=7,<10" python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_diagnostics_benchmark.py -v --tb=short ``` @@ -23,7 +27,7 @@ python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test 先为基线建立独立 checkout。以下 `997d2aa` 是本 PR 加入诊断实现之前的历史版本； rebase 后与最新上游比较时，应改用对应的 PR base SHA。 -```powershell +```bash git worktree add --detach ../ScratchV-dsl-baseline 997d2aa python -m benchmarks.bench_dsl_diagnostics --baseline-root ../ScratchV-dsl-baseline --json-output benchmark_reports/dsl_diagnostics.json --markdown benchmark_reports/dsl_diagnostics.md --html benchmark_reports/dsl_diagnostics.html ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" index dd5e243..2346026 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -507,6 +507,9 @@ scratchv bad.dsl -o output.s 2> error.txt ## 16. 测试计划 +本节复现命令统一使用 Linux / Bash，在仓库根目录和 Python 3.12 虚拟环境中执行； +依赖安装见 [CI 与 Benchmark](09-DSL错误提示美化器-CI与Benchmark.md#专项测试)。 + ### 16.1 快速测试 开发过程中每个小步骤运行： @@ -526,12 +529,6 @@ python -m pytest \ tests/test_dsl_diagnostics_cli.py -q ``` -Windows PowerShell 可把路径放在同一行执行。 - -```powershell -python -m pytest tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py -q -``` - ### 16.3 项目验证 ```bash diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" index 9fed5ea..bd89c20 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -286,20 +286,32 @@ convert_onnx_to_llvm(model) → LLVM IR (866K lines, 183MB) ## 7. 使用方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境和依赖： + +```bash +python3.12 -m venv .venv-regalloc +source .venv-regalloc/bin/activate +python -m pip install -e ".[all]" +``` + +CNN 用例还需要 `models/graph/cnn.onnx`；请先确认模型文件存在。LLVM 对比依赖 +`llvmlite`，报告中的 `llvm_available=false` 表示该项未执行，不能作为 LLVM 对比通过。 + ```bash -# 运行全部 4 项 benchmark，生成三格式报告（Windows PowerShell） -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json report.json ` - --output-html report.html ` - --output-md report.md +# 运行全部 4 项 benchmark，生成三格式报告 +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ + --output-md benchmark_reports/regalloc_bench.md # 单独运行某项 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_simple --repeats 100 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_dense --repeats 50 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_cnn ` +python -m benchmarks.test_regalloc.bench_simple --repeats 100 +python -m benchmarks.test_regalloc.bench_dense --repeats 50 +python -m benchmarks.test_regalloc.bench_cnn \ --cnn-path models/graph/cnn.onnx --repeats 30 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_pseudo --repeats 30 +python -m benchmarks.test_regalloc.bench_pseudo --repeats 30 ``` --- diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" index 8f5e0f3..a4c2dda 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" @@ -137,13 +137,16 @@ ### 5.2 30 次运行结果 -命令： - -```powershell -python -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json benchmark_reports/regalloc_bench.json ` - --output-html benchmark_reports/regalloc_bench.html ` +以下是原验证命令的 Linux / Bash 等价写法，在仓库根目录和已安装依赖的 Python 3.12 +虚拟环境中执行。环境准备见 [Benchmark 设计文档](17-寄存器分配-Benchmark设计文档.md#7-使用方式)。 +下表保留原历史测量，不代表使用这份 Linux 命令重新运行过。 + +```bash +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ --output-md benchmark_reports/regalloc_bench.md ``` diff --git a/probes/w1_qwen3_export/README.md b/probes/w1_qwen3_export/README.md index 02d5da2..f72c019 100644 --- a/probes/w1_qwen3_export/README.md +++ b/probes/w1_qwen3_export/README.md @@ -15,24 +15,19 @@ for blocked positions. Short-input predictions use the last valid token. ## Environment -Use Python 3.12. The existing `requirements/qwen3-small-probe.txt` CPU +The published reproduction target is Ubuntu 24.04 x86_64 with Bash and Python 3.12. See the [Linux reproduction contract](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md). The existing `requirements/qwen3-small-probe.txt` CPU environment also satisfies this gate. For a separate reproducible export environment: -```sh -python -m venv output/qwen3-export-venv -source output/qwen3-export-venv/bin/activate +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate python -m pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cpu python -m pip install -r requirements/qwen3-export.txt ``` -Keep this environment activated for all `python` commands below. On Windows -PowerShell, create it with `py -3.12 -m venv output/qwen3-export-venv`, then -activate it with `& .\output\qwen3-export-venv\Scripts\Activate.ps1` and run -the two installation commands above. If activation is unavailable, replace -every later `python` with `& .\output\qwen3-export-venv\Scripts\python.exe` -(or `output/qwen3-export-venv/bin/python` on Linux); put multiline commands on -one line in PowerShell rather than using shell backslash continuations. +Keep this environment activated for all `python` commands below; the interpreter is `.venv-linux/bin/python`. Record the checkout SHA and actual dependency versions with each new run. Verify/download mode enforces NumPy 2.2.6, ONNX 1.18.0, ONNX Runtime 1.22.1 and protobuf 5.29.5; @@ -41,7 +36,7 @@ export mode additionally enforces the complete fixed export stack, including ## Verify the fixed release -```sh +```bash python probes/w1_qwen3_export/run.py --mode download \ --model-dir output/qwen3-full-model \ --output-dir output/qwen3-full-probe --threads 2 @@ -59,7 +54,7 @@ uses a previously downloaded ZIP with the same size/hash checks. For an already available complete model directory, skip network access: -```sh +```bash python probes/w1_qwen3_export/run.py --mode verify \ --model-dir models/qwen3-0.6b-onnx \ --output-dir output/qwen3-full-probe --threads 2 @@ -78,7 +73,7 @@ ScratchV's full-model execution or language quality. Obtain the official `Qwen/Qwen3-0.6B` Hugging Face snapshot at revision `c1899de289a04d12100db370d81485cdf75e47ca`. For example, in the pinned environment: -```sh +```bash python -c "from huggingface_hub import snapshot_download; snapshot_download('Qwen/Qwen3-0.6B', revision='c1899de289a04d12100db370d81485cdf75e47ca', local_dir='output/qwen3-source')" python probes/w1_qwen3_export/run.py --mode export \ --source-dir output/qwen3-source \ @@ -110,7 +105,7 @@ measurement. Export mode also saves `export.log` and its reference artifacts. Source fingerprints are included in failed reports as well as successful ones. Upload the report directory as CI evidence, not the multi-gigabyte model. -```sh +```bash python -m pytest tests/test_w1_qwen3_export.py -q ``` diff --git a/probes/w2_backend_ops/README.md b/probes/w2_backend_ops/README.md index a535d14..dd26184 100644 --- a/probes/w2_backend_ops/README.md +++ b/probes/w2_backend_ops/README.md @@ -1,9 +1,11 @@ # W2 前端模式与基础后端算子验收 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 准备并激活环境。下文 `python` 为 `.venv-linux/bin/python`。 + `unit:frontend-ops` 明确检查 RMSNorm、RoPE、SwiGLU、GQA 四种分解图模式。 这些模式由现有 ONNX 基础算子表达，不声明已有独立融合 Attention/RMSNorm IR。 -```powershell +```bash python -m pytest tests/test_qwen3_frontend_patterns.py -q ``` @@ -11,12 +13,12 @@ python -m pytest tests/test_qwen3_frontend_patterns.py -q 交叉编译为 RV64GC ELF，然后在 QEMU `virt` 裸机执行。16 个用例分别运行 `none`、`all` 两个优化级别，共 32 次目标执行；不下载模型权重。 -```powershell -python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc path/to/zig --qemu path/to/qemu-system-riscv64 +```bash +python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" python -m pytest tests/test_w2_backend_ops_gate.py -q ``` -Linux 可省略工具路径，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 +也可省略工具参数，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 使用现有 `requirements/qwen3-small-probe.txt` 中的 NumPy/ONNX/ORT 环境即可， 本探测不需要 PyTorch。输出目录须不存在或为空；依赖、工具、编译、执行或 比较失败均非零退出，不将跳过当通过。 diff --git a/probes/w2_qwen3_parse/README.md b/probes/w2_qwen3_parse/README.md index 23a69e7..d303efe 100644 --- a/probes/w2_qwen3_parse/README.md +++ b/probes/w2_qwen3_parse/README.md @@ -6,29 +6,18 @@ ## 准备环境 -使用 Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 -Windows PowerShell： - -```powershell -py -3.12 -m venv output/qwen3-parse-venv -$parsePython = ".\output\qwen3-parse-venv\Scripts\python.exe" -& $parsePython -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" -& $parsePython -m pip check -& $parsePython -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE -``` - -Linux shell： +Linux Bash： ```bash +set -euo pipefail python3.12 -m venv output/qwen3-parse-venv . output/qwen3-parse-venv/bin/activate python -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" python -m pip check python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify \ --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -echo $? ``` 所有命令在仓库根目录执行。`--model-dir` 必填，应指向同目录保存 `model.onnx` 和三个真实 external-data 分片的完整产物，不能只有 ONNX 文件或 LFS pointer。`verify` 是默认模式，不访问网络；不存在完整模型时，将命令中的 `--mode verify` 改成 `--mode download`，由入口获取并校验 W1 固定发布产物。不要用新导出或其他 revision 的文件替换固定产物。 @@ -78,10 +67,10 @@ python -m pip install "pytest==9.1.1" python -m pytest tests/test_qwen3_full_structure.py tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py -q ``` -PowerShell 将上面命令中的 `python` 替换为 `& $parsePython`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 +命令中的 `python` 指已激活的 `output/qwen3-parse-venv/bin/python`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 [LLM 工作流](../../.github/workflows/llm-deploy.yml) 的 PR 路径接入上述三组测试，并在独立 `full-qwen3-frontend` job 以 `--mode download` 实跑完整固定图。手动/定时同样运行该解析任务。解析有独立 Summary，并以 `always()` 上传 `output/qwen3-parse/`，artifact 名为 `qwen3-full-frontend-parse-output`。下载约 1.24 GB，解析 worker 峰值 RSS 约 5 GB；runner 的实际容量与耗时仍须远端验证。 `w2-acceptance` 汇总任务要求数值任务和完整解析任务均实际成功，失败、取消、跳过或缺失都不能通过。手动/定时的 `full-qwen3-onnx` job 独立保留 W1 完整 ONNX ORT 验证；解析不依赖它成功。并行的两个完整模型任务会各自下载资产。 -PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存本地实跑与 Review 修复结果，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 +PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存 Linux 复现入口与已记录的 Linux CI 证据，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 diff --git a/probes/w2_qwen3_small/README.md b/probes/w2_qwen3_small/README.md index 66ccc2b..bda658d 100644 --- a/probes/w2_qwen3_small/README.md +++ b/probes/w2_qwen3_small/README.md @@ -27,20 +27,22 @@ RV64GC ELF，在 QEMU `virt` 裸机执行。下文“不是 RISC-V 验证”的 ONNX 输入为 `input_ids: INT64 [1,256]` 与 `attention_mask: FP32 [1,1,256,256]`。mask 是加性因果和 key-padding 掩码（允许位置为 0，屏蔽位置为 FP32 最小有限值），不是 tokenizer 常用的二维 0/1 mask；位置编号固定为 `0..255`。普通图输出为 `logits: FP32 [1,256,128]`。 -## 本地运行 - -先确认用于建环境的 `python` 是 Python 3.12。下面是 PowerShell 命令；Linux/macOS 可将 `& $probePython` 换成 `output/qwen3-probe-venv/bin/python`。 - -```powershell -python -m venv output/qwen3-probe-venv -$probePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $probePython -m pip install --upgrade pip -& $probePython -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu -& $probePython -m pip install -r requirements/qwen3-small-probe.txt -& $probePython -m pip install --no-deps -e . -& $probePython -m pip check -& $probePython probes/w2_qwen3_small/run.py --output-dir output/qwen3-small -& $probePython -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q +## Linux 复现 + +正式交付使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12。通用工具准备见 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)；在仓库根目录执行： + +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate +python -m pip install --upgrade pip +python -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu +python -m pip install -r requirements/qwen3-small-probe.txt +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +python probes/w2_qwen3_small/run.py --output-dir output/qwen3-small +python -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q ``` CPU PyTorch 必须在安装依赖文件前单独安装；随后 `torch==2.7.1` 会保留已安装的 `2.7.1+cpu`。不要只执行依赖文件安装而让 pip 从默认索引另选 CUDA 依赖。导出相关包采用已验证的固定组合，避免环境升级改变导出图或数值基线。 @@ -121,15 +123,13 @@ ScratchV 承担 ONNX/IR、IR 优化、张量循环生成、内存规划与输入 Linux 工具准备（Python 导出依赖仍按前面的固定版本安装）： ```bash +sudo apt-get update sudo apt-get install --no-install-recommends qemu-system-misc python -m pip install ziglang==0.14.1 export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" ``` -Windows 可使用便携 Zig 和 QEMU，设置 `SCRATCHV_CC` 为 `zig.exe` 的完整路径、 -`SCRATCHV_QEMU` 为 `qemu-system-riscv64.exe` 的完整路径。也会自动搜索仓库的 -`output/tools/`。不需要 WSL、Linux 镜像或系统安装；QEMU Windows 分发来源见 -[QEMU 下载页](https://www.qemu.org/download/#windows)。工具仅在本机准备，脚本不会隐式联网安装。 +设置 `SCRATCHV_QEMU="$(command -v qemu-system-riscv64)"`；所有工具均为 Linux 可执行文件。脚本不会隐式联网安装工具。 先验证矩阵乘，再导出模型并运行完整验收；输出目录必须为空，避免覆盖旧证据： diff --git a/probes/w2_runtime/README.md b/probes/w2_runtime/README.md index 462324a..3ed2242 100644 --- a/probes/w2_runtime/README.md +++ b/probes/w2_runtime/README.md @@ -8,18 +8,10 @@ 使用 [两层模型指南](../w2_qwen3_small/README.md) 中的 Python 3.12 固定 CPU 环境；其中已包含 `numpy==2.2.6`、`tokenizers==0.21.4`、`transformers==4.51.3`。仅使用生产 tokenizer 时可安装 `pip install -e ".[llm]"`，无需 Torch；复现 gate 仍使用固定版本和官方参考依赖。 -在仓库根目录运行，PowerShell： - -```powershell -$runtimePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $runtimePython -B -X utf8 probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime -$LASTEXITCODE -``` - -Linux： +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活环境，在仓库根目录运行： ```bash -output/qwen3-probe-venv/bin/python -B -X utf8 probes/w2_runtime/run.py \ +.venv-linux/bin/python -B -X utf8 probes/w2_runtime/run.py \ --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime ``` diff --git a/probes/w2_runtime_model/README.md b/probes/w2_runtime_model/README.md index 794fa0e..d58cd4f 100644 --- a/probes/w2_runtime_model/README.md +++ b/probes/w2_runtime_model/README.md @@ -4,9 +4,9 @@ 这是 W2 的接口集成证据；权重随机，不代表预训练 Qwen3-0.6B 的语言能力。执行单步前向与一个 greedy token，不执行生成循环、KV cache 或 QEMU。 -## 本地复现 +## Linux 复现 -使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活 `.venv-linux`。使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 ```bash python probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime diff --git a/scripts/check_linux_repro_docs.py b/scripts/check_linux_repro_docs.py new file mode 100644 index 0000000..dfe8b6f --- /dev/null +++ b/scripts/check_linux_repro_docs.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Reject host-specific reproduction instructions; optionally parse Bash blocks. + +This checks documentation, not execution or numerical acceptance. Historical +platform descriptions and portable implementation details remain valid. +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import shutil +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] +FENCE = re.compile(r"^\s*(`{3,}|~{3,})\s*([\w+-]*)\s*$") +HOST_PATTERNS = ( + (re.compile(r"(?= len(active) and not match.group(2): + if language in ("bash", "sh", "shell"): + blocks.append((start, "\n".join(body) + "\n")) + active = None + elif active: + body.append(line) + if language in ("bash", "sh", "shell") and re.search(r"\b[\w.-]+\.exe\b", line, re.I): + problems.append(f"{path}:{number}: non-Linux executable in shell example") + if active: + problems.append(f"{path}:{start}: unterminated fenced block") + if bash: + for start, content in blocks: + try: + result = subprocess.run([bash, "--noprofile", "--norc", "-n"], input=content, + capture_output=True, text=True, encoding="utf-8", timeout=10) + except (OSError, subprocess.TimeoutExpired) as exc: + problems.append(f"{path}:{start}: Bash validation could not run: {exc}") + continue + if result.returncode: + problems.append(f"{path}:{start}: invalid Bash: {result.stderr.strip()}") + return problems, len(blocks) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--check-bash", action="store_true", help="Require Bash and syntax-check every shell block") + parser.add_argument("--bash", help="Explicit Bash executable; defaults to PATH") + args = parser.parse_args(argv) + bash = (args.bash or shutil.which("bash")) if args.check_bash else None + if args.check_bash and not bash: + parser.exit(1, "FAIL: Bash is required for --check-bash\n") + paths = document_paths(args.root.resolve()) + if not paths: + parser.exit(1, "FAIL: no reproduction documents found\n") + problems, blocks = [], 0 + for path in paths: + findings, count = inspect_document(path, bash=bash) + problems.extend(findings) + blocks += count + for problem in problems: + print(problem) + print(f"{'FAIL' if problems else 'PASS'}: {len(paths)} documents, {blocks} shell blocks; " + f"Bash syntax {'checked' if bash else 'not requested'}; no model execution") + return int(bool(problems)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_linux_repro_docs.py b/tests/test_linux_repro_docs.py new file mode 100644 index 0000000..b528a79 --- /dev/null +++ b/tests/test_linux_repro_docs.py @@ -0,0 +1,67 @@ +"""Check rejection of unusable public reproduction commands, not doc wording.""" +import shutil + +import pytest + +from scripts import check_linux_repro_docs as docs + + +@pytest.mark.parametrize("content", [ + "```powershell\npython run.py\n```\n", + "~~~pwsh\npython run.py\n~~~\n", + "Use D:/private/models for the checkpoint\n", + "python C:\\private\\run.py\n", + "$env:OMP_NUM_THREADS = '1'\n", + "$LASTEXITCODE\n", + "venv/Scripts/python.exe\n", + "```bash\nzig.exe version\n```\n", + "```bash\necho unfinished\n", +]) +def test_rejects_non_linux_or_broken_examples(tmp_path, content): + path = tmp_path / "README.md" + path.write_text(content, encoding="utf-8") + assert docs.inspect_document(path)[0] + + +def test_linux_examples_and_truthful_history_are_allowed(tmp_path): + path = tmp_path / "README.md" + path.write_text('Historical Windows results are not Linux evidence.\n' + 'https://example.com/source\n' + '```bash\nset -euo pipefail\n' + '"$SCRATCHV_PYTHON" run.py --model-dir /srv/models/qwen3\n```\n', encoding="utf-8") + assert docs.inspect_document(path) == ([], 1) + + +def test_required_bash_cannot_silently_skip(monkeypatch): + monkeypatch.setattr(docs.shutil, "which", lambda _: None) + with pytest.raises(SystemExit) as exc: + docs.main(["--check-bash"]) + assert exc.value.code == 1 + + +def test_empty_input_is_not_success(tmp_path): + with pytest.raises(SystemExit) as exc: + docs.main(["--root", str(tmp_path)]) + assert exc.value.code == 1 + + +def test_inventory_covers_shared_guides_and_benchmarks(tmp_path): + names = ("README.md", "CONTRIBUTING.md", "docs/guide/setup.md", + "docs/topics/topic/reproduce.md", "probes/w4/example.md", "benchmarks/sample/README.md") + for name in names: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("Linux reproduction\n", encoding="utf-8") + assert {p.relative_to(tmp_path).as_posix() for p in docs.document_paths(tmp_path)} == set(names) + + +def test_actual_bash_rejects_bad_syntax_without_executing(tmp_path): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is required; Linux CI exercises this case") + path = tmp_path / "README.md" + marker = tmp_path / "must-not-exist" + path.write_text(f'```bash\ntouch "{marker.as_posix()}"\nif then\n```\n', encoding="utf-8") + problems, count = docs.inspect_document(path, bash=bash) + assert count == 1 and any("invalid Bash" in item for item in problems) + assert not marker.exists() diff --git a/tests/test_tensor_c_codegen.py b/tests/test_tensor_c_codegen.py index 23dd265..06b037f 100644 --- a/tests/test_tensor_c_codegen.py +++ b/tests/test_tensor_c_codegen.py @@ -1,7 +1,7 @@ """Execute generated tensor C against ORT, plus static/numeric rejection cases. -Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Repository -local output/tools/zig-python/ziglang/zig.exe is also recognized on Windows. +Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Optional +repository-local tools are resolved for the actual host platform. Validation tests do not require a C toolchain; execution tests skip explicitly when no compiler is available rather than reporting unexecuted cases as passes. """ @@ -35,7 +35,7 @@ def builder(*params): @pytest.fixture(scope="module") def compiler(): zig = os.environ.get("SCRATCHV_ZIG") - local = ROOT / "output/tools/zig-python/ziglang/zig.exe" + local = ROOT / "output/tools/zig-python/ziglang" / ("zig.exe" if os.name == "nt" else "zig") if zig or local.is_file(): return [zig or str(local), "cc"] for name in ("clang", "gcc", "cc"): From f7ebbfa745ca5be31aea857425b0e97e5ce614c2 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Tue, 6 Oct 2026 17:27:11 +0800 Subject: [PATCH 4/8] docs: align shared setup guide with Linux reproduction --- ...55\345\273\272\346\214\207\345\215\227.md" | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" index 952e2bd..868d42c 100644 --- "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" +++ "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" @@ -4,6 +4,10 @@ > **预计时间**: 15-30 分钟 > **前置要求**: 会用终端（命令行），电脑能上网 +本指南面向 Linux/Bash，推荐 Ubuntu 24.04。Qwen3 部署和 W1 及后续阶段的独立复现 +须使用 [Linux 固定环境指南](../llm-deploy-v1.0/LINUX_REPRODUCTION.md)，不要用本页的 +通用可选依赖替代该项目固定版本。个人开发环境不作为 Linux 验收依据。 + --- ## 目录 @@ -29,23 +33,14 @@ ### 如果你还没有安装 -**Ubuntu / Debian / WSL**: +**Ubuntu / Debian**: ```bash sudo apt update sudo apt install python3 python3-pip python3-venv python3-dev git build-essential ``` -**macOS**: -```bash -# 先安装 Homebrew（如果还没有）: https://brew.sh -brew install python@3.12 git -``` - -**Windows**: -- 推荐安装 [WSL2](https://learn.microsoft.com/zh-cn/windows/wsl/install)（Windows Subsystem for Linux），然后在 WSL 里按 Ubuntu 的方式操作 -- 或者从 [python.org](https://www.python.org/downloads/) 下载 Python 安装包 - -> 💡 **为什么推荐 WSL？** ScratchV 的 RISC-V 仿真工具（TinyFive、Spike）主要在 Linux 上测试过。WSL 让你在 Windows 上拥有一个完整的 Linux 环境，体验最好。 +在 Linux 主机上安装和验证编译器及仿真工具。复现记录须保存发行版、CPU、 +Python 和工具版本；其他个人环境的运行结果不能直接记作 Linux 通过。 --- @@ -87,7 +82,7 @@ remote: Counting objects: 100% (1234/1234), done. python3 -m venv .venv # 激活虚拟环境 -source .venv/bin/activate # Linux / macOS / WSL +source .venv/bin/activate ``` 激活成功后，你的终端提示符前面会出现 `(.venv)`： From 8fcc6ea5f7407c5e74f267dc4b5a34513400d2bf Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 11 Oct 2026 17:48:43 +0800 Subject: [PATCH 5/8] fix(w2): backport compact probe summaries to acceptance CI --- .github/workflows/llm-deploy.yml | 35 +++- docs/llm-deploy-v1.0/W2/README.md | 10 ++ probes/numeric_summary.py | 119 +++++++++++++ probes/w2_backend_ops/run.py | 22 ++- probes/w2_qwen3_small/diagnostics.py | 8 + probes/w2_qwen3_small/riscv.py | 217 +++++++++++++---------- probes/w2_qwen3_small/run.py | 207 ++++++++++++--------- probes/w2_runtime/run.py | 9 +- probes/w2_runtime_model/run.py | 39 +++- scratchv/verification/numeric_metrics.py | 140 +++++++++++++++ tests/test_numeric_metrics.py | 150 ++++++++++++++++ tests/test_numeric_summary.py | 123 +++++++++++++ tests/test_qwen3_small_reports.py | 119 +++++++++++++ tests/test_w2_actions_summary.py | 69 +++++++ tests/test_w2_runtime_model.py | 2 +- 15 files changed, 1075 insertions(+), 194 deletions(-) create mode 100644 probes/numeric_summary.py create mode 100644 scratchv/verification/numeric_metrics.py create mode 100644 tests/test_numeric_metrics.py create mode 100644 tests/test_numeric_summary.py create mode 100644 tests/test_qwen3_small_reports.py create mode 100644 tests/test_w2_actions_summary.py diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index 4771007..d2bb224 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -47,6 +47,8 @@ on: - 'tests/test_qwen3_tokenizer.py' - 'tests/test_llm_inputs.py' - 'tests/test_w2_*.py' + - 'tests/test_numeric_metrics.py' + - 'tests/test_numeric_summary.py' - '.github/workflows/llm-deploy.yml' schedule: # Nightly, off the hour: 19:23 UTC = 03:23 北京时间. @@ -132,7 +134,8 @@ jobs: tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py \ tests/test_llm_inputs.py tests/test_w2_backend_ops_gate.py \ tests/test_w2_runtime_gate.py tests/test_w2_runtime_metadata.py \ - tests/test_w2_runtime_reporting.py tests/test_w2_acceptance.py tests/test_w2_ci_results.py -q + tests/test_w2_runtime_reporting.py tests/test_w2_acceptance.py tests/test_w2_ci_results.py \ + tests/test_numeric_metrics.py tests/test_numeric_summary.py tests/test_w2_actions_summary.py -q - name: Repeated Linux timeout cleanup regression timeout-minutes: 5 @@ -260,7 +263,7 @@ jobs: "$QWEN3_PY" -m pytest \ tests/test_qwen3_small_probe.py \ tests/test_qwen3_small_model.py \ - tests/test_qwen3_small_gate.py -q \ + tests/test_qwen3_small_gate.py tests/test_qwen3_small_reports.py -q \ --junit-xml=output/qwen3-small/tests.xml - name: "numeric:qemu-qwen3-small" @@ -288,9 +291,19 @@ jobs: echo "unit:backend-ops: $BACKEND_OUTCOME" echo "unit:runtime: $RUNTIME_OUTCOME" echo "integration:runtime-model: $RUNTIME_MODEL_OUTCOME" - for report in output/w2-backend-ops/report.md output/w2-runtime/report.md output/w2-runtime-model/report.md; do - if [ -f "$report" ]; then cat "$report"; fi - done + emit_report() { + report="$1"; outcome="$2" + if [ "$outcome" = "skipped" ] || [ -z "$outcome" ]; then + echo "Not run / 未运行: $report. No numerical acceptance result." + elif [ -f "$report" ]; then + cat "$report" + else + echo "Report unavailable / 未生成报告: $report (step: $outcome). No numerical acceptance result; inspect logs." + fi + } + emit_report output/w2-backend-ops/report.md "$BACKEND_OUTCOME" + emit_report output/w2-runtime/report.md "$RUNTIME_OUTCOME" + emit_report output/w2-runtime-model/report.md "$RUNTIME_MODEL_OUTCOME" echo "These gates do not execute full-model IR/QEMU or text generation." } >> "$GITHUB_STEP_SUMMARY" @@ -326,8 +339,12 @@ jobs: { echo "## numeric:qemu-qwen3-small" echo "Outcome: $PROBE_OUTCOME" - if [ -f output/qwen3-riscv/report.md ]; then + if [ "$PROBE_OUTCOME" = "skipped" ] || [ -z "$PROBE_OUTCOME" ]; then + echo "Not run / 未运行. No QEMU numerical acceptance result." + elif [ -f output/qwen3-riscv/report.md ]; then cat output/qwen3-riscv/report.md + else + echo "Report unavailable / 未生成报告. No QEMU numerical acceptance result; inspect logs." fi } >> "$GITHUB_STEP_SUMMARY" @@ -343,10 +360,12 @@ jobs: echo "Probe: $PROBE_OUTCOME; regression tests: $TEST_OUTCOME" echo "Explicit artifact integration (requires 3 passed / 0 skipped): $ARTIFACT_TEST_OUTCOME" echo "" - if [ -f output/qwen3-small/report.md ]; then + if [ "$PROBE_OUTCOME" = "skipped" ] || [ -z "$PROBE_OUTCOME" ]; then + echo "Not run / 未运行. No IR numerical acceptance result." + elif [ -f output/qwen3-small/report.md ]; then cat output/qwen3-small/report.md else - echo "No report was produced; inspect the dependency/probe step logs." + echo "Report unavailable / 未生成报告. No IR numerical acceptance result; inspect logs." fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index 0db79de..1a8b36a 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -56,6 +56,16 @@ git status --short 总报告位于本次 `--output-dir` 下的 `report.json`、`report.md`、`report.html`；子目录保存各门禁原始报告、命令、日志、数组和模型/ELF 身份。源码指纹核对的是实际执行文件，不能把别处安装的模块误当当前 checkout。 +数值报告首页按链路展示普通图的最终 logits（模型输出）：ORT 对 PyTorch、原生 NumPy IR 对 ORT，以及生成 C 后经 RISC-V/QEMU 执行的输出对 ORT。最后一项同时验证 C 生成和目标端执行，并不是独立的宿主 C Transformer 测试，也不验证旧标量汇编后端。中间检查点用于定位误差，单独折叠，不混入最终输出的精度。 + +每次实际数组比较新增余弦相似度与相对 L2 误差：前者为两输出向量夹角的余弦，越接近 1 越一致；后者为 `||实际输出−参考输出||₂ / ||参考输出||₂`，越接近 0 越一致。汇总取各次比较中最小余弦、最大相对 L2 和最大绝对误差，它们可能来自不同用例；原有最大绝对误差门槛保持不变。零范数、非有限值等无法定义的指标明确标注原因，不填成 0；旧报告缺少新增指标时显示未记录。 + +逐用例、检查点、计时和完整 JSON 默认折叠。对应表下解释普通图/诊断图及 IR 优化等级；QEMU 秒数仅为进程墙钟时长。报告精简不减少实际测试：七组输入包含两组满长随机输入与五组边界/扰动用例，不是七个随机种子。 + +Actions Summary 直接展示上述精简报告。步骤被跳过时明确显示“未运行”，不使用残留报告；步骤已运行但报告缺失时显示“未生成报告”，不能据此认定数值验收通过。 + +**PPL / zero-shot 暂不纳入当前验收。** PPL（困惑度）衡量模型预测真实文本下一个 token 的能力；zero-shot 衡量不提供示例时完成指定任务的能力。W2 使用随机权重小模型，这两项不能说明其语言质量。后续应在真实预训练权重上固定语料、Tokenizer、上下文窗口和任务评分方式，再比较编译前后质量；本轮明确标记未评测。 + - 保存 checkout SHA、dirty/diff、Linux 发行版、CPU、Python/NumPy/ORT/Torch、Zig/QEMU 版本。 - 保存每项实际退出码、PASS/FAIL/PARTIAL/未执行、原始误差、首次失败阶段及报告位置。 - QEMU 时长是进程墙钟，包含启动、guest 执行、UART 和退出；不是目标硬件性能或纯 forward 时间。 diff --git a/probes/numeric_summary.py b/probes/numeric_summary.py new file mode 100644 index 0000000..3410d83 --- /dev/null +++ b/probes/numeric_summary.py @@ -0,0 +1,119 @@ +"""Compact numerical evidence tables; never invent metrics for older reports. + +The ``passed`` flag remains the existing numerical gate's result. The two +additional metrics are observations, not new pass/fail thresholds. A table row +is an aggregation of tensor comparisons, not a comparison of concatenated data. +""" +from __future__ import annotations + +from collections import Counter +from html import escape +import math +from numbers import Real + + +METRIC_EXPLANATION = ( + "最大绝对误差是单个元素最坏的偏差；相对 L2 = ||实际−参考||₂ / ||参考||₂，越小越好；" + "余弦相似度衡量两个输出向量方向的一致性，越接近 1 越好。" + "先逐次比较，再取最大绝对误差、最大相对 L2、最小余弦（可能来自不同用例，不拼接计算）。" + "原有通过门槛不变，其他检查仍生效；新增两项仅供观察，缺失或未定义明确标注。" +) + +_METRICS = ("max_abs", "relative_l2", "cosine_similarity") +_HEADERS = ("比较", "最大绝对误差", "相对 L2 误差", "余弦相似度", "结果") + + +def _metric_summary(rows, key): + values, missing, reasons = [], 0, Counter() + for row in rows: + if key not in row: + missing += 1 + continue + value = row[key] + reason = row.get(key + "_reason") or row.get("reason") + valid = isinstance(value, Real) and not isinstance(value, bool) + try: + number = float(value) if valid else None + valid = valid and math.isfinite(number) + except (OverflowError, ValueError): + valid = False + if valid and (number < 0 if key != "cosine_similarity" else not -1 <= number <= 1): + valid = False + if valid: + values.append(number) + else: + reasons[str(reason or ("未提供原因" if value is None else "数值无效"))] += 1 + worst = (min(values) if key == "cosine_similarity" else max(values)) if values else None + return dict(value=worst, valid_count=len(values), total=len(rows), + missing=missing, undefined=sum(reasons.values()), reasons=reasons) + + +def _metric_text(summary, key): + parts = [] + if summary["value"] is not None: + # 17 significant digits distinguish adjacent binary64 cosine values. + # Scientific notation preserves even the smallest nonzero errors. + number = summary["value"] + parts.append(format(number, ".17g") if key == "cosine_similarity" else format(number, ".16e")) + if summary["missing"]: + parts.append(f"未记录 {summary['missing']}/{summary['total']}") + if summary["undefined"]: + reasons = "；".join(f"{reason} ×{count}" for reason, count in summary["reasons"].items()) + parts.append(f"未定义 {summary['undefined']}/{summary['total']}（{reasons}）") + if summary["value"] is not None and (summary["missing"] or summary["undefined"]): + parts[0] += f"（已记录 {summary['valid_count']}/{summary['total']}）" + return "；".join(parts) or "未记录（无比较）" + + +def _table_rows(comparisons): + """Preserve caller ordering and distinguish gate results from coverage.""" + result = [] + for label, records in comparisons: + rows = list(records) + if not all(isinstance(row, dict) for row in rows): + raise TypeError("comparison rows must be dictionaries") + metrics = [_metric_summary(rows, key) for key in _METRICS] + passed = sum(row.get("passed") is True for row in rows) + failed = sum(row.get("passed") is False for row in rows) + unknown = len(rows) - passed - failed + if not rows: + status = "未测（0 条比较）" + elif passed == len(rows): + status = f"门槛通过（{passed}/{len(rows)}）" + else: + status = f"门槛未通过（通过 {passed}/{len(rows)}，失败 {failed}，未确认 {unknown}）" + if rows and any(item["missing"] or item["undefined"] for item in metrics): + status += "；指标不完整" + result.append([str(label), *(_metric_text(item, key) for item, key in zip(metrics, _METRICS)), status]) + return result + + +def _markdown_cell(value): + # Escape HTML as well: GitHub renders HTML inside Markdown table cells. + return escape(value).replace("|", "|").replace("\r", " ").replace("\n", " ") + + +def compact_metric_table(comparisons): + """Render ``[(label, [tensor_comparison, ...]), ...]`` as Markdown. + + Only individual comparison flags determine this table's gate status. The + caller must report its overall probe status (including execution failures) + separately; this table is not a replacement for that status. + """ + rows = _table_rows(comparisons) + if not rows: + return "暂无数值比较记录。" + def render(row): + return "| " + " | ".join(_markdown_cell(value) for value in row) + " |" + + return "\n".join([render(_HEADERS), render(["---"] * len(_HEADERS)), *(render(row) for row in rows)]) + + +def compact_metric_table_html(comparisons): + """HTML equivalent of :func:`compact_metric_table`, using identical rules.""" + rows = _table_rows(comparisons) + if not rows: + return "

暂无数值比较记录。

" + header = "".join(f"{escape(value)}" for value in _HEADERS) + body = "".join("" + "".join(f"{escape(value)}" for value in row) + "" for row in rows) + return f"{header}{body}
" diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py index c801a31..53fc22f 100644 --- a/probes/w2_backend_ops/run.py +++ b/probes/w2_backend_ops/run.py @@ -27,6 +27,7 @@ from probes.w2_backend_ops.cases import build_cases from probes.w2_qwen3_small.diagnostics import tensor_diff +from probes.numeric_summary import compact_metric_table, compact_metric_table_html, METRIC_EXPLANATION from scratchv.analysis.ir_verifier import verify_ir from scratchv.compiler import CompilerConfig, CompilerDriver from scratchv.frontend.onnx_parser import ONNXParser @@ -68,7 +69,8 @@ def interpret(program, bindings, feed, level): def evidence(): sources = [Path(__file__), ROOT / "probes/w2_backend_ops/cases.py", - ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "scratchv/compiler.py", + ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "probes/numeric_summary.py", + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "scratchv/compiler.py", ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/pass_manager.py", ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/verification/ir_interpreter.py", @@ -177,7 +179,8 @@ def _report_markdown(report): for item in case["executions"]: lines.append(f"| {case['name']} | {item['optimization']} | {'PASS' if item['passed'] else 'FAIL'} | " f"{item.get('qemu_vs_ort', {}).get('max_abs')} | {item['qemu_process_wall_seconds']} |") - lines.extend(["", "Timing includes QEMU startup, guest execution, UART transfer and exit; not guest cycles.", + lines.extend(["", "Level 的 none/all 表示关闭/开启全部 ScratchV IR 优化；C 交叉编译均使用 -O2。" + "QEMU process seconds 为进程墙钟秒数，包含启动、目标程序计算、输出传输和退出，不是纯计算耗时。", "", "Full diagnostics, commands, environment and source hashes:", "", "```json", payload.rstrip(), "```", ""]) return "\n".join(lines) @@ -209,9 +212,20 @@ def record_failure(): def views(): markdown = _report_markdown(report) - write("report.md", markdown) + summary = f"W2 基础后端算子检查：{'PASS' if report['passed'] else 'FAIL'}（展开详情）" + comparisons = [("基础算子 C→RISC-V/QEMU 对 ORT", [ + next((execution.get("qemu_vs_ort", {}) for execution in case.get("executions", []) + if execution.get("optimization") == level), {}) + for case in report.get("cases", []) for level in LEVELS])] + description = ("基础后端检查 MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA 七类算子或组合；" + "16 个用例分别关闭/开启 IR 优化，计划 32 次 QEMU 执行。不是完整 Transformer 模型。") + write("report.md", f"
\n{summary}\n\n{description}\n\n" + + compact_metric_table(comparisons) + "\n\n" + METRIC_EXPLANATION + + f"\n\n{markdown}\n
\n\n") write("report.html", "W2 backend operators" - "

W2 backend operators

" + escape(markdown) + "
") + "
" + escape(summary) + "

" + escape(description) + "

" + + compact_metric_table_html(comparisons) + "

" + escape(METRIC_EXPLANATION) + "

" + escape(markdown)
+              + "
") views() if failures: diff --git a/probes/w2_qwen3_small/diagnostics.py b/probes/w2_qwen3_small/diagnostics.py index 5766f66..dbb8b83 100644 --- a/probes/w2_qwen3_small/diagnostics.py +++ b/probes/w2_qwen3_small/diagnostics.py @@ -15,6 +15,8 @@ import onnx from onnx import TensorProto, helper, numpy_helper +from scratchv.verification.numeric_metrics import numeric_metrics, undefined_numeric_metrics + def _names(names, label): names = list(names) @@ -154,14 +156,18 @@ def tensor_diff(actual, expected, atol=1e-5) -> dict: actual_finite=finite[0], expected_finite=finite[1], finite=all(finite), empty=bool(comparable and actual.size == expected.size == 0), worst_index=None, actual_value=None, expected_value=None, reason=None, atol=atol) + report.update(undefined_numeric_metrics("comparison is invalid")) if not comparable: report["reason"] = "outputs must be NumPy arrays" + report.update(undefined_numeric_metrics(report["reason"])) return report if not shape_matches or not dtype_matches: report["reason"] = "shape mismatch" if not shape_matches else "dtype mismatch" + report.update(undefined_numeric_metrics(report["reason"])) return report if actual.dtype.kind not in "biuf": report["reason"] = "outputs must have numeric dtypes" + report.update(undefined_numeric_metrics(report["reason"])) return report if not all(finite): invalid = ~np.isfinite(actual) | ~np.isfinite(expected) @@ -169,7 +175,9 @@ def tensor_diff(actual, expected, atol=1e-5) -> dict: report.update(worst_index=[int(dim) for dim in index], actual_value=_json_scalar(actual[index]), expected_value=_json_scalar(expected[index]), reason="nonfinite output or reference") + report.update(undefined_numeric_metrics(report["reason"])) return report + report.update(numeric_metrics(actual, expected)) if not actual.size: report.update(passed=True, max_abs=0.0) return report diff --git a/probes/w2_qwen3_small/riscv.py b/probes/w2_qwen3_small/riscv.py index 6268024..a43e91a 100644 --- a/probes/w2_qwen3_small/riscv.py +++ b/probes/w2_qwen3_small/riscv.py @@ -24,7 +24,10 @@ import onnxruntime as ort from probes.w2_qwen3_small.diagnostics import compare_outputs, tensor_diff, unpack_trace -from probes.w2_qwen3_small.run import ATOL, arrays_sha256, input_cases +from probes.w2_qwen3_small.run import ( + ATOL, _details_views, _page_html, _table_views, arrays_sha256, input_cases, +) +from probes.numeric_summary import METRIC_EXPLANATION, compact_metric_table, compact_metric_table_html from scratchv.analysis.ir_verifier import verify_ir from scratchv.compiler import CompilerConfig, CompilerDriver from scratchv.frontend.onnx_parser import ONNXParser @@ -201,6 +204,7 @@ def run_probe(model_dir, out, report, *, cc=None, qemu=None, timeout=180): report["model_sha256"] = evidence["model_sha256"] report["schema_sha256"] = evidence["schema_sha256"] sources = [ROOT / "scratchv/compiler.py", Path(__file__), + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "probes/numeric_summary.py", *sorted((ROOT / "scratchv/optimizer").glob("*.py")), ROOT / "scratchv/backend/tensor_c_codegen.py", ROOT / "scratchv/runtime/riscv_tensor.py"] @@ -292,102 +296,131 @@ def run_probe(model_dir, out, report, *, cc=None, qemu=None, timeout=180): report["stage"] = "complete" -def write_reports(out, report): - report["timing"] = summarize_timing(report) +def _report_views(report): timing = report["timing"] counts = timing["status_counts"] - (out / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") - lines = ["# Two-layer Qwen3: ScratchV IR → C tensor kernels → RV64GC → QEMU", "", - f"Result: **{'PASS' if report['passed'] else 'FAIL'}**; stage: {report['stage']}", "", - "## 仿真耗时", "", - f"- 探测流程总耗时：**{format_seconds(timing['pipeline_seconds'])} 秒**（不含报告渲染）。", - f"- 已记录 QEMU 进程耗时合计：**{format_seconds(timing['qemu_process_wall_seconds_total'])} 秒**。", - f"- 成功交叉编译记录合计：**{format_seconds(timing['cross_compile_seconds_total'])} 秒**；" - f"已记录 {timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次构建，" - "不含 ScratchV 解析/IR 优化/代码生成或失败构建的耗时。", - f"- 计划 {timing['planned_count']} 次，已尝试 {timing['attempted_count']} 次；" - f"通过 {counts['success']}，数值失败 {counts['numeric_failed']}，" - f"运行失败 {counts['runtime_error']}，超时 {counts['timeout']}，" - f"未启动 {counts['not_started']}，未尝试 {timing['not_attempted_count']}。", - f"- 记录范围：{'部分结果，尚未完成全部执行/数值核验' if timing['incomplete'] else '全部执行/数值核验已有记录'}；" - "失败样本与通过样本分别统计，未测得的时长不按零计。", "", - "QEMU 进程墙钟时长包含启动、guest 计算、输出打包/UART 和退出；超时还包括进程清理。" - "不包含主机输入准备或输出解码。diagnostic 包含检查点开销，须与 normal 分开。" - "这些数据仅用于监控，不代表纯前向时长或目标硬件性能，也不作为性能通过阈值。", "", - "| 图/优化级别 | 已尝试/计划 | 通过/失败 | 未尝试 | 通过平均/最大(s) | 失败平均/最大(s) | 已测合计(s) |", - "|---|---:|---:|---:|---:|---:|---:|"] - group_rows = [] + cases = report.get("cases", []) + comparisons = [] + for field, levels, label in ( + ("optimized_ir", ("none", "basic", "all"), "IR 原生 NumPy（优化回归）/ ORT"), + ("qemu", ("none", "all"), "生成 C → RV64/QEMU / ORT"), + ): + rows = [] + for case in cases + [{}] * max(0, QEMU_CASE_COUNT - len(cases)): + for level in levels: + rows.append(next((row for row in case.get(field, []) + if row.get("graph") == "normal" and row.get("optimization") == level), {})) + comparisons.append((label, rows)) + title = "两层 Qwen3：编译到 RISC-V 的数值精度" + status = "PASS" if report.get("passed") else "FAIL" + description = ("真实 Qwen3 结构的两层缩小模型，固定随机权重，FP32、L=256、无 KV cache。" + "同一 ONNX 模型、权重和输入分别经 ORT、原生 NumPy IR，以及 ScratchV IR → C 张量代码 → " + "C 交叉编译器 → RV64GC → QEMU 执行。C 行测的是 RISC-V 上的实际输出，不是单独的主机 C 测试。") + scope = ("核心表只汇总普通图最终 logits 的最坏指标（全部位置含 padding），比较按“被测 / 参考”排列。" + "七个输入场景覆盖两份满长输入、1/17/255 token 边界、因果与补齐隔离。" + "普通/诊断图、29 个检查点及隔离检查仍须全部通过严格 MaxAbs < 1e-5（rtol=0）。") + timing_line = (f"仿真耗时：探测流程 {format_seconds(timing['pipeline_seconds'])} 秒；" + f"已记录 QEMU 进程累计 {format_seconds(timing['qemu_process_wall_seconds_total'])} 秒" + "（含启动和传输，不代表纯前向或目标硬件性能）。") + coverage = (f"执行覆盖：{timing['attempted_count']}/{timing['planned_count']} 次已尝试，" + f"{counts['success']} 次通过；" + + ("部分结果，尚未完成全部执行。" if timing['incomplete'] else "全部执行完成。")) + markdown = [f"# {title}", "", f"Result: **{status}**; stage: {report.get('stage')}", "", + description, "", scope, "", compact_metric_table(comparisons), "", METRIC_EXPLANATION, + "", coverage, "", timing_line] + html = [f"

{escape(title)}

Result: {status}; stage: {escape(str(report.get('stage')))}

", + f"

{escape(description)}

{escape(scope)}

", compact_metric_table_html(comparisons), + f"

{escape(METRIC_EXPLANATION)}

{escape(coverage)}

{escape(timing_line)}

"] + failures = [] + if report.get("error"): + failures.append(f"阶段 {report.get('stage')}: {report['error']}") + for case in cases: + if case.get("passed") is not True: + failures.append(f"{case.get('name')}: FAIL；有检查失败或未完成") + for row in case.get("qemu", []): + if row.get("passed") is not True: + reason = row.get("error") or row.get("reason") or "尚未完成数值核验" + first = row.get("checkpoints", {}).get("first_divergence") + failures.append(f"{case.get('name')} / {row.get('graph')}/{row.get('optimization')}: " + f"{row.get('status')}; {reason}" + (f"；首次超限检查点 {first}" if first else "")) + for row in case.get("optimized_ir", []): + if row.get("passed") is not True: + failures.append(f"{case.get('name')} IR {row.get('graph')}/{row.get('optimization')}: " + f"{row.get('reason') or 'FAIL'}") + for row in report.get("invariants", []): + if row.get("passed") is not True: + failures.append(f"{row.get('name')} / {row.get('graph')}/{row.get('optimization')}: 隔离检查 FAIL") + if not report.get("passed") and not failures: + failures.append("尚未完成全部验收，请检查 report.json。") + if failures: + markdown += ["", "**失败／未完成项**", "", *[f"- {escape(item)}" for item in failures]] + html.append("

失败／未完成项

    " + + "".join(f"
  • {escape(item)}
    • " for item in failures) + "
") + + def append_detail(title, headers, rows, note): + md, body = _table_views(headers, rows) + md, body = _details_views(title, md + "\n\n" + note, body + f"

{escape(note)}

") + markdown.extend(["", md]) + html.append(body) + + graph_note = ("图指模型计算步骤及其连接关系：normal 只输出最终 logits；diagnostic 增加中间检查点以定位误差，含额外打包与传输开销。" + "none/basic/all 指 ScratchV IR 优化：none 不优化；basic 提前计算常量并删除不影响输出的计算；" + "all 再加入局部指令简化、乘加融合和循环内重复计算外提。它们不是 C 编译优化等级，本探测的 C 编译固定使用 -O2。") + time_note = (graph_note + " QEMU 墙钟时间包含启动、目标程序计算、输出打包/串口传输和退出；超时还含清理，" + "不含主机输入准备或输出解码。未测得的时长不按零计；平均/最大值仅统计有测量值的样本。" + "七种输入不是重复采样，不能据此推导稳定加速比，也不设置性能通过阈值。" + f"成功交叉编译记录累计 {format_seconds(timing['cross_compile_seconds_total'])} 秒" + f"（{timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次），" + "不含 ScratchV 解析、IR 优化、代码生成及失败构建；流程时间不含报告渲染。") + groups = [] for group in timing["groups"]: successful, failed = group["successful"], group["failed"] - failed_count = group["attempted_count"] - group["status_counts"]["success"] - cells = [f"{group['graph']}/{group['optimization']}", - f"{group['attempted_count']}/{group['planned_count']}", - f"{group['status_counts']['success']}/{failed_count}", str(group['not_attempted_count']), - f"{format_seconds(successful['mean_seconds'])}/{format_seconds(successful['max_seconds'])}", - f"{format_seconds(failed['mean_seconds'])}/{format_seconds(failed['max_seconds'])}", - format_seconds(group['all_measured']['total_seconds'])] - group_rows.append(cells) - lines.append("| " + " | ".join(cells) + " |") - lines += ["", "平均/最大值只统计有实测时长的样本；失败数包含尚未启动但已尝试的条目。" - "七种输入不是重复采样，不据此宣称稳定加速比。", "", - "FP32, L=256, random weights, 29 checkpoints, strict max absolute error < 1e-5.", - "The C cross-compiler is Zig/LLVM; this does not validate the legacy scalar assembly selector.", - "IR optimization levels: none/basic/all. QEMU: none/all, normal and diagnostic graphs.", ""] - if report.get("error"): - lines += [f"Error: {report['error']}", ""] - lines += ["| Case | Graph | IR optimization | QEMU进程(s) | 超时设定(s) | Max abs | First divergent checkpoint | Status |", - "|---|---|---|---:|---:|---:|---|---|"] - execution_rows = [] - for case in report.get("cases", []): + groups.append([f"{group['graph']}/{group['optimization']}", + f"{group['attempted_count']}/{group['planned_count']}", + f"{group['status_counts']['success']}/{group['attempted_count'] - group['status_counts']['success']}", + group['not_attempted_count'], + f"{format_seconds(successful['mean_seconds'])}/{format_seconds(successful['max_seconds'])}", + f"{format_seconds(failed['mean_seconds'])}/{format_seconds(failed['max_seconds'])}", + format_seconds(group['all_measured']['total_seconds'])]) + append_detail("图与优化级别：仿真耗时明细", + ["图/优化级别", "已尝试/计划", "通过/失败", "未尝试", "通过平均/最大(s)", "失败平均/最大(s)", "已测合计(s)"], + groups, time_note) + executions = [] + for case in cases: for row in case.get("qemu", []): - cells = [case['name'], row['graph'], row['optimization'], - format_seconds(row.get('qemu_process_wall_seconds')), - format_seconds(row.get('timeout_seconds')), str(row.get('max_abs')), - row.get('checkpoints', {}).get('first_divergence') or 'none', row['status']] - execution_rows.append(cells) - lines.append("| " + " | ".join(cells) + " |") - (out / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8") - details = [] - for case in report.get("cases", []): - details.append(f"
{escape(case['name'])}: " - f"{'PASS' if case['passed'] else 'FAIL'}
"
-                       f"{escape(json.dumps(case, indent=2))}
") - - def html_table(headers, rows): - header = "".join(f"{escape(value)}" for value in headers) - body = "".join("" + "".join(f"{escape(value)}" for value in row) + "" - for row in rows) - return f"
{header}{body}
" - - overview = ("

仿真耗时

" - f"

探测流程总耗时：{format_seconds(timing['pipeline_seconds'])} 秒；" - f"已记录 QEMU 进程合计：{format_seconds(timing['qemu_process_wall_seconds_total'])} 秒；" - f"成功交叉编译记录合计：{format_seconds(timing['cross_compile_seconds_total'])} 秒" - f"（{timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次）。

" - f"

计划 {timing['planned_count']} 次，已尝试 {timing['attempted_count']} 次；" - f"通过 {counts['success']}，数值失败 {counts['numeric_failed']}，" - f"运行失败 {counts['runtime_error']}，超时 {counts['timeout']}，" - f"未启动 {counts['not_started']}，未尝试 {timing['not_attempted_count']}。" - f"{'部分结果' if timing['incomplete'] else '完整执行记录'}。

" - "

QEMU 时长包含进程启动、guest 计算、输出打包/UART 和退出，超时包含进程清理；" - "不含主机输入准备或输出解码。diagnostic 含检查点开销，与 normal 分开统计。" - "流程总耗时不含报告渲染；交叉编译仅统计成功记录，不含 ScratchV 解析/IR 优化/代码生成。" - "未测得不按零计，失败与通过样本分别计算平均/最大值。七种输入不是重复采样，" - "仅用于监控，不代表纯前向时长、稳定加速比或目标硬件性能，无性能通过阈值。

") - overview += html_table(["图/优化", "已尝试/计划", "通过/失败", "未尝试", "通过平均/最大(s)", - "失败平均/最大(s)", "已测合计(s)"], group_rows) - overview += "

逐次执行

" + html_table( - ["Case", "Graph", "IR优化", "QEMU进程(s)", "超时设定(s)", "Max abs", "首次偏差", "Status"], - execution_rows) - if report.get("error"): - overview += f"

Error: {escape(report['error'])}

" - (out / "report.html").write_text( - "Qwen3 RV64 probe" - "" - f"

Qwen3 RV64 QEMU: {'PASS' if report['passed'] else 'FAIL'}

" - + overview + "".join(details) + "", encoding="utf-8") + executions.append([case.get('name'), row.get('graph'), row.get('optimization'), + format_seconds(row.get('qemu_process_wall_seconds')), + format_seconds(row.get('timeout_seconds')), row.get('max_abs', '未统计'), + row.get('checkpoints', {}).get('first_divergence') or 'none', row.get('status')]) + append_detail("逐场景 QEMU 执行明细", + ["场景", "图", "IR 优化", "QEMU 进程(s)", "超时设定(s)", "本图输出 MaxAbs", "首次超限检查点", "结果"], + executions, graph_note + " 此明细中的 diagnostic MaxAbs 包含全部打包检查点；首页核心表只使用 normal logits。" + "超时设定是限制，QEMU 进程(s) 才是实测墙钟时间。") + checks = [] + for case in cases: + for row in case.get('optimized_ir', []): + checks.append([case.get('name'), f"IR {row.get('graph')}/{row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + for row in case.get('normal_vs_trace', []): + checks.append([case.get('name'), f"normal/diagnostic logits {row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + for row in report.get('invariants', []): + checks.append([row.get('name'), f"{row.get('graph')}/{row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + append_detail("优化 IR、普通/诊断图一致性与隔离检查", ["场景/检查", "路径", "MaxAbs", "结果"], checks, + "所有这些检查继续参与验收，完整逐检查点误差保存在 report.json。") + footer = ("完整数值明细、命令、环境、源码指纹、C/ELF 与原始数组随 CI artifact 提供；结构化报告为 report.json。" + "这条路径不验证旧的标量汇编选择器，也不声称完整预训练模型已在 QEMU 运行。") + markdown.extend(["", footer]) + html.append(f"

{escape(footer)}

") + return {"report.md": "\n".join(markdown) + "\n", "report.html": _page_html(title, "".join(html))} + + +def write_reports(out, report): + report["timing"] = summarize_timing(report) + (out / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") + for name, content in _report_views(report).items(): + (out / name).write_text(content, encoding="utf-8") def main(argv=None): diff --git a/probes/w2_qwen3_small/run.py b/probes/w2_qwen3_small/run.py index 35fb1ac..223f490 100644 --- a/probes/w2_qwen3_small/run.py +++ b/probes/w2_qwen3_small/run.py @@ -31,6 +31,10 @@ build_diagnostic_model, compare_outputs, tensor_diff, unpack_trace, ) +from probes.numeric_summary import ( # noqa: E402 + METRIC_EXPLANATION, compact_metric_table, compact_metric_table_html, +) + ATOL = 1e-5 SEQ = 256 VOCAB = 128 @@ -158,7 +162,9 @@ def provenance() -> dict: for path in [*sorted(Path(__file__).parent.glob("*.py")), ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/verification/ir_interpreter.py", - ROOT / "scratchv/verification/ir_numpy_ops.py"]} + ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/verification/numeric_metrics.py", + ROOT / "probes/numeric_summary.py"]} return result @@ -304,93 +310,132 @@ def reference(feed): report.pop("current_case", None) -def write_reports(out: Path, report: dict) -> None: - (out / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", - encoding="utf-8") - lines = ["# Real two-layer Qwen3 numerical probe", "", - f"Result: **{'PASS' if report['passed'] else 'FAIL'}**", "", - "Official transformers 4.51.3 Qwen3, seeded random weights, FP32, L=256, no KV cache.", - "All checkpoint positions must pass max absolute error < 1e-5 (rtol=0).", - "Padding queries are also checked and are reported separately in JSON.", - "Diagnostic outputs observe the same forward; ordinary logits-only IR is checked separately.", - "This probe does not validate pretrained language quality or RISC-V execution.", ""] +def _table_views(headers, rows): + """Render the same small, escaped table in both report formats.""" + def markdown_cell(value): + return escape(str(value)).replace("|", "|").replace("\n", "
") + + markdown = ["| " + " | ".join(map(markdown_cell, headers)) + " |", + "|" + "---|" * len(headers)] + markdown += ["| " + " | ".join(map(markdown_cell, row)) + " |" for row in rows] + header = "".join(f"{escape(str(value))}" for value in headers) + body = "".join("" + "".join(f"{escape(str(value))}" for value in row) + + "" for row in rows) + return "\n".join(markdown), f"{header}{body}
" + + +def _details_views(title, markdown, html, *, opened=False): + attribute = " open" if opened else "" + heading = f"{escape(title)}" + return f"{heading}\n\n{markdown}\n\n", f"{heading}{html}" + + +def _page_html(title, content): + css = ("body{font:16px/1.5 system-ui,sans-serif;max-width:1100px;margin:32px auto;padding:0 20px;color:#182536}" + "details{margin:16px 0;padding:12px;border:1px solid #dce3eb;overflow:auto}" + "summary{cursor:pointer;font-weight:650}table{border-collapse:collapse;width:100%;font-size:14px}" + "th,td{text-align:left;padding:8px;border:1px solid #dce3eb}" + "pre{white-space:pre-wrap;overflow-wrap:anywhere}.failure{color:#a42020;font-weight:bold}") + return ("" + "" + f"{escape(title)}{content}") + + +def _host_report_views(report): + cases = report.get("cases", []) + comparisons = [] + for name, label in (("pytorch_vs_ort", "ORT / PyTorch（Transformer 导出）"), + ("ir_vs_ort", "IR 原生 NumPy / ORT")): + rows = [case.get("ordinary_logits", {}).get(name, {}) for case in cases] + # Missing cases or old metric fields must not look like full coverage. + rows += [{}] * max(0, 7 - len(cases)) + comparisons.append((label, rows)) + description = ("官方 Qwen3 结构的两层缩小模型，固定随机权重，FP32、L=256、无 KV cache。" + "同一模型、同一权重和输入分别执行 PyTorch、ONNX Runtime（ORT）与 ScratchV IR 解释器。") + scope = ("核心表仅汇总普通图的最终 logits，覆盖全部位置（含 padding）。" + "比较名称按“被测 / 参考”排列；七个输入场景包括两份满长输入、1/17/255 token 边界、因果与补齐隔离检查。" + "验收仍要求所有检查点及普通/诊断图一致性通过严格 MaxAbs < 1e-5（rtol=0）。") + limitation = "本报告验证数值计算，不运行 C/RISC-V，也不评估随机权重模型的语言能力（PPL、zero-shot 未启用）。" + status = "PASS" if report.get("passed") else "FAIL" + title = "两层 Qwen3：Transformer 导出与 IR 精度" + markdown = [f"# {title}", "", f"Result: **{status}**", "", description, "", scope, "", + compact_metric_table(comparisons), "", METRIC_EXPLANATION, "", limitation] + html = [f"

{escape(title)}

Result: {status}

", + f"

{escape(description)}

{escape(scope)}

", compact_metric_table_html(comparisons), + f"

{escape(METRIC_EXPLANATION)}

{escape(limitation)}

"] + failures = [] if report.get("error"): - lines += [f"Failed stage: `{report['stage']}`", "", "```text", report["error"], "```", ""] - lines += ["| Case | Valid tokens | PyTorch / ORT max error | IR / ORT max error | First divergence | Result |", - "|---|---:|---:|---:|---|---|"] - for case in report.get("cases", []): - pairs = [case.get(name, {}) for name in ("pytorch_vs_ort", "ir_vs_ort")] - maxima = [max((row["max_abs"] for row in pair.get("checkpoints", []) - if row.get("max_abs") is not None), default=None) for pair in pairs] - values = [f"{value:.3e}" if value is not None else "n/a" for value in maxima] - first = next((pair.get("first_divergence") for pair in pairs if pair.get("first_divergence")), "none") - lines.append(f"| {case['name']} | {case['valid_length']} | {values[0]} | {values[1]} | " - f"{first} | {'PASS' if case['passed'] else 'FAIL'} |") - for case in report.get("cases", []): - lines += ["", f"
{case['name']}: checkpoint comparisons", "", - "| Comparison | Checkpoint | Max absolute error | Result |", "|---|---|---:|---|"] + failures.append(f"阶段 {report.get('stage', 'unknown')}: {report['error']}") + for case in cases: + if case.get("passed") is not True: + first = next((case.get(pair, {}).get("first_divergence") for pair in + ("pytorch_vs_ort", "ir_vs_ort") if case.get(pair, {}).get("first_divergence")), None) + failures.append(f"{case.get('name', 'unknown')}: FAIL；" + + (f"首次超限检查点 {first}" if first else "有检查失败或尚未完成，请展开明细")) + for check in report.get("invariants", []): + if check.get("passed") is not True: + failures.append(f"{check.get('name')} / {check.get('backend')}: 隔离检查 FAIL") + if not report.get("passed") and not failures: + failures.append(f"阶段 {report.get('stage', 'unknown')}: 尚未完成全部验收") + if failures: + markdown += ["", "**失败／未完成项**", "", *[f"- {escape(item)}" for item in failures]] + html.append("

失败／未完成项

    " + + "".join(f"
  • {escape(item)}
    • " for item in failures) + "
") + + def append_detail(title, headers, rows, note="", *, opened=False): + md, body = _table_views(headers, rows) + if note: + md += "\n\n" + note + body += f"

{escape(note)}

" + md, body = _details_views(title, md, body, opened=opened) + markdown.extend(["", md]) + html.append(body) + + case_rows = [] + for case in cases: + ordinary = case.get("ordinary_logits", {}) + case_rows.append([case.get("name"), case.get("valid_length"), + ordinary.get("pytorch_vs_ort", {}).get("max_abs", "未统计"), + ordinary.get("ir_vs_ort", {}).get("max_abs", "未统计"), + "PASS" if case.get("passed") else "FAIL"]) + append_detail("输入场景明细", ["场景", "有效 token", "ORT / PyTorch MaxAbs", "IR / ORT MaxAbs", "全部检查"], case_rows) + checkpoint_rows = [] + for case in cases: for pair in ("pytorch_vs_ort", "ir_vs_ort"): for row in case.get(pair, {}).get("checkpoints", []): - lines.append(f"| {pair} | {row['name']} | {row.get('max_abs')} | " - f"{'PASS' if row['passed'] else 'FAIL'} |") - lines += ["", "
"] - lines += ["", "| Invariant | Backend | Max absolute error | Result |", "|---|---|---:|---|"] - for check in report.get("invariants", []): - lines.append(f"| {check['name']} | {check['backend']} | {check.get('max_abs')} | " - f"{'PASS' if check['passed'] else 'FAIL'} |") - (out / "report.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + checkpoint_rows.append([case.get("name"), pair, row.get("name"), row.get("max_abs"), + "PASS" if row.get("passed") else row.get("reason") or "FAIL"]) + for name, row in case.get("ordinary_logits", {}).items(): + if "diagnostic_vs_ordinary" in name: + checkpoint_rows.append([case.get("name"), name, "logits", row.get("max_abs"), + "PASS" if row.get("passed") else row.get("reason") or "FAIL"]) + append_detail("检查点与普通/诊断图一致性", ["场景", "比较", "检查点", "MaxAbs", "结果"], checkpoint_rows, + "normal 为只输出最终 logits 的普通图；diagnostic 额外输出中间检查点以定位误差。两种图都参与验收。") + append_detail("因果与补齐隔离检查", ["检查", "执行路径", "MaxAbs", "结果"], + [[row.get("name"), row.get("backend"), row.get("max_abs"), + "PASS" if row.get("passed") else "FAIL"] for row in report.get("invariants", [])]) + footer = "完整诊断、位置明细、命令、环境与源码指纹保存在 report.json；原始数组随 CI artifact 提供。" + markdown.extend(["", footer]) + html.append(f"

{escape(footer)}

") + return "\n".join(markdown) + "\n", _page_html(title, "".join(html)) + + +def _report_markdown(report: dict) -> str: + return _host_report_views(report)[0] + + +def _report_html(report: dict) -> str: + return _host_report_views(report)[1] + + +def write_reports(out: Path, report: dict) -> None: + (out / "report.json").write_text(json.dumps(report, indent=2, allow_nan=False) + "\n", encoding="utf-8") + (out / "report.md").write_text(_report_markdown(report), encoding="utf-8") write_html(out, report) def write_html(out: Path, report: dict) -> None: - """A portable, dependency-free view of the same numeric evidence.""" - def cell(value): - return escape(str(value)) - - def table(rows, columns): - header = "".join(f"{cell(column)}" for column in columns) - body = "".join("" + "".join(f"{cell(value)}" for value in row) + "" - for row in rows) - return f"{header}{body}
" - - status = "PASS" if report["passed"] else "FAIL" - sections = [f"

Real two-layer Qwen3 probe

{status}

", - "

FP32 · L=256 · 29 checkpoints · max absolute error < 1e-5

", - "

Official Qwen3 with random weights. All positions, including padding queries, are checked. " - "Ordinary logits and diagnostic graphs must both pass. This is not a RISC-V or pretrained quality test.

"] - if report.get("error"): - sections.append(f"
{cell(report['stage'])}: {cell(report['error'])}
") - for case in report.get("cases", []): - case_status = "PASS" if case["passed"] else "FAIL" - details = [] - for pair in ("pytorch_vs_ort", "ir_vs_ort"): - comparison = case.get(pair, {}) - details.append(f"

{cell(pair)}

First divergence: " - f"{cell(comparison.get('first_divergence') or 'none')}

") - rows = [[row["name"], row.get("max_abs"), row.get("worst_index"), - row.get("actual_value"), row.get("expected_value"), - "PASS" if row["passed"] else row.get("reason")] - for row in comparison.get("checkpoints", [])] - details.append(table(rows, ["Checkpoint", "Max abs", "Worst index", "Actual", "Reference", "Result"])) - auxiliary = {"capture_preserves_logits": case.get("capture_preserves_logits"), - "ordinary_logits": case.get("ordinary_logits"), - "attention_checks": case.get("attention_checks")} - details.append(f"

Other checks

{cell(json.dumps(auxiliary, indent=2))}
") - opened = " open" if not case["passed"] else "" - sections.append(f"{cell(case['name'])} · " - f"{case['valid_length']} valid tokens · {case_status}{''.join(details)}") - rows = [[check["name"], check["backend"], check.get("max_abs"), - "PASS" if check["passed"] else "FAIL"] for check in report.get("invariants", [])] - sections.append("

Metamorphic checks

" + table(rows, ["Invariant", "Backend", "Max abs", "Result"])) - sections.append("

Full environment, schemas, hashes and valid/padding breakdowns are in report.json.

") - css = ("body{font:16px/1.5 system-ui,sans-serif;max-width:1200px;margin:40px auto;padding:0 24px;color:#182536;background:#f7f9fc}" - "h1{font-size:30px}h3{font-size:18px}details{background:white;margin:16px 0;padding:16px;border:1px solid #dce3eb;border-radius:8px;overflow:auto}" - "summary{cursor:pointer;font-weight:650}table{border-collapse:collapse;width:100%;font-size:13px;background:white}" - "th,td{text-align:left;padding:8px;border-bottom:1px solid #dce3eb}pre{white-space:pre-wrap;overflow-wrap:anywhere;font-size:12px}" - ".pass{color:#126538;font-weight:bold}.fail{color:#a42020;font-weight:bold}") - document = ("" - f"Qwen3 small probe · {status}{''.join(sections)}") - (out / "report.html").write_text(document, encoding="utf-8") + (out / "report.html").write_text(_report_html(report), encoding="utf-8") def main(argv=None) -> int: diff --git a/probes/w2_runtime/run.py b/probes/w2_runtime/run.py index 7ec2e08..f6b93ec 100644 --- a/probes/w2_runtime/run.py +++ b/probes/w2_runtime/run.py @@ -280,17 +280,24 @@ def write(name, data): "Fixed official tokenizer assets; encode/decode vs Transformers fast + slow and golden corpus.", "NFC normalization is official behavior; equivalence does not mean arbitrary raw-text byte preservation.", "No pretrained weights, full-model inference or text generation is executed.", "", + "
Tokenizer 与输入准备逐项检查（展开详情）", "", "| Case | Checks | Passed |", "|---|---:|---|"] for row in report["tokenizer_cases"] + report["runtime_cases"]: lines.append(f"| {row['name']} | {row.get('checks', 1)} | {row['passed']} |") + lines.extend(["", "
", ""]) if "error" in report: lines.extend(["", "Error: " + report["error"]]) markdown = "\n".join(lines) + "\n" + intro, detail = markdown.split("
", 1) + detail, suffix = detail.split("
", 1) + caption, detail = detail.split("", 1) + caption = caption.removeprefix("") failures = [] documents = { "report.md": markdown, "report.html": 'W2 runtime gate
'
-        + html.escape(markdown) + "
", + + html.escape(intro) + '
' + html.escape(caption) + + '
' + html.escape(detail) + '
' + html.escape(suffix) + '
', } for name, content in documents.items(): if error := write(name, content): diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py index 05f6f29..ae8d091 100644 --- a/probes/w2_runtime_model/run.py +++ b/probes/w2_runtime_model/run.py @@ -24,7 +24,9 @@ from probes.w2_runtime.run import MANIFEST, checkout_evidence, require_environment, sha256, verify_assets from probes.w2_qwen3_small.run import arrays_sha256, environment +from probes.numeric_summary import compact_metric_table, compact_metric_table_html, METRIC_EXPLANATION from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token, prepare_inputs +from scratchv.verification.numeric_metrics import numeric_metrics SEQ, VOCAB, ATOL = 256, 151936, 1e-5 LEVELS = ("none", "all") @@ -79,7 +81,7 @@ def compare_logits(actual, expected, valid_length, *, vocab_size=VOCAB): maxima["valid_max_abs"] = max(maxima["valid_max_abs"], float(difference[:boundary].max())) if boundary < stop - start: maxima["padding_max_abs"] = max(maxima["padding_max_abs"], float(difference[boundary:].max())) - return {**maxima, "worst_index": worst, "elements": int(actual.size), + return {**maxima, **numeric_metrics(actual, expected), "worst_index": worst, "elements": int(actual.size), "passed": maxima["max_abs"] < ATOL, "atol": ATOL, "rtol": 0} @@ -154,7 +156,8 @@ def check_onnx_contract(model): def source_evidence(): - paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", + paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", ROOT / "probes/numeric_summary.py", + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "probes/w2_qwen3_small/model.py", ROOT / "probes/w2_qwen3_small/run.py", ROOT / "probes/w1_qwen3_export/manifest.json", ROOT / "scratchv/runtime/qwen3_tokenizer.py", ROOT / "scratchv/runtime/llm_inputs.py", @@ -334,11 +337,33 @@ def save_reports(out, report): failures = [] def views(): - text = ("# W2 runtime/model integration\n\nResult: **" + ("PASS" if report["passed"] else "FAIL") - + "**\n\nOfficial two-layer Qwen3, random weights, full vocabulary; one forward and greedy token.\n" - + "Not pretrained inference or a generation loop. All positions use strict max_abs < 1e-5.\n\n```json\n" - + json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n```\n") - return {"report.md": text, "report.html": '
' + html.escape(text) + "
"} + status = "PASS" if report["passed"] else "FAIL" + description = ("两层随机权重 Qwen3，FP32、L=256、完整词表；两个文本场景使用同一权重和输入分别运行 " + "PyTorch、ORT、原生 NumPy IR，比较全部最终 logits（最大绝对误差 < 1e-5，rtol=0），" + "并核对最后有效位置选择分数最高 token 的结果。不是预训练模型或多步生成。") + comparisons = [] + for engine, label, reference in (("ort", "ORT 对 PyTorch", "vs_torch"), + ("ir-none", "IR 原生 NumPy（无优化）对 ORT", "vs_ort"), + ("ir-all", "IR 原生 NumPy（全部优化）对 ORT", "vs_ort")): + rows = [next((r.get(reference, {}) for r in case.get("executions", []) + if r.get("engine") == engine), {}) for case in report.get("cases", [])] + rows += [{}] * max(0, len(CASES) - len(rows)) + comparisons.append((label, rows)) + note = "PPL / zero-shot：未评测；随机权重模型仅用于验证计算和接口，不评价语言能力。" + error = str(report.get("error", "")) + payload = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + text = (f"# W2 runtime/model integration\n\nResult: **{status}**\n\n{description}\n\n" + + compact_metric_table(comparisons) + f"\n\n{METRIC_EXPLANATION}\n\n{note}\n\n" + + (f"失败原因：{error}\n\n" if error else "") + + "
文本、采样及完整诊断（展开详情）\n\n```json\n" + + payload + "\n```\n\n
\n\n") + page = ('

W2 runtime/model integration

' + + f'

Result: {status}

{html.escape(description)}

' + + compact_metric_table_html(comparisons) + f'

{html.escape(METRIC_EXPLANATION)}

' + + f'

{html.escape(note)}

' + (f'

失败原因：{html.escape(error)}

' if error else "") + + '
文本、采样及完整诊断（展开详情）
'
+                + html.escape(payload) + '
') + return {"report.md": text, "report.html": page} def failed(): report["report_write_errors"] = failures diff --git a/scratchv/verification/numeric_metrics.py b/scratchv/verification/numeric_metrics.py new file mode 100644 index 0000000..5a47965 --- /dev/null +++ b/scratchv/verification/numeric_metrics.py @@ -0,0 +1,140 @@ +"""Descriptive tensor metrics; these never decide numerical acceptance. + +Norms are accumulated in float64 relative to a power-of-two scale. Iteration +is bounded even for noncontiguous arrays and memory-mapped model outputs. +""" +from __future__ import annotations + +import math + +import numpy as np + + +def undefined_numeric_metrics(reason): + """JSON-safe fields for a comparison that has no numerical meaning.""" + return dict(relative_l2=None, cosine_similarity=None, + relative_l2_reason=reason, cosine_similarity_reason=reason) + + +class _ScaledSquares: + def __init__(self): + self.exponent = 0 + self.squares = 0.0 + + def add(self, values, exponent_shift=0): + maximum = float(np.max(np.abs(values), initial=0.0)) + if maximum == 0: + return + _, exponent = math.frexp(maximum) + normalized = np.ldexp(values, -exponent) + squares = float(np.sum(normalized * normalized, dtype=np.float64)) + exponent += exponent_shift + if self.squares == 0: + self.exponent, self.squares = exponent, squares + elif exponent > self.exponent: + self.squares = math.ldexp(self.squares, 2 * (self.exponent - exponent)) + squares + self.exponent = exponent + else: + self.squares += math.ldexp(squares, 2 * (exponent - self.exponent)) + + +def _chunks(actual, expected, block_elements): + return np.nditer([actual, expected], flags=["external_loop", "buffered"], + op_flags=[["readonly"], ["readonly"]], order="C", + buffersize=block_elements) + + +def numeric_metrics(actual, expected, *, block_elements=262144): + """Return cosine and ||actual-expected||_2 / ||expected||_2. + + Shapes and dtypes must match, with real numeric (at most FP64) data. Empty + and nonfinite tensors have undefined metrics. Cosine is undefined if either + norm is zero. A zero reference gives relative L2 zero only when actual is + also zero; otherwise it is undefined. Unrepresentable relative L2 values + are reported as null with a reason, never as infinity or an invented zero. + """ + if type(block_elements) is not int or block_elements < 1: + raise ValueError("block_elements must be a positive integer") + if not isinstance(actual, np.ndarray) or not isinstance(expected, np.ndarray): + return undefined_numeric_metrics("outputs must be NumPy arrays") + if actual.shape != expected.shape: + return undefined_numeric_metrics("shape mismatch") + if actual.dtype != expected.dtype: + return undefined_numeric_metrics("dtype mismatch") + if actual.dtype.kind not in "biuf" or (actual.dtype.kind == "f" and actual.dtype.itemsize > 8): + return undefined_numeric_metrics("metrics require real numeric dtypes up to float64") + if not actual.size: + return undefined_numeric_metrics("empty tensor") + + actual_norm, expected_norm, difference_norm = (_ScaledSquares() for _ in range(3)) + identical = True + for a, b in _chunks(actual, expected, block_elements): + if not np.isfinite(a).all() or not np.isfinite(b).all(): + return undefined_numeric_metrics("nonfinite output or reference") + equal = bool(np.array_equal(a, b)) + identical = identical and equal + a64, b64 = a.astype(np.float64), b.astype(np.float64) + actual_norm.add(a64) + expected_norm.add(b64) + if equal: + continue + if actual.dtype.kind in "biu": + # Subtract before rounding to FP64: int64/uint64 neighbours above + # 2**53 must retain their nonzero difference. Copies stay bounded. + difference = (a.astype(object) - b.astype(object)).astype(np.float64) + difference_norm.add(difference) + else: + with np.errstate(over="ignore", invalid="ignore"): + difference = a64 - b64 + overflow = ~np.isfinite(difference) + if overflow.any(): + difference_norm.add(difference[~overflow]) + # Only overflowing differences are halved, so unrelated tiny + # values are not lost to a premature division by two. + difference_norm.add(a64[overflow] * 0.5 - b64[overflow] * 0.5, 1) + else: + difference_norm.add(difference) + + result = dict(relative_l2=None, cosine_similarity=None, + relative_l2_reason=None, cosine_similarity_reason=None) + if expected_norm.squares == 0: + if actual_norm.squares == 0: + result["relative_l2"] = 0.0 + else: + result["relative_l2_reason"] = "zero reference norm with nonzero actual norm" + elif difference_norm.squares == 0: + result["relative_l2"] = 0.0 + else: + factor = math.sqrt(difference_norm.squares / expected_norm.squares) + try: + relative = math.ldexp(factor, difference_norm.exponent - expected_norm.exponent) + except OverflowError: + relative = math.inf + if not math.isfinite(relative): + result["relative_l2_reason"] = "relative L2 exceeds float64 range" + elif relative == 0: + result["relative_l2_reason"] = "nonzero relative L2 is below float64 range" + else: + result["relative_l2"] = relative + + if actual_norm.squares == 0 or expected_norm.squares == 0: + result["cosine_similarity_reason"] = "cosine is undefined for a zero norm" + elif identical: + result["cosine_similarity"] = 1.0 + else: + # Scale each vector independently; squaring unscaled FP64 values can + # overflow, and using one shared scale can erase the smaller vector. + dot, correction = 0.0, 0.0 + for a, b in _chunks(actual, expected, block_elements): + a64 = np.ldexp(a.astype(np.float64), -actual_norm.exponent) + b64 = np.ldexp(b.astype(np.float64), -expected_norm.exponent) + term = float(np.sum(a64 * b64, dtype=np.float64)) + total = dot + term + correction += ((dot - total) + term if abs(dot) >= abs(term) + else (term - total) + dot) + dot = total + cosine = ((dot + correction) / math.sqrt(actual_norm.squares) + / math.sqrt(expected_norm.squares)) + # Cauchy-Schwarz bounds the exact result; trim only FP64 roundoff. + result["cosine_similarity"] = min(1.0, max(-1.0, cosine)) + return result diff --git a/tests/test_numeric_metrics.py b/tests/test_numeric_metrics.py new file mode 100644 index 0000000..488f2d6 --- /dev/null +++ b/tests/test_numeric_metrics.py @@ -0,0 +1,150 @@ +"""Independent metric formulas, extreme ranges and unchanged gate decisions.""" +import json +import math + +import numpy as np +import pytest + +from scratchv.verification.numeric_metrics import numeric_metrics + + +@pytest.mark.parametrize("actual,reference,cosine,relative", [ + ([3, 4], [3, 4], 1, 0), + ([6, 8], [3, 4], 1, 1), + ([-3, -4], [3, 4], -1, 2), + ([1, 0], [0, 1], 0, math.sqrt(2)), + ([0, 0], [3, 4], None, 1), + ([0, 0], [0, 0], None, 0), + ([3, 4], [0, 0], None, None), +]) +def test_analytic_vectors(actual, reference, cosine, relative): + result = numeric_metrics(np.array(actual, dtype=np.float64), + np.array(reference, dtype=np.float64), block_elements=1) + for key, value in (("cosine_similarity", cosine), ("relative_l2", relative)): + if value is None: + assert result[key] is None and result[key + "_reason"] + else: + assert result[key] == pytest.approx(value, rel=2e-15, abs=1e-15) + assert result[key + "_reason"] is None + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("scale", [np.finfo(np.float64).max, 1e-300, + np.nextafter(np.float64(0), np.float64(1))]) +def test_squares_and_differences_do_not_overflow_or_erase_tiny_values(scale): + reference = np.array([scale, scale], dtype=np.float64) + opposite = numeric_metrics(-reference, reference, block_elements=1) + assert opposite["relative_l2"] == pytest.approx(2) + assert opposite["cosine_similarity"] == pytest.approx(-1) + orthogonal = numeric_metrics(np.array([scale, -scale]), reference) + assert orthogonal["relative_l2"] == pytest.approx(math.sqrt(2)) + assert orthogonal["cosine_similarity"] == 0 + + +def test_each_vector_has_its_own_scale_for_cosine(): + maximum = np.finfo(np.float64).max + minimum = np.nextafter(np.float64(0), np.float64(1)) + result = numeric_metrics(np.array([maximum]), np.array([minimum])) + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] is None + assert "exceeds" in result["relative_l2_reason"] + result = numeric_metrics(np.array([maximum, minimum]), np.array([maximum, 0])) + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] is None + assert "below" in result["relative_l2_reason"] + + +@pytest.mark.parametrize("dtype,base", [(np.int64, 2**63 - 2), (np.uint64, 2**64 - 2)]) +def test_integer_neighbours_keep_nonzero_difference_above_fp64_exact_range(dtype, base): + actual, expected = np.array([base + 1], dtype), np.array([base], dtype) + result = numeric_metrics(actual, expected) + assert result["relative_l2"] == pytest.approx(1 / base, rel=2e-15, abs=0) + assert result["relative_l2"] > 0 + assert result["cosine_similarity"] == 1 + + +def test_signed_integer_subtraction_cannot_wrap(): + limits = np.iinfo(np.int64) + result = numeric_metrics(np.array([limits.min]), np.array([limits.max])) + assert result["relative_l2"] == pytest.approx((2**64 - 1) / (2**63 - 1)) + assert result["cosine_similarity"] == -1 + + +@pytest.mark.parametrize("actual,expected,reason", [ + (None, np.ones(1), "NumPy arrays"), + (np.ones(2), np.ones(1), "shape mismatch"), + (np.ones(1, np.float32), np.ones(1, np.float64), "dtype mismatch"), + (np.ones(1, complex), np.ones(1, complex), "real numeric"), + (np.empty((2, 0)), np.empty((2, 0)), "empty"), + (np.array([1, np.nan]), np.ones(2), "nonfinite"), + (np.array([1, np.inf]), np.array([1, np.inf]), "nonfinite"), +]) +def test_undefined_inputs_have_explicit_reasons(actual, expected, reason): + result = numeric_metrics(actual, expected) + for key in ("relative_l2", "cosine_similarity"): + assert result[key] is None and reason in result[key + "_reason"] + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("block", [0, -1, True, 1.5]) +def test_bad_block_size_is_rejected(block): + with pytest.raises(ValueError, match="positive integer"): + numeric_metrics(np.ones(1), np.ones(1), block_elements=block) + + +@pytest.mark.parametrize("block", [1, 7, 262144]) +def test_strided_memmap_matches_independent_vector_formula(tmp_path, block): + source = np.lib.format.open_memmap(tmp_path / "arrays.npy", mode="w+", + dtype=np.float32, shape=(2, 23, 37)) + source[:] = np.random.default_rng(487).normal(size=source.shape) + actual, expected = source[0, ::2, ::-3].T, source[1, ::2, ::-3].T + assert not actual.flags.c_contiguous + a, b = actual.astype(np.float64).ravel(), expected.astype(np.float64).ravel() + result = numeric_metrics(actual, expected, block_elements=block) + assert result["relative_l2"] == pytest.approx(np.linalg.norm(a - b) / np.linalg.norm(b), rel=3e-15) + assert result["cosine_similarity"] == pytest.approx(np.dot(a, b) / np.linalg.norm(a) / np.linalg.norm(b), + rel=3e-14, abs=1e-16) + + +def test_iteration_never_exceeds_requested_buffer(monkeypatch): + import scratchv.verification.numeric_metrics as module + original = module._chunks + lengths = [] + + def observed(*args): + for a, b in original(*args): + lengths.append(a.size) + yield a, b + + monkeypatch.setattr(module, "_chunks", observed) + array = np.arange(20000, dtype=np.float32).reshape(100, 200).T[:, ::2] + result = numeric_metrics(array, array + 1, block_elements=127) + assert result["relative_l2"] > 0 + assert len(lengths) > 2 and max(lengths) <= 127 + + +def test_new_metrics_do_not_weaken_absolute_gate(): + from probes.w2_qwen3_small.diagnostics import tensor_diff + compare = tensor_diff + actual = np.array([2, 4], dtype=np.float32) + expected = np.array([1, 2], dtype=np.float32) + result = compare(actual, expected) + assert not result["passed"] and result["max_abs"] == 2 + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] == 1 + passed = compare(actual, actual) + assert passed["passed"] and passed["max_abs"] == 0 + assert passed["cosine_similarity"] == 1 and passed["relative_l2"] == 0 + invalid = compare(actual, expected.astype(np.float64)) + assert not invalid["passed"] + for key in ("relative_l2", "cosine_similarity"): + assert invalid[key] is None and invalid[key + "_reason"] + + +def test_empty_gate_policies_are_preserved(): + from probes.w2_qwen3_small.diagnostics import tensor_diff + array = np.empty(0, dtype=np.float32) + row = tensor_diff(array, array) + assert row["passed"] + assert row["relative_l2"] is None and row["cosine_similarity"] is None + assert row["relative_l2_reason"] and row["cosine_similarity_reason"] diff --git a/tests/test_numeric_summary.py b/tests/test_numeric_summary.py new file mode 100644 index 0000000..432055f --- /dev/null +++ b/tests/test_numeric_summary.py @@ -0,0 +1,123 @@ +"""Keep concise reports truthful when evidence is missing or undefined.""" +from html import unescape +import re + +import pytest + +from probes.numeric_summary import ( + METRIC_EXPLANATION, + compact_metric_table, + compact_metric_table_html, +) + + +def record(**updates): + row = dict(passed=True, max_abs=1e-6, relative_l2=2e-7, cosine_similarity=0.999) + row.update(updates) + return row + + +def test_worst_metrics_are_aggregated_independently_without_mutating_evidence(): + rows = [record(), {"passed": True, "max_abs": 3e-6, "relative_l2": 1e-8, "cosine_similarity": 0.9}] + original = [dict(row) for row in rows] + result = compact_metric_table([("IR vs ORT", rows)]) + assert format(3e-6, ".16e") in result + assert format(2e-7, ".16e") in result + assert format(0.9, ".17g") in result + assert "门槛通过（2/2）" in result + assert rows == original + + +@pytest.mark.parametrize("render", [compact_metric_table, compact_metric_table_html]) +def test_legacy_pass_does_not_claim_complete_metric_evidence(render): + result = render([("legacy", [{"passed": True, "max_abs": 0.0}])]) + assert "门槛通过（1/1）；指标不完整" in result + assert result.count("未记录 1/1") == 2 + + +@pytest.mark.parametrize("rows", [[], [{}], [{"passed": "true"}], [{"passed": 1}]]) +def test_empty_missing_and_nonboolean_status_never_pass(rows): + result = compact_metric_table([("unexecuted", rows)]) + assert "门槛通过" not in result + assert "未测" in result or "未确认 1" in result + + +def test_failures_are_not_hidden_by_passing_rows(): + failed = record() + failed["passed"] = False + result = compact_metric_table([("mixed", [record(), failed, {}])]) + assert "门槛未通过（通过 1/3，失败 1，未确认 1）" in result + assert "已记录 2/3" in result + assert "未记录 1/3" in result + + +def test_undefined_values_show_reason_and_coverage_not_zero(): + undefined = dict(passed=True, max_abs=0.0, relative_l2=None, cosine_similarity=None, + relative_l2_reason="zero reference norm", cosine_similarity_reason="zero norm") + result = compact_metric_table([("partial", [record(), undefined, {}])]) + assert "未定义 1/3（zero reference norm ×1）" in result + assert "未定义 1/3（zero norm ×1）" in result + assert "已记录 1/3" in result + assert "未记录 1/3" in result + + +def test_undefined_general_reason_is_preserved(): + result = compact_metric_table([("bad", [{"passed": False, "max_abs": None, "reason": "shape mismatch"}])]) + assert "未定义 1/1（shape mismatch ×1）" in result + + +@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), -1.0, True, "0"]) +def test_invalid_error_values_cannot_be_reported_as_numeric_evidence(invalid): + row = record() + row["relative_l2"] = invalid + result = compact_metric_table([("bad", [row])]) + assert "未定义 1/1（数值无效 ×1）" in result + assert "指标不完整" in result + + +def test_small_nonzero_errors_and_near_one_cosines_remain_distinguishable(): + row = dict(passed=True, max_abs=5e-324, relative_l2=1e-100, cosine_similarity=0.9999999999999999) + result = compact_metric_table([("tiny", [row])]) + assert format(5e-324, ".16e") in result + assert format(1e-100, ".16e") in result + assert "0.99999999999999989" in result + + +def test_cosine_domain_and_existing_gate_flags_are_not_reinterpreted(): + # Cosine observes directional similarity; it does not impose a new gate. + result = compact_metric_table([("opposite", [record(cosine_similarity=-1.0)])]) + assert "门槛通过（1/1）" in result and "未定义" not in result + invalid = compact_metric_table([("invalid", [record(cosine_similarity=1.1)])]) + assert "未定义 1/1（数值无效 ×1）" in invalid + + +def test_entirely_undefined_metrics_do_not_acquire_a_numeric_value(): + row = dict(passed=False, max_abs=None, relative_l2=None, cosine_similarity=None) + result = compact_metric_table([("undefined", [row])]) + assert result.count("未定义 1/1（未提供原因 ×1）") == 3 + assert "已记录" not in result + + +def test_html_and_markdown_have_same_values_order_and_safe_untrusted_text(): + labels = ["second " + front = riscv._report_views(report)[filename].split("" not in front + assert "<script>" in front + + +def test_report_rendering_does_not_mutate_numeric_evidence(): + for report, render in ((host_report(), run._host_report_views), (qemu_report(), riscv._report_views)): + before = deepcopy(report) + render(report) + assert report == before diff --git a/tests/test_w2_actions_summary.py b/tests/test_w2_actions_summary.py new file mode 100644 index 0000000..040d68d --- /dev/null +++ b/tests/test_w2_actions_summary.py @@ -0,0 +1,69 @@ +"""Execute the workflow summary shell against completed, skipped and absent evidence.""" +import os +from pathlib import Path +import shutil +import subprocess +import textwrap + +import pytest + + +WORKFLOW = Path(__file__).parents[1] / ".github/workflows/llm-deploy.yml" +SUMMARIES = ( + ("Summarize W2 unit gates", ("w2-backend-ops", "w2-runtime", "w2-runtime-model")), + ("Summarize RISC-V numeric gate", ("qwen3-riscv",)), + ("Summarize Qwen3 small probe", ("qwen3-small",)), +) + + +def shell_block(name): + # Only read the literal run block: never evaluate Actions expressions. + source = WORKFLOW.read_text(encoding="utf-8") + step = source.split(" - name: " + name + "\n", 1)[1].split("\n - ", 1)[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +def execute(tmp_path, name, directories, outcome, have_report): + bash = os.environ.get("SCRATCHV_BASH") or shutil.which("bash") + if not bash: + pytest.skip("Bash is required to exercise GitHub Actions summary commands") + for directory in directories: + report = tmp_path / "output" / directory / "report.md" + report.parent.mkdir(parents=True) + if have_report: + report.write_text("SENTINEL measured report\n", encoding="utf-8") + summary = tmp_path / "summary.md" + env = dict(os.environ, GITHUB_STEP_SUMMARY=summary.as_posix()) + for key in ("FRONTEND_OUTCOME", "BACKEND_OUTCOME", "RUNTIME_OUTCOME", "RUNTIME_MODEL_OUTCOME", + "PROBE_OUTCOME", "TEST_OUTCOME", "ARTIFACT_TEST_OUTCOME"): + env[key] = outcome + result = subprocess.run([bash, "-eu", "-c", shell_block(name)], cwd=tmp_path, + env=env, capture_output=True, text=True, encoding="utf-8", timeout=10) + assert result.returncode == 0, result.stderr + return summary.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_ran_probe_keeps_available_report(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, True) + assert summary.count("SENTINEL measured report") == len(directories) + assert outcome in summary + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["skipped", ""]) +def test_unrun_probe_never_reuses_stale_report(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, True) + assert summary.count("Not run / 未运行") == len(directories) + assert "SENTINEL" not in summary + assert "No " in summary and "acceptance result" in summary + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) +def test_absent_report_does_not_invent_acceptance(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, False) + assert summary.count("Report unavailable / 未生成报告") == len(directories) + assert "No " in summary and "acceptance result" in summary + assert "SENTINEL" not in summary diff --git a/tests/test_w2_runtime_model.py b/tests/test_w2_runtime_model.py index 99969f4..333147b 100644 --- a/tests/test_w2_runtime_model.py +++ b/tests/test_w2_runtime_model.py @@ -169,7 +169,7 @@ def replace(path, dest): assert not (tmp_path / ".report.json.tmp").exists() for name in ("report.md", "report.html"): if name != failure: - assert "FAIL" in (tmp_path / name).read_text() + assert "FAIL" in (tmp_path / name).read_text(encoding="utf-8") def test_main_preserves_existing_evidence(tmp_path): From 611417afa3566d223c46eaeea64669c77302a0f3 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 9 Oct 2026 14:31:17 +0800 Subject: [PATCH 6/8] docs: disambiguate the kernel architecture guide --- .../kernels/{ARCHITECTURE.md => KERNEL_ARCHITECTURE.md} | 2 +- scratchv/backend/kernels/USAGE.md | 4 ++-- scratchv/backend/kernels/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename scratchv/backend/kernels/{ARCHITECTURE.md => KERNEL_ARCHITECTURE.md} (99%) diff --git a/scratchv/backend/kernels/ARCHITECTURE.md b/scratchv/backend/kernels/KERNEL_ARCHITECTURE.md similarity index 99% rename from scratchv/backend/kernels/ARCHITECTURE.md rename to scratchv/backend/kernels/KERNEL_ARCHITECTURE.md index 98eb639..18e8e44 100644 --- a/scratchv/backend/kernels/ARCHITECTURE.md +++ b/scratchv/backend/kernels/KERNEL_ARCHITECTURE.md @@ -293,7 +293,7 @@ cost**，否则「结果对但慢一倍」会被判通过。 ``` scratchv/backend/kernels/ - ARCHITECTURE.md USAGE.md DEVELOPMENT.md OPTIMIZATION.md + KERNEL_ARCHITECTURE.md USAGE.md DEVELOPMENT.md OPTIMIZATION.md __init__.py 包声明（必须有，否则 python -m 起不来） target.py TargetDesc + TARGETS（rv32im / rv32imf） dtypes.py DtypePolicy + POLICIES + mac_instrs + zero_acc diff --git a/scratchv/backend/kernels/USAGE.md b/scratchv/backend/kernels/USAGE.md index 9f7b5c4..5655bf4 100644 --- a/scratchv/backend/kernels/USAGE.md +++ b/scratchv/backend/kernels/USAGE.md @@ -5,7 +5,7 @@ > > | 你想知道 | 看哪 | > |---|---| -> | 为什么这么分层、哪些东西做了哪些没做 | `ARCHITECTURE.md` | +> | 为什么这么分层、哪些东西做了哪些没做 | `KERNEL_ARCHITECTURE.md` | > | **怎么做一遍**（含全部代码） | **`DEVELOPMENT.md`** | > | 平台有哪些硬约束、字段含义、报错怎么查 | **本文档** | > | 性能从哪来、现在差在哪 | `OPTIMIZATION.md` | @@ -298,5 +298,5 @@ PROBLEMS = { |---|---| | `DEVELOPMENT.md` | **教程**（2000+ 行）：从零写出六题，含全部可运行代码、实测数据、检查点 | | 本文档 | **参考手册**：平台契约、字段表、命令行、排查 | -| `ARCHITECTURE.md` | **设计**：四轴分层、pass 清单与状态、目录结构、分期 | +| `KERNEL_ARCHITECTURE.md` | **设计**：四轴分层、pass 清单与状态、目录结构、分期 | | `OPTIMIZATION.md` | **性能**：代价分解、实测数据、优化入口清单 | diff --git a/scratchv/backend/kernels/__init__.py b/scratchv/backend/kernels/__init__.py index 24eac7b..4fd5982 100644 --- a/scratchv/backend/kernels/__init__.py +++ b/scratchv/backend/kernels/__init__.py @@ -6,5 +6,5 @@ python -m scratchv.backend.kernels --list python -m scratchv.backend.kernels --problem add-fp32 -o add_fp32.s -文档见同目录 ARCHITECTURE.md / DEVELOPMENT.md / USAGE.md / OPTIMIZATION.md。 +文档见同目录 KERNEL_ARCHITECTURE.md / DEVELOPMENT.md / USAGE.md / OPTIMIZATION.md。 """ From bf9caa11fbc0927d36bdd3e14c999db6560ef5a3 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 11 Oct 2026 17:53:43 +0800 Subject: [PATCH 7/8] docs: make architecture command examples valid Bash --- docs/reference/ARCHITECTURE_DESIGN.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/ARCHITECTURE_DESIGN.md b/docs/reference/ARCHITECTURE_DESIGN.md index 1e1711f..506efeb 100644 --- a/docs/reference/ARCHITECTURE_DESIGN.md +++ b/docs/reference/ARCHITECTURE_DESIGN.md @@ -542,8 +542,8 @@ grep -rn "riscv64\|riscv32\|+mattr" scratchv/ --include=*.py # 内核层与主线是否真的没有调用关系（§6 内核调用钩子） grep -rn "backend.kernels" scratchv/ tests/ tools/ --include=*.py -# LLVM 成本模型可切集合（附录 B.2） -llc --cost-kind= +# LLVM 成本模型（附录 B.2）；也可选 latency、code-size、size-latency +llc --cost-kind=throughput # 附录 B.3 的两条实测 ./scratchv_env/bin/python -m scratchv my_examples/hello.dsl --backend llvm -o /tmp/o.ll @@ -558,8 +558,9 @@ llc -march=riscv32 -mattr=+m /tmp/o.ll -o /tmp/o.s grep -n "backend == " scratchv/compiler.py # 平台侧实测口径（附录 B.4） +# 运行前把下方带引号的 和 替换为实际参数。 cd /root/riscv-ai-compiler-platform set -a && . /root/.riscv_platform_env && set +a PLATFORM_ENABLE_SANDBOX=0 ./.venv/bin/python \ - /root/workspace/riscv_matmul/measure.py + /root/workspace/riscv_matmul/measure.py '' '' ``` From 701ac22eb3d04d8cddccb326a279fbe9acf55514 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 11 Oct 2026 20:15:26 +0800 Subject: [PATCH 8/8] ci: consolidate PR validation and summaries into one workflow --- .github/workflows/ci.yml | 60 +++- .github/workflows/linux-reproduction-docs.yml | 23 +- .github/workflows/llm-deploy.yml | 37 +- .github/workflows/topic06-benchmark.yml | 10 +- docs/CI.md | 18 + docs/llm-deploy-v1.0/W2/README.md | 2 + ...00\345\217\221\350\256\241\345\210\222.md" | 2 + scripts/ci_scope.py | 127 +++++++ scripts/ci_scopes.json | 45 +++ scripts/summarize_ci.py | 62 ++++ tests/test_ci_routing.py | 329 ++++++++++++++++++ tests/test_ci_workflows.py | 73 ++++ 12 files changed, 740 insertions(+), 48 deletions(-) create mode 100644 docs/CI.md create mode 100644 scripts/ci_scope.py create mode 100644 scripts/ci_scopes.json create mode 100644 scripts/summarize_ci.py create mode 100644 tests/test_ci_routing.py create mode 100644 tests/test_ci_workflows.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f294d8..f26009c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,11 @@ on: pull_request: branches: [main] +# Only newer PR revisions cancel older PR runs. Pushes retain their own run. +concurrency: + group: ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true @@ -13,6 +18,39 @@ permissions: contents: read jobs: + scope: + name: Select validation scope + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + llm: ${{ steps.scope.outputs.llm }} + docs: ${{ steps.scope.outputs.docs }} + w4: ${{ steps.scope.outputs.w4 }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + fetch-depth: 0 + - name: Preserve deployment and documentation path scopes + id: scope + run: python3 scripts/ci_scope.py + + llm: + name: Deployment acceptance + needs: scope + if: github.event_name == 'pull_request' && needs.scope.outputs.llm == 'true' + uses: ./.github/workflows/llm-deploy.yml + + docs: + name: Documentation checks + needs: scope + if: github.event_name == 'pull_request' && needs.scope.outputs.docs == 'true' + uses: ./.github/workflows/linux-reproduction-docs.yml + + topic06: + name: Topic 06 benchmark + uses: ./.github/workflows/topic06-benchmark.yml + # ═════════════════════════════════════════════════════════════════════════ # 课题功能测试：所有14个模块的单元测试 + 集成测试 # ═════════════════════════════════════════════════════════════════════════ @@ -92,7 +130,7 @@ jobs: echo "Using python: $($PY -c 'import sys; print(sys.executable, sys.version)')" $PY -m pip install --upgrade pip setuptools wheel $PY -m pip install -e ".[all]" - $PY -m pip install "pytest>=7,<10" + $PY -m pip install "pytest>=7,<10" "PyYAML==6.0.3" - name: Run Topic 04 PassManager regressions id: topic04_pass_manager @@ -640,6 +678,10 @@ jobs: - name: Write job summary if: always() run: | + echo "## Compiler benchmarks" >> "$GITHUB_STEP_SUMMARY" + echo "Detailed measurements and diagnostics are retained below and in the benchmark artifact." >> "$GITHUB_STEP_SUMMARY" + echo "
Compiler benchmark details" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" if [ -f benchmark_reports/ir_interpreter.md ]; then cat benchmark_reports/ir_interpreter.md >> "$GITHUB_STEP_SUMMARY" fi @@ -692,6 +734,7 @@ jobs: if [ -f benchmark_reports/ir_verifier.md ]; then cat benchmark_reports/ir_verifier.md >> "$GITHUB_STEP_SUMMARY" fi + echo "
" >> "$GITHUB_STEP_SUMMARY" # ═════════════════════════════════════════════════════════════════════════ # GitHub Pages 部署 (仅 main 分支) @@ -710,3 +753,18 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 + + acceptance: + name: CI acceptance + if: always() + needs: [scope, test, benchmark, topic06, docs, llm] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Summarize and require every selected check + env: + CI_JOB_RESULTS: ${{ toJSON(needs) }} + run: python3 scripts/summarize_ci.py >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/linux-reproduction-docs.yml b/.github/workflows/linux-reproduction-docs.yml index 68eccb2..b746a62 100644 --- a/.github/workflows/linux-reproduction-docs.yml +++ b/.github/workflows/linux-reproduction-docs.yml @@ -1,18 +1,7 @@ name: Linux reproduction docs on: - pull_request: - paths: - - 'docs/**' - - 'probes/**/*.md' - - 'probes/**/*.txt' - - 'benchmarks/**/*.md' - - 'benchmarks/**/*.txt' - - 'README.md' - - 'CONTRIBUTING.md' - - 'scripts/check_linux_repro_docs.py' - - 'tests/test_linux_repro_docs.py' - - '.github/workflows/linux-reproduction-docs.yml' + workflow_call: workflow_dispatch: permissions: @@ -36,3 +25,13 @@ jobs: set -euo pipefail python -m pip install pytest==9.1.1 python -m pytest tests/test_linux_repro_docs.py -q + - name: Summarize Linux documentation checks + if: always() + env: + DOCS_STATUS: ${{ job.status }} + run: | + { + echo "## Linux reproduction documentation" + echo "Result: $DOCS_STATUS. Public instructions, Bash syntax and rejection tests." + echo "No model execution or second-person reproduction is claimed." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index d2bb224..c7bbaba 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -2,9 +2,9 @@ name: LLM Deploy v1.0 # CI for the Qwen3-0.6B-on-RISC-V project (docs/llm-deploy-v1.0/). # -# Separate from ci.yml on purpose: that one tests the ScratchV compiler, this -# one tests the deployment project. A failure here should not read as "the -# compiler is broken". +# PR validation is called by ci.yml so compiler and deployment evidence share +# one Actions run. Separate jobs retain their dependencies and failure identity. +# Manual and scheduled entry points remain available for explicit full runs. # # PRs execute small numeric gates and a separate complete frontend audit. # Both are mandatory for the W2 acceptance result. Full-model ORT remains @@ -20,36 +20,7 @@ name: LLM Deploy v1.0 # added when their week arrives, from the plan's §4.3 table. on: - pull_request: - paths: - - 'docs/llm-deploy-v1.0/**' - - 'probes/**' - - 'scripts/check_llm_deploy_docs.py' - - 'scripts/check_linux_repro_docs.py' - - 'tests/test_linux_repro_docs.py' - - 'scripts/check_w1_repro_env.py' - - 'scripts/run_w2_acceptance.py' - - 'scripts/check_w2_ci_results.py' - - 'export_qwen3_onnx.py' - - 'scratchv/**' - - 'requirements/**' - - 'pyproject.toml' - - 'tests/test_qwen3_small*.py' - - 'tests/test_qwen3_full*.py' - - 'tests/test_tensor*.py' - - 'tests/test_riscv_tensor*.py' - - 'tests/test_optimizer*.py' - - 'tests/qwen_artifacts.py' - - 'tests/test_qwen_artifact_selection.py' - - 'tests/test_w1_qwen3*.py' - - 'tests/test_w1_repro_env.py' - - 'tests/test_qwen3_frontend_patterns.py' - - 'tests/test_qwen3_tokenizer.py' - - 'tests/test_llm_inputs.py' - - 'tests/test_w2_*.py' - - 'tests/test_numeric_metrics.py' - - 'tests/test_numeric_summary.py' - - '.github/workflows/llm-deploy.yml' + workflow_call: schedule: # Nightly, off the hour: 19:23 UTC = 03:23 北京时间. - cron: '23 19 * * *' diff --git a/.github/workflows/topic06-benchmark.yml b/.github/workflows/topic06-benchmark.yml index 5f3ab84..e4cba6b 100644 --- a/.github/workflows/topic06-benchmark.yml +++ b/.github/workflows/topic06-benchmark.yml @@ -1,8 +1,11 @@ name: Topic 06 Compiler Benchmark on: - push: - pull_request: + workflow_call: + workflow_dispatch: + +permissions: + contents: read jobs: topic06-benchmark: @@ -41,8 +44,11 @@ jobs: if: always() run: | if [ -f benchmark_reports/topic06/report.md ]; then + echo "
Topic 06 benchmark details" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" cat benchmark_reports/topic06/report.md >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" echo "> 完整 HTML、JSON、PNG 和单用例报告请下载本次运行的 \`topic06-benchmark-reports\` Artifact。" >> "$GITHUB_STEP_SUMMARY" else echo "Topic 06 report was not generated." >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..117c214 --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,18 @@ +# PR 验收与 Actions 报告入口 + +普通 PR 由 **CI** 统一运行通用测试、编译器 benchmark、Topic 06、相关部署验收和 Linux 文档检查。各部分保留自己的 job、固定依赖和 artifact，在同一个 Actions 运行页查看；已有测试与数值门槛不因合并入口而减少。 + +**AI Code Review** 保持独立：它从基础分支读取配置，使用仓库密钥和评论权限。执行 PR 代码的 CI 只授予 `contents: read`，不继承这些权限。因此每次普通 PR 更新预期触发两个工作流，而不是把 AI Review 合并进测试环境。 + +| 查看位置 | 内容 | +| --- | --- | +| CI 的 `CI overview` | 每项检查的实际结果、是否在本次范围内；选中的任务失败、取消或跳过都会使总验收失败 | +| 同页部署 job summary | 本次小模型数值与完整前端解析数据；详细内容折叠 | +| 同页 benchmark summary | 折叠的编译器与 Topic 06 详细报告，完整产物保留在 artifacts | +| 手动完整验证运行页 | 已交付阶段的完整模型实测指标和原始数组；不拿历史报告填充本次未运行项目 | + +部署和文档路径条件集中在 `scripts/ci_scopes.json`。路由根据 PR base/head 的完整 merge-base diff 判断，删除和改名移出目录也会触发原范围；改动公共 CI 配置会检查所有已交付范围。范围判断失败不能默认为无需验收。相同 PR 的新提交取消旧 CI，避免重复占用 runner。 + +普通 PR 不运行完整 Qwen3 IR 或完整 QEMU 前向。保留 `LLM Deploy v1.0` 手动入口及各阶段已有完整验证选项；在包含 W4 的分支，W4 的可复用流程默认 `run_full: false`，只有显式完整调用才传 `true`。Nightly 的原有显式启用条件不变。范围外项目显示 **NOT RUN**，不代表通过。 + +同一运行页中的多个 job 不等于多个 workflow run；job 分开可保留环境隔离和失败定位。此前运行记录仍保留，新的合并入口不会删除历史日志与证据。若仓库另有按旧 check 名称设置的分支保护，维护者应核对新的 Checks 名称；`CI acceptance` 是统一的失败闭合验收入口，不能绕过其失败结果。 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index 1a8b36a..0627c78 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -1,5 +1,7 @@ # W2 Linux 独立复现与验收 +普通 PR 的 W2 验收现在归入统一 **CI** 运行页，关键数据仍由部署 job 写入同页 Summary；详见 [Actions 入口说明](../../CI.md)。下文历史运行链接保留原身份。 + 正式交付、CI 和第二人复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。完整环境准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。Linux 为宿主环境；QEMU 内的 RV64GC/LP64D 程序仍使用当前裸机 ABI。 W2 的范围是完整前端解析、小模型 IR 数值、基础后端和 Host 单步接口。完整 28 层模型的 IR 数值前向属于 W3；完整模型 QEMU 与生成循环属于后续阶段。W1 的人工接口决议继续见 [W1 记录](../W1/README.md) 和 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92)。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 8c6182e..a3ebed1 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -149,6 +149,8 @@ Host 端循环： ### 4.1 CI 分层设计 +普通 PR 的通用测试、Topic 06、部署验收和文档检查由 `ci.yml` 统一编排，在一个 Actions 运行页展示；AI Review 因权限边界保持独立。路径筛选、未运行标记与报告入口见 [CI 说明](../CI.md)。完整模型仍由各阶段的显式手动/定时入口执行。 + | 层级 | 触发 | 用途 | 超时 | 阻塞级别 | |---|---|---|---|---| | PR 快速门禁 | 每次 PR | 单元测试、小模型 IR、QEMU 小算子 | 15 min | 阻塞合并 | diff --git a/scripts/ci_scope.py b/scripts/ci_scope.py new file mode 100644 index 0000000..19658dc --- /dev/null +++ b/scripts/ci_scope.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Select optional PR checks from the complete, local merge-base diff.""" +from __future__ import annotations + +from fnmatch import fnmatchcase +from functools import lru_cache +import json +import os +from pathlib import Path +import re +import subprocess +import sys + + +SCOPES = ("llm", "docs", "w4") +SHARED_PATHS = (".github/workflows/**", "scripts/ci_*", "scripts/summarize_ci.py", "tests/test_ci_*") +SHA = re.compile(r"[0-9a-fA-F]{40}\Z") +ROOT = Path(__file__).resolve().parents[1] + + +def read_config(path: Path) -> dict[str, list[str]]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict) or set(value) != set(SCOPES): + raise ValueError("scope configuration must define llm, docs and w4") + for patterns in value.values(): + if not isinstance(patterns, list) or any( + not isinstance(pattern, str) or not pattern for pattern in patterns + ): + raise ValueError("scope configuration values must be lists of nonempty patterns") + return value + + +def pr_commits(event_path: Path) -> tuple[str, str]: + event = json.loads(event_path.read_text(encoding="utf-8")) + try: + base = event["pull_request"]["base"]["sha"] + head = event["pull_request"]["head"]["sha"] + except (KeyError, TypeError): + raise ValueError("pull request event must contain base and head SHA") from None + if any(not isinstance(value, str) or not SHA.fullmatch(value) for value in (base, head)): + raise ValueError("pull request SHAs must each contain exactly 40 hexadecimal characters") + return base, head + + +def changed_paths(root: Path, base: str, head: str) -> list[str]: + # A rename out of a scope still changes its old path. Disable rename folding + # so both the removed and added names participate in routing. + result = subprocess.run( + ["git", "diff", "--no-ext-diff", "--no-textconv", "--no-renames", + "--name-only", "-z", f"{base}...{head}", "--"], + cwd=root, capture_output=True, check=True, timeout=60, + ) + if result.stdout and not result.stdout.endswith(b"\0"): + raise ValueError("git did not produce a complete NUL-delimited path list") + return [value.decode("utf-8", errors="surrogateescape") + for value in result.stdout.split(b"\0") if value] + + +def matches_path(path: str, pattern: str) -> bool: + """Match path segments: * stays in one segment, ** spans zero or more.""" + segments, patterns = path.split("/"), pattern.split("/") + + @lru_cache(maxsize=None) + def match(position: int, pattern_position: int) -> bool: + if pattern_position == len(patterns): + return position == len(segments) + current = patterns[pattern_position] + if current == "**": + return match(position, pattern_position + 1) or ( + position < len(segments) and match(position + 1, pattern_position) + ) + return (position < len(segments) and fnmatchcase(segments[position], current) + and match(position + 1, pattern_position + 1)) + + return match(0, 0) + + +def select_scopes(config: dict[str, list[str]], paths: list[str]) -> dict[str, bool]: + shared = any(matches_path(path, pattern) for path in paths for pattern in SHARED_PATHS) + return { + name: bool(patterns) and (shared or any( + matches_path(path, pattern) for path in paths for pattern in patterns + )) + for name, patterns in config.items() + } + + +def detect_scopes(root: Path, event_name: str, event_path: Path | None, + config_path: Path) -> tuple[dict[str, bool], int]: + config = read_config(config_path) + if event_name not in {"pull_request", "push", "schedule", "workflow_dispatch"}: + raise ValueError("unsupported or missing GitHub event name") + if event_name != "pull_request": + return dict.fromkeys(SCOPES, False), 0 + if event_path is None: + raise ValueError("GITHUB_EVENT_PATH is required for pull requests") + base, head = pr_commits(event_path) + paths = changed_paths(root, base, head) + return select_scopes(config, paths), len(paths) + + +def main() -> int: + try: + event_path = os.environ.get("GITHUB_EVENT_PATH") + scopes, count = detect_scopes( + ROOT, os.environ.get("GITHUB_EVENT_NAME", ""), + Path(event_path) if event_path else None, ROOT / "scripts/ci_scopes.json", + ) + destination = os.environ.get("GITHUB_OUTPUT") + if not destination: + raise ValueError("GITHUB_OUTPUT is required") + lines = "".join(f"{name}={str(scopes[name]).lower()}\n" for name in SCOPES) + with open(destination, "a", encoding="utf-8") as stream: + stream.write(lines) + print(f"Changed path count: {count}") + print(lines, end="") + return 0 + except (OSError, ValueError, subprocess.SubprocessError): + # Event contents, Git stderr and filenames may be attacker-controlled. + # Keep the public diagnostic bounded and do not echo those values. + print("CI scope detection failed: check configuration, event SHAs and fetched Git history.", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci_scopes.json b/scripts/ci_scopes.json new file mode 100644 index 0000000..c4f84d9 --- /dev/null +++ b/scripts/ci_scopes.json @@ -0,0 +1,45 @@ +{ + "llm": [ + "docs/llm-deploy-v1.0/**", + "probes/**", + "scripts/check_llm_deploy_docs.py", + "scripts/check_linux_repro_docs.py", + "tests/test_linux_repro_docs.py", + "scripts/check_w1_repro_env.py", + "scripts/run_w2_acceptance.py", + "scripts/check_w2_ci_results.py", + "export_qwen3_onnx.py", + "scratchv/**", + "requirements/**", + "pyproject.toml", + "tests/test_qwen3_small*.py", + "tests/test_qwen3_full*.py", + "tests/test_tensor*.py", + "tests/test_riscv_tensor*.py", + "tests/test_optimizer*.py", + "tests/qwen_artifacts.py", + "tests/test_qwen_artifact_selection.py", + "tests/test_w1_qwen3*.py", + "tests/test_w1_repro_env.py", + "tests/test_qwen3_frontend_patterns.py", + "tests/test_qwen3_tokenizer.py", + "tests/test_llm_inputs.py", + "tests/test_w2_*.py", + "tests/test_numeric_metrics.py", + "tests/test_numeric_summary.py", + ".github/workflows/llm-deploy.yml" + ], + "docs": [ + "docs/**", + "probes/**/*.md", + "probes/**/*.txt", + "benchmarks/**/*.md", + "benchmarks/**/*.txt", + "README.md", + "CONTRIBUTING.md", + "scripts/check_linux_repro_docs.py", + "tests/test_linux_repro_docs.py", + ".github/workflows/linux-reproduction-docs.yml" + ], + "w4": [] +} diff --git a/scripts/summarize_ci.py b/scripts/summarize_ci.py new file mode 100644 index 0000000..76bdd38 --- /dev/null +++ b/scripts/summarize_ci.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Summarize the unified PR run and fail closed on missing required evidence.""" +from __future__ import annotations + +import json +import os +import sys + + +REQUIRED = ("scope", "test", "benchmark", "topic06") +LABELS = {"success": "PASS", "failure": "FAIL", "cancelled": "CANCELLED", "skipped": "NOT RUN"} + + +def render_summary(results: object) -> tuple[str, bool]: + valid = isinstance(results, dict) + jobs = results if valid else {} + scope = jobs.get("scope") + outputs = scope.get("outputs") if isinstance(scope, dict) else None + # A delivered/requested W4 gate must not disappear from the dependency + # list. Only an explicit false output permits branches without W4. + w4_flag = outputs.get("w4") if isinstance(outputs, dict) else None + include_w4 = "w4" in jobs or w4_flag != "false" + rows = [] + for name in (*REQUIRED, "docs", "llm", *(("w4",) if include_w4 else ())): + job = jobs.get(name) + outcome = job.get("result") if isinstance(job, dict) else None + requested = True + if name not in REQUIRED: + flag = outputs.get(name) if isinstance(outputs, dict) else None + requested = {"true": True, "false": False}.get(flag) if isinstance(flag, str) else None + expected = "success" if requested else "skipped" + passed = requested is not None and outcome == expected + valid = valid and passed + label = LABELS.get(outcome, "INVALID") if isinstance(outcome, str) else "INVALID" + if outcome is None: + label = "MISSING" + requirement = "Required" if requested else "Not requested" + if requested is None: + requirement = "Invalid/missing scope output" + rows.append(f"| {name} | {label} | {requirement}{'' if passed else ' — FAIL'} |") + text = "\n".join([ + "## CI overview", "", f"**{'PASS' if valid else 'FAIL'}**", "", + "| Check | Result | Scope |", "| --- | --- | --- |", *rows, "", + "Numerical evidence and detailed reports appear in the corresponding job summaries and artifacts in this run.", + "Full W3 IR and W4 QEMU validation are not run by ordinary PR CI; use the explicit manual validation entry. " + "NOT RUN does not count as numerical acceptance, independent reproduction or team sign-off.", "", + ]) + return text, valid + + +def main() -> int: + try: + results = json.loads(os.environ.get("CI_JOB_RESULTS", "{}")) + except (ValueError, TypeError): + results = None + text, passed = render_summary(results) + print(text, end="") + return 0 if passed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_ci_routing.py b/tests/test_ci_routing.py new file mode 100644 index 0000000..5e7a98c --- /dev/null +++ b/tests/test_ci_routing.py @@ -0,0 +1,329 @@ +"""Exercise PR routing against real Git history and the final fail-closed gate.""" +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name): + spec = importlib.util.spec_from_file_location(name, ROOT / f"scripts/{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +scope = load_script("ci_scope") +summary = load_script("summarize_ci") +CONFIG = {"llm": ["scratchv/**", "probes/**"], "docs": ["docs/**"], "w4": []} + + +def git(root, *args): + return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, + text=True, encoding="utf-8").stdout.strip() + + +def write(root, path, text="test\n"): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + + +def commit(root, message): + git(root, "add", "-A") + git(root, "commit", "-qm", message) + return git(root, "rev-parse", "HEAD") + + +@pytest.fixture +def repository(tmp_path): + root = tmp_path / "repo" + root.mkdir() + git(root, "init", "-q") + git(root, "config", "user.name", "CI routing test") + git(root, "config", "user.email", "ci-routing@example.invalid") + git(root, "config", "core.autocrlf", "false") + write(root, "README.md") + write(root, "docs/old.md") + write(root, "scratchv/delete.py") + base = commit(root, "base") + config = tmp_path / "scopes.json" + config.write_text(json.dumps(CONFIG), encoding="utf-8") + event = tmp_path / "event.json" + return root, base, config, event + + +def set_event(event, base, head): + event.write_text(json.dumps({"pull_request": {"base": {"sha": base}, "head": {"sha": head}}}), + encoding="utf-8") + + +def test_real_diff_includes_rename_source_deletion_and_space_names(repository): + root, base, config, event = repository + (root / "scratchv/delete.py").unlink() + (root / "docs/old.md").rename(root / "moved outside docs.md") + write(root, "unrelated name with spaces.txt") + head = commit(root, "rename and delete") + set_event(event, base, head) + paths = scope.changed_paths(root, base, head) + assert set(paths) == {"scratchv/delete.py", "docs/old.md", "moved outside docs.md", + "unrelated name with spaces.txt"} + assert scope.detect_scopes(root, "pull_request", event, config) == ( + {"llm": True, "docs": True, "w4": False}, 4) + + +def test_merge_base_diff_excludes_base_branch_only_changes(repository): + root, base, config, event = repository + write(root, "scratchv/base_only.py") + advanced_base = commit(root, "advance base") + git(root, "checkout", "--detach", base) + write(root, "README.md", "head change\n") + head = commit(root, "PR branch") + set_event(event, advanced_base, head) + assert scope.detect_scopes(root, "pull_request", event, config) == ( + {"llm": False, "docs": False, "w4": False}, 1) + + +@pytest.mark.parametrize("path,expected", [ + ("scratchv/backend/test.py", (True, False, False)), + ("probes/model/run.py", (True, False, False)), + ("docs/nested/readme.md", (False, True, False)), + ("README.md", (False, False, False)), + (".github/workflows/test.yml", (True, True, False)), + ("scripts/ci_scopes.json", (True, True, False)), + ("scripts/summarize_ci.py", (True, True, False)), + ("tests/test_ci_routing.py", (True, True, False)), +]) +def test_path_scopes_and_shared_configuration(path, expected): + assert tuple(scope.select_scopes(CONFIG, [path]).values()) == expected + + +@pytest.mark.parametrize("path,pattern,expected", [ + ("probes/README.md", "probes/**/*.md", True), + ("probes/model/README.md", "probes/**/*.md", True), + ("probes/model/deeper/README.md", "probes/**/*.md", True), + ("probes/model/README.md", "probes/*.md", False), + ("README.md", "**/*.md", True), + ("docs/README.md", "**/*.md", True), + ("tests/nested/test_ci_routing.py", "tests/test_ci_*", False), + ("tests/test_ci_routing.py", "tests/test_ci_*", True), + ("probes/README.txt", "probes/**/*.md", False), + ("probes/readme.MD", "probes/**/*.md", False), +]) +def test_github_path_globs_keep_segment_boundaries(path, pattern, expected): + assert scope.matches_path(path, pattern) is expected + + +def test_delivered_w4_scope_receives_matching_and_shared_changes(): + config = {**CONFIG, "w4": ["probes/w4_*/**"]} + assert scope.select_scopes(config, ["probes/w4_full/run.py"])["w4"] + assert scope.select_scopes(config, ["scripts/ci_scope.py"])["w4"] + + +@pytest.mark.parametrize("event_name", ["push", "schedule", "workflow_dispatch"]) +def test_non_pr_never_activates_scopes_or_needs_event(repository, event_name): + root, _, config, event = repository + event.write_text("invalid JSON", encoding="utf-8") + assert scope.detect_scopes(root, event_name, event, config) == ( + {"llm": False, "docs": False, "w4": False}, 0) + + +@pytest.mark.parametrize("event_name", ["pull_request_target", "", "unexpected", "pull_request\n"]) +def test_unknown_or_missing_event_fails_closed(repository, event_name): + root, _, config, event = repository + with pytest.raises(ValueError): + scope.detect_scopes(root, event_name, event, config) + + +@pytest.mark.parametrize("value", [None, [], {}, {"llm": []}, {**CONFIG, "unexpected": []}, + {**CONFIG, "docs": "docs/**"}, {**CONFIG, "docs": [""]}, + {**CONFIG, "docs": [None]}]) +def test_invalid_scope_configuration_fails(tmp_path, value): + path = tmp_path / "scope.json" + path.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(ValueError): + scope.read_config(path) + + +@pytest.mark.parametrize("reader", [scope.read_config, scope.pr_commits]) +def test_malformed_json_fails_before_routing(tmp_path, reader): + path = tmp_path / "invalid.json" + path.write_text("{invalid JSON", encoding="utf-8") + with pytest.raises(ValueError): + reader(path) + + +@pytest.mark.parametrize("value", [None, [], {}, {"pull_request": None}, + {"pull_request": {"base": {}, "head": {}}}]) +def test_invalid_event_shape_fails(tmp_path, value): + event = tmp_path / "event.json" + event.write_text(json.dumps(value), encoding="utf-8") + with pytest.raises(ValueError): + scope.pr_commits(event) + + +@pytest.mark.parametrize("value", ["main", "a" * 39, "a" * 41, "g" * 40, + "a" * 40 + "\n", "--help", None, True]) +@pytest.mark.parametrize("side", ["base", "head"]) +def test_invalid_sha_fails_without_git_execution(tmp_path, value, side): + event = tmp_path / "event.json" + payload = {"pull_request": {"base": {"sha": "a" * 40}, "head": {"sha": "b" * 40}}} + payload["pull_request"][side]["sha"] = value + event.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError): + scope.pr_commits(event) + + +def test_missing_history_and_event_fail(repository): + root, base, config, event = repository + set_event(event, base, "f" * 40) + with pytest.raises(subprocess.CalledProcessError): + scope.detect_scopes(root, "pull_request", event, config) + with pytest.raises(ValueError): + scope.detect_scopes(root, "pull_request", None, config) + + +def test_cli_writes_only_boolean_scopes_and_count(repository, monkeypatch, capsys): + root, base, config, event = repository + write(root, "docs/private filename.md") + head = commit(root, "docs") + set_event(event, base, head) + write(root, "scripts/ci_scopes.json", config.read_text(encoding="utf-8")) + monkeypatch.setattr(scope, "ROOT", root) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + output = root.parent / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert scope.main() == 0 + assert output.read_text(encoding="utf-8") == "llm=false\ndocs=true\nw4=false\n" + captured = capsys.readouterr() + assert "Changed path count: 1" in captured.out + assert "private" not in captured.out and not captured.err + + +def test_cli_failure_does_not_publish_false_success(repository, monkeypatch, capsys): + root, base, config, event = repository + write(root, "scripts/ci_scopes.json", config.read_text(encoding="utf-8")) + set_event(event, base, "f" * 40) + monkeypatch.setattr(scope, "ROOT", root) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + output = root.parent / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + assert scope.main() == 1 + captured = capsys.readouterr() + assert not output.exists() and not captured.out + assert "CI scope detection failed" in captured.err + + +def test_git_timeout_is_bounded_and_never_publishes_success(repository, monkeypatch, capsys): + root, base, config, event = repository + write(root, "scripts/ci_scopes.json", config.read_text(encoding="utf-8")) + set_event(event, base, base) + monkeypatch.setattr(scope, "ROOT", root) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + output = root.parent / "github-output" + monkeypatch.setenv("GITHUB_OUTPUT", str(output)) + + def timeout(command, **kwargs): + assert kwargs["timeout"] == 60 + raise subprocess.TimeoutExpired(command, 60) + + monkeypatch.setattr(scope.subprocess, "run", timeout) + assert scope.main() == 1 + captured = capsys.readouterr() + assert not output.exists() and not captured.out + assert "CI scope detection failed" in captured.err + + +def successful_jobs(*, requested=True, w4=False): + results = {name: {"result": "success"} for name in summary.REQUIRED} + results["scope"]["outputs"] = {name: str(requested).lower() for name in ("llm", "docs", "w4")} + if not w4: + results["scope"]["outputs"]["w4"] = "false" + for name in ("llm", "docs", *(("w4",) if w4 else ())): + results[name] = {"result": "success" if requested else "skipped"} + return results + + +@pytest.mark.parametrize("requested", [True, False]) +@pytest.mark.parametrize("w4", [True, False]) +def test_complete_summary_accepts_only_requested_success_or_unrequested_skip(requested, w4): + text, passed = summary.render_summary(successful_jobs(requested=requested, w4=w4)) + assert passed and "**PASS**" in text + assert ("NOT RUN" in text.split("Numerical evidence")[0]) is (not requested) + assert "explicit manual validation entry" in text + assert len(text) < 1600 + + +@pytest.mark.parametrize("name", [*summary.REQUIRED, "llm", "docs", "w4"]) +@pytest.mark.parametrize("outcome", ["failure", "cancelled", "skipped", "pending", None, True, {}]) +def test_requested_jobs_never_accept_failure_missing_or_skip(name, outcome): + results = successful_jobs(w4=True) + results[name]["result"] = outcome + text, passed = summary.render_summary(results) + assert not passed and "**FAIL**" in text + + +@pytest.mark.parametrize("name", [*summary.REQUIRED, "llm", "docs"]) +def test_missing_required_jobs_fail(name): + results = successful_jobs() + del results[name] + assert not summary.render_summary(results)[1] + + +@pytest.mark.parametrize("flag", ["true", "", None, True, []]) +def test_w4_cannot_disappear_from_needs_unless_explicitly_unrequested(flag): + results = successful_jobs() + results["scope"]["outputs"]["w4"] = flag + text, passed = summary.render_summary(results) + assert not passed and "| w4 | MISSING |" in text + + +@pytest.mark.parametrize("name", ["llm", "docs", "w4"]) +@pytest.mark.parametrize("flag", [None, True, False, "TRUE", "", "1", []]) +def test_missing_or_invalid_scope_outputs_fail_closed(name, flag): + results = successful_jobs(w4=True) + results["scope"]["outputs"][name] = flag + text, passed = summary.render_summary(results) + assert not passed and "Invalid/missing scope output" in text + + +@pytest.mark.parametrize("outputs", [None, [], "true", {}, {"llm": "true"}, {"docs": "false"}]) +def test_missing_or_malformed_output_object_fails_closed(outputs): + results = successful_jobs() + results["scope"]["outputs"] = outputs + assert not summary.render_summary(results)[1] + + +@pytest.mark.parametrize("name", ["llm", "docs", "w4"]) +@pytest.mark.parametrize("outcome", ["success", "failure", "cancelled", None]) +def test_unrequested_jobs_must_really_be_skipped(name, outcome): + results = successful_jobs(requested=False, w4=True) + results[name]["result"] = outcome + assert not summary.render_summary(results)[1] + + +@pytest.mark.parametrize("payload", ["invalid JSON", "null", "[]", "{}"]) +def test_summary_cli_malformed_input_fails_with_table(payload): + env = dict(os.environ, CI_JOB_RESULTS=payload) + result = subprocess.run([sys.executable, "-B", "-X", "utf8", str(ROOT / "scripts/summarize_ci.py")], + env=env, capture_output=True, text=True, encoding="utf-8", timeout=10) + assert result.returncode == 1 and "**FAIL**" in result.stdout + assert "| Check | Result | Scope |" in result.stdout + + +def test_summary_cli_returns_success_only_for_complete_result(): + env = dict(os.environ, CI_JOB_RESULTS=json.dumps(successful_jobs(requested=False))) + result = subprocess.run([sys.executable, "-B", "-X", "utf8", str(ROOT / "scripts/summarize_ci.py")], + env=env, capture_output=True, text=True, encoding="utf-8", timeout=10) + assert result.returncode == 0 and "**PASS**" in result.stdout diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py new file mode 100644 index 0000000..208e18d --- /dev/null +++ b/tests/test_ci_workflows.py @@ -0,0 +1,73 @@ +"""Keep one PR validation run without dropping gates or running full models.""" +import json +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = ROOT / '.github/workflows' + + +def workflow(name): + return yaml.load((WORKFLOWS / name).read_text(encoding='utf-8'), Loader=yaml.BaseLoader) + + +def test_only_ci_and_privileged_ai_review_automatically_handle_prs(): + triggers = {} + for path in WORKFLOWS.glob('*.yml'): + data = workflow(path.name) + events = set(data['on']) + if events & {'pull_request', 'pull_request_target'}: + triggers[path.name] = events & {'pull_request', 'pull_request_target'} + assert triggers == {'ci.yml': {'pull_request'}, 'ai-code-review.yml': {'pull_request_target'}} + assert workflow('ci.yml')['permissions'] == {'contents': 'read'} + assert workflow('ai-code-review.yml')['permissions']['pull-requests'] == 'write' + + +def test_reusable_jobs_share_the_callers_run_and_same_commit(): + jobs = workflow('ci.yml')['jobs'] + for name, filename in [('llm', 'llm-deploy.yml'), ('docs', 'linux-reproduction-docs.yml'), + ('topic06', 'topic06-benchmark.yml')]: + assert jobs[name]['uses'] == './.github/workflows/' + filename + assert 'workflow_call' in workflow(filename)['on'] + assert not ({'pull_request', 'push'} & set(workflow(filename)['on'])) + for name in ('llm', 'docs'): + assert jobs[name]['needs'] == 'scope' + assert jobs[name]['if'] == f"github.event_name == 'pull_request' && needs.scope.outputs.{name} == 'true'" + assert workflow('llm-deploy.yml')['jobs']['full-qwen3-onnx']['if'] == "github.event_name != 'pull_request'" + assert 'workflow_dispatch' in workflow('llm-deploy.yml')['on'] + + +def test_scope_and_acceptance_are_fail_closed_and_keep_all_required_dependencies(): + data = workflow('ci.yml') + jobs = data['jobs'] + assert jobs['scope']['outputs'] == {name: '${{ steps.scope.outputs.' + name + ' }}' + for name in ('llm', 'docs', 'w4')} + checkout = jobs['scope']['steps'][0] + assert checkout['with']['fetch-depth'] == '0' + assert checkout['with']['persist-credentials'] == 'false' + gate = jobs['acceptance'] + assert gate['if'] == 'always()' + assert set(gate['needs']) == {'scope', 'test', 'benchmark', 'topic06', 'docs', 'llm'} | ({'w4'} if 'w4' in jobs else set()) + assert gate['steps'][-1]['env']['CI_JOB_RESULTS'] == '${{ toJSON(needs) }}' + assert gate['steps'][-1]['run'] == 'python3 scripts/summarize_ci.py >> "$GITHUB_STEP_SUMMARY"' + assert data['concurrency']['cancel-in-progress'] == "${{ github.event_name == 'pull_request' }}" + assert 'github.run_id' in data['concurrency']['group'] + + +def test_scopes_keep_required_frontend_and_docs_coverage(): + scopes = json.loads((ROOT / 'scripts/ci_scopes.json').read_text(encoding='utf-8')) + assert {'scratchv/**', 'probes/**', 'requirements/**'} <= set(scopes['llm']) + assert {'docs/**', 'README.md', 'CONTRIBUTING.md'} <= set(scopes['docs']) + assert bool(scopes['w4']) == (WORKFLOWS / 'w4-full-numeric.yml').exists() + + +def test_long_compiler_and_topic_reports_are_folded_without_dropping_artifacts(): + for filename, job, step_name in [('ci.yml', 'benchmark', 'Write job summary'), + ('topic06-benchmark.yml', 'topic06-benchmark', 'Publish benchmark summary')]: + steps = workflow(filename)['jobs'][job]['steps'] + summary = next(step for step in steps if step.get('name') == step_name) + assert summary['if'] == 'always()' + assert summary['run'].count('
') == summary['run'].count('
') == 1 + assert any('actions/upload-artifact@' in step.get('uses', '') for step in steps)