From 65616a8cde6a581661ac3734378da78a9fcbe5ed Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sat, 3 Oct 2026 20:56:04 +0800 Subject: [PATCH 01/30] feat(w2): validate full Qwen3 frontend and runtime contracts --- .github/workflows/llm-deploy.yml | 164 +++++- docs/llm-deploy-v1.0/W1/README.md | 8 +- ...47\350\241\214\350\256\241\345\210\222.md" | 2 +- ...42\346\265\213\346\226\271\346\241\210.md" | 4 +- docs/llm-deploy-v1.0/W1/interfaces.md | 21 +- docs/llm-deploy-v1.0/W1/risks.md | 2 +- docs/llm-deploy-v1.0/W2/README.md | 262 ++++++++++ ...14\346\224\266\346\212\245\345\221\212.md" | 125 +++++ ...21\345\270\203\350\257\264\346\230\216.md" | 27 + ...00\345\217\221\350\256\241\345\210\222.md" | 10 +- probes/w2_backend_ops/README.md | 45 ++ probes/w2_backend_ops/cases.py | 174 +++++++ probes/w2_backend_ops/run.py | 258 ++++++++++ probes/w2_qwen3_parse/README.md | 87 ++++ probes/w2_qwen3_parse/audit.py | 417 +++++++++++++++ probes/w2_qwen3_parse/run.py | 367 +++++++++++++ probes/w2_qwen3_parse/validation.py | 427 +++++++++++++++ probes/w2_qwen3_small/README.md | 7 + probes/w2_runtime/README.md | 59 +++ probes/w2_runtime/corpus.json | 487 ++++++++++++++++++ probes/w2_runtime/run.py | 379 ++++++++++++++ probes/w2_runtime_model/README.md | 42 ++ probes/w2_runtime_model/run.py | 402 +++++++++++++++ pyproject.toml | 1 + scratchv/backend/tensor_c_codegen.py | 9 + scratchv/runtime/llm_inputs.py | 179 +++++++ scratchv/runtime/qwen3_tokenizer.py | 223 ++++++++ scratchv/runtime/riscv_tensor.py | 68 ++- scripts/check_w2_ci_results.py | 38 ++ scripts/run_w2_acceptance.py | 447 ++++++++++++++++ tests/test_llm_inputs.py | 228 ++++++++ tests/test_qwen3_frontend_patterns.py | 56 ++ tests/test_qwen3_full_audit.py | 375 ++++++++++++++ tests/test_qwen3_full_parse.py | 422 +++++++++++++++ tests/test_qwen3_full_structure.py | 327 ++++++++++++ tests/test_qwen3_tokenizer.py | 221 ++++++++ tests/test_riscv_tensor_runtime.py | 90 +++- tests/test_tensor_c_codegen.py | 42 ++ tests/test_w2_acceptance.py | 296 +++++++++++ tests/test_w2_backend_ops_gate.py | 220 ++++++++ tests/test_w2_ci_results.py | 49 ++ tests/test_w2_runtime_gate.py | 192 +++++++ tests/test_w2_runtime_metadata.py | 103 ++++ tests/test_w2_runtime_model.py | 252 +++++++++ tests/test_w2_runtime_reporting.py | 120 +++++ 45 files changed, 7695 insertions(+), 39 deletions(-) create mode 100644 docs/llm-deploy-v1.0/W2/README.md create mode 100644 "docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" create mode 100644 "docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" create mode 100644 probes/w2_backend_ops/README.md create mode 100644 probes/w2_backend_ops/cases.py create mode 100644 probes/w2_backend_ops/run.py create mode 100644 probes/w2_qwen3_parse/README.md create mode 100644 probes/w2_qwen3_parse/audit.py create mode 100644 probes/w2_qwen3_parse/run.py create mode 100644 probes/w2_qwen3_parse/validation.py create mode 100644 probes/w2_runtime/README.md create mode 100644 probes/w2_runtime/corpus.json create mode 100644 probes/w2_runtime/run.py create mode 100644 probes/w2_runtime_model/README.md create mode 100644 probes/w2_runtime_model/run.py create mode 100644 scratchv/runtime/llm_inputs.py create mode 100644 scratchv/runtime/qwen3_tokenizer.py create mode 100644 scripts/check_w2_ci_results.py create mode 100644 scripts/run_w2_acceptance.py create mode 100644 tests/test_llm_inputs.py create mode 100644 tests/test_qwen3_frontend_patterns.py create mode 100644 tests/test_qwen3_full_audit.py create mode 100644 tests/test_qwen3_full_parse.py create mode 100644 tests/test_qwen3_full_structure.py create mode 100644 tests/test_qwen3_tokenizer.py create mode 100644 tests/test_w2_acceptance.py create mode 100644 tests/test_w2_backend_ops_gate.py create mode 100644 tests/test_w2_ci_results.py create mode 100644 tests/test_w2_runtime_gate.py create mode 100644 tests/test_w2_runtime_metadata.py create mode 100644 tests/test_w2_runtime_model.py create mode 100644 tests/test_w2_runtime_reporting.py diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index 7052655..492177c 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -6,12 +6,14 @@ name: LLM Deploy v1.0 # one tests the deployment project. A failure here should not read as "the # compiler is broken". # -# The PR job executes small numeric gates. A separate manual/nightly job -# downloads and validates the pinned full-model release with ORT. +# PRs execute small numeric gates and a separate complete frontend audit. +# Both are mandatory for the W2 acceptance result. Full-model ORT remains +# manual/nightly; frontend parsing never claims full IR execution. # # docs:interfaces docs:risks # probe:qemu-matmul probe:small-transformer probe:qwen3-onnx -# numeric:ir-qwen3-small +# numeric:ir-qwen3-small frontend:parse-qwen3 +# unit:frontend-ops unit:backend-ops unit:runtime integration:runtime-model # # Later weeks' gates (numeric:ir-full-qwen3 at W3, numeric:qemu-full-qwen3 at # W4, e2e:qemu-paris at W5, release:gate at W8) are NOT stubbed here. They get @@ -24,10 +26,14 @@ on: - 'probes/**' - 'scripts/check_llm_deploy_docs.py' - 'scripts/check_w1_repro_env.py' + - 'scripts/run_w2_acceptance.py' + - 'scripts/check_w2_ci_results.py' - 'export_qwen3_onnx.py' - 'scratchv/**' - 'requirements/**' + - 'pyproject.toml' - 'tests/test_qwen3_small*.py' + - 'tests/test_qwen3_full*.py' - 'tests/test_tensor*.py' - 'tests/test_riscv_tensor*.py' - 'tests/test_optimizer*.py' @@ -35,6 +41,10 @@ on: - 'tests/test_qwen_artifact_selection.py' - 'tests/test_w1_qwen3*.py' - 'tests/test_w1_repro_env.py' + - 'tests/test_qwen3_frontend_patterns.py' + - 'tests/test_qwen3_tokenizer.py' + - 'tests/test_llm_inputs.py' + - 'tests/test_w2_*.py' - '.github/workflows/llm-deploy.yml' schedule: # Nightly, off the hour: 19:23 UTC = 03:23 北京时间. @@ -50,11 +60,12 @@ env: PROBE_TRANSFORMER: probes/w1_tiny_transformer/run.py PROBE_QWEN3_EXPORT: probes/w1_qwen3_export/run.py PROBE_QWEN3_SMALL: probes/w2_qwen3_small/run.py + PROBE_QWEN3_PARSE: probes/w2_qwen3_parse/run.py QWEN3_PY: output/qwen3-probe-venv/bin/python jobs: llm-deploy: - name: "LLM Deploy v1.0 / W1" + name: "LLM Deploy v1.0 / W1 + W2" runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -108,7 +119,15 @@ jobs: - name: Tensor compiler and runtime regressions run: | - python3 -m pytest tests/test_tensor_c_codegen.py tests/test_tensor_compiler.py tests/test_riscv_tensor_runtime.py tests/test_optimizer_numeric_semantics.py tests/test_qwen_artifact_selection.py tests/test_w1_qwen3_export.py tests/test_qwen3_small_riscv_gate.py tests/test_w1_repro_env.py -q + python3 -m pytest \ + tests/test_tensor_c_codegen.py tests/test_tensor_compiler.py \ + tests/test_riscv_tensor_runtime.py tests/test_optimizer_numeric_semantics.py \ + tests/test_qwen_artifact_selection.py tests/test_w1_qwen3_export.py tests/test_qwen3_small_riscv_gate.py \ + tests/test_w1_repro_env.py tests/test_qwen3_full_structure.py \ + tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py \ + tests/test_llm_inputs.py tests/test_w2_backend_ops_gate.py \ + tests/test_w2_runtime_gate.py tests/test_w2_runtime_metadata.py \ + tests/test_w2_runtime_reporting.py tests/test_w2_acceptance.py tests/test_w2_ci_results.py -q - name: Repeated Linux timeout cleanup regression timeout-minutes: 5 @@ -123,6 +142,18 @@ jobs: --junit-xml="output/process-cleanup-repeats/attempt-$attempt.xml" done + - name: "unit:frontend-ops" + id: frontend_ops + run: | + python3 -m pytest tests/test_qwen3_frontend_patterns.py -q \ + --junit-xml=output/w2-frontend-ops.xml + + - name: "unit:backend-ops" + id: backend_ops + timeout-minutes: 15 + run: | + python3 probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops + # ── probe:small-transformer ─────────────────────────────────────── - name: "probe:small-transformer" id: transformer @@ -151,6 +182,34 @@ jobs: "$QWEN3_PY" -X utf8 scripts/check_w1_repro_env.py \ --require-clean --output-dir output/w1-repro-env + - name: Runtime unit regressions in pinned environment + run: | + "$QWEN3_PY" -m pytest \ + tests/test_llm_inputs.py tests/test_qwen3_tokenizer.py \ + tests/test_w2_runtime_gate.py tests/test_w2_runtime_metadata.py \ + tests/test_w2_runtime_reporting.py tests/test_w2_runtime_model.py \ + -q --junit-xml=output/w2-runtime-tests.xml + + - name: "unit:runtime" + id: runtime + timeout-minutes: 10 + run: | + "$QWEN3_PY" probes/w2_runtime/run.py --mode download \ + --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime + + - name: "integration:runtime-model" + id: runtime_model + timeout-minutes: 15 + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + HF_HUB_OFFLINE: "1" + TRANSFORMERS_OFFLINE: "1" + run: | + "$QWEN3_PY" probes/w2_runtime_model/run.py \ + --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime-model + - name: "numeric:ir-qwen3-small" id: qwen3_small timeout-minutes: 15 @@ -210,6 +269,39 @@ jobs: "$QWEN3_PY" probes/w2_qwen3_small/riscv.py \ --model-dir output/qwen3-small --output-dir output/qwen3-riscv + - name: Summarize W2 unit gates + if: always() + env: + FRONTEND_OUTCOME: ${{ steps.frontend_ops.outcome }} + BACKEND_OUTCOME: ${{ steps.backend_ops.outcome }} + RUNTIME_OUTCOME: ${{ steps.runtime.outcome }} + RUNTIME_MODEL_OUTCOME: ${{ steps.runtime_model.outcome }} + run: | + { + echo "## W2 frontend, backend and host runtime" + echo "unit:frontend-ops: $FRONTEND_OUTCOME" + echo "unit:backend-ops: $BACKEND_OUTCOME" + echo "unit:runtime: $RUNTIME_OUTCOME" + echo "integration:runtime-model: $RUNTIME_MODEL_OUTCOME" + for report in output/w2-backend-ops/report.md output/w2-runtime/report.md output/w2-runtime-model/report.md; do + if [ -f "$report" ]; then cat "$report"; fi + done + echo "These gates do not execute full-model IR/QEMU or text generation." + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload W2 unit evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: w2-unit-gates-output + path: | + output/w2-backend-ops/ + output/w2-runtime/ + output/w2-runtime-model/report.* + output/w2-frontend-ops.xml + output/w2-runtime-tests.xml + if-no-files-found: ignore + - name: Upload RISC-V tensor evidence if: always() uses: actions/upload-artifact@v4 @@ -352,3 +444,65 @@ jobs: name: qwen3-full-onnx-probe-output path: output/qwen3-full-probe/ if-no-files-found: ignore + + + full-qwen3-frontend: + name: "W2 / complete Qwen3 frontend" + # Required on PRs as well as manual/nightly runs. No full-model ORT or IR inference. + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Install pinned frontend audit dependencies + run: | + python3 -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" + python3 -m pip check + - name: "frontend:parse-qwen3" + id: full_frontend + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + run: | + set -euo pipefail + python3 "$PROBE_QWEN3_PARSE" --mode download \ + --model-dir output/qwen3-full-model \ + --output-dir output/qwen3-parse --timeout 1200 + - name: Summarize complete Qwen3 frontend gate + if: always() + env: + PROBE_OUTCOME: ${{ steps.full_frontend.outcome }} + run: | + { + echo "## frontend:parse-qwen3" + echo "Outcome: $PROBE_OUTCOME" + if [ -f output/qwen3-parse/report.md ]; then + cat output/qwen3-parse/report.md + fi + echo "Complete graph and binding audit only; no full IR or QEMU numerical claim." + } >> "$GITHUB_STEP_SUMMARY" + - name: Upload complete Qwen3 frontend evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: qwen3-full-frontend-parse-output + path: output/qwen3-parse/ + if-no-files-found: ignore + + w2-acceptance: + name: "W2 / required acceptance" + needs: [llm-deploy, full-qwen3-frontend] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Require actual success from every W2 job + env: + W2_JOB_RESULTS: ${{ toJSON(needs) }} + run: python3 scripts/check_w2_ci_results.py >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/llm-deploy-v1.0/W1/README.md b/docs/llm-deploy-v1.0/W1/README.md index ecc1738..661d413 100644 --- a/docs/llm-deploy-v1.0/W1/README.md +++ b/docs/llm-deploy-v1.0/W1/README.md @@ -2,7 +2,7 @@ 本指南用于另一名成员从目标提交重新生成参考结果、编译产物并运行探测。当前已知本地结果见 [执行计划](W1-执行计划.md)，接口候选见 [interfaces.md](interfaces.md)，风险见 [risks.md](risks.md)。**Mastttttter 已于 2026-10-03 完成 `df18b02f` 的两层数值链路独立复现并本人确认；团队接口确认和 W1 出口评审仍待完成。** 原始证据见 [独立复现报告](https://github.com/ScratchV-Compiler/ScratchV/pull/91#issuecomment-5958157573)。该确认归属所列提交，不自动覆盖后续修复。 -本次 PR #91 集成此前本地收尾修复、复现预检和 QEMU 时长记录；历史本地结果与待决策问题见 [收尾与审查报告](W1-本地收尾与审查报告.md)。本次提交的实际测试、CI 与精确 SHA 以 PR 说明和对应运行报告为准，不用历史结果替代。 +PR #91 已合并，最终 head 为 `5903c1381d57f24062de7395f24385a59e279325`，合并提交为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。报告落盘失败和源码身份归属修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)；发布前整合工作树记录见 [本轮修复与验收报告](../W2/W1修复与W2本地验收报告.md)。人工确认和完整模型第二人复现继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。下方原 PR 修复数据均为历史记录。 所有命令在仓库根目录执行。先记录 `git rev-parse HEAD` 和 `git status --short`；干净checkout便于关联提交，存在本地改动则保留diff并明确标记。不要使用第一人的预生成参考数组代替自己的运行。下述输出目录用完后保留，下一轮使用不同名称，不删除旧证据。 @@ -12,7 +12,7 @@ ## 0. 确定复现版本与领取任务 -`5ea22ecc7fd314025f6453c556dbb3e3f16a3175` 是已有 Linux CI 证据的**历史基线**，不包含本次新增的环境预检和时长汇总。复现本次更新时，应先从 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91) 确认维护者/作者约定的完整 head SHA，再获取 PR head 并核对；PR 分支可能继续更新，不能仅记录分支名。 +`5ea22ecc7fd314025f6453c556dbb3e3f16a3175` 是较早的 Linux CI **历史基线**。复现已合并 W1 时可使用下述固定合并 SHA；若复现本次尚未提交的修复，则必须在该基线上另取得完整增量文件，并核对验收报告中的源码哈希，不能把仅检出的合并提交称为已包含本地修复。 在新目录获取本次版本的示例（需已安装 Git 和 Git LFS，PowerShell/bash 均可逐条执行）： @@ -20,11 +20,11 @@ git clone --no-checkout https://github.com/ScratchV-Compiler/ScratchV.git ScratchV-w1-repro cd ScratchV-w1-repro git lfs install --local -git fetch origin pull/91/head +git fetch origin 3bb88e81498d6d9293aed70e25a4e5a2104465d0 git rev-parse FETCH_HEAD ``` -先确认输出与约定的 PR head SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 +先确认输出与约定的合并 SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 ```bash git checkout --detach FETCH_HEAD diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" index ac6abc8..4814a0f 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" @@ -3,7 +3,7 @@ > 上游：[开发计划.md](../开发计划.md) §4.3 W1。负责人：E2 主导，全员参与。 > 状态（2026-10-03）：**Mastttttter 已完成 `df18b02f` 的合成图、MatMul、真实两层 IR/QEMU 独立复现并本人确认。完整模型第二人复现、团队确认与 W1 出口评审仍待完成。** [原始报告](https://github.com/ScratchV-Compiler/ScratchV/pull/91#issuecomment-5958157573)保留首轮 1 项失败、3 项跳过及后续复跑；本次修复退出状态误判与产物路径缺口，新提交验证单独记录。下文 `5ea22ec`、`545e696` CI 均为历史证据。 > 本文记录实现事实和待办，不代表已召开冻结会议。命令与签字模板见 [独立复现指南](README.md)。 -> 本次将暂留的本地收尾与仿真计时改动集成到 PR #91；上述远端结果只对应所列提交，新增改动须另行验证，不继承远端 PASS。 +> PR #91 已合并为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。本地继续补齐报告发布与源码身份修复，结果见 [本轮报告](../W2/W1修复与W2本地验收报告.md)。W1 报告修复单独发布于 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付；人工确认和完整模型第二人复现由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 继续跟踪，旧 CI 不覆盖本地增量。 ## 1. 目标和验收边界 diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" index 8112c1f..465338c 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" @@ -1,9 +1,9 @@ # W1 探测方案与结果记录 > 上游：[开发计划.md](../开发计划.md) §4.3 W1、[执行计划](W1-执行计划.md)。 -> 更新：2026-10-02。**`5ea22ec` 主 CI 和小模型 Linux 部署任务已通过；完整 ONNX 重型证据来自验证源码未变的 `545e696`。独立复现和团队确认待完成**。 +> 更新：2026-10-03。**PR #91 已合并；Mastttttter 已确认 `df18b02f` 的两层独立复现。完整模型第二人复现、团队确认和出口评审仍待完成**。下文 `5ea22ec`、`545e696` 等远端结果均是历史证据。 > 完整命令和验收模板见 [README](README.md)。保留各探测的范围，不用替代模型或解析成功冒充原数值门槛。 -> 本次将暂留的本地收尾与仿真计时改动集成到 PR #91；新增结果与上述远端基线分开记录，不能把旧 CI 结果当作新增修改的验证。 +> 合并后的本地修复与 W2 增量见 [本轮验收报告](../W2/W1修复与W2本地验收报告.md)；其中 W1 报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付，远端 CI 以对应提交实际执行为准。人工待办继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。 ## 0. 总览 diff --git a/docs/llm-deploy-v1.0/W1/interfaces.md b/docs/llm-deploy-v1.0/W1/interfaces.md index dedf913..bfdf308 100644 --- a/docs/llm-deploy-v1.0/W1/interfaces.md +++ b/docs/llm-deploy-v1.0/W1/interfaces.md @@ -1,7 +1,7 @@ # 接口基线 v1.1：待团队冻结确认 > 上游：[开发计划.md](../开发计划.md) §2.2、§4.3 W1。 -> 状态（2026-10-02）：**按当前实现核对的候选基线，团队确认未完成**。 +> 状态（2026-10-03）：**按当前实现核对的候选基线，团队确认未完成**。PR #91 已合并；W2 运行时扩展与 W1 报告修复分开交付，D1–D8 不因此自动通过。 > `docs:interfaces` 只检查文档结构；“有版本号”不等于冻结会议已通过。正式决议见 §7。 ## 0. 四类接口与路线 @@ -13,7 +13,7 @@ | 三 | 后端 | `CompilerDriver(backend="tensor-c")` → 显式张量循环 C → Zig/LLVM → RV64GC ELF | | 四 | 运行时 | 指针数组模型 ABI；QEMU virt 裸机原始输入装载与 UART 输出协议 | -上述新路径已经本地及 `5ea22ec` 的 Linux 数值验证；本次收尾与仿真计时改动一起集成到 PR #91；新增改动须以对应新提交的检查为准，不能继承该提交的 CI PASS。原 `riscv` 标量选择器及旧 `llvm` 路径不由本探测证明；旧选择器的 MatMul 占位/张量与 FP32 覆盖缺口仍保留。不能将“仓库有 LLVM 后端”或 CNN standalone 可运行等同于 Qwen FP32 已支持。 +上述新路径已有本地及历史提交的 Linux 数值验证，收尾与仿真计时已随 PR #91 合并。合并后的本地修复与 W2 扩展须以本轮报告及未来对应提交的检查为准，不能继承旧 CI PASS。原 `riscv` 标量选择器及旧 `llvm` 路径不由本探测证明；旧选择器的 MatMul 占位/张量与 FP32 覆盖缺口仍保留。不能将“仓库有 LLVM 后端”或 CNN standalone 可运行等同于 Qwen FP32 已支持。 ## 1. 冻结与变更规则 @@ -107,11 +107,19 @@ int scratchv_run(const void *const inputs[], void *output); 当前 QEMU 使用 virt/TCG、512 MiB RAM、无 BIOS/OS，自带启动汇编和链接脚本。输入通过 raw loader 写入保留区域（地址 `0x9c000000`，容量 64 MiB），输出通过 UART 二进制帧返回。帧包含标识、状态、长度、校验和和结束标记；解析拒绝截断/损坏/异常状态，超时失败并清理本次子进程。该传输是现有探测协议，不应误写为通用 Linux FFI 或 mmap。 -### 5.3 后续运行时边界 +### 5.3 W2 Host 运行时辅助接口 -Tokenizer、采样、生成循环和完整模型权重装载仍待实现/验收。当前小模型权重静态嵌入；完整 0.6B 外部分片合计 2,384,201,728 bytes（约 2.22 GiB），不应直接套用 512 MiB guest 配置。W4 需设计代码与权重分离，明确名称/偏移/dtype/shape 描述、装载策略、容量和地址重叠检查、生命周期及分块输出；选择 Linux mmap 时须另建 Linux guest/user-mode 路线，裸机不具备该系统调用。完整 logits 对照仍是原数值门槛，后续生成专用“最后有效位置 logits”接口需独立验收，不能悄悄替换原输出。 +`scratchv/runtime/qwen3_tokenizer.py` 的 `Qwen3Tokenizer.from_directory` 读取已校验的官方本地配置，提供编码/解码和特殊 token 信息，不下载权重、不依赖 Torch/Transformers、不渲染聊天模板。基础词表、含 added tokens 的实际 ID 集合与模型 logits 宽度是不同契约；Qwen3-0.6B 分别为 151643、151669 和 151936。未知或空余模型 ID 解码时显式报错。普通文本编码不自动加入 BOS/EOS；NFC 规范化遵循官方 tokenizer JSON。 -固定长度的 host 生成循环须选择最后一个有效 token 的 logits，不能选择右侧 padding 的最后物理位置。ID 范围、mask、空输入、EOS 和有效长度达到 256 的行为需要显式测试；超长输入不静默截断。上述生成策略是后续实现要求，不代表当前探测已经提供生成 API。 +`scratchv/runtime/llm_inputs.py` 提供 `prepare_inputs`、`build_attention_mask`、`last_valid_logits`、`greedy_next_token` 和 `generation_stop_reason`。输入维持 batch=1、L=256、INT64 IDs、FP32 加性 mask；空/超长输入失败，prompt 中出现 pad ID 不影响有效长度。greedy 选择最后有效位置、平局取最小 ID，拒绝错误形状/dtype和非有限输出。停止判断区分生成 EOS、上下文容量和生成预算，prompt 中的 EOS 不自行停止。 + +这些接口是 W2 辅助能力，未改变模型 ABI 或交付生成循环。验收以 [运行时探测](../../../probes/w2_runtime/README.md) 的固定语料与官方 fast/slow 对照为准；不替代 D1–D8 团队确认。 + +### 5.4 后续运行时边界 + +生成循环和完整模型权重装载仍待实现/验收。当前小模型权重静态嵌入；完整 0.6B 外部分片合计 2,384,201,728 bytes（约 2.22 GiB），不应直接套用 512 MiB guest 配置。W4 需设计代码与权重分离，明确名称/偏移/dtype/shape 描述、装载策略、容量和地址重叠检查、生命周期及分块输出；选择 Linux mmap 时须另建 Linux guest/user-mode 路线，裸机不具备该系统调用。完整 logits 对照仍是原数值门槛，后续生成专用“最后有效位置 logits”接口需独立验收，不能悄悄替换原输出。 + +固定长度的 host 生成循环须复用已验收的最后有效位置、mask、ID 范围、空输入、EOS 和长度边界辅助函数，不能选择右侧 padding 的最后物理位置或静默截断。后续仍需将这些能力与完整模型前向组合验证；当前探测未提供端到端生成 API。 ## 6. 能力与剩余缺口 @@ -122,7 +130,8 @@ Tokenizer、采样、生成循环和完整模型权重装载仍待实现/验收 | tensor-c / RV64 运行器 | 本地及 `5ea22ec` 的[Linux 部署任务](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)通过，含 28 次两层 QEMU 执行；该提交主 CI 也通过 | E3/E4/E5：第二人独立复现、后续修改的受影响门禁；不外推完整模型 | | 完整模型 IR 数值与容量 | 未由小模型验收 | E2/E4/E5：逐步扩大配置及预训练权重验证 | | 完整 ONNX 门禁 | 历史导出、本地 verify，以及 `545e696` 的[Linux download/ORT 重型任务](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767)均通过；三项验证源码至 `5ea22ec` 未变 | E1/E5：第二人复现及受影响门禁；不冒称重新导出或 `5ea22ec` 重跑重型任务 | -| Tokenizer / 生成 / 大权重加载 | 当前探测不覆盖 | E4：W2 基础运行时、W4 权重装载、W5 生成集成及专项验收 | +| Tokenizer / mask / greedy | W2 基础模块与独立门禁已交付，结果按当前提交检查 | E4/E5：第二人复现，后续集成验收 | +| 生成 / 大权重加载 | 当前探测不覆盖 | E4：W4 权重装载、W5 生成集成及专项验收 | ## 7. 团队待决议与确认 diff --git a/docs/llm-deploy-v1.0/W1/risks.md b/docs/llm-deploy-v1.0/W1/risks.md index 26def2c..acd810a 100644 --- a/docs/llm-deploy-v1.0/W1/risks.md +++ b/docs/llm-deploy-v1.0/W1/risks.md @@ -1,7 +1,7 @@ # 风险清单 v1.1 候选 > 上游：[开发计划.md](../开发计划.md) §5。 -> 状态（2026-10-02）：**已按本地实测和 `5ea22ec` Linux CI 回填；责任人及处置路线待团队确认。本次收尾与仿真计时改动集成到 PR #91，验证须对应新提交，不能继承旧 CI PASS**。 +> 状态（2026-10-03）：**PR #91 已合并，责任人及处置路线仍待团队确认**。下文旧 SHA 的数值结果是历史证据；合并后的本地修复与 W2 新结果见 [本轮报告](../W2/W1修复与W2本地验收报告.md)。人工待办由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪，不能继承旧 CI PASS。 > 对应 `docs:risks` 的结构检查：21 条风险，每条有触发信号和 Plan B。自动检查不替代人工评审。 ## 0. 阅读约定 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md new file mode 100644 index 0000000..e233014 --- /dev/null +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -0,0 +1,262 @@ +# W2：完整前端解析、基础后端与 Host 运行时 + +本目录记录开发计划中的 W2 验收：完整 28 层 Qwen3-0.6B 的前端解析与审计、四类模型模式回归、基础算子 QEMU，以及 Tokenizer/输入/mask/greedy。已有真实两层模型的 IR/ORT/QEMU 门禁继续保留。 + +完整图的范围仍止于前端验收。完整 IR 与 ORT 的数值对照、真实维度预训练子图、权重独立装载、完整 QEMU 前向和生成循环仍是后续工作。本轮也不自动关闭 W1 的第二人复现、接口确认和出口评审。 + +**当前交付状态（2026-10-03）**：本次按 W1/W2 拆分为独立分支。W2 直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含另一 W1 分支的两层报告/参考测试修复。当前分支的范围、重新执行的证据及 PR 归属见 [分支验收与发布说明](W2-分支验收与发布说明.md)。下方原本地整合记录及历史 CI 按各自源码归属，不代表本分支已通过远端 CI。 + +## W2 独立技术验收入口 + +使用已有的 Python 3.12 固定 CPU 环境（[依赖](../../../requirements/qwen3-small-probe.txt)）、Zig 0.14.1、QEMU RV64，以及固定完整 ONNX 和官方 Tokenizer 资产。在仓库或完整源码副本根目录执行： + +```powershell +$w2Python = 'D:/path/to/qwen3-venv/Scripts/python.exe' +& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` + --python $w2Python ` + --output-dir output/w2-acceptance-new ` + --tokenizer-dir D:/path/to/qwen3-tokenizer ` + --model-dir D:/path/to/qwen3-0.6b-onnx ` + --cc D:/path/to/zig.exe ` + --qemu D:/path/to/qemu-system-riscv64.exe +$LASTEXITCODE +``` + +Linux 使用相同参数，将解释器和工具替换为本机路径即可。程序不安装依赖；默认两个资产目录均使用 `verify` 离线校验。无资产时可显式传 `--tokenizer-mode download` 或 `--model-mode download`，后者需要下载约 1.24 GB 的完整模型发布包。所有结果使用本次全新的输出目录，不能复用已有目录，即使为空。 + +| 执行顺序 | 独立门禁 | 要求 | +|---|---|---| +| 1 | `frontend` | 22 项 RMSNorm/RoPE/SwiGLU/GQA 模式单测，不接受 skip | +| 2 | `backend` | 16 个小图 × none/all，32 次实际 RV64 QEMU | +| 3 | `runtime` | 官方 Tokenizer 20 组/480 对照，以及 5 组输入、mask、采样与停止条件 | +| 4 | `runtime-model` | 官方文本 ID 直入完整词表的真实两层小配置，Torch/ORT/IR none/all 单步接口集成 | +| 5 | `small-ir` | 从固定种子重新导出原两层普通/诊断图，7 组输入、29 检查点与不变性验证 | +| 6 | `small-qemu` | 使用本轮新导出的模型与报告，28 次两层实际 QEMU | +| 7 | `full-parse` | 固定 28 层完整图解析、逐节点/权重绑定审计与 IR verifier | + +只有七项全部实际执行并通过才返回 **PASS / 退出 0**。`--gates frontend backend` 等子集支持开发期间定位，但最多返回 **PARTIAL / 退出 2**；失败或阻断返回 **FAIL / 退出 1**。small-qemu 自动包含 small-ir，runtime-model 自动包含 runtime。默认每门禁截止 1800 秒、每次 QEMU 截止 180 秒、完整解析 worker 截止 300 秒，可用 `--gate-timeout`、`--qemu-timeout`、`--parse-timeout` 明确调整。 + +总报告在 `output/w2-acceptance-new/report.json/md/html`，每项有独立目录和 `logs/.log`。总入口核对退出码、报告身份、必要计数、实际输出和源码指纹；运行期间源文件变化也会失败。报告记录 HEAD/dirty diff（可用时）及本次源码 SHA256；没有 `.git` 的源码包允许运行，但必须明确标为未知 Git 身份，不能借用父目录仓库的提交。 + +新 [单步模型集成探测](../../../probes/w2_runtime_model/README.md) 保留完整词表 151936，hidden32/FFN96/两层随机权重。原文本 ID 不裁剪、不取模、不重新映射。逐元素检查完整 logits（含 padding），校验最后有效位置、原始 greedy ID 和官方解码；不要求随机权重生成有意义的文本。本地会保存较大的参考数组（约 614 MiB），CI 仅上传摘要报告，不上传这些参考数组或权重。该探测不执行完整 0.6B、完整词表 QEMU 或多步生成。 + +当前本地 workflow 已加入单步集成、完整 28 层解析必跑任务和汇总检查。`W2 / required acceptance` 要求轻量/数值任务与完整解析任务均为 success；失败、取消、跳过、缺失结果均不通过。完整 ONNX 的 ORT 前向仍是独立的手动/夜间 W1 任务。配置随本 W2 分支发布，远端 CI 以对应提交的实际执行为准；分支保护规则未变更。源码隔离复现共用本机 Python/工具链时，只能证明不依赖旧源码目录与旧产物，不能代替第二人或另一台机器复现。 + +统一验收的 Linux 超时路径会先发 SIGTERM，让运行器清理独立会话内的编译器、QEMU 或解析 worker，再执行进程组兜底清理。Windows 继续使用进程树/Job 清理。Linux 专属进程集成回归在 Windows 上明确跳过，发布前须在 Linux 实跑；不能用 Windows 结果替代它。 + +## 本轮继续交付与最终实测（2026-10-02，本地未提交） + +本轮新增完整词表的单步模型接口集成、七项统一验收入口及回归，并补齐原两层 IR 门禁的报告原子写入：MD/HTML 或 JSON 发布失败时必须返回失败，保留原数值错误，不遗留成功 JSON。此前四条报告路径的修复继续保留。 + +从当前工作树复制 **553 个源码文件**到全新的 `output/w2-independent-source-v2`，包含未提交改动，不复制 `.git`、历史 output、Python 环境或模型产物；从该目录真实运行统一入口，最终 **7/7 PASS，退出 0**。固定 ONNX/Tokenizer 资产重新验证哈希，Python 环境与 Zig/QEMU 工具仍共用本机现有安装。导入路径确认来自新副本，模型重新导出、ELF 重新编译、所有结果重新执行；总报告的 **134 项源码指纹与最终工作树一致**。 + +总证据：`output/w2-independent-source-v2/output/w2-acceptance/report.json/md/html`；每项明细、误差位置、日志和实际命令位于同一目录下对应子目录。快照来源及逐文件哈希在 `output/w2-independent-source-v2-manifest.json`，外层日志在 `output/w2-independent-acceptance-v2.log`。这些本机产物不进入版本库。 + +| 门禁 | 最终实测 | +|---|---| +| 前端四模式 | **22/22** 通过，未跳过 | +| 基础后端 | **32/32 QEMU**；最大绝对误差 **2.384185791015625e-7** | +| 运行时 | **20/20** 官方语料、**480/480** 编码/解码对照、**5/5** 输入与采样场景 | +| 单步模型集成 | **2 个原始文本、8 次模型调用**（2 Torch + 2 ORT + 4 IR）；完整输出 `[1,256,151936]`；最大绝对误差 **2.8312206268310547e-7**，采样 ID/解码一致 | +| 两层 IR | **7 组输入、29 检查点、6 项不变性**全部通过；所有数值比较最大绝对误差 **1.9073486328125e-6** | +| 两层 QEMU | **28/28**，普通/诊断图及 none/all 全通过；普通 logits 最大误差 **3.129243850708008e-7**，诊断检查点最大误差 **1.6689300537109375e-6** | +| 完整前端 | **28/28 层、7,847 节点、8,161 value、2,148 绑定**全部通过；5,910 条 IR 指令、verifier 无错误；绑定数据 **2,384,216,772 bytes** | + +统一流程 **264.340 秒**；两层 QEMU 进程累计 **157.642 秒**，其门禁全流程 **172.264 秒**；完整解析门禁 **9.538 秒**，其中 parser **2.031 秒**，worker 峰值 RSS **5,064,372,224 bytes（约 4.72 GiB）**。QEMU 计时包含进程启动、guest 运算和输出传输，诊断图包含检查点工作；这些是单次主机观察，不是纯推理性能，也不能据此声称提速。 + +首次源码副本实跑保存在 `output/w2-independent-source`，总结果正确为 FAIL：前端 pytest 临时目录的父目录未创建，导致 22 个 fixture setup 错误，其余六项通过。修复为使用统一输出根下的临时目录，新增真实 pytest 子进程回归后从新副本完整重跑，没有复用第一次的 PASS。另修复源码包向上查找 Git 的身份错误：运行时、两层 IR 和完整解析入口检查自身 `.git`；缺失时保留源码指纹并明确未知提交。最终实跑不依赖 `GIT_CEILING_DIRECTORIES` 等外部补救设置。 + +专项回归先执行 **631 passed / 0 skipped**（`output/w2-closeout-combined-tests.xml`）。最终修正上述入口后，对受影响两组分别复验 **72 passed / 0 skipped**（`output/w2-source-identity-all-tests.xml`）与 **47 passed / 0 skipped**（`output/w2-acceptance-tests-d.xml`）；批次重叠，不相加。工作流 YAML、25 段 Bash 语法、文档链接和 diff 格式检查通过。本轮未执行全仓测试或远端 CI，未提交、未推送、未更新 PR #91。 + +本轮补齐的是 W2 的独立技术验收和单步接口集成。第二人独立复现、团队接口确认以及发布后对应源码的远端 CI 仍需完成；完整预训练模型的 IR 数值、真实维度后端、独立权重装载和多步生成仍属于后续阶段。 + +## 前一轮：拆分后的本地 Review（2026-10-02） + +本轮审查完整解析、四模式前端、基础后端、Tokenizer、输入/mask/greedy，以及本地分支携带的两层 QEMU 修复。发现一类 P2 验收漏洞并修复四条报告路径：必需的报告写入失败时，可能留下可解析的 `passed=true` JSON。它会使仅消费 artifact 的脚本误判，不能以数值计算正确为理由忽略。 + +| 入口 | 复现与修复 | +|---|---| +| `probes/w2_backend_ops/run.py` | 原来先写 PASS JSON，随后 MD/HTML 失败；改为保存视图结果后发布 JSON，保存失败使门禁非零退出，保留原错误 | +| `probes/w2_qwen3_parse/run.py` | 同类先发布成功状态问题；保留验证阶段、failure_context 和主错误，报告保存异常单列，JSON 最后原子发布 | +| `probes/w2_runtime/run.py` | 虽然原来已最后写 JSON，完整内容写入后关闭/flush 抛异常仍可留下 PASS；改为同目录临时文件成功关闭后原子替换，失败不发布成功 JSON | +| `probes/w2_qwen3_small/riscv.py` | 既有两层 QEMU 门禁存在同类问题；视图与 JSON 原子写入，JSON 最后发布，保留原执行错误与时间证据。此修复只在本地，未同步 PR #91 | + +新增 **30 项**失败注入回归，覆盖 MD/HTML/JSON 无法保存、完整写入后报错、JSON 发布失败、原探测已失败及 CLI 状态。没有修改数值算法、精度阈值或模型配置。在本轮实际核对的路径和样本中，未发现新的前端/MatMul/Tokenizer 计算错误；这不构成完整模型数值正确性的证明。 + +| 本轮验证 | 结果与证据 | +|---|---| +| W2 与两层报告相关专项 | **396 passed / 0 skipped**；`output/w2-review-combined-tests.xml` | +| 前端模式、C 后端与 RISC-V 运行时相关回归 | **138 passed / 0 skipped**；`output/w2-review-compiler-regression.xml`。与上一行有重叠，不相加 | +| 基础后端实际 RV64 QEMU | **32/32**；对 ORT/独立 NumPy 最大绝对误差均 **2.384185791015625e-7**；流程 **23.588 秒**，QEMU 累计 **5.800 秒**；`output/w2-review-backend-real/report.json` | +| 官方 Tokenizer 离线探测 | **20/20** 语料、**480/480** 对照、**5/5** 输入与采样场景；流程 **3.739 秒**；`output/w2-review-runtime-offline-fixed/report.json` | + +已核对上述后端 30 项、运行时 5 项源码 SHA256 与修复后代码一致。额外 500 组随机 Unicode/特殊 token 文本编码和 4,000 次解码对照也通过；它们是本轮探索性检查，不算新增 CI 用例。本轮未重跑完整 28 层模型、整两层 QEMU 或全仓测试，也没有远端 CI；历史结果仍按原 SHA 归属。计时为单次观察，不用于声称性能改善。 + +### 可以继续的工作，按优先级 + +1. **W2 交付收尾**：第二人用自己的环境执行统一验收入口；未来按用户指示整理独立 W2 PR，明确哪些两层门禁修复应单独回补 W1。当前 PR #91 的绿色状态不是本地 W2 的验收；同机源码副本复现也不能代替第二人验收。 +2. **真实维度与预训练子图**：先验证投影 MatMul、RMSNorm、RoPE、Attention、SwiGLU，保留 Torch/ORT/IR 和适当小子图的 QEMU 误差。固定源配置为 hidden=1024、head_dim=128、16 个 Q head/8 个 KV head、FFN=3072。现有两层缩小配置和 3-token GQA case 不能证明这些长累加维度满足误差要求。优先建立基线，再决定 MatMul 分块/布局优化。 +3. **参数化中等模型与逐层定位（W3）**：将两层配置和固定 29 检查点推广成配置驱动，跑计划中的六层中等模型，记录首个偏差、最大绝对误差、耗时与峰值内存。Attention 可先沿用现有分解 IR，融合不是正确性前置。 +4. **完整 IR 内存准备（W3）**：当前解释器会复制 initializer，并在 `values` 中保留中间结果，没有最后使用后回收。先测权重、活跃张量与 RSS，设计只读权重复用和按生命周期回收；再执行完整 IR/ORT，沿用计划中的 `<1e-4` 门槛。 +5. **完整后端与生成集成（W4/W5）**：当前常量/工作区默认各 256 MiB，QEMU 为 512 MiB，权重展开为 C 字面量；仅真实 embedding 就约 593.5 MiB。需要代码与权重分离、装载 ABI、地址布局和容量验证，再接完整前向与生成循环。Host helper 与完整词表两层随机模型的单步调用现已集成；真实预训练模型调用、逐 token 循环、chat template 与流式解码仍未完成。 + +建议下一轮从第 2 项的真实维度数值基线开始，同时准备第 3 项的可配置探测，避免直接以完整模型试跑替代可定位的小范围验证。 + +## W2 基础能力补齐（2026-10-02，历史提交 ff193c7） + +该次在已通过完整解析和两层数值探测的 `dfc9437` 上补齐三个独立入口，曾更新 PR #91，现已按 W1/W2 拆分要求撤出该 PR 并保留在本地。表中“通过”必须读取对应提交的报告或 CI；本页不把旧提交或跳过项计作当前本地修改通过。 + +| W2 门禁 | 当前入口与范围 | +|---|---| +| `unit:frontend-ops` | `tests/test_qwen3_frontend_patterns.py`：RMSNorm、RoPE、SwiGLU、GQA 四类模式，含错误 epsilon、旋转符号、门控分支与 KV 重复顺序反例 | +| `unit:backend-ops` | `probes/w2_backend_ops/run.py`：16 小图，MatMul/逐元素/Softmax/RMSNorm/RoPE/SwiGLU/GQA 七类；none/all 共 32 次实际 RV64 QEMU；独立 NumPy 公式和 ORT 双参考 | +| `unit:runtime` | `probes/w2_runtime/run.py`：固定官方七份 tokenizer/配置文件，20 组语料、fast/slow 双参考与 golden IDs/解码文本；mask/最后有效位置/greedy/EOS/容量/预算 | +| `numeric:ir-small` | 既有 `probes/w2_qwen3_small/run.py`：真实两层缩小配置随机权重，继续验收 IR 对照；另保留两层 QEMU 门禁 | +| `frontend:parse-qwen3` | 既有 `probes/w2_qwen3_parse/run.py`：完整 28 层固定产物解析与绑定审计；手动/定时完整模型 CI 实际执行 | + +具体命令见 [基础后端门禁](../../../probes/w2_backend_ops/README.md) 和 [运行时门禁](../../../probes/w2_runtime/README.md)。生产新增 `scratchv/runtime/llm_inputs.py` 与 `scratchv/runtime/qwen3_tokenizer.py`；Tokenizer 只依赖可选 `tokenizers`，不在生产模块引入 Torch/Transformers。纯文本编码、完整模型输入准备与单步采样为可复用函数，尚未组合成端到端生成循环。 + +交叉 review 修复了非默认 prefix/special-token 配置被静默接受而与官方语义不同、缺省 unk token 偏离参考的问题；门禁也增加词表/特殊 ID 元数据与默认参数对照，并修正报告写入失败遗留 PASS 的路径。官方 NFC 规范化是保留行为，不把组合 Unicode 归一化误判成编译器错误。未知或空余 logits 行解码时失败，不静默丢弃或改变 greedy 候选。 + +本次 CI 增加明确的三个门禁、固定环境单测、独立 Summary 与 `w2-unit-gates-output` artifact。运行时下载约 15.2 MiB 的 tokenizer/config/许可证，按固定 revision 和既有 manifest SHA256 校验，不下载模型权重。源码和测试文件进入 Git，模型、下载缓存、ELF 与本地 output 不进入 Git。 + +### 本轮本地验收 + +| 验证 | 结果 | 本地证据 | +|---|---|---| +| 新增专项测试（固定 CPU 环境） | **228 passed / 0 skipped**，含四类前端模式 22 项、后端门禁 18 项、运行时 188 项 | `output/w2-final-unit-tests.xml` | +| 全仓回归（通用开发环境） | **2,732 passed / 41 skipped / 0 failed** | `output/w2-final-full-ready-tests.xml` | +| 基础后端真实 RV64 QEMU | **32 / 32** 通过；对 ORT 与独立 NumPy 参考的最大绝对误差均为 **2.384185791015625e-7** | `output/w2-backend-ops-final/report.json/md/html` | +| 官方 Tokenizer 下载与运行时门禁 | **20 / 20** 语料、**480 / 480** 比较、**5 / 5** 输入与采样场景通过 | `output/w2-runtime-final-download/report.json/md/html` | +| 同一执行身份离线复验 | 禁用 Hugging Face 在线访问，复用经过哈希校验的本地资源；上述 **20 / 480 / 5** 全部通过 | `output/w2-runtime-final-offline-same-user/report.json/md/html` | + +41 个跳过项包括本机缺少 `qemu-riscv32` 的 38 项，以及通用环境没有 Torch/Tokenizers 的 3 个模块；新 Tokenizer 模块已在上述固定 CPU 环境执行。两个既有 Torch 模块在本轮未本地重跑，PR CI 的固定环境继续执行原两层数值探测。各批次测试有重叠，不能相加成通过总数。 + +首次全仓运行因子进程未获得 LLVM 工具路径失败，修正测试进程的 PATH 后得到上表结果；首次跨执行身份的离线复验因 Windows 目录读取权限失败，同一身份复验通过。失败报告仍保留，未通过修改数值门槛或跳过检查消除失败。 + +后端探测全流程 **22.194 秒**，32 次 QEMU 进程墙钟累计 **5.660 秒**；Tokenizer 下载门禁 **14.824 秒**，同一身份离线复验 **4.198 秒**。均为本机单次观察，不是性能改善结论。已核对后端 30 项、运行时 5 项源码指纹与本次源码一致。仓库指定的本地 L2 harness 文件缺失，未宣称执行；本轮以实际专项、全仓和真实探测结果交付。 + +## 完整前端解析的本地复现 + +详细环境、Windows/Linux 命令、下载模式和报告说明见 [探测入口 README](../../../probes/w2_qwen3_parse/README.md)。所有命令在仓库根目录执行；使用 Python 3.12 和固定 NumPy/ONNX/ORT/protobuf 环境，无需 torch、Zig 或 QEMU。 + +已有完整模型时： + +```bash +python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +``` + +PowerShell 可使用实际虚拟环境解释器： + +```powershell +& .\output\qwen3-parse-venv\Scripts\python.exe -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +$LASTEXITCODE +``` + +没有完整模型时，将 `verify` 改成 `download`，仍使用同一固定 W1 manifest。模型必须包含真实 external-data 分片。每次使用新的报告目录，失败时保留已有文件与日志。 + +## 如何判断本阶段完成 + +| 项目 | 应保留的证据 | +|---|---| +| 源码身份 | checkout SHA、是否有本地改动、实际 diff 与新增文件 | +| 固定模型身份 | 发布产物和 external-data 文件校验结果 | +| 完整结构与层覆盖 | 本次结构审计摘要及 28 层检查结果 | +| 实际节点转换 | `nodes.json` 与审计摘要 | +| initializer 绑定 | `bindings.json` 与审计摘要 | +| 共享 IR | `ir.txt` 与 IR 验证器结果 | +| 本次执行 | 命令、退出码、`report.json/md/html`、实测时间及内存（如平台可测） | +| CI | 对应提交和实际 run URL；未执行时明确写未执行 | + +退出 0 和 `passed=true` 必须来自本次执行。轻量 fixture 测试、历史 ORT 通过或一个可打开的报告文件都不足以代替完整图审计。失败发生较早时，某些明细可能尚未产生；按实际阶段和错误定位，不将缺失记录补成成功。 + +本地 W2 workflow 现在让 PR、手动和定时任务均执行独立的 `full-qwen3-frontend` 完整解析任务，无须等待完整 ONNX 的 ORT 前向。该任务下载约 1.24 GB 的固定发布包，解析时需约 5 GB RSS；手动/夜间同时运行 ORT 任务时，两台 runner 会各自下载。汇总任务拒绝失败或跳过的依赖任务。PR #91 已合并；未来 W2 远端结果须关联其独立提交和实际 run，不能继承旧提交的 PASS。 + +## 完整解析 Review 与实测（2026-10-02，历史提交 dfc9437） + +该次同时复核本地完整解析门禁和 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91)。核对时 PR head 为 `df18b02f152878b1ea838bbf6073def01d367456`，base 为 `d1c2b5f04cad817acf0b2f14d3bfa4ff56dbfa11`，当时差异涉及 52 个文件。审查覆盖编译入口、IR 优化、tensor-c 后端、RISC-V 运行时、数值探测、CI 和复现说明。下述完整解析与 Review 修复曾在 `dfc9437` 纳入 PR #91，现已撤出该 PR 并保留于本地；当时实际执行源码由报告中的 SHA256 确定。 + +### 完整解析门禁修复 + +| 问题 | 修复与验证 | +|---|---| +| 属性审计复用生产 parser 的属性解码，可能让相同错误在转换和审计两端同时通过 | 直接从原 ONNX protobuf 独立读取属性，拒绝重复属性；增加错误 Transpose 轴和错误 epsilon 的反例 | +| 未检查函数参数名称与常量标记 | 对照原 ONNX 输入校验名称、顺序、shape/dtype，并拒绝被标记为常量的输入 | +| 标量辅助 `LOAD_CONST` 校验不完整 | 对 initializer 和 Constant 两条路径核对 opcode、目标 Value、空操作数、shape/dtype、常量标记、常量值及指令属性 | +| 审计时隐式 `np.asarray` 可能掩盖错误的绑定类型 | 要求实际绑定是 `numpy.ndarray`；list 等错误类型直接失败，合法只读 ndarray 仍接受 | +| 日志写入失败可能覆盖 worker 原始错误 | stdout/stderr 独立尝试保存；保留原错误、阶段及退出码，附加 `log_errors`；成功路径缺少日志也不能报 PASS | +| 文档链接指向被忽略的本地输出，干净 checkout 中无法访问 | 改为明确的本地输出路径说明，并增加模拟缺少 output 目录的文档链接回归测试 | + +进一步修正失败明细写入的同类问题：保存 `nodes.partial.json` 等失败证据时，写入异常记录到 `evidence_errors`，不覆盖原始解析或绑定错误；成功路径缺少必需证据仍失败。supervisor 在启动 worker 前保存 `attempt`，记录命令、模型路径、解释器、checkout 和源码指纹，便于定位过早超时、worker 未能启动等情况；该字段不代表验证已经执行。 + +### PR #91 修复 + +| 问题 | 修复与验证 | +|---|---| +| tensor-c 接受与 IR 标量常量值不同的外部绑定，可能与解释器行为不一致 | 校验绑定与常量声明一致；增加 FP32/INT32/INT64 错值、错误向量形状和合法标量执行的回归测试 | +| 编译或 QEMU 失败时，保存日志的异常可能掩盖编译错误、超时或 guest 错误 | 独立保存两路日志；保留主错误及原耗时，并附加日志错误。覆盖编译失败/成功与 QEMU 超时/失败/成功等组合 | +| RISC-V 报告不足以确定本次 ORT/IR 对照的实际执行环境和完整源码身份 | 顶层 `environment` 记录当前进程，`export_evidence.environment` 保留导出时环境；源码指纹覆盖 parser、IR、优化、后端和运行时，共 30 项 | + +未发现本次审查范围内需要额外修改的 MatMul 实现问题；下列结果只证明已测试输入和路径。未调整精度门槛、模型权重或 D8 等团队待确认决策。 + +### 修复后的实测 + +| 验证 | 当前结果 | 本地证据 | +|---|---|---| +| 全仓回归 | 2,563 passed / 40 skipped / 0 failed | `output/review-fix-full-tests.xml`、同名 `.log` | +| 固定 CPU 环境最终相关测试 | 286 passed / 0 skipped，包含全部 114 项完整解析测试 | `output/review-fix-pinned-tests.xml`、同名 `.log` | +| 真实 Qwen3 结构两层小模型重新导出与 PyTorch/ORT/IR 对照 | 7 / 7 组输入通过；全部数值比较最大绝对误差 `1.9073486328125e-6` | `output/review-fix-qwen3-small/report.json` | +| RISC-V QEMU 两层探测 | 7 组输入 × 普通/诊断图 × none/all 优化，共 28 / 28 执行通过；QEMU 对照最大绝对误差 `1.6689300537109375e-6`，门槛 `atol=1e-5, rtol=0` | `output/review-fix-qwen3-riscv/report.json`、`report.md` | +| QEMU MatMul | 7 / 7 通过，最大绝对误差 `2.384185791015625e-7` | `output/review-fix-matmul/report.json` | +| 完整 Qwen3 前端解析 | 28 / 28 层；7,847 节点、23 类算子；5,910 条 IR；2,148 项绑定；IR verifier 0 问题 | `output/review-fix-full-qwen3-final/report.json`、`report.md`、`report.html` | + +全仓回归中的 38 个 RV32 执行测试因缺少 `qemu-riscv32` 跳过，另有 2 个依赖 torch 的模块因通用开发环境不含 torch 而跳过；最终固定 CPU 环境相关测试已覆盖这两个模块。全仓测试收集后补充了 3 项失败证据回归测试，均包含在最后的 286 项测试内；没有把不同批次或重叠测试相加为一个通过总数。RV64 tensor 真实运行时集成测试和上述实际 QEMU 探测已执行。 + +本次 QEMU 进程墙钟时间累计 **160.450 秒**，探测全流程 **174.634 秒**，4 个成功构建累计 **6.586 秒**。普通图每次平均约 **1.143 / 1.147 秒**（none/all），诊断图约 **10.138 / 10.494 秒**。QEMU 时间包含进程启动、guest 计算、输出打包/UART 和退出；诊断图还包含中间张量输出，不能当作纯前向耗时或真实硬件性能。 + +完整模型本次纯解析 **3.636 秒**，入口至验证及日志保存结束 **12.772 秒**，worker 峰值 RSS **5,064,331,264 bytes，约 4.72 GiB**。完整模型输出静态契约为 FP32 `[1,256,151936]`，绑定共 **2,384,216,772 bytes**；这些是解析与绑定审计结果，本次未执行完整模型数值前向。以上耗时均为本机单次运行，期间存在并行测试，不用于声称性能改善。 + +已核对完整解析报告的 20 项、RISC-V 报告的 30 项源码 SHA256 与当前源码一致；完整解析的 `nodes.json`、`bindings.json`、`ir.txt` 的文件大小和 SHA256 均与报告一致。测试后仅补充说明文档，执行源码未再修改。输出目录被 Git 忽略，仅在本机保存；他人复现需运行入口生成自己的报告。 + +最终还实跑了 0.1 秒截止和模型缺失两个失败路径：均正确退出 1，分别记录 `timeout`、`hashes` 阶段，且保留启动前的 20 项源码指纹；报告在 `output/review-fix-final-expected-timeout/` 和 `output/review-fix-final-expected-missing/`。文档检查包含尚未跟踪的新文档，共检查 97 个文件、196 个相对链接，0 个失效；LLM workflow YAML 与 19 段 shell 命令语法检查通过，`git diff --check` 通过。 + +当前没有已确认且尚未修复的本轮阻断性问题。完整 IR/ORT 数值与容量验收、完整模型 QEMU 前向、权重独立装载、W1 第二人复现和接口签字仍未由本次工作完成。本页实测均为本地证据，远端 CI 状态及 run 链接以 PR 描述和 Checks 为准。 + +## 首次交付结果（2026-10-02，历史记录） + +首次交付工作在独立本地分支 `codex/w2-full-qwen3-parse`，基于 PR #91 的 `df18b02f152878b1ea838bbf6073def01d367456`。实际工作目录为 `D:/cyq/code/ScratchV/output/w1-pr89-review`；当时修改未暂存、未提交、未推送，尚未更新 PR #91。上层原工作目录中的已有改动未纳入本轮。 + +首次交付的完整实物执行退出 0，结果 **PASS**。本地证据保存在 `output/w2-full-qwen3-parse-complete/`：`report.md`、`report.json`、`nodes.json`、`bindings.json` 和 `ir.txt`。这些是该次 checkout 的本地输出，`output/` 不进入版本库；其他机器需执行入口重新生成，不能依赖这些历史路径存在。 + +| 验收项 | 首次交付实测 | +|---|---| +| 固定模型 | Qwen3-0.6B，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 / L=256 / 无 KV Cache | +| 完整层链 | 28 / 28 层，含最终归一化与共享 embedding / LM Head | +| ONNX 节点 | 7,847 / 7,847；23 类基础算子；全部可达 logits | +| ONNX value | 8,161 项静态 shape / dtype 审计通过 | +| 实际 IR | 5,910 条指令；IR verifier 0 个问题 | +| 真实绑定 | 312 个 initializer + 1,836 个 Constant，共 2,148 项；逐项内容哈希通过 | +| 绑定字节数 | 2,384,216,772 bytes | +| 合法别名 | 112 个 Identity，均核对实际对象映射 | +| 返回值 | `logits`，FP32 `[1,256,151936]` | +| 纯解析耗时 | 2.493 秒 | +| 全入口耗时 | 10.533 秒，包含检查、解析、审计与子进程开销 | +| 验证进程峰值 RSS | 5,063,430,144 bytes，约 4.72 GiB，Windows PeakWorkingSetSize | + +耗时来自单次本地运行，不是平均值或性能承诺；本轮没有做解析性能优化。报告记录运行环境、实际源码 SHA256 和产物哈希。工作树有本地改动，不能将此结果仅归属于基准 HEAD。 + +首次交付的轻量测试 **91 / 91 通过**，在固定 CPU 环境和现有开发环境各执行一次，分别记录于 `output/w2-gates-pinned-delivery.xml` 和 `output/w2-gates-general-delivery.xml`。测试含真实两层 fixture 经文件哈希、结构检查、生产 parser、IR verifier 和绑定审计的贯通，以及漏节点、错权重、错 shape/dtype、外部数据损坏、解析异常、缺报告与超时等反例。当时全仓回归 **2,521 passed / 40 skipped**，记录于 `output/w2-regression-final.xml`；首次交付最后修正后重跑上述全部 91 项相关测试。当前修复后的验证以本页上方结果为准；跳过项不计为通过。 + +实际进程反例也已执行：模型缺失在 `hashes` 阶段退出 1；0.1 秒截止测试在 `timeout` 阶段退出 1，实测总耗时约 0.184 秒，均保存 FAIL 报告。记录分别位于 `output/w2-expected-missing-model/` 和 `output/w2-expected-timeout/`。 + +首次交付新增三个探测模块、三组测试与复现文档，并更新本地 LLM CI 配置。没有发现这份完整固定图需要新增前端算子，当时未修改通用 ONNX parser 或编译后端。首次互审修正了验收实现中的两个重点问题：控制参数必须从原 ONNX 独立求值，避免解析器错误自证通过；超时日志写入失败、进程异常退出及 verifier 失败时应尽量保留原始错误和节点证据。后续 Review 的更多修复见本页上方。 + +本阶段已具备完整前端解析的本地验收证据；远端 CI、第二人复现、完整模型 IR 数值及完整模型 QEMU 前向尚未由本轮验证。 + +## 后续衔接 + +1. 前端审计通过后，依据实际图和权重规模建立完整 IR 数值与峰值内存基线，逐步扩大预训练模型验证范围。 +2. 设计代码与权重分离、权重绑定 ABI 和内存布局；当前 C 常量展开及小模型 guest 配置不能直接承担完整 0.6B 模型。 +3. 完整 IR 数值验收通过后，再验证完整 QEMU 单次前向；随后将已有 Tokenizer、输入准备、最后有效 token logits、采样和停止条件组合成生成循环，并验证真实文本输出。 + +前端解析通过不能跳过这些阶段，也不能据此宣布完整 Qwen3 已在 ScratchV 上运行。开发范围见 [总开发计划](../开发计划.md)，W1 人工验收状态见 [W1 独立复现记录](../W1/README.md)。 diff --git "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" new file mode 100644 index 0000000..e08da37 --- /dev/null +++ "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -0,0 +1,125 @@ +# W1 修复与 W2 本地验收报告 + +日期：2026-10-03。**历史整合工作树报告**：记录的是发布前 W1+W2 合并在本地的验证；其中“未提交”等为当时状态。后来按 W1/W2 拆分独立发布，当前 W2 分支结果见 [分支验收与发布说明](W2-分支验收与发布说明.md)，不能将下述组合源码哈希当作当前分支哈希。 + +## 1. 结论与源码位置 + +W1 的报告可靠性与源码归属修复已补入；W2 已在 PR #91 合并基线上整合，并重新执行七项技术验收，结果 **7/7 PASS**。完整图的通过范围是前端解析、结构和权重绑定审计；完整 0.6B 的 IR 数值、QEMU 前向与生成循环仍未完成。本报告不替代团队冻结接口、第二人复现或远端 CI。 + +本轮源码在 `D:/cyq/code/ScratchV/output/w1-w2-local-closeout`，本地分支 `codex/w1-w2-local-closeout`。基线是 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，所有增量均未提交。原 `output/w1-pr89-review` 的 W2 修改、`output/w1-pr91-repro-fixes` 和最外层工作目录的代码保持原样。 + +下述 `output/...` 均相对于本轮源码目录。它们是本机证据，不是仓库发布资产；复现者应使用新的目录生成自己的报告。 + +## 2. W1 补完了什么 + +| 修复 | 原问题 | 当前行为与主要文件 | +|---|---|---| +| 两层报告完整发布 | MD/HTML 写入失败或 JSON 发布失败可能留下成功证据 | 视图先写、JSON 最后原子发布；失败返回非零，尽力保存 FAIL，保留原始数值错误。`probes/w2_qwen3_small/run.py`、`riscv.py` | +| 源码与环境归属 | 无 `.git` 的源码副本可能向父目录查找 Git，误认其他 checkout；QEMU 对照环境可能与导出环境混淆 | 无本目录元数据时明确未知 Git 身份，保留源码 SHA256；分别记录当前执行与导出环境，并补齐 parser/IR/优化/运行时指纹 | +| 回归覆盖 | 成功路径测试不足以验证失败时的证据 | 故障注入覆盖写入、关闭、替换失败与原始错误保留；补源码副本及当前环境归属回归 | +| 原有算子用例与固定 ORT 环境兼容 | ORT 1.22.1 无 DOUBLE Cos 内核，ReduceMean 的 omitted axes + noop=1 路径报内核错误 | 三项用例显式使用 ONNX 官方 ReferenceEvaluator，继续实际运行生产 parser/IR，ReduceMean 再检查与输入完全相等；其余用例继续 ORT 对照，不捕获或忽略数值错误 | +| 共享后端与运行时防错 | 外部绑定可能覆盖标量 IR 常量；日志 I/O 失败可能遮蔽原编译/QEMU 错误 | `tensor_c_codegen.py` 校验常量绑定一致；`riscv_tensor.py` 独立保存两路日志，保留主错误和耗时；一并整合之前 W2 分支暂存的修复 | +| 文档状态 | 仍使用 PR 未合并、旧提交或“所有第二人复现待做”的描述 | 固定已合并基线，保留 `df18b02f` 第二人两层复现事实，区分完整模型复现与接口签字待办 | + +在整合 W2 前，W1 四个源码/测试文件的专项测试已执行 **120 passed / 0 skipped**，证据为 `output/w1-fix-tests-v2.xml`。W1 独立检查点在 `output/w1-fix-checkpoint/`：包含补丁、四文件快照与哈希，便于后续按 W1/W2 分拆提交。这次没有重新打开或更新已合并的 PR #91。 + +最终共享修复的建议拆分补丁为 `output/w1-final-review/w1-fixes.patch`，含上述报告/归属、标量绑定、运行时清理/日志及 ORT 用例兼容的 10 个源码/测试文件，配有 `manifest.json`；它是供审阅的本地补丁，不是已暂存或已提交的改动。 + +最后还单独实跑原 W1 入口：MatMul **7/7**（4×4 小数最大误差 `1.4901161193847656e-8`，七例最大 `2.384185791015625e-7`）；PR #89 合成两层图的 IR vs ORT 最大误差约 `7.153e-7`，ORT vs 独立 NumPy 约 `1.097e-6`。证据在 `output/w1-matmul-final/`、`output/w1-transformer-final/`。完整 0.6B ONNX 的 ORT 前向本轮没有重跑，其历史验收仍按原记录归属。 + +W1 人工出口继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪：团队 D1–D8 接口决议、风险处置责任与 Plan B、完整模型第二人复现和最终出口评审。本轮没有代签或关闭这些事项。 + +## 3. W2 工作与开发计划逐项对照 + +| 开发计划要求 | 当前实现与本轮证据 | 状态边界 | +|---|---|---| +| `unit:frontend-ops`：RMSNorm/RoPE/SwiGLU/GQA | 22 项真实 ONNX 模式测试全部通过 | 通过基础算子组合解析，未新增同名融合 IR 指令 | +| `unit:backend-ops`：基础算子 QEMU | MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA，7 类/16 小图 × none/all，共 32 次通过 | 超过表中六类最低覆盖；仅证明这些 shape 与输入 | +| `numeric:ir-small`：两层 IR vs ORT | 真实 Qwen3 类的两层缩小配置，7 输入、29 检查点、6 项不变性通过；none/basic/all 优化对照 | 随机固定权重，不是预训练权重切片 | +| Attention IR 定义 | 候选接口文档明确基础算子组合、Q/K 独立 RMSNorm、全维 RoPE、GQA 分组与加性 mask 语义 | 当前不强制新增融合 ATTENTION opcode；这项接口决议仍需团队确认 | +| `unit:runtime`：Tokenizer、greedy、mask | 固定官方 Tokenizer 20 组语料/480 对照，5 组输入、mask、采样与停止条件通过 | 往返要求按官方 tokenizer 的特殊 token/规范化语义校验，不声称任意文本字节级恒等 | +| `frontend:parse-qwen3`：完整 28 层 | 固定 FP32/L=256 产物完整解析；7,847 节点、8,161 value、2,148 绑定、5,910 IR 指令通过审计，verifier 无问题 | 不代表完整 IR 数值或 QEMU 已运行 | +| 单步运行时与模型接口集成 | 2 个文本、完整词表 151936；2 Torch + 2 ORT + 4 IR，共 8 次调用；原始 token ID、最后有效位置、greedy ID 与解码一致 | hidden32/FFN96 两层随机小配置；未执行完整词表 QEMU 或多步生成 | +| W1 数值链路不退化 | 两层普通/诊断图 × none/all × 7 输入，28 次真实 QEMU 全通过 | 原 W1 的 20 次 Linux 超时回归与 3 项显式产物集成 CI 配置继续保留 | +| CI 全绿 | 本地配置增加完整 28 层解析 PR 必跑任务与汇总检查，拒绝跳过/失败/缺失 | **尚未提交，所以没有本轮远端 CI 通过结论** | + +本轮整合原先仅本地保存的完整解析、后端和运行时工作，新增修复 CI 验收缺口与嵌套进程超时清理。生产后端还保留审查所得的标量常量绑定一致性检查，拒绝调用方用不同绑定覆盖 IR 常量；日志写入失败保留原编译/QEMU 错误与耗时。这些此前 W2 分支的修复均已在本轮基线上重新验证。 + +## 4. 本轮重新执行的验收数据 + +总入口：`scripts/run_w2_acceptance.py`。总报告：`output/w2-acceptance-20261003/report.json`，另有同名 Markdown/HTML。每项包含自己的原始报告、日志、命令、输出数组、模型或 ELF 身份。总入口核对 **136 项源码指纹**，拒绝旧目录、缺失门禁、skip、非有限误差及运行中代码变化。 + +| 门禁 | 结果 | 最大绝对误差 | 顶层观测耗时 | +|---|---|---:|---:| +| frontend | 22/22 | 不适用 | 1.157 秒 | +| backend | 32/32 QEMU | `2.384185791015625e-7` | 23.769 秒 | +| runtime | 20/20、480/480、5/5 | 离散结果精确对照 | 4.905 秒 | +| runtime-model | 2 文本、8 次调用 | `2.8312206268310547e-7` | 14.889 秒 | +| small-ir | 7 输入/29 检查点/6 不变性 | `1.9073486328125e-6` | 6.971 秒 | +| small-qemu | 28/28 QEMU | 普通图 `3.129243850708008e-7`；诊断图 `1.6689300537109375e-6` | 153.658 秒 | +| full-parse | 28/28 层、逐节点/绑定审计通过 | 本项不执行数值前向 | 11.111 秒 | + +数值门槛保持 `max_abs < 1e-5`、`rtol=0`，没有放宽精度、裁剪 token ID 或用历史产物替代本次执行。small-qemu 报告也包含主机 IR 参考，其全报告最大误差 `1.9073486328125e-6` 不应误写成 QEMU 自身误差。 + +整轮 **216.597 秒（约 3 分 37 秒）**。基础后端 32 次 QEMU 进程墙钟累计 **5.545 秒**；两层 28 次累计 **140.312 秒**，4 次 C→RV64 编译累计 **6.189 秒**。两层普通图平均约 1.06/1.07 秒（none/all），诊断图约 8.94/8.97 秒；诊断输出多，不能当作纯前向速度。 + +完整前端 worker 的纯解析 **2.579 秒**，验证流程 **10.818 秒**，峰值 RSS **5,064,474,624 bytes，约 4.72 GiB**；绑定数据 **2,384,216,772 bytes**。这些是本机单次观察，不是优化前后基准对比，也不等同真实硬件性能。 + +## 5. Review、回归及 CI 检查 + +本轮 Review 先修复两个新发现的问题： + +1. 完整解析原来只在手动/夜间任务运行，PR 绿色无法证明 W2 全部门禁。现在 `full-qwen3-frontend` 在匹配路径的 PR 上必跑；`w2-acceptance` 对所有依赖要求 success。对应 21 项结果校验测试覆盖失败、跳过、取消、缺失和畸形 JSON。分支保护设置未修改。 +2. 顶层探测的 Linux 超时直接杀进程组，可能遗留 `_run_process` 创建的独立编译器/QEMU 会话。现在先 SIGTERM 触发运行器清理，再强制清理原组；增加实际进程中断回归和 Linux 嵌套会话回归。Windows 路径沿用 Job/进程树处理。 + +扩大回归时还修复了上文三项 ORT 参考内核兼容问题；相关四个测试文件最终 **251 passed / 0 skipped**，证据 `output/review-regression-fixes.xml`。没有修改编译器计算算法或数值门槛来适应参考执行器。 + +超时与 CI 专项结果 **128 passed / 1 skipped**，证据 `output/timeout-ci-tests.xml`；跳过的是 Linux 专属嵌套会话集成用例，本机 Windows 无法执行，不能计作通过。W2 集成/报告/CI 结果校验前一轮为 **111 passed / 0 skipped**，证据 `output/w2-integration-tests.xml`。各批次有重叠，不能相加。 + +最终 `tests/` 全量回归 **3020 passed / 39 skipped / 0 failed，177.66 秒**，证据为 `output/final-all-tests-v4.xml` 和同名 `.log`。39 个 skip 明确分为：本机无 `qemu-riscv32` 的 38 个旧 RV32 用户态执行测试，以及 1 个 Linux 专属嵌套会话清理测试。真实 RV64 tensor 运行时测试、三项显式产物优化集成及本轮所有可在 Windows 执行的 W1/W2 测试均已执行；不能把 39 个 skip 写成通过。 + +最终工作流 YAML、29 段 Bash 语法、文档相对链接、W1 四类接口/21 条风险及 Plan B 的结构检查、`git diff --check` 通过；详见 `output/final-static-review.json`。七项验收记录的 136 项源码指纹仍与最终执行代码一致。报告及状态文档在数值执行结束后补写，不改变该数值结果对应的执行代码。项目约定的 `.claude/harness/verify/run.py` 不存在，未宣称运行 L2 harness。 + +失败尝试也保留：W1 首次测试因自定义 pytest 临时目录的父目录尚未创建而产生 82 个 setup error，创建目录后 120 项通过；全仓第一次在收集阶段发现固定模型环境缺少可选 `llvmlite`。后者使用已有开发环境的 llvmlite 0.49.0 与 tinyfive 1.0.0 的本地副本补入仅本轮测试的路径，未改固定模型环境或全局安装。 + +中间全仓尝试 `final-all-tests-v2` 为 228 failed / 2791 passed / 40 skipped：224 项是 LLVM 工具未发现，3 项是上述 ORT 参考限制，1 项文档链接失败发生在报告文件尚未写完时。`v3` 显式指定 llvm-mca 后为 5 failed / 3014 passed / 40 skipped，剩余 5 项均因 clang/ld.lld 未从 PATH 找到。最终通过 `output/run_local_regressions.py` 在 Python 进程内配置路径，并确认三项 LLVM 工具实际可运行；不把这些中间失败隐藏或计算成通过。 + +## 6. 尚待完成与后续工作 + +- 本轮 Linux 嵌套会话清理集成回归、修改后的 GitHub workflow 和完整解析 runner 的实际容量，需要发布后取得对应源码的 Linux CI 证据。源码/配置本地通过不等于远端已通过。 +- W2 需要第二人用自己的环境和全新输出目录运行七项统一验收；W1 已确认的是旧 SHA 的两层链路，不能自动覆盖这次新增修复或 W2。 +- W1 接口冻结与风险责任决议保持候选，继续按 Issue #92 讨论；不要因技术门禁通过代替团队确认。 +- 后续按 W3 先做真实维度、预训练权重子图和更深层 IR 数值/内存基线，再做权重与代码分离、权重加载 ABI、内存复用和完整模型 QEMU 前向；最后接多步生成。原 `backend=riscv` 标量选择器并未因此支持大模型，本路线使用 `tensor-c`。 + +## 7. 本机复现命令 + +在本轮源码目录执行，输出目录必须是全新的。下面使用现有工具与固定模型资产，不自动安装或下载。 + +```powershell +$w2Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' +& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` + --python $w2Python ` + --output-dir output/w2-acceptance-review-new ` + --tokenizer-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` + --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` + --cc D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/zig-python/ziglang/zig.exe ` + --qemu D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/qemu/qemu-system-riscv64.exe +$LASTEXITCODE +``` + +人工首先检查本报告、七项总报告和文件清单，再决定后续拆分提交方式。本轮没有创建、修改或关闭 GitHub PR/Issue。 + +## 8. 修改文件审阅顺序 + +| 范围 | 主要文件 | +|---|---| +| W1 两层证据可靠性 | `probes/w2_qwen3_small/run.py`、`riscv.py` 及对应 gate 单测 | +| 编译/运行时基础修复 | `scratchv/backend/tensor_c_codegen.py`、`scratchv/runtime/riscv_tensor.py` 及对应单测 | +| 固定 ORT 测试兼容 | `tests/test_onnx_qwen_numeric_ops.py`、`tests/test_onnx_shape_ops.py` | +| 完整图审计 | `probes/w2_qwen3_parse/{run,validation,audit}.py`、三组 `test_qwen3_full_*.py` | +| 基础后端与前端模式 | `probes/w2_backend_ops/`、`tests/test_qwen3_frontend_patterns.py`、后端 gate 测试 | +| Host 运行时与单步模型集成 | `scratchv/runtime/{qwen3_tokenizer,llm_inputs}.py`、`probes/w2_runtime/`、`probes/w2_runtime_model/` 及对应测试 | +| 验收入口与 CI | `scripts/run_w2_acceptance.py`、`scripts/check_w2_ci_results.py`、`.github/workflows/llm-deploy.yml` 及对应测试 | +| 文档与依赖声明 | W1/W2 指南、执行/探测计划、接口/风险候选、总开发计划、各 probe README、`pyproject.toml` 的可选 tokenizer 依赖 | + +最终全文件清单与哈希保存在 `output/final-review-manifest.json`；完整已跟踪文件差异在 `output/final-tracked.patch`，新增文件保留在上表目录。模型、参考数组、ELF、可选测试依赖副本均在被忽略目录，未纳入源码差异。 diff --git "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" new file mode 100644 index 0000000..4e95d1e --- /dev/null +++ "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" @@ -0,0 +1,27 @@ +# W2 独立分支验收与发布说明 + +W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。本分支直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含独立 W1 PR 中的六个两层报告/参考测试文件。两部分可分别审阅和合并。共享后端的标量绑定、运行时日志和取消清理加固随 W2 交付。 + +## 范围 + +- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和权重绑定审计。 +- RMSNorm/RoPE/SwiGLU/GQA 前端模式，七类基础后端的 16 小图/32 次 QEMU。 +- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层小配置的单步模型集成。 +- 七项统一验收入口，完整前端 PR 必跑任务，以及拒绝失败、取消、跳过、缺失的 CI 汇总。 +- 标量 IR 常量外部绑定一致性；日志写入失败保留主错误；取消时清理独立编译器/QEMU 会话。 + +本分支 Windows 固定 CPU 环境重新执行：**629 passed / 1 skipped**（仅 Linux 嵌套会话清理用例跳过）；**七项统一验收 7/7 PASS**，实际 32 次基础后端 + 28 次两层 QEMU 全通过。完整图为 28 层、7,847 节点、2,148 绑定、5,910 条 IR，verifier 与逐项审计通过。运行时为 20 组语料/480 对照/5 场景；完整词表单步集成为 2 文本、8 次调用。136 项源码指纹与发布前代码一致。 + +总入口耗时 251.494 秒，两层 QEMU 进程累计 154.125 秒，基础后端 QEMU 累计 6.194 秒；并行验收时的本机单次观察，不用作性能对比。精度最大误差：基础后端 `2.384185791015625e-7`，单步集成 `2.8312206268310547e-7`，两层 IR `1.9073486328125e-6`，两层目标诊断 `1.6689300537109375e-6`，均小于 `1e-5`。 + +另外，以本分支新生成的模型显式执行 none/basic/all 三项产物优化集成，**3/3 通过、0 跳过**，证据为 `output/w2-publish-artifact.xml`。 + +证据在 `output/w2-publish-tests.xml`、`output/w2-publish-acceptance/report.json/md/html` 及各子目录。工作流 YAML、29 段 Bash、文档链接及 diff 格式检查通过。本分支没有重跑全仓 tests；原整合工作树 3020 项通过属于下述历史记录，不能用其代替本分支验证。固定 ORT 环境下三个原有算子参考用例的兼容修复归属 PR #93。项目的本地 L2 harness 文件缺失，未声称运行。此前 [W1 修复与 W2 本地验收报告](W1修复与W2本地验收报告.md) 记录的是两部分合在一起的历史工作树，不应冒充本分支的精确源码证据。 + +## 复现与限制 + +运行 [统一入口](README.md) 的七项门禁；每次使用新的输出目录。源文件哈希、模型/ELF 身份、原始命令、误差和仿真墙钟时间均保存在生成的 JSON/Markdown/HTML 中。 + +完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合，无新增融合 opcode。第二人复现、接口冻结、完整模型权重独立装载和生成循环仍需后续完成。 + +Linux 嵌套会话清理用例在 Windows 上明确跳过；CI 配置已经提供入口，但只有对应提交的实际远端执行才能证明 Linux 通过。完整 ONNX ORT 前向继续由独立手动/夜间任务验证。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 3f84dec..030275d 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -7,7 +7,7 @@ **周期**：8 周 **验收方式**：CI 门禁驱动 -**实施基线更新（2026-10-02）**：本文保留 v1 的范围、周次目标和数值门槛，按提交 `5ea22ec` 已验证的实现校正路线。当前小模型采用 `tensor-c → Zig/LLVM → RV64GC/LP64D → QEMU virt 裸机`，接口仍为候选基线，未代替团队冻结决议。W1 技术探测已通过，第二人独立复现、接口/风险确认和出口评审仍待完成，详见 [W1 执行计划](W1/W1-执行计划.md) 和 [独立复现指南](W1/README.md)。后文 W2–W8 是待验收计划，不因小模型成功自动完成。 +**实施基线更新（2026-10-03）**：本文保留 v1 的范围、周次目标和数值门槛。PR #91 已合并为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，小模型采用 `tensor-c → Zig/LLVM → RV64GC/LP64D → QEMU virt 裸机`。W1 已有 `df18b02f` 两层链路的第二人独立复现；完整模型第二人复现、接口/风险确认和出口评审仍待完成，详见 [W1 执行计划](W1/W1-执行计划.md) 和 [独立复现指南](W1/README.md)。合并后的 W1 报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 单独交付；发布前整合工作树实测与限制见 [本轮验收报告](W2/W1修复与W2本地验收报告.md)。W2–W8 按各自出口验收，不因小模型成功自动完成。 --- @@ -47,7 +47,7 @@ ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译 ### 2.1 整体数据流 -下图是完整模型的目标数据流。当前已实现的是 Python host 装载输入、启动裸机 QEMU 和读取 UART 结果；Tokenizer、采样、token 循环与完整模型仍待交付。图中的 FFI 表示模型调用边界：host 通过 QEMU 装载/传输访问 guest，不能在宿主机直接调用 RISC-V 函数。实际模型 ABI 为 `int scratchv_run(const void *const inputs[], void *output)`，详见 [候选接口](W1/interfaces.md)。 +下图是完整模型的目标数据流。当前已实现 Python host 装载输入、启动裸机 QEMU 和读取 UART 结果；W2 另提供固定官方 Tokenizer、输入/mask、最后有效位置 greedy 及停止边界辅助接口，见 [W2 交付记录](W2/README.md)。完整 token 循环与完整模型仍待交付。图中的 FFI 表示模型调用边界：host 通过 QEMU 装载/传输访问 guest，不能在宿主机直接调用 RISC-V 函数。实际模型 ABI 为 `int scratchv_run(const void *const inputs[], void *output)`，详见 [候选接口](W1/interfaces.md)。 ``` ┌─────────────────────────────────────────────────────────────────┐ @@ -77,11 +77,11 @@ ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译 | 阶段 | 已验证能力（两层小模型范围） | 后续需验收 / 扩展 | 负责模块 | |---|---|---|---| -| 前端 | Qwen3 基础算子解析、属性/shape/dtype 和权重绑定 | 完整 28 层导出图逐节点与权重验收；模式识别按优化需要添加 | `scratchv/frontend/` | +| 前端 | Qwen3 基础算子解析；固定完整 28 层图的属性/shape/dtype、逐节点转换和权重绑定已验收 | 预训练完整图数值；模式融合按优化需要添加 | `scratchv/frontend/` | | IR | 共享 SSA、verifier、NumPy 解释器可表达基础 Attention/RoPE/SwiGLU 图 | 完整 IR 数值与峰值内存；融合节点不是正确性前置 | `scratchv/ir/`、`scratchv/verification/` | | 优化 | none/basic/all 后 IR 对照；修复 FP32 舍入和 SSA 问题 | 更大配置回归；融合 Attention 等作为可选优化逐项验收 | `scratchv/optimizer/` | | 后端 | tensor-c 显式 FP32 张量循环，经 Zig/LLVM 生成 RV64GC 机器码 | 大权重与代码分离、完整容量、MatMul 分块/布局；旧标量选择器缺口另行决定是否补齐 | `scratchv/backend/` | -| 运行时 | 固定输入装载、裸机 QEMU、UART 张量输出与错误处理 | Tokenizer、采样、生成循环、完整权重装载与分块输出 | `scratchv/runtime/` | +| 运行时 | 固定输入装载、裸机 QEMU、UART 张量输出与错误处理；Tokenizer/mask/greedy/停止边界辅助接口 | 生成循环、完整权重装载与分块输出、端到端集成 | `scratchv/runtime/` | ### 2.3 Qwen3-0.6B 关键配置 @@ -188,6 +188,8 @@ Host 端循环： | **CI 验收** | `unit:frontend-ops`：四类模型模式所用基础算子及边界用例通过
`unit:backend-ops`：基础算子 QEMU 单测，误差 < 1e-5
`numeric:ir-small`：2 层小模型 IR vs ORT，误差 < 1e-5
`unit:runtime`：固定语料 tokenizer 编码/解码与官方配置一致，greedy、最后有效位置和掩码正确
`frontend:parse-qwen3`：完整 Qwen3 解析无错误，28 层和权重绑定完整 | | **出口标准** | 小模型 IR 误差 < 1e-5；完整 Qwen3 解析无缺失算子；CI 全绿 | +**W2 实施补记**：已新增四模式前端独立回归、七类基础后端的 16 个小图/32 次 QEMU 门禁，以及官方 tokenizer fast/slow/golden 对照和 host 输入/采样门禁。具体命令、实测与 CI 状态见 [W2 交付记录](W2/README.md)。Tokenizer 会遵循官方 NFC 规范化，“往返正确”按官方输出一致验收；模型 151936 个 logits 行不等于 151669 个可解码 token。融合 Attention 不是本轮正确性前置，完整 IR 数值仍按 W3 单独验收。 + #### W3：完整 Qwen3 IR 数值验证（硬门槛） | 项目 | 内容 | diff --git a/probes/w2_backend_ops/README.md b/probes/w2_backend_ops/README.md new file mode 100644 index 0000000..a535d14 --- /dev/null +++ b/probes/w2_backend_ops/README.md @@ -0,0 +1,45 @@ +# W2 前端模式与基础后端算子验收 + +`unit:frontend-ops` 明确检查 RMSNorm、RoPE、SwiGLU、GQA 四种分解图模式。 +这些模式由现有 ONNX 基础算子表达，不声明已有独立融合 Attention/RMSNorm IR。 + +```powershell +python -m pytest tests/test_qwen3_frontend_patterns.py -q +``` + +`unit:backend-ops` 从真实 ONNX 图走 production parser、优化器、tensor-c 后端， +交叉编译为 RV64GC ELF，然后在 QEMU `virt` 裸机执行。16 个用例分别运行 +`none`、`all` 两个优化级别，共 32 次目标执行；不下载模型权重。 + +```powershell +python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc path/to/zig --qemu path/to/qemu-system-riscv64 +python -m pytest tests/test_w2_backend_ops_gate.py -q +``` + +Linux 可省略工具路径，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 +使用现有 `requirements/qwen3-small-probe.txt` 中的 NumPy/ONNX/ORT 环境即可， +本探测不需要 PyTorch。输出目录须不存在或为空；依赖、工具、编译、执行或 +比较失败均非零退出，不将跳过当通过。 + +| 类别 | 用例与针对的问题 | +|---|---| +| MatMul | 4×4、四维 batch 广播、向量 dot 标量返回 | +| 逐元素 | Add/Sub/Mul/Div 串联，标量与尾维广播 | +| Softmax | axis=-1/0/1，大 logits 稳定性及非末轴 | +| RMSNorm | hidden/Q/K 的不同 head 数，全零/极小输入、权重、1e-6/1e-3 epsilon | +| RoPE | 全 head_dim rotate_half、Q/K head 广播、position=0/1/255 及非零起点 | +| SwiGLU | 两组不同 gate/up 权重，SiLU 仅作用于 gate，再经 down 投影 | +| GQA | KV 连续重复顺序、FP32 最小有限值因果 mask、attention context，重复比 2 与 MHA 比 1 | + +NumPy 参考使用独立数学公式及双精度中间值，最后转 FP32；ORT 单独执行 ONNX。 +IR 与 QEMU 都同时对照这两种参考，要求 shape、dtype 一致、全部有限，且 +`max_abs < 1e-5`、`rtol=0`。前端单测还注入错误 epsilon、旋转符号、门控分支、 +KV head 顺序，确认用例可以发现这些语义错误。 + +`report.json`、`report.md`、`report.html` 含逐例误差、失败阶段、环境及源码哈希， +还有编译命令、ELF 哈希和每次 QEMU 进程实测墙钟时长。 +`qemu_process_wall_seconds` 包括启动、guest 计算、UART 输出与退出，不是 guest +纯计算时间或周期数。每例同时保存模型、输入、NumPy/ORT/IR/QEMU 张量和工具日志， +可定位第一次不一致发生于参考、IR 还是后端。 + +门禁验证静态 FP32 小图，不代表完整 0.6B 的内存容量、性能或完整模型 QEMU 前向已通过。 diff --git a/probes/w2_backend_ops/cases.py b/probes/w2_backend_ops/cases.py new file mode 100644 index 0000000..49d182b --- /dev/null +++ b/probes/w2_backend_ops/cases.py @@ -0,0 +1,174 @@ +"""Small, deterministic ONNX graphs with independent NumPy formula references. + +These are semantic pattern tests, not fused RMSNorm/RoPE/SwiGLU/GQA operators. +The graph uses the same decomposed primitives accepted by the Qwen3 frontend. +""" +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +import onnx +from onnx import TensorProto as T, helper, numpy_helper + + +@dataclass(frozen=True) +class Case: + name: str + family: str + model: onnx.ModelProto + feed: dict[str, np.ndarray] + expected: np.ndarray + purpose: str + + +def graph(name, family, nodes, feed, expected, purpose, constants=None): + expected = np.asarray(expected, dtype=np.float32) + model = helper.make_model(helper.make_graph( + nodes, name, + [helper.make_tensor_value_info(key, T.FLOAT, value.shape) for key, value in feed.items()], + [helper.make_tensor_value_info("y", T.FLOAT, expected.shape)], + [numpy_helper.from_array(np.asarray(value), key) for key, value in (constants or {}).items()], + ), opset_imports=[helper.make_opsetid("", 18)], ir_version=10) + onnx.checker.check_model(model) + return Case(name, family, model, feed, expected, purpose) + + +def rmsnorm_case(kind="hidden", epsilon=1e-6): + shape = (1, 3, 16) if kind == "hidden" else ((1, 4, 3, 8) if kind == "q" else (1, 2, 3, 8)) + x = np.linspace(-0.003, 0.004, np.prod(shape), dtype=np.float32).reshape(shape) + x[..., 0] = 0 + x[..., 0, :] = 0 + weight = np.linspace(0.4, 1.6, shape[-1], dtype=np.float32) + # Accumulate independently in double precision, then round only the result. + expected = (x.astype(np.float64) / np.sqrt(np.mean(x.astype(np.float64) ** 2, + axis=-1, keepdims=True) + np.float32(epsilon))) * weight + nodes = [helper.make_node("Pow", ["x", "two"], ["square"]), + helper.make_node("ReduceMean", ["square", "last"], ["variance"], keepdims=1), + helper.make_node("Add", ["variance", "epsilon"], ["stabilized"]), + helper.make_node("Sqrt", ["stabilized"], ["denominator"]), + helper.make_node("Div", ["x", "denominator"], ["normalized"]), + helper.make_node("Mul", ["normalized", "weight"], ["y"])] + return graph(f"rmsnorm_{kind}_eps_{epsilon:g}", "rmsnorm", nodes, {"x": x}, expected, + "Normalize the last dimension with learned weights; small inputs expose epsilon changes", + {"two": np.array(2, np.int64), "last": np.array([-1], np.int64), + "epsilon": np.array(epsilon, np.float32), "weight": weight}) + + +def rope_case(positions=(0, 1, 255), heads=4): + dim = 8 + x = np.linspace(-1.1, 1.2, heads * len(positions) * dim, dtype=np.float32).reshape(1, heads, -1, dim) + position = np.array(positions, np.float32).reshape(1, -1, 1) + inv = (1.0 / 1e6 ** (np.arange(0, dim, 2, dtype=np.float64) / dim)).astype(np.float32) + # Direct complex rotation of each pair of halves, not graph Slice/Concat. + angle = np.asarray(positions, np.float64)[:, None] * inv.astype(np.float64)[None, :] + first, second = x[..., :dim // 2].astype(np.float64), x[..., dim // 2:].astype(np.float64) + expected = np.empty_like(x) + expected[..., :dim // 2] = first * np.cos(angle) - second * np.sin(angle) + expected[..., dim // 2:] = second * np.cos(angle) + first * np.sin(angle) + nodes = [helper.make_node("Mul", ["position", "inv_freq"], ["angle"]), + helper.make_node("Concat", ["angle", "angle"], ["full_angle"], axis=-1), + helper.make_node("Cos", ["full_angle"], ["cos_3d"]), + helper.make_node("Sin", ["full_angle"], ["sin_3d"]), + helper.make_node("Unsqueeze", ["cos_3d", "head_axis"], ["cos"]), + helper.make_node("Unsqueeze", ["sin_3d", "head_axis"], ["sin"]), + helper.make_node("Slice", ["x", "zero", "half", "last"], ["first"]), + helper.make_node("Slice", ["x", "half", "end", "last"], ["second"]), + helper.make_node("Neg", ["second"], ["negative_second"]), + helper.make_node("Concat", ["negative_second", "first"], ["rotated"], axis=-1), + helper.make_node("Mul", ["x", "cos"], ["real"]), + helper.make_node("Mul", ["rotated", "sin"], ["imaginary"]), + helper.make_node("Add", ["real", "imaginary"], ["y"])] + return graph(f"rope_h{heads}_p{'_'.join(map(str, positions))}", "rope", nodes, + {"x": x, "position": position}, expected, + "Full head dimension rotate_half with broadcast positions, including nonzero offsets", + {"inv_freq": inv.reshape(1, 1, -1), "head_axis": np.array([1], np.int64), + "zero": np.array([0], np.int64), "half": np.array([4], np.int64), + "end": np.array([8], np.int64), "last": np.array([-1], np.int64)}) + + +def swiglu_case(seed=11): + rng = np.random.default_rng(seed) + x = rng.normal(size=(1, 3, 4)).astype(np.float32) + weights = {name: (rng.normal(size=shape) / 4).astype(np.float32) for name, shape in + (("gate_weight", (4, 7)), ("up_weight", (4, 7)), ("down_weight", (7, 4)))} + gate = x.astype(np.float64) @ weights["gate_weight"].astype(np.float64) + up = x.astype(np.float64) @ weights["up_weight"].astype(np.float64) + expected = ((gate / (1 + np.exp(-gate))) * up) @ weights["down_weight"].astype(np.float64) + nodes = [helper.make_node("MatMul", ["x", "gate_weight"], ["gate"]), + helper.make_node("MatMul", ["x", "up_weight"], ["up"]), + helper.make_node("Sigmoid", ["gate"], ["activation"]), + helper.make_node("Mul", ["gate", "activation"], ["silu"]), + helper.make_node("Mul", ["silu", "up"], ["hidden"]), + helper.make_node("MatMul", ["hidden", "down_weight"], ["y"])] + return graph(f"swiglu_seed_{seed}", "swiglu", nodes, {"x": x}, expected, + "Different gate/up projections, SiLU on gate only, and down projection", weights) + + +def gqa_case(repeats=2): + rng = np.random.default_rng(131) + kv_heads, length, dim = 2, 3, 4 + q = (rng.normal(size=(1, kv_heads * repeats, length, dim)) / 2).astype(np.float32) + k = (rng.normal(size=(1, kv_heads, length, dim)) / 2).astype(np.float32) + v = (np.arange(kv_heads * length * dim).reshape(1, kv_heads, length, dim) / 13).astype(np.float32) + mask = np.where(np.tril(np.ones((length, length), dtype=bool)), + 0, np.finfo(np.float32).min).astype(np.float32) + expected = np.empty_like(q) + # Explicit head mapping detects tile-vs-repeat mistakes; masked softmax is + # calculated independently in double precision for every query head. + for head in range(q.shape[1]): + kv = head // repeats + scores = q[0, head].astype(np.float64) @ k[0, kv].astype(np.float64).T / np.sqrt(dim) + mask + probabilities = np.exp(scores - scores.max(axis=-1, keepdims=True)) + probabilities /= probabilities.sum(axis=-1, keepdims=True) + expected[0, head] = probabilities @ v[0, kv].astype(np.float64) + nodes = [] + for name in ("k", "v"): + nodes.extend([helper.make_node("Unsqueeze", [name, "repeat_axis"], [f"{name}_5d"]), + helper.make_node("Expand", [f"{name}_5d", "expanded_shape"], [f"{name}_expanded"]), + helper.make_node("Reshape", [f"{name}_expanded", "repeated_shape"], [f"{name}_repeated"])]) + nodes.extend([helper.make_node("Transpose", ["k_repeated"], ["kt"], perm=[0, 1, 3, 2]), + helper.make_node("MatMul", ["q", "kt"], ["unscaled"]), + helper.make_node("Mul", ["unscaled", "scale"], ["scores"]), + helper.make_node("Add", ["scores", "mask"], ["masked"]), + helper.make_node("Softmax", ["masked"], ["probabilities"], axis=-1), + helper.make_node("MatMul", ["probabilities", "v_repeated"], ["y"])]) + return graph(f"gqa_repeat_{repeats}", "gqa", nodes, {"q": q, "k": k, "v": v}, expected, + "Contiguous KV head repetition, causal mask and attention context, including MHA ratio 1", + {"repeat_axis": np.array([2], np.int64), "scale": np.array(1 / np.sqrt(dim), np.float32), + "mask": mask, "expanded_shape": np.array([1, kv_heads, repeats, length, dim], np.int64), + "repeated_shape": np.array([1, kv_heads * repeats, length, dim], np.int64)}) + + +def build_cases(): + rng = np.random.default_rng(314159) + cases = [] + for name, left, right in (("matrix", (4, 4), (4, 4)), + ("batch_broadcast", (2, 1, 3, 5), (1, 4, 5, 2)), + ("vector_dot", (5,), (5,))): + a, b = [(rng.normal(size=shape) / 3).astype(np.float32) for shape in (left, right)] + cases.append(graph(f"matmul_{name}", "matmul", [helper.make_node("MatMul", ["a", "b"], ["y"])], + {"a": a, "b": b}, a.astype(np.float64) @ b.astype(np.float64), + "Matrix, batched broadcasting and scalar dot-product outputs")) + x = np.linspace(-2, 2, 24, dtype=np.float32).reshape(2, 3, 4) + bias = np.array([0.1, -0.3, 0.7, 1.1], np.float32) + denominator = np.array(0.75, np.float32) + cases.append(graph("elementwise_broadcast", "elementwise", [ + helper.make_node("Add", ["x", "bias"], ["sum"]), + helper.make_node("Sub", ["sum", "denominator"], ["difference"]), + helper.make_node("Mul", ["difference", "bias"], ["product"]), + helper.make_node("Div", ["product", "denominator"], ["y"])], + {"x": x, "bias": bias, "denominator": denominator}, + (x.astype(np.float64) + bias - denominator) * bias / denominator, + "Scalar and trailing-dimension broadcast through Add/Sub/Mul/Div")) + for axis in (-1, 0, 1): + data = np.arange(24, dtype=np.float32).reshape(2, 3, 4) + 1000 + expected = np.exp(data.astype(np.float64) - data.max(axis=axis, keepdims=True)) + expected /= expected.sum(axis=axis, keepdims=True) + cases.append(graph(f"softmax_axis_{axis}", "softmax", [ + helper.make_node("Softmax", ["x"], ["y"], axis=axis)], {"x": data}, expected, + "Numerically stable softmax with large logits and non-last dimensions")) + cases.extend([rmsnorm_case("hidden"), rmsnorm_case("q"), rmsnorm_case("k", 1e-3), + rope_case(), rope_case((7, 19, 255), heads=2), + swiglu_case(), swiglu_case(23), gqa_case(), gqa_case(1)]) + return cases diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py new file mode 100644 index 0000000..c801a31 --- /dev/null +++ b/probes/w2_backend_ops/run.py @@ -0,0 +1,258 @@ +"""W2 operator gate: independent formulas -> ORT -> IR -> tensor C -> RV64 QEMU. + +All seven families and both optimization levels are mandatory. Missing tools, +nonfinite tensors, shape/type changes or max_abs >= 1e-5 fail the gate. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +from html import escape +import json +import math +from pathlib import Path +import platform +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import numpy as np +import onnx +import onnxruntime as ort + +from probes.w2_backend_ops.cases import build_cases +from probes.w2_qwen3_small.diagnostics import tensor_diff +from scratchv.analysis.ir_verifier import verify_ir +from scratchv.compiler import CompilerConfig, CompilerDriver +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.pass_manager import create_optimization_pass_manager +from scratchv.runtime.riscv_tensor import build_riscv_tensor, discover_toolchain, run_riscv_tensor +from scratchv.verification.ir_interpreter import IRInterpreter + +ATOL = 1e-5 +LEVELS = ("none", "all") +FAMILIES = ("matmul", "elementwise", "softmax", "rmsnorm", "rope", "swiglu", "gqa") +FRONTEND_FAMILIES = ("rmsnorm", "rope", "swiglu", "gqa") + + +def require_valid(program): + passed, issues = verify_ir(program) + if not passed: + raise ValueError("; ".join(str(issue) for issue in issues)) + + +def reference_case(case, path): + onnx.save(case.model, path) + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + expected = ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"]).run(None, case.feed)[0] + parser = ONNXParser() + program = parser.parse(str(path)) + require_valid(program) + return expected, parser, program + + +def interpret(program, bindings, feed, level): + manager = create_optimization_pass_manager(level) + manager.before_pass = lambda p, value: require_valid(value) + manager.after_pass = lambda p, value: require_valid(value) + optimized = manager.run_pipeline(copy.deepcopy(program)).data + require_valid(optimized) + return IRInterpreter(optimized).run(feed, initializers=bindings).return_value + + +def evidence(): + sources = [Path(__file__), ROOT / "probes/w2_backend_ops/cases.py", + ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "scratchv/compiler.py", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", + ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/pass_manager.py", + ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/verification/ir_interpreter.py", + ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/backend/tensor_c_codegen.py", ROOT / "scratchv/runtime/riscv_tensor.py", + *sorted((ROOT / "scratchv/analysis").glob("*.py")), + *sorted((ROOT / "scratchv/optimizer").glob("*.py"))] + return {"environment": {"python": platform.python_version(), "executable": sys.executable, + "platform": platform.platform(), "numpy": np.__version__, + "onnx": onnx.__version__, "onnxruntime": ort.__version__}, + "source_sha256": {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sources}} + + +def run_probe(out, report, *, cc=None, qemu=None, timeout=120): + cases = build_cases() + if (not cases or len({case.name for case in cases}) != len(cases) + or set(case.family for case in cases) != set(FAMILIES)): + raise ValueError("Cases must have unique names and cover every required operator family") + report.update(evidence(), stage="toolchain", families=list(FAMILIES), levels=list(LEVELS), + planned_executions=len(cases) * len(LEVELS), cases=[]) + tools = discover_toolchain(cc=cc, qemu=qemu) + report["toolchain"] = {"cc": list(tools.cc), "qemu": tools.qemu} + report["stage"] = "execution" + for case in cases: + folder = out / case.name + folder.mkdir() + row = {"name": case.name, "family": case.family, "purpose": case.purpose, + "passed": False, "stage": "reference", "executions": []} + report["cases"].append(row) + try: + path = folder / "model.onnx" + expected, parser, program = reference_case(case, path) + np.savez(folder / "inputs.npz", **case.feed) + np.save(folder / "numpy.npy", case.expected) + np.save(folder / "ort.npy", expected) + row["model_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + row["input_sha256"] = {name: hashlib.sha256(array.tobytes()).hexdigest() + for name, array in case.feed.items()} + row["ort_vs_numpy"] = tensor_diff(expected, case.expected, ATOL) + if not row["ort_vs_numpy"]["passed"]: + raise ValueError("ORT disagrees with independent NumPy formula") + for level in LEVELS: + target = folder / level + target.mkdir() + execution = {"optimization": level, "passed": False, "stage": "ir", + "qemu_process_wall_seconds": None} + row["executions"].append(execution) + try: + ir_output = interpret(program, parser.initializers, case.feed, level) + np.save(target / "ir.npy", ir_output) + execution["ir_vs_ort"] = tensor_diff(ir_output, expected, ATOL) + execution["ir_vs_numpy"] = tensor_diff(ir_output, case.expected, ATOL) + if not all(execution[key]["passed"] for key in ("ir_vs_ort", "ir_vs_numpy")): + raise ValueError("IR disagrees with ORT or independent NumPy formula") + execution["stage"] = "compile" + driver = CompilerDriver(CompilerConfig(backend="tensor-c", optimize_level=level, verify_ir=True)) + result = driver.compile(str(path), str(target / "model.c")) + if not result.success: + raise RuntimeError("; ".join(result.errors)) + executable = build_riscv_tensor(driver.tensor_artifact, target / "build", tools) + execution.update(elf_sha256=executable.elf_sha256, compile_command=executable.compile_command, + compile_seconds=executable.compile_seconds, + workspace_bytes=executable.workspace_bytes, tool_versions=executable.tool_versions, + stage="qemu") + actual = run_riscv_tensor(executable, case.feed, target / "run", timeout=timeout) + execution.update(qemu_process_wall_seconds=actual.elapsed_s, command=actual.command, stage="numeric") + np.save(target / "qemu.npy", actual.output) + execution["qemu_vs_ort"] = tensor_diff(actual.output, expected, ATOL) + execution["qemu_vs_numpy"] = tensor_diff(actual.output, case.expected, ATOL) + execution["passed"] = all(execution[key]["passed"] for key in ("qemu_vs_ort", "qemu_vs_numpy")) + execution["stage"] = "complete" + except Exception as exc: + execution["error"] = f"{type(exc).__name__}: {exc}" + if getattr(exc, "elapsed_s", None) is not None: + execution.update(qemu_process_wall_seconds=exc.elapsed_s, + command=getattr(exc, "command", None), + status=getattr(exc, "status", "runtime_error")) + print(f"[{case.name}/{level}] {'PASS' if execution['passed'] else 'FAIL'} " + f"max_abs={execution.get('qemu_vs_ort', {}).get('max_abs')} " + f"stage={execution['stage']}", flush=True) + row["passed"] = len(row["executions"]) == len(LEVELS) and all(item["passed"] for item in row["executions"]) + row["stage"] = "complete" + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + executions = [execution for case in report["cases"] for execution in case["executions"]] + report["completed_executions"] = sum(item["stage"] == "complete" for item in executions) + report["passed_executions"] = sum(item["passed"] for item in executions) + measured = [item["qemu_process_wall_seconds"] for item in executions if item["qemu_process_wall_seconds"] is not None] + report["qemu_process_wall_seconds"] = {"measured_count": len(measured), + "total": sum(measured) if measured else None} + report["passed"] = (len(executions) == report["planned_executions"] + and all(case["passed"] for case in report["cases"])) + report["stage"] = "complete" + + +def _report_markdown(report): + payload = json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n" + lines = ["# W2 backend operator gate", "", f"Result: {'PASS' if report['passed'] else 'FAIL'}", + "", "Strict max_abs < 1e-5; rtol=0; exact shape/dtype; finite values.", + "", "| Case | Level | Result | QEMU vs ORT max_abs | QEMU process seconds |", + "|---|---|---|---|---|"] + for case in report.get("cases", []): + if not case["executions"]: + lines.append(f"| {case['name']} | not started | FAIL | - | - |") + for item in case["executions"]: + lines.append(f"| {case['name']} | {item['optimization']} | {'PASS' if item['passed'] else 'FAIL'} | " + f"{item.get('qemu_vs_ort', {}).get('max_abs')} | {item['qemu_process_wall_seconds']} |") + lines.extend(["", "Timing includes QEMU startup, guest execution, UART transfer and exit; not guest cycles.", + "", "Full diagnostics, commands, environment and source hashes:", "", "```json", payload.rstrip(), "```", ""]) + return "\n".join(lines) + + +def write_reports(out, report): + """Keep primary errors and publish JSON only after required evidence succeeds.""" + failures = [] + + def write(name, content, *, atomic=False): + path = out / name + pending = out / (name + ".tmp") if atomic else path + try: + pending.write_text(content, encoding="utf-8") + if atomic: + pending.replace(path) + except OSError as exc: + failures.append(f"{name}: {type(exc).__name__}: {exc}") + finally: + if atomic: + try: + pending.unlink(missing_ok=True) + except OSError: + pass + + def record_failure(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete backend gate evidence") + + def views(): + markdown = _report_markdown(report) + write("report.md", markdown) + write("report.html", "W2 backend operators" + "

W2 backend operators

" + escape(markdown) + "
") + + views() + if failures: + record_failure() + views() + # Write and close a sibling temporary file before publishing JSON. A late + # write/close failure must not leave a valid PASS artifact for CI consumers. + before_json = len(failures) + write("report.json", json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + "\n", atomic=True) + if len(failures) != before_json: + record_failure() + views() + if failures: + print("[unit:backend-ops] FAIL; report write errors: " + "; ".join(failures), file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--output-dir", type=Path, default=ROOT / "output/w2-backend-ops") + cli.add_argument("--cc") + cli.add_argument("--qemu") + cli.add_argument("--timeout", type=float, default=120) + args = cli.parse_args(argv) + if not math.isfinite(args.timeout) or args.timeout <= 0: + cli.error("--timeout must be finite and positive") + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + cli.error("--output-dir must be empty") + out.mkdir(parents=True, exist_ok=True) + report = {"passed": False, "stage": "setup", "backend": "ScratchV tensor-c -> RV64GC -> QEMU", + "atol": ATOL, "rtol": 0, "timeout_seconds": args.timeout} + start = time.perf_counter() + try: + run_probe(out, report, cc=args.cc, qemu=args.qemu, timeout=args.timeout) + except Exception as exc: + report["error"] = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + report["seconds"] = time.perf_counter() - start + write_reports(out, report) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_qwen3_parse/README.md b/probes/w2_qwen3_parse/README.md new file mode 100644 index 0000000..23a69e7 --- /dev/null +++ b/probes/w2_qwen3_parse/README.md @@ -0,0 +1,87 @@ +# W2：完整 Qwen3 ONNX 前端解析探测 + +入口 `probes/w2_qwen3_parse/run.py` 面向固定发布的完整 28 层 Qwen3-0.6B 图，验证 ScratchV 前端解析与结构、节点及权重绑定审计。使用 W1 已固定的 FP32、batch=1、L=256、无 KV Cache 的 ONNX 产物和 [manifest](../w1_qwen3_export/manifest.json)。 + +本入口不执行完整模型 IR，不生成完整模型 RISC-V，不运行 QEMU，也不重新导出模型或比较 PyTorch/ORT 数值。完整模型的 ORT 执行继续由 [W1 导出门禁](../w1_qwen3_export/README.md) 负责；两项门禁分别报告结果。 + +## 准备环境 + +使用 Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 + +Windows PowerShell： + +```powershell +py -3.12 -m venv output/qwen3-parse-venv +$parsePython = ".\output\qwen3-parse-venv\Scripts\python.exe" +& $parsePython -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" +& $parsePython -m pip check +& $parsePython -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +$LASTEXITCODE +``` + +Linux shell： + +```bash +python3.12 -m venv output/qwen3-parse-venv +. output/qwen3-parse-venv/bin/activate +python -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" +python -m pip check +python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify \ + --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +echo $? +``` + +所有命令在仓库根目录执行。`--model-dir` 必填，应指向同目录保存 `model.onnx` 和三个真实 external-data 分片的完整产物，不能只有 ONNX 文件或 LFS pointer。`verify` 是默认模式，不访问网络；不存在完整模型时，将命令中的 `--mode verify` 改成 `--mode download`，由入口获取并校验 W1 固定发布产物。不要用新导出或其他 revision 的文件替换固定产物。 + +`--output-dir` 默认是 `output/qwen3-parse`，必须不存在或为空，每次使用新的目录以保留失败日志与首次报告。`--timeout` 默认 300 秒，限制整个验证子进程，包含文件检查、解析、审计，以及 download 模式的下载；慢网络应据实调整此上限。它是超时设置，不是执行耗时或纯解析耗时。 + +固定 ZIP 约 1.24 GB，解包后需要数 GiB 磁盘空间，完整权重解析也需要数 GiB 内存。解析通过不表示当前 tensor-c 的常量上限或 QEMU 内存布局能够装载完整模型；不要从小型 fixture 的内存消耗推断完整图容量。 + +## 验收范围与证据 + +检查顺序及具体失败阶段以 `report.json` 为准。完整门禁覆盖固定产物身份、完整图结构与 28 层覆盖、实际 ONNX→共享 IR 解析、节点转换与 initializer 绑定审计，以及共享 IR 验证器检查。节点数量、算子统计、绑定数量和实际耗时应读取本次报告，不使用旧报告代替执行。 + +结构检查沿实际输入依赖追踪模型，而不是只统计节点名中的层号： + +- 核对全部 initializer 的名字、shape、dtype，以及两个静态输入和 logits 输出。 +- 核对每层投影和 RMSNorm、Q/K 归一化、attention 的缩放/掩码/Softmax/V/O 路径、RoPE 位置依赖和 GQA 扩展。 +- 核对 SwiGLU、两个残差连接和完整层顺序，最后归一化连接共享 LM Head。 +- 要求全部节点和 initializer 都参与返回 logits 的依赖，避免断开的装饰节点被当作完整模型。 + +这些是固定图的结构规则，不是对算子数值实现或所有可能 ONNX 导出形式的证明。 + +转换审计记录实际 parser 的每个节点，核对 IR 指令区间、opcode、输入 Value、输出映射和静态属性，允许合法的 Constant 绑定与 Identity 别名而不把它们误报为漏转换。形状、切片和轴等控制参数从原 ONNX 小型控制子图独立求值，不复用被检查解析器的常量缓存。所有 ONNX value 的静态 shape/dtype、全局张量集合与唯一 RETURN 也会核对。权重和 Constant 绑定逐张量比较 shape、dtype 与内容 SHA256；外部分片按块读取校验，不为审计再保存一套权重。该审计采用明确支持的基础算子转换规则，未知转换方式应报错，由新增规则和测试后再接受。 + +| 文件 | 用途 | +|---|---| +| `report.json` | 机器可读的本次结果、阶段、来源与审计摘要 | +| `report.md`、`report.html` | 便于 review 的摘要和可浏览报告 | +| `nodes.json` | ONNX 节点与转换审计证据 | +| `bindings.json` | initializer / IR 绑定审计证据 | +| `ir.txt` | 本次解析得到的共享 IR 文本 | +| `worker.stdout`、`worker.stderr`、`worker-progress.json` | 子进程日志和最后进度，用于定位超时或异常退出 | + +报告目录不包含权重分片。CI 上传该目录，不上传模型目录。失败可能发生在报告明细生成之前，应结合报告与命令日志判断缺失文件，不能把缺失当作通过。 + +`attempt` 保存 supervisor 启动前的命令、输入路径、解释器与源码身份；它用于定位早期失败，不代表 worker 已初始化或验证通过。`log_errors` 和 `evidence_errors` 分别保存附加日志、失败明细的写入问题，不覆盖原始解析/审计错误。成功路径的必需证据或日志无法保存时仍失败。 + +`seconds` 记录 supervisor 溯源开始至验证与日志保存结束的耗时，不含随后读取最后进度与生成最终三种报告的时间；各阶段和纯解析耗时另行记录。峰值 RSS 覆盖验证子进程，不包含 supervisor 父进程。平台无法提供的测量值保持未测得。超时后只报告已经保存的进度，不能将 300 秒上限当作实际解析耗时或补全缺失的峰值内存。 + +成功需要命令退出 0 且本次 `report.json` 的 `passed` 为 true。失败时保留退出码、失败阶段、错误、输入模型身份和已经产生的明细。记录实际 checkout SHA；有未提交修改时还应保存完整 diff 和新增文件，单独记录 HEAD 不能确定所执行代码。 + +`passed=true` 只证明这份固定产物完成本门禁的前端解析与审计。它不证明完整 IR 数值正确、优化后语义等价、完整权重部署、文本生成或目标硬件性能。下一阶段仍需逐步开展完整模型 IR/ORT 数值对照和容量验证。 + +## 轻量测试与 CI + +```bash +python -m pip install "pytest==9.1.1" +python -m pytest tests/test_qwen3_full_structure.py tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py -q +``` + +PowerShell 将上面命令中的 `python` 替换为 `& $parsePython`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 + +[LLM 工作流](../../.github/workflows/llm-deploy.yml) 的 PR 路径接入上述三组测试，并在独立 `full-qwen3-frontend` job 以 `--mode download` 实跑完整固定图。手动/定时同样运行该解析任务。解析有独立 Summary，并以 `always()` 上传 `output/qwen3-parse/`，artifact 名为 `qwen3-full-frontend-parse-output`。下载约 1.24 GB，解析 worker 峰值 RSS 约 5 GB；runner 的实际容量与耗时仍须远端验证。 + +`w2-acceptance` 汇总任务要求数值任务和完整解析任务均实际成功，失败、取消、跳过或缺失都不能通过。手动/定时的 `full-qwen3-onnx` job 独立保留 W1 完整 ONNX ORT 验证；解析不依赖它成功。并行的两个完整模型任务会各自下载资产。 + +PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存本地实跑与 Review 修复结果，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 diff --git a/probes/w2_qwen3_parse/audit.py b/probes/w2_qwen3_parse/audit.py new file mode 100644 index 0000000..cc8a891 --- /dev/null +++ b/probes/w2_qwen3_parse/audit.py @@ -0,0 +1,417 @@ +"""Read-only evidence for complete ONNX -> IR translation and weight binding. + +This deliberately audits the fixed, single-output Qwen graph. It does not run +the model or claim numerical equivalence. External tensors are hashed in chunks +without loading another copy of the complete weights. +""" + +from __future__ import annotations + +import hashlib +import math +from pathlib import Path, PureWindowsPath + +import numpy as np +import onnx + +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.ir.types import DataType, OpCode +from probes.w2_qwen3_parse.validation import make_control_constant_reader + + +class AuditError(ValueError): + """Translation or binding evidence is incomplete or inconsistent.""" + + +_TYPES = {1: DataType.FLOAT32, 6: DataType.INT32, + 7: DataType.INT64, 11: DataType.FLOAT64} +_CHUNK_BYTES = 1024 * 1024 + + +def _require(condition, message): + if not condition: + raise AuditError(message) + + +def _value_record(value): + return {"ir_name": value.name, "shape": list(value.shape), + "dtype": value.dtype.value} + + +class AuditedONNXParser(ONNXParser): + """The production parser, with per-node observations rather than new lowering.""" + + def __init__(self, progress=None): + super().__init__() + self.progress = progress + self.node_records = [] + self.current_context = {"stage": "not_started"} + self._node_instructions = [] + + def parse(self, model_path): + self.node_records = [] + self._node_instructions = [] + self.current_context = {"stage": "parse", "model_path": str(model_path)} + if self.progress: + self.progress(dict(self.current_context)) + program = super().parse(model_path) + self.current_context = {"stage": "parsed", "node_count": len(self.node_records)} + if self.progress: + self.progress(dict(self.current_context)) + return program + + def _translate_node(self, node, output_names): + index = len(self.node_records) + block = self.builder.current_block + start = len(block.instructions) + self.current_context = {"stage": "translate_node", "node_index": index, + "node_name": node.name, "op_type": node.op_type, + "inputs": list(node.input), "outputs": list(node.output)} + if self.progress: + self.progress(dict(self.current_context)) + record = {**self.current_context, "ir_start": start, "status": "running", + "input_values": {name: _value_record(self._value_map[name]) + for name in node.input if name in self._value_map}} + try: + super()._translate_node(node, output_names) + except Exception as exc: + record.update(status="failed", error=f"{type(exc).__name__}: {exc}") + self.node_records.append(record) + raise + instructions = block.instructions[start:] + record.update(status="translated", ir_end=len(block.instructions), + output_values={name: _value_record(self._value_map[name]) + for name in node.output if name in self._value_map}, + instructions=[{"opcode": inst.opcode.value, + "dest": inst.dest.name if inst.dest else None, + "operands": [value.name for value in inst.operands], + "attrs": dict(inst.attrs)} for inst in instructions]) + self.node_records.append(record) + self._node_instructions.append(instructions) + + +def _array_hash(data): + """Hash logical C-order little-endian bytes with bounded temporary buffers.""" + digest = hashlib.sha256() + array = np.asarray(data) + little_dtype = array.dtype.newbyteorder("<") + if array.flags.c_contiguous and array.dtype == little_dtype: + view = memoryview(array.reshape(-1)).cast("B") + for offset in range(0, len(view), _CHUNK_BYTES): + digest.update(view[offset:offset + _CHUNK_BYTES]) + else: + iterator = np.nditer(array, flags=["external_loop", "buffered", "zerosize_ok"], + op_flags=["readonly"], op_dtypes=[little_dtype], + order="C", buffersize=max(1, _CHUNK_BYTES // array.itemsize)) + for chunk in iterator: + digest.update(chunk.tobytes(order="C")) + return digest.hexdigest() + + +def _external_hash(tensor, model_dir, expected_bytes): + entries = list(tensor.external_data) + fields = {item.key: item.value for item in entries} + _require(len(fields) == len(entries), f"{tensor.name}: duplicate external-data keys") + location = fields.get("location", "") + windows = PureWindowsPath(location) + _require(bool(location) and "\x00" not in location and not windows.drive + and not windows.is_absolute() and not Path(location).is_absolute(), + f"{tensor.name}: invalid external-data location") + root = Path(model_dir).resolve() + path = (root / location.replace("\\", "/")).resolve() + _require(path.is_relative_to(root) and path.is_file(), + f"{tensor.name}: external data missing or outside model directory: {location}") + for key in ("offset", "length"): + if key in fields: + _require(fields[key].isascii() and fields[key].isdecimal(), + f"{tensor.name}: invalid external-data {key}") + offset = int(fields.get("offset", 0)) + size = path.stat().st_size + length = int(fields.get("length", size - offset)) + _require(offset <= size and length == expected_bytes and offset + length <= size, + f"{tensor.name}: external-data range/length mismatch") + digest = hashlib.sha256() + remaining = length + with path.open("rb") as stream: + stream.seek(offset) + while remaining: + chunk = stream.read(min(_CHUNK_BYTES, remaining)) + _require(bool(chunk), f"{tensor.name}: truncated external data") + digest.update(chunk) + remaining -= len(chunk) + return digest.hexdigest(), {"location": location, "offset": offset, "length": length} + + +def _tensor_spec(tensor, model_dir): + _require(tensor.data_type in _TYPES, f"{tensor.name}: unsupported tensor dtype") + dtype = np.dtype(onnx.helper.tensor_dtype_to_np_dtype(tensor.data_type)) + shape = tuple(tensor.dims) + _require(all(dim >= 0 for dim in shape), f"{tensor.name}: negative tensor dimension") + size = math.prod(shape) * dtype.itemsize + if tensor.data_location == onnx.TensorProto.EXTERNAL: + sha, external = _external_hash(tensor, model_dir, size) + else: + sha = _array_hash(onnx.numpy_helper.to_array(tensor)) + external = None + return dtype, shape, size, sha, external + + +def _onnx_attributes(node): + """Read source protobuf directly, independently of production lowering.""" + attributes = {attribute.name: onnx.helper.get_attribute_value(attribute) + for attribute in node.attribute} + _require(len(attributes) == len(node.attribute), f"{node.name}: duplicate ONNX attributes") + return attributes + + +def _constant_spec(node, model_dir): + attrs = _onnx_attributes(node) + _require(len(attrs) == 1, f"{node.name}: Constant requires exactly one attribute") + key, data = next(iter(attrs.items())) + if key == "value": + return _tensor_spec(data, model_dir) + if key in ("value_int", "value_ints"): + array = np.asarray(data, dtype=np.int64) + elif key in ("value_float", "value_floats"): + array = np.asarray(data, dtype=np.float32) + else: + raise AuditError(f"{node.name}: unsupported Constant attribute {key}") + return array.dtype, array.shape, array.nbytes, _array_hash(array), None + + +def _same_literal(left, right): + if left == right: + return left != 0 or math.copysign(1, left) == math.copysign(1, right) + return (isinstance(left, float) and isinstance(right, float) + and math.isnan(left) and math.isnan(right)) + + +def _audit_scalar_load(instruction, data, dtype, context): + """Cover auxiliary LOAD_CONST results which have no ONNX value-map entry.""" + destination = instruction.dest + _require(instruction.opcode == OpCode.LOAD_CONST and destination is not None + and not instruction.operands and not instruction.target + and destination.dtype == dtype and destination.shape == () + and destination.is_constant is True + and _same_literal(destination.const_value, data.item()) + and set(instruction.attrs) == {"value"} + and _same_literal(instruction.attrs.get("value"), data.item()), + f"{context}: scalar instruction metadata mismatch") + + +def _expected_lowering(node, parser, read_control): + """Independent structural contract for the operations in the fixed graph.""" + attrs = _onnx_attributes(node) + names = [name for name in node.input if name] + args = [parser._value_map[name] for name in names] + op = node.op_type + ir_attrs = {} + operands = args + if op in {"Abs", "Add", "Cast", "Cos", "Div", "Exp", "Gather", "MatMul", + "Mul", "Neg", "Pow", "Reciprocal", "Relu", "Sigmoid", "Sin", + "Softmax", "Sqrt", "Sub", "Transpose", "Concat", "Reshape", + "Expand", "Unsqueeze", "Slice", "ReduceMean"}: + opcode = "reduce_mean" if op == "ReduceMean" else op.lower() + else: + raise AuditError(f"{node.name}: no independent lowering contract for {op}") + + def integers(index): + # Never ask the parser/IR/cache to certify its own shape evaluation. + # This reader resolves the original ONNX graph with bounded NumPy ops. + name = node.input[index] + array = read_control(name) + _require(array.ndim == 1 and array.dtype.kind in "iu", + f"{node.name}: {name} is not a constant integer vector") + return tuple(int(value) for value in array) + + if op == "Cast": + _require(set(attrs) == {"to"} and attrs["to"] in _TYPES, + f"{node.name}: unsupported Cast attributes") + # CAST encodes its target in dest.dtype, not an instruction attribute. + ir_attrs = {} + elif op in ("Gather", "Softmax"): + ir_attrs = {"axis": attrs.get("axis", 0 if op == "Gather" else -1)} + elif op == "MatMul": + a, b = args + if len(a.shape) == len(b.shape) == 2 and all(d > 0 for d in a.shape + b.shape): + ir_attrs = {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1]} + elif op in ("Reshape", "Expand"): + _require(not attrs.get("allowzero", 0), f"{node.name}: unsupported allowzero") + ir_attrs = {"shape": integers(1)} + operands = args[:1] + elif op == "Unsqueeze": + ir_attrs = {"axes": integers(1) if len(args) > 1 else tuple(attrs["axes"])} + operands = args[:1] + elif op == "Slice": + for index, key in enumerate(("starts", "ends", "axes", "steps"), 1): + if len(node.input) > index and node.input[index]: + ir_attrs[key] = integers(index) + elif key in attrs: + ir_attrs[key] = tuple(attrs[key]) + operands = args[:1] + elif op == "ReduceMean": + axes = integers(1) if len(args) > 1 else attrs.get("axes") + if not axes and attrs.get("noop_with_empty_axes", 0): + return None, args[:1], {} + ir_attrs = {"keepdims": bool(attrs.get("keepdims", 1))} + if axes is not None: + ir_attrs["axes"] = tuple(axes) + operands = args[:1] + elif op == "Transpose" and "perm" in attrs: + ir_attrs = {"perm": tuple(attrs["perm"])} + elif op == "Concat": + ir_attrs = {"axis": attrs["axis"]} + return opcode, operands, ir_attrs + + +def audit_parsed_model(model, parser, program, model_dir): + """Raise AuditError on any gap; otherwise return serializable evidence.""" + graph = model.graph + _require(len(graph.output) == 1, "fixed-graph audit requires exactly one graph output") + _require(program is parser.builder.program, "program is not the observed parser result") + _require(len(program.functions) == 1 and len(program.functions[0].blocks) == 1, + "fixed-graph audit requires one function and one basic block") + function = program.functions[0] + instructions = function.blocks[0].instructions + _require(len(parser.node_records) == len(graph.node) + and len(parser._node_instructions) == len(graph.node), + "translated node count does not cover the ONNX graph") + read_control = make_control_constant_reader(model) + inferred = onnx.shape_inference.infer_shapes(model, strict_mode=True) + metadata = {value.name: value.type.tensor_type for value in + [*inferred.graph.input, *inferred.graph.value_info, *inferred.graph.output]} + expected_names = {value.name for value in [*graph.input, *graph.initializer]} + initializer_names = {value.name for value in graph.initializer} + _require(len(initializer_names) == len(graph.initializer), "duplicate initializer names") + def progress(stage, **context): + parser.current_context = {"stage": stage, **context} + if parser.progress: + parser.progress(dict(parser.current_context)) + + bindings = [] + for value in graph.initializer: + progress("audit_source_tensor", tensor_name=value.name, source="initializer") + bindings.append((value.name, "initializer", _tensor_spec(value, model_dir))) + for node in graph.node: + _require(len(node.output) == 1 and bool(node.output[0]), + f"{node.name}: audit requires one nonempty node output") + _require(node.output[0] not in expected_names, f"{node.name}: duplicate ONNX definition") + expected_names.add(node.output[0]) + if node.op_type == "Constant": + progress("audit_source_tensor", tensor_name=node.output[0], source="Constant") + bindings.append((node.output[0], "Constant", _constant_spec(node, model_dir))) + _require(set(parser._value_map) == expected_names, + "ONNX -> IR value mapping has missing or unexpected names") + _require(graph.output[0].name in expected_names, + "graph output does not have an ONNX definition") + for name in expected_names - initializer_names: + info = metadata.get(name) + _require(info is not None and info.HasField("shape") + and all(dim.HasField("dim_value") for dim in info.shape.dim), + f"{name}: missing static ONNX shape") + _require(info.elem_type in _TYPES, f"{name}: unsupported ONNX dtype") + value = parser._value_map[name] + _require(value.shape == tuple(dim.dim_value for dim in info.shape.dim) + and value.dtype == _TYPES[info.elem_type], f"{name}: IR shape/dtype mismatch") + + globals_by_name = {value.name: value for value in program.global_values} + binding_names = {name for name, _, _ in bindings} + _require(len(globals_by_name) == len(program.global_values) + and set(globals_by_name) == binding_names, + "IR global definitions are missing, duplicated, or unexpected") + _require(set(parser.initializers) == binding_names, + "initializer bindings are missing or unexpected") + tensor_rows = [] + for name, source, (dtype, shape, size, expected_sha, external) in bindings: + progress("audit_bound_tensor", tensor_name=name, source=source) + array = parser.initializers[name] + _require(isinstance(array, np.ndarray), f"{name}: binding must be a NumPy array") + value = globals_by_name[name] + _require(value is parser._value_map[name], f"{name}: global/value mapping identity mismatch") + _require(array.dtype == dtype and array.shape == shape and value.shape == shape + and value.dtype == _TYPES[onnx.helper.np_dtype_to_tensor_dtype(dtype)], + f"{name}: tensor binding shape/dtype mismatch") + _require(value.is_constant == (len(shape) == 0), f"{name}: scalar metadata mismatch") + if len(shape) == 0: + _require(_same_literal(value.const_value, array.item()), f"{name}: scalar value mismatch") + actual_sha = _array_hash(array) + _require(actual_sha == expected_sha, f"{name}: bound tensor content SHA256 mismatch") + tensor_rows.append({"name": name, "source": source, "shape": list(shape), + "dtype": str(dtype), "bytes": size, "sha256": actual_sha, + "external": external}) + expected_params = [parser._value_map[value.name] for value in graph.input + if value.name not in initializer_names] + _require(len(function.params) == len(expected_params) + and all(a is b for a, b in zip(function.params, expected_params)), + "function parameters do not match ONNX inputs") + expected_param_names = [value.name for value in graph.input if value.name not in initializer_names] + _require([value.name for value in function.params] == expected_param_names, + "function parameter names do not match ONNX input bindings") + _require(not any(value.is_constant for value in function.params), + "function parameters must not be constants") + + scalar_initializers = [value for value in graph.initializer if len(value.dims) == 0] + cursor = len(scalar_initializers) + for inst, tensor in zip(instructions[:cursor], scalar_initializers): + data = parser.initializers[tensor.name] + _audit_scalar_load(inst, data, parser._value_map[tensor.name].dtype, + f"{tensor.name}: scalar initializer") + alias_rows = [] + for index, (node, record, observed) in enumerate(zip( + graph.node, parser.node_records, parser._node_instructions)): + progress("audit_node", node_index=index, node_name=node.name, op_type=node.op_type) + context = f"node #{index} {node.name} ({node.op_type})" + _require(record.get("status") == "translated" and record.get("node_index") == index + and record.get("node_name") == node.name and record.get("op_type") == node.op_type + and record.get("inputs") == list(node.input) and record.get("outputs") == list(node.output), + f"{context}: node observation mismatch") + end = record.get("ir_end") + _require(record.get("ir_start") == cursor and isinstance(end, int) + and end == cursor + len(observed) + and all(a is b for a, b in zip(instructions[cursor:end], observed)) + and len(instructions[cursor:end]) == len(observed), + f"{context}: emitted IR range changed or has a gap") + cursor = end + output = parser._value_map[node.output[0]] + if node.op_type == "Constant": + _require(not node.input, f"{context}: Constant has inputs") + array = parser.initializers[node.output[0]] + _require(len(observed) == (1 if array.ndim == 0 else 0), + f"{context}: Constant emitted unexpected instructions") + if observed: + _audit_scalar_load(observed[0], array, output.dtype, f"{context}: Constant") + continue + if node.op_type == "Identity": + _require(len(node.input) == 1, f"{context}: invalid Identity inputs") + opcode, operands, ir_attrs = None, [parser._value_map[node.input[0]]], {} + else: + opcode, operands, ir_attrs = _expected_lowering(node, parser, read_control) + if opcode is None: + _require(not observed and output is operands[0], f"{context}: incorrect alias binding") + alias_rows.append({"onnx_name": node.output[0], "input": node.input[0], + "ir_name": output.name}) + else: + _require(len(observed) == 1, f"{context}: node did not emit exactly one instruction") + inst = observed[0] + _require(inst.opcode.value == opcode and inst.dest is output, + f"{context}: wrong IR opcode or output binding") + _require(len(inst.operands) == len(operands) + and all(a is b for a, b in zip(inst.operands, operands)), + f"{context}: wrong IR dependency bindings") + _require(inst.attrs == ir_attrs, f"{context}: IR attributes mismatch") + _require(len(instructions) == cursor + 1, "unexpected instructions outside node ranges and RETURN") + returns = [inst for inst in instructions if inst.opcode == OpCode.RETURN] + output_name = graph.output[0].name + _require(len(returns) == 1 and instructions[-1] is returns[0] + and len(returns[0].operands) == 1 + and returns[0].operands[0] is parser._value_map[output_name], + "actual RETURN does not bind the unique ONNX graph output") + progress("audit_complete", node_count=len(graph.node), binding_count=len(bindings)) + return {"passed": True, "node_count": len(graph.node), "value_count": len(expected_names), + "ir_instruction_count": len(instructions), "global_count": len(globals_by_name), + "initializer_count": len(graph.initializer), + "constant_count": len(bindings) - len(graph.initializer), + "binding_count": len(bindings), "binding_bytes": sum(row["bytes"] for row in tensor_rows), + "tensor_bindings": tensor_rows, "alias_count": len(alias_rows), "aliases": alias_rows, + "return": {"onnx_name": output_name, **_value_record(parser._value_map[output_name])}, + "node_records": parser.node_records} diff --git a/probes/w2_qwen3_parse/run.py b/probes/w2_qwen3_parse/run.py new file mode 100644 index 0000000..1cf2366 --- /dev/null +++ b/probes/w2_qwen3_parse/run.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Audit complete, pinned Qwen3 ONNX -> ScratchV IR parsing and weight binding. + +This gate does not execute the full IR, generate RISC-V code, or run QEMU. +Large-model work runs in a bounded child process; failure preserves reports. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +from html import escape +import json +import math +from pathlib import Path +import subprocess +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w1_qwen3_export import run as artifacts + +GATE = "frontend:parse-qwen3" +SCOPE = ("Complete pinned 28-layer Qwen3 ONNX parsing, graph connectivity, IR verification " + "and actual weight/constant bindings. No full-model IR numerical comparison, " + "code generation or QEMU execution.") + + +def write_json(path, value): + path = Path(path) + # Readers must not observe a half-written progress or result file. + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2, allow_nan=False) + "\n", + encoding="utf-8") + temporary.replace(path) + + +def source_fingerprints(): + files = [*sorted(Path(__file__).parent.glob("*.py")), + ROOT / "probes/w1_qwen3_export/run.py", ROOT / "probes/w1_qwen3_export/manifest.json", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", + ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/runtime/riscv_tensor.py", + *sorted((ROOT / "scratchv/analysis").glob("*.py"))] + return {p.relative_to(ROOT).as_posix(): artifacts.sha256(p) for p in files} + + +def checkout_evidence(): + result = {"head": None, "status": "unknown", "clean": None} + if not (ROOT / ".git").exists(): + # Worktrees have a .git file; source archives have neither a file nor + # a directory here and must not inherit another checkout's identity. + result["reason"] = "Source directory has no .git metadata" + return result + try: + for key, arguments in (("head", ["rev-parse", "HEAD"]), + ("status", ["status", "--porcelain=v1", "--untracked-files=all"])): + process = subprocess.run(["git", *arguments], cwd=ROOT, capture_output=True, + text=True, encoding="utf-8", errors="replace", timeout=10, + check=True) + result[key] = process.stdout.rstrip("\r\n") + result["clean"] = not bool(result["status"]) + except (OSError, subprocess.SubprocessError) as exc: + result.update(head=None, status="unknown", clean=None, + error=f"{type(exc).__name__}: {exc}", + reason="Checkout metadata unavailable") + return result + + +def initial_report(mode): + return {"schema_version": 1, "gate": GATE, "passed": False, "stage": "initialization", + "created_at": datetime.now(timezone.utc).isoformat(), "mode": mode, "scope": SCOPE, + "model_id": artifacts.MANIFEST["model_id"], "revision": artifacts.MANIFEST["revision"], + "stages_seconds": {}} + + +def save_failure_evidence(path, value, report): + """A secondary evidence failure must not replace the active validation error.""" + try: + write_json(path, value) + except (OSError, TypeError, ValueError) as exc: + report.setdefault("evidence_errors", []).append( + f"{path.name}: {type(exc).__name__}: {exc}") + + +def run_validation(model_dir, out, report, *, mode="verify", manifest=None, progress=None): + """Run real parsing. Small test fixtures may explicitly supply a manifest. + + The CLI always uses the committed full-model manifest and architecture. + This function never replaces parsing with a handler inventory or ORT run. + """ + import onnx + from scratchv.analysis.ir_verifier import verify_ir + from probes.w2_qwen3_parse.audit import AuditedONNXParser, audit_parsed_model + from probes.w2_qwen3_parse.validation import audit_graph_structure + + manifest = artifacts.MANIFEST if manifest is None else manifest + model_dir, out = Path(model_dir).resolve(), Path(out).resolve() + report["model_dir"] = str(model_dir) + + def stage(name, operation): + report["stage"] = name + if progress: + progress({"stage": name}) + started = time.perf_counter() + try: + return operation() + finally: + report["stages_seconds"][name] = time.perf_counter() - started + + if mode == "download": + report["acquisition"] = stage("acquire", lambda: artifacts.acquire_model(model_dir, manifest=manifest)) + report["hashes"] = stage("hashes", lambda: artifacts.verify_files(model_dir, manifest)) + report["onnx_structure"] = stage("onnx-structure", lambda: artifacts.inspect_model(model_dir, report["hashes"])) + model = stage("graph-load", lambda: onnx.load(str(model_dir / "model.onnx"), load_external_data=False)) + report["graph"] = stage("graph-connectivity", lambda: audit_graph_structure(model)) + if report["graph"].get("passed") is not True: + raise ValueError("Complete Qwen3 graph connectivity did not pass") + parser = AuditedONNXParser(progress=progress) + try: + program = stage("scratchv-parse", lambda: parser.parse(str(model_dir / "model.onnx"))) + except Exception: + report["failure_context"] = dict(parser.current_context) + report["nodes_completed"] = sum(row.get("status") == "translated" for row in parser.node_records) + save_failure_evidence(out / "nodes.partial.json", parser.node_records, report) + raise + report["ir"] = {"functions": len(program.functions), "globals": len(program.global_values), + "blocks": sum(len(f.blocks) for f in program.functions), + "instructions": sum(len(b.instructions) for f in program.functions for b in f.blocks), + "operators": dict(sorted(Counter(i.opcode.value for f in program.functions + for b in f.blocks for i in b.instructions).items()))} + (out / "ir.txt").write_text(program.dump(), encoding="utf-8") + write_json(out / "nodes.json", parser.node_records) + verified, issues = stage("ir-verifier", lambda: verify_ir(program, stage="full-qwen3-after-parse")) + report["ir_verifier"] = {"passed": verified, "issue_count": len(issues), + "issues": [str(issue) for issue in issues]} + if not verified: + raise ValueError("ScratchV IR verifier rejected the complete model; see ir_verifier.issues") + try: + audit = stage("ir-and-bindings", lambda: audit_parsed_model(model, parser, program, model_dir)) + except Exception: + report["failure_context"] = dict(parser.current_context) + save_failure_evidence(out / "nodes.json", parser.node_records, report) + raise + nodes = audit.pop("node_records", parser.node_records) + bindings = audit.pop("tensor_bindings") + write_json(out / "nodes.json", nodes) + write_json(out / "bindings.json", bindings) + report["audit"] = audit + if audit.get("passed") is not True: + raise ValueError("IR/weight binding audit did not pass") + report["evidence"] = {name: {"sha256": artifacts.sha256(out / name), + "bytes": (out / name).stat().st_size} + for name in ("nodes.json", "bindings.json", "ir.txt")} + report["passed"] = True + report["stage"] = "complete" + + +def worker_main(args): + report = initial_report(args.mode) + out = args.output_dir.resolve() + started = time.perf_counter() + last_checkpoint = -math.inf + + def checkpoint(context): + nonlocal last_checkpoint + now = time.perf_counter() + # Stage boundaries are always recorded; frequent node progress is sampled. + if set(context) == {"stage"} or now - last_checkpoint >= 0.5: + write_json(out / "worker-progress.json", {"context": context, + "elapsed_seconds": now - started, "peak_memory": artifacts.peak_memory()}) + last_checkpoint = now + + try: + report["stage"] = "environment" + checkpoint({"stage": "environment"}) + report["environment"] = artifacts.environment() + report["checkout"] = checkout_evidence() + report["source_sha256"] = source_fingerprints() + run_validation(args.model_dir, out, report, mode=args.mode, progress=checkpoint) + except Exception as exc: + report["passed"] = False + report["error"] = f"{type(exc).__name__}: {exc}" + traceback.print_exc() + report["validation_seconds"] = time.perf_counter() - started + report["peak_memory"] = artifacts.peak_memory() + report["peak_memory"]["scope"] = "Validation worker process, including loading and real ScratchV parsing; parent excluded" + write_json(out / "worker-report.json", report) + return 0 if report["passed"] else 1 + + +def worker_command(args): + return [sys.executable, "-B", "-X", "utf8", str(Path(__file__).resolve()), "--worker", + "--mode", args.mode, "--model-dir", str(args.model_dir.resolve()), + "--output-dir", str(args.output_dir.resolve())] + + +def launch_worker(args): + from scratchv.runtime.riscv_tensor import _run_process + return _run_process(worker_command(args), cwd=ROOT, timeout=args.timeout) + + +def number(value): + return "未测得" if value is None else f"{value:.3f}" + + +def save_worker_logs(out, report, stdout, stderr): + """Keep reporting the validation result even when a log file is locked.""" + for name, content in (("worker.stdout", stdout), ("worker.stderr", stderr)): + try: + (out / name).write_bytes(content or b"") + except OSError as exc: + report.setdefault("log_errors", []).append(f"{name}: {type(exc).__name__}: {exc}") + if report.get("log_errors") and report.get("passed"): + report.update(passed=False, stage="log-write", error="Could not preserve validation worker logs") + + +def _render_reports(report): + graph, audit, ir = report.get("graph", {}), report.get("audit", {}), report.get("ir", {}) + layers = graph.get("layers", []) + peak = report.get("peak_memory", {}) + lines = ["# 完整 Qwen3 ONNX → ScratchV IR 解析验收", "", + f"Result: **{'PASS' if report['passed'] else 'FAIL'}**；阶段：`{report['stage']}`", "", + "只验证完整解析、结构、IR 和实际权重绑定；不证明完整 IR 数值或 QEMU 前向通过。", "", + f"- 模型：`{report['model_id']}`，revision `{report['revision']}`。", + f"- 源码 HEAD：`{report.get('checkout', {}).get('head', 'unavailable')}`；" + f"工作树 clean：`{report.get('checkout', {}).get('clean', 'unavailable')}`。", + "- dirty 工作树的结果须结合 source_sha256 和本地 diff，不能只归属 HEAD。", + f"- 完整层链：{len(layers) if isinstance(layers, list) else layers} 层；" + f"ONNX 节点：{report.get('onnx_structure', {}).get('operators') and sum(report['onnx_structure']['operators'].values()) or '未完成'}。", + f"- IR：{ir.get('instructions', '未完成')} 条指令、{ir.get('globals', '未完成')} 个 globals；" + f"verifier：{report.get('ir_verifier', {}).get('passed', '未完成')}。", + f"- Supervisor 总耗时：{number(report.get('seconds'))} 秒；" + f"实际解析：{number(report.get('stages_seconds', {}).get('scratchv-parse'))} 秒。", + f"- 验证进程峰值 RSS：{peak.get('bytes', '未测得')} bytes；{peak.get('method', '')}。", "", + "| 阶段 | 耗时（秒） |", "|---|---:|"] + lines += [f"| {name} | {number(seconds)} |" for name, seconds in report.get("stages_seconds", {}).items()] + if report.get("error"): + lines += ["", "## 失败信息", "", report["error"], "", + "```json", json.dumps(report.get("failure_context", report.get("last_progress", {})), + ensure_ascii=False, indent=2), "```"] + if report.get("log_errors"): + lines += ["", "子进程日志落盘失败：", *report["log_errors"]] + if report.get("evidence_errors"): + lines += ["", "失败明细保存异常（原始验证错误保留）：", *report["evidence_errors"]] + if report.get("report_write_errors"): + lines += ["", "报告保存异常（原始验证错误保留）：", *report["report_write_errors"]] + concise_audit = {key: value for key, value in audit.items() if key != "aliases"} + lines += ["", "## 审计摘要", "", "```json", json.dumps(concise_audit, ensure_ascii=False, indent=2), "```", "", + "`nodes.json` 保存节点翻译记录；`bindings.json` 保存逐张量绑定和内容哈希；" + "`ir.txt` 保存 IR 文本，均不包含原始权重数组。", "", + "缺文件、解析错误、审计失败、超时或进程异常均失败；未执行的检查不能视为通过。" + "耗时和内存为观察数据，没有新增性能通过阈值。"] + markdown = "\n".join(lines) + "\n" + details = json.dumps({"graph": graph, "audit": audit, "ir_verifier": report.get("ir_verifier")}, + ensure_ascii=False, indent=2) + html = ( + "Qwen3 完整解析验收" + "" + f"

frontend:parse-qwen3 — {'PASS' if report['passed'] else 'FAIL'}

" + f"
{escape(markdown)}
结构与绑定详情" + f"
{escape(details)}
") + return {"report.md": markdown, "report.html": html} + + +def write_reports(out, report): + """Publish JSON last, so incomplete evidence cannot leave a stale PASS.""" + failures = [] + + def record_failure(name, exc): + failures.append(f"{name}: {type(exc).__name__}: {exc}") + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", + error="Could not preserve complete validation reports") + + def write_views(*, record_errors): + for name, content in _render_reports(report).items(): + try: + (out / name).write_text(content, encoding="utf-8") + except OSError as exc: + if record_errors: + record_failure(name, exc) + + write_views(record_errors=True) + if failures: + # A view written before another failed may still contain PASS. Refresh + # every writable view, preserving the original validation failure. + write_views(record_errors=False) + try: + write_json(out / "report.json", report) + except OSError as exc: + record_failure("report.json", exc) + write_views(record_errors=False) + for error in failures: + print(error, file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--mode", choices=("verify", "download"), default="verify") + cli.add_argument("--model-dir", type=Path, required=True) + cli.add_argument("--output-dir", type=Path, default=ROOT / "output/qwen3-parse") + cli.add_argument("--timeout", type=float, default=300) + cli.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + args = cli.parse_args(argv) + if not math.isfinite(args.timeout) or args.timeout <= 0: + cli.error("--timeout must be finite and positive") + if args.worker: + return worker_main(args) + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + cli.error("--output-dir must be absent or empty; preserve previous evidence") + out.mkdir(parents=True, exist_ok=True) + report = initial_report(args.mode) + report.update(stage="worker-launch", timeout_seconds=args.timeout) + started = time.perf_counter() + result = None + try: + # Keep the attempted run identifiable even if the child never starts or + # exits before it can validate dependencies and write its own report. + report["attempt"] = {"command": worker_command(args), "cwd": str(ROOT), + "model_dir": str(args.model_dir.resolve()), + "python": sys.version, "checkout": checkout_evidence(), + "source_sha256": source_fingerprints(), + "scope": "Supervisor metadata; not proof of worker initialization or validation"} + result = launch_worker(args) + report["worker_returncode"] = result.returncode + worker = json.loads((out / "worker-report.json").read_text(encoding="utf-8")) + if worker.get("gate") != GATE or worker.get("schema_version") != 1: + raise ValueError("Invalid validation worker report") + report.update(worker) + if result.returncode or report.get("passed") is not True or report.get("stage") != "complete": + report["passed"] = False + report.setdefault("error", f"Validation worker did not pass (exit {result.returncode})") + elif any(not (out / name).is_file() for name in ("nodes.json", "bindings.json", "ir.txt")): + raise ValueError("Validation worker omitted required evidence files") + except subprocess.TimeoutExpired as exc: + report.update(passed=False, stage="timeout", error=f"Validation worker exceeded {args.timeout} seconds") + save_worker_logs(out, report, exc.stdout, exc.stderr) + except Exception as exc: + report["passed"] = False + report["error"] = f"{type(exc).__name__}: {exc}" + finally: + if result is not None: + save_worker_logs(out, report, result.stdout, result.stderr) + report["seconds"] = time.perf_counter() - started + if not report["passed"] and (out / "worker-progress.json").is_file(): + try: + report["last_progress"] = json.loads((out / "worker-progress.json").read_text(encoding="utf-8")) + except (OSError, ValueError): + pass + write_reports(out, report) + print(f"[{GATE}] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; " + f"seconds={report['seconds']:.3f}; report={out / 'report.json'}", flush=True) + if report.get("error"): + print(report["error"], file=sys.stderr) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_qwen3_parse/validation.py b/probes/w2_qwen3_parse/validation.py new file mode 100644 index 0000000..dfc3dfe --- /dev/null +++ b/probes/w2_qwen3_parse/validation.py @@ -0,0 +1,427 @@ +"""Structural acceptance of the pinned, decomposed Qwen3 ONNX graph. + +This deliberately recognizes the exported architecture, not arbitrary graphs +whose node names contain ``layers.N``. It follows actual operands from the +weights through both residuals in every decoder layer to the returned logits. +No external tensor data is read and no full-model numerical claim is made. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +import math + +import numpy as np +import onnx + + +QWEN3_06B_CONFIG = { + "num_hidden_layers": 28, + "hidden_size": 1024, + "intermediate_size": 3072, + "num_attention_heads": 16, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 151936, + "sequence_length": 256, + "rms_norm_eps": 1e-6, +} + + +class Qwen3StructureError(ValueError): + """A missing or incorrectly connected part of the Qwen3 architecture.""" + + +def _require(condition, message): + if not condition: + raise Qwen3StructureError(message) + + +def _attrs(node): + return {a.name: onnx.helper.get_attribute_value(a) for a in node.attribute} + + +def _one(nodes, context): + _require(len(nodes) == 1, f"{context}: expected exactly one node, found {len(nodes)}") + return nodes[0] + + +class _Graph: + def __init__(self, graph): + self.graph = graph + self.initializers = {t.name: t for t in graph.initializer} + self.nodes = list(graph.node) + self.producers = {} + self.consumers = defaultdict(list) + self.by_op = defaultdict(list) + self.bits = {} + self.ancestors = {} + self.constants = {} + self.indices = {} + roots = list(dict.fromkeys([*(v.name for v in graph.input), *self.initializers])) + for index, name in enumerate(roots): + self.bits[name] = self.ancestors[name] = 1 << index + for index, node in enumerate(self.nodes): + label = node.name or f"node[{index}]" + self.indices[id(node)] = index + _require(node.domain in ("", "ai.onnx"), f"{label}: unsupported domain {node.domain!r}") + _require(len(node.output) == 1 and node.output[0], + f"{label}: this pinned graph requires one nonempty output per node") + output = node.output[0] + _require(output not in self.ancestors, f"{label}: duplicate definition {output!r}") + bit = 1 << (len(roots) + index) + inherited = bit + for value in node.input: + if not value: + continue + _require(value in self.ancestors, + f"{label}: input {value!r} has no preceding definition") + inherited |= self.ancestors[value] + self.consumers[value].append(node) + self.bits[output] = bit + self.ancestors[output] = inherited + self.producers[output] = node + self.by_op[node.op_type].append(node) + + def depends(self, output, value): + return bool(self.ancestors.get(output, 0) & self.bits.get(value, 0)) + + def strip(self, value, extra=()): + while value in self.producers: + node = self.producers[value] + if node.op_type not in ("Identity", "Cast", *extra): + break + value = node.input[0] + return value + + def producer(self, value, op, context): + node = self.producers.get(self.strip(value)) + _require(node is not None and node.op_type == op, + f"{context}: expected {op} producer for {value!r}") + return node + + def same(self, left, right, extra=()): + return self.strip(left, extra) == self.strip(right, extra) + + def constant(self, value): + """Resolve only tiny control constants; never open external weights.""" + if value in self.constants: + return self.constants[value] + if value in self.initializers: + tensor = self.initializers[value] + _require(not tensor.external_data and math.prod(tensor.dims) <= 1024, + f"constant {value!r}: external or large tensor is not a control constant") + result = onnx.numpy_helper.to_array(tensor) + else: + node = self.producers.get(value) + _require(node is not None, f"constant {value!r}: missing definition") + attrs = _attrs(node) + op = node.op_type + if op == "Constant": + if "value" in attrs: + tensor = attrs["value"] + _require(not tensor.external_data and math.prod(tensor.dims) <= 1024, + f"constant {value!r}: external or large tensor") + result = onnx.numpy_helper.to_array(tensor) + elif "value_int" in attrs: + result = np.asarray(attrs["value_int"], dtype=np.int64) + elif "value_ints" in attrs: + result = np.asarray(attrs["value_ints"], dtype=np.int64) + elif "value_float" in attrs: + result = np.asarray(attrs["value_float"], dtype=np.float32) + elif "value_floats" in attrs: + result = np.asarray(attrs["value_floats"], dtype=np.float32) + else: + raise Qwen3StructureError(f"constant {value!r}: unsupported Constant encoding") + elif op in ("Identity", "Cast", "Abs"): + result = self.constant(node.input[0]) + if op == "Cast": + result = result.astype(onnx.helper.tensor_dtype_to_np_dtype(attrs["to"])) + elif op == "Abs": + result = np.abs(result) + elif op == "Reshape": + data = self.constant(node.input[0]) + shape = self.constant(node.input[1]).reshape(-1).tolist() + shape = [data.shape[i] if d == 0 else d for i, d in enumerate(shape)] + result = data.reshape(shape) + elif op == "Concat": + result = np.concatenate([self.constant(v) for v in node.input], axis=attrs["axis"]) + elif op == "Unsqueeze": + axes = self.constant(node.input[1]).reshape(-1).tolist() + result = np.expand_dims(self.constant(node.input[0]), tuple(axes)) + else: + raise Qwen3StructureError(f"constant {value!r}: unexpected {op} dependency") + _require(result.size <= 1024, f"constant {value!r}: exceeds control-constant size limit") + self.constants[value] = result + return result + + def scalar(self, value, expected, context): + data = self.constant(value) + _require(data.size == 1 and np.isfinite(data).all() + and math.isclose(float(data.reshape(-1)[0]), expected, rel_tol=1e-6, abs_tol=0), + f"{context}: {value!r} must be scalar {expected}") + + def projection(self, weight, context): + candidates = [] + for node in self.by_op["MatMul"]: + if len(node.input) != 2: + continue + transposed = self.producers.get(self.strip(node.input[1])) + if (transposed is not None and transposed.op_type == "Transpose" + and self.same(transposed.input[0], weight)): + _require(tuple(_attrs(transposed).get("perm", (1, 0))) == (1, 0), + f"{context}: weight transpose must use perm=[1, 0]") + candidates.append(node) + return _one(candidates, f"{context}, weight {weight!r}") + + def norm(self, weight, epsilon, context): + node = _one([n for n in self.by_op["Mul"] + if len(n.input) == 2 and any(self.same(v, weight) for v in n.input)], + f"{context}, weight {weight!r}") + normalized = _one([v for v in node.input if not self.same(v, weight)], + f"{context}: normalized activation") + multiply = self.producer(normalized, "Mul", context) + reciprocals = [v for v in multiply.input + if self.producers.get(self.strip(v)) is not None + and self.producers[self.strip(v)].op_type == "Reciprocal"] + reciprocal_value = _one(reciprocals, f"{context}: RMS inverse square root") + reciprocal = self.producer(reciprocal_value, "Reciprocal", context) + raw = next(v for v in multiply.input if v != reciprocal_value) + sqrt = self.producer(reciprocal.input[0], "Sqrt", context) + add = self.producer(sqrt.input[0], "Add", context) + means = [v for v in add.input if self.producers.get(self.strip(v)) is not None + and self.producers[self.strip(v)].op_type == "ReduceMean"] + mean_value = _one(means, f"{context}: RMS mean") + self.scalar(next(v for v in add.input if v != mean_value), epsilon, context) + mean = self.producer(mean_value, "ReduceMean", context) + attrs = _attrs(mean) + axes = (self.constant(mean.input[1]).reshape(-1).tolist() + if len(mean.input) == 2 else list(attrs.get("axes", []))) + _require(axes == [-1] and attrs.get("keepdims", 1) == 1, + f"{context}: RMS ReduceMean must use axes=[-1], keepdims=1") + power = self.producer(mean.input[0], "Pow", context) + _require(self.same(power.input[0], raw), f"{context}: RMS numerator/denominator input mismatch") + self.scalar(power.input[1], 2.0, context) + return node, self.strip(raw) + + def residual(self, branch, bypass, context): + candidates = [n for n in self.by_op["Add"] if len(n.input) == 2 + and ((self.same(n.input[0], branch) and self.same(n.input[1], bypass)) + or (self.same(n.input[1], branch) and self.same(n.input[0], bypass)))] + return _one(candidates, context) + + +def make_control_constant_reader(model): + """Return ``reader(onnx_value_name) -> ndarray`` for small control values. + + Resolution uses the original ONNX graph, never the parser, its cached + constants, or the emitted IR. Every intermediate is limited to 1,024 + elements and external control tensors are rejected. This is the bounded + constant subset needed by the pinned export's axes, slices and shapes, + not a general ONNX interpreter or a way to load model weights. + """ + return _Graph(model.graph).constant + + +def _metadata(graph, config): + hidden, head = config["hidden_size"], config["head_dim"] + query, kv = config["num_attention_heads"] * head, config["num_key_value_heads"] * head + middle = config["intermediate_size"] + per_layer = { + "self_attn.q_proj.weight": (query, hidden), + "self_attn.k_proj.weight": (kv, hidden), + "self_attn.v_proj.weight": (kv, hidden), + "self_attn.o_proj.weight": (hidden, query), + "self_attn.q_norm.weight": (head,), + "self_attn.k_norm.weight": (head,), + "mlp.gate_proj.weight": (middle, hidden), + "mlp.up_proj.weight": (middle, hidden), + "mlp.down_proj.weight": (hidden, middle), + "input_layernorm.weight": (hidden,), + "post_attention_layernorm.weight": (hidden,), + } + expected = {f"model.model.layers.{i}.{name}": (shape, onnx.TensorProto.FLOAT) + for i in range(config["num_hidden_layers"]) for name, shape in per_layer.items()} + expected.update({ + "model.model.norm.weight": ((hidden,), onnx.TensorProto.FLOAT), + "model.lm_head.weight": ((config["vocab_size"], hidden), onnx.TensorProto.FLOAT), + "positions": ((1, config["sequence_length"]), onnx.TensorProto.INT64), + "model.model.rotary_emb.inv_freq": ((head // 2,), onnx.TensorProto.FLOAT), + }) + actual = {t.name: t for t in graph.initializer} + _require(len(actual) == len(graph.initializer), "initializers: duplicate weight names") + _require(set(actual) == set(expected), + f"initializers: missing={sorted(set(expected) - set(actual))}, " + f"unexpected={sorted(set(actual) - set(expected))}") + for name, (shape, dtype) in expected.items(): + tensor = actual[name] + _require(tuple(tensor.dims) == shape and tensor.data_type == dtype, + f"weight {name!r}: expected shape={shape}, dtype={dtype}; " + f"got shape={tuple(tensor.dims)}, dtype={tensor.data_type}") + length = config["sequence_length"] + signature = {"input_ids": ((1, length), onnx.TensorProto.INT64), + "attention_mask": ((1, 1, length, length), onnx.TensorProto.FLOAT), + "logits": ((1, length, config["vocab_size"]), onnx.TensorProto.FLOAT)} + _require([v.name for v in graph.input] == ["input_ids", "attention_mask"], + "graph inputs must be input_ids and attention_mask, in that order") + _require([v.name for v in graph.output] == ["logits"], "graph output must be logits") + for value in [*graph.input, *graph.output]: + tensor = value.type.tensor_type + shape, dtype = signature[value.name] + _require(all(d.HasField("dim_value") for d in tensor.shape.dim) + and tuple(d.dim_value for d in tensor.shape.dim) == shape + and tensor.elem_type == dtype, f"{value.name}: incorrect static shape or dtype") + + +def _attention(index, q, k, v, out, q_norm, k_norm, config, context): + qv, kv = q_norm.output[0], k_norm.output[0] + softmax = _one([n for n in index.by_op["Softmax"] + if index.depends(out.input[0], n.output[0]) + and index.depends(n.input[0], qv) and index.depends(n.input[0], kv)], + f"{context}: attention Softmax") + _require(_attrs(softmax).get("axis", -1) == -1, + f"{context}: attention Softmax must use axis=-1") + scores = _one([n for n in index.by_op["MatMul"] + if index.depends(softmax.input[0], n.output[0]) + and index.depends(n.input[0], qv) and index.depends(n.input[1], kv)], + f"{context}: QK attention MatMul") + _require(not index.depends(scores.input[0], kv) and not index.depends(scores.input[1], qv), + f"{context}: Q and K score operands must retain separate paths") + weighted = _one([n for n in index.by_op["MatMul"] + if index.depends(out.input[0], n.output[0]) + and index.same(n.input[0], softmax.output[0]) + and index.depends(n.input[1], v.output[0])], + f"{context}: attention probabilities times V") + mask_add = index.producer(softmax.input[0], "Add", context) + masks = [value for value in mask_add.input + if index.depends(value, "attention_mask") and not index.depends(value, scores.output[0])] + mask = _one(masks, f"{context}: additive attention mask") + scale = index.producer(next(value for value in mask_add.input if value != mask), "Mul", context) + scale_inputs = [value for value in scale.input if not index.same(value, scores.output[0])] + _require(len(scale_inputs) == 1, f"{context}: scores must be multiplied by a scalar scale") + index.scalar(scale_inputs[0], config["head_dim"] ** -0.5, f"{context}: attention scale") + for op in ("Cos", "Sin"): + rope = _one([n for n in index.by_op[op] + if index.depends(scores.input[0], n.output[0]) + and index.depends(scores.input[1], n.output[0])], f"{context}: shared RoPE {op}") + _require(index.depends(rope.input[0], "positions") + and index.depends(rope.input[0], "model.model.rotary_emb.inv_freq"), + f"{context}: RoPE {op} must use positions and inv_freq") + if config["num_attention_heads"] != config["num_key_value_heads"]: + expected = (1, config["num_key_value_heads"], + config["num_attention_heads"] // config["num_key_value_heads"], + config["sequence_length"], config["head_dim"]) + key_transpose = index.producer(scores.input[1], "Transpose", f"{context}: K score transpose") + _require(tuple(_attrs(key_transpose).get("perm", ())) == (0, 1, 3, 2), + f"{context}: K score transpose must use perm=[0, 1, 3, 2]") + for value, source, label in ((key_transpose.input[0], kv, "K"), + (weighted.input[1], v.output[0], "V")): + reshape = index.producer(value, "Reshape", f"{context}: GQA {label} flattened heads") + shape = tuple(index.constant(reshape.input[1]).reshape(-1).tolist()) + target = (1, config["num_attention_heads"], config["sequence_length"], config["head_dim"]) + _require(shape == target, f"{context}: GQA {label} head reshape {shape} != {target}") + expand = index.producer(reshape.input[0], "Expand", f"{context}: GQA {label} expansion") + _require(index.depends(expand.input[0], source), + f"{context}: GQA {label} expansion does not consume its projection") + shape = tuple(index.constant(expand.input[1]).reshape(-1).tolist()) + _require(shape == expected, f"{context}: GQA {label} expansion shape {shape} != {expected}") + source_value = index.strip(expand.input[0], ("Slice",)) + unsqueeze = index.producer(source_value, "Unsqueeze", f"{context}: GQA {label} group axis") + axes = tuple(index.constant(unsqueeze.input[1]).reshape(-1).tolist()) + _require(axes == (2,), f"{context}: GQA {label} repeat group must be axis 2") + return {"scores": scores.name, "softmax": softmax.name, "weighted_values": weighted.name} + + +def audit_graph_structure(model, config=None): + """Check the complete decoder dataflow without loading external weights. + + ``config`` overrides architecture dimensions for small test fixtures. This + gate recognizes the pinned export's decomposed RMSNorm/RoPE/GQA/SwiGLU; + a different exporter representation needs a reviewed new pattern. + """ + config = dict(QWEN3_06B_CONFIG) | dict(config or {}) + for name, value in config.items(): + if name == "rms_norm_eps": + _require(isinstance(value, (float, int)) and not isinstance(value, bool) + and math.isfinite(value) and value > 0, f"config: invalid {name}") + else: + _require(isinstance(value, int) and not isinstance(value, bool) and value > 0, + f"config: invalid {name}") + _require(config["head_dim"] % 2 == 0, "config: head_dim must be even for RoPE") + _require(config["num_attention_heads"] % config["num_key_value_heads"] == 0, + "config: attention heads must be divisible by KV heads") + _metadata(model.graph, config) + index = _Graph(model.graph) + shared_weight = "model.lm_head.weight" + embedding = _one([n for n in index.by_op["Gather"] + if list(n.input) == [shared_weight, "input_ids"]], + "embedding: tied LM Head weight and input_ids Gather") + _require(_attrs(embedding).get("axis", 0) == 0, "embedding: Gather axis must be zero") + previous = embedding.output[0] + rows = [] + for layer in range(config["num_hidden_layers"]): + context = f"layer {layer}" + prefix = f"model.model.layers.{layer}." + projections = {name: index.projection(prefix + name + ".weight", f"{context} {name}") + for name in ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj", + "self_attn.o_proj", "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj")} + norms = {name: index.norm(prefix + name + ".weight", config["rms_norm_eps"], f"{context} {name}") + for name in ("input_layernorm", "post_attention_layernorm", "self_attn.q_norm", "self_attn.k_norm")} + input_norm, raw = norms["input_layernorm"] + _require(index.same(raw, previous), f"{context}: input norm must consume previous layer residual") + q, k, v, out = (projections[f"self_attn.{name}_proj"] for name in ("q", "k", "v", "o")) + _require(all(index.same(n.input[0], input_norm.output[0]) for n in (q, k, v)), + f"{context}: Q/K/V projections must share this layer's input norm") + q_norm, q_raw = norms["self_attn.q_norm"] + k_norm, k_raw = norms["self_attn.k_norm"] + _require(index.same(q_raw, q.output[0], ("Reshape",)) + and index.same(k_raw, k.output[0], ("Reshape",)), + f"{context}: Q/K norm must consume the matching projection") + attention = _attention(index, q, k, v, out, q_norm, k_norm, config, context) + first_residual = index.residual(out.output[0], previous, f"{context}: attention residual") + post_norm, post_raw = norms["post_attention_layernorm"] + _require(index.same(post_raw, first_residual.output[0]), + f"{context}: post-attention norm must consume attention residual") + gate, up, down = (projections[f"mlp.{name}_proj"] for name in ("gate", "up", "down")) + _require(all(index.same(n.input[0], post_norm.output[0]) for n in (gate, up)), + f"{context}: gate/up projections must share post-attention norm") + product = index.producer(down.input[0], "Mul", f"{context}: SwiGLU") + activation_values = [value for value in product.input if not index.same(value, up.output[0])] + _require(len(activation_values) == 1, f"{context}: SwiGLU must multiply by up projection") + activation = index.producer(activation_values[0], "Mul", f"{context}: SiLU") + sigmoid_values = [value for value in activation.input if not index.same(value, gate.output[0])] + _require(len(sigmoid_values) == 1, f"{context}: SiLU must multiply gate by sigmoid(gate)") + sigmoid = index.producer(sigmoid_values[0], "Sigmoid", f"{context}: SiLU") + _require(index.same(sigmoid.input[0], gate.output[0]), f"{context}: SiLU sigmoid input must be gate") + second_residual = index.residual(down.output[0], first_residual.output[0], + f"{context}: MLP residual") + previous = second_residual.output[0] + rows.append({"layer": layer, "passed": True, "input": raw, "output": previous, + "projections": {name: node.name for name, node in projections.items()}, + "norms": {name: node.name for name, (node, _) in norms.items()}, + "attention": attention, + "residuals": [first_residual.name, second_residual.name]}) + final_norm, raw = index.norm("model.model.norm.weight", config["rms_norm_eps"], "final norm") + _require(index.same(raw, previous), "final norm must consume the last layer residual") + head = index.projection(shared_weight, "LM Head") + _require(index.same(head.input[0], final_norm.output[0], ("Slice",)), + "LM Head must consume final norm (allowing the exporter's logits slices)") + _require(index.same("logits", head.output[0]), "logits must return the tied LM Head projection") + live = [node for node in index.nodes if index.depends("logits", node.output[0])] + dead = [node.name or node.output[0] for node in index.nodes if not index.depends("logits", node.output[0])] + _require(not dead, f"graph has nodes disconnected from logits: {dead[:8]}") + for name in index.initializers: + _require(index.depends("logits", name), f"initializer {name!r} is disconnected from logits") + return {"passed": True, "config": config, + "graph": {"node_count": len(index.nodes), "initializer_count": len(index.initializers), + "operators": dict(sorted(Counter(n.op_type for n in index.nodes).items())), + "inputs": [v.name for v in model.graph.input], "outputs": ["logits"]}, + "layers": rows, "layer_count": len(rows), "live_node_count": len(live), + "embedding": embedding.name, "final_norm": final_norm.name, "lm_head": head.name, + "tied_embeddings": True, + "checks": ["exact weight inventory, shapes and dtypes", "static input/output signature", + "RMSNorm paths and constants", "Q/K/V and GQA attention dataflow", + "shared RoPE position/frequency dependencies", "SwiGLU dataflow", + "both residuals and ordered layer chain", "final norm and tied LM Head", + "every node and initializer reaches logits"], + "scope": "Structural acceptance of the pinned export; no numerical execution or external weight reads."} diff --git a/probes/w2_qwen3_small/README.md b/probes/w2_qwen3_small/README.md index 44779a7..66ccc2b 100644 --- a/probes/w2_qwen3_small/README.md +++ b/probes/w2_qwen3_small/README.md @@ -152,12 +152,19 @@ padding 隔离。每个比较仍要求 FP32、形状一致、有限值、最大 失败、输入目录混用或哈希不一致应直接失败，不能只靠当前 ORT 比较通过。该绑定 用于避免意外混用产物，不是第三方认证；参考计算仍须在指定环境重新生成。 +当前执行环境与完整源码归属的报告增强单独见 [W1 PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。 +W2 分支未包含该报告代码修复；未合并前不能把原导出环境当成本次 QEMU 对照的执行环境。 + `output/qwen3-riscv/` 包含 `report.json/md/html`、生成的 C、启动汇编与链接脚本、 ELF、编译器/QEMU 命令与日志、每次执行的输入和 UART 二进制以及 QEMU/ORT 数组。 JSON 记录源码/模型/输入/ELF 哈希、工具版本、优化统计、工作区大小、最差元素与 首次发生偏差的检查点。缺工具、编译失败、trap、超时、损坏/截断的 UART、错误 退出或任一数值不匹配都会失败，不允许跳过后返回成功。 +编译器或 QEMU 的日志文件保存失败时，仍分别尝试保存 stdout 和 stderr。 +已有的编译错误、guest 状态、协议错误或超时保持为主要错误，日志保存错误另行附加； +执行成功但必需日志无法保存时，入口仍返回失败，避免把不完整证据当作完整验收。 + ### 查看执行时长 使用上面的 `--output-dir output/qwen3-riscv` 命令即可生成时长报告，无需另开计时 diff --git a/probes/w2_runtime/README.md b/probes/w2_runtime/README.md new file mode 100644 index 0000000..462324a --- /dev/null +++ b/probes/w2_runtime/README.md @@ -0,0 +1,59 @@ +# W2 Host 运行时验收 + +`unit:runtime` 验证文本 tokenizer、固定输入准备、最后有效位置 greedy 采样及停止边界，不执行完整模型或生成循环。生产模块只依赖 NumPy 与可选的 `tokenizers`；Transformers 只用于此门禁的官方对照。 + +## 固定资产与环境 + +模型为 `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`。复用 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个 `source_files` 哈希：配置、tokenizer JSON、merges、vocab 与许可证，合计约 15.2 MiB。下载模式只获取这些文件，不下载模型权重。已有资产必须全部匹配哈希；损坏目录保留并失败，不静默覆盖。 + +使用 [两层模型指南](../w2_qwen3_small/README.md) 中的 Python 3.12 固定 CPU 环境；其中已包含 `numpy==2.2.6`、`tokenizers==0.21.4`、`transformers==4.51.3`。仅使用生产 tokenizer 时可安装 `pip install -e ".[llm]"`，无需 Torch；复现 gate 仍使用固定版本和官方参考依赖。 + +在仓库根目录运行，PowerShell： + +```powershell +$runtimePython = ".\output\qwen3-probe-venv\Scripts\python.exe" +& $runtimePython -B -X utf8 probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +$LASTEXITCODE +``` + +Linux： + +```bash +output/qwen3-probe-venv/bin/python -B -X utf8 probes/w2_runtime/run.py \ + --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +``` + +已有七个文件时使用 `--mode verify`，该模式不访问网络。输出目录必须为空或不存在，每次复现选择新的目录。CLI 退出 0 且本次 `report.json` 的 `passed=true` 才代表通过。报告写入不完整、依赖缺失、下载或哈希校验失败均为失败。 + +## 验收内容 + +- 固定 20 组语料涵盖中英混合、空白、换行、代码、数字、CJK/RTL、emoji、组合字符和特殊 token；保存固定 IDs 与解码文本作为 golden corpus。 +- 每组均对照官方 `Qwen2TokenizerFast` 和独立 Python `Qwen2Tokenizer`，覆盖默认参数、`add_special_tokens`、`skip_special_tokens` 与空格清理组合。官方 JSON 的 NFC 规范化会把 `e` 加组合重音转换为 `é`；“100%”指与官方输出完全一致，不是任意原始 Unicode 字节可逆。 +- 分开核对基础词表 151643、含 added tokens 的词表 151669、模型输出维度 151936；模型中空余行不等于可解码 ID。未知 ID 报错，不丢弃；不静默屏蔽 logits 或改变 greedy。 +- tokenizer EOS 为 151645，generation EOS 为 `(151645,151643)`，pad 为 151643。普通文本编码不注入 BOS/EOS，不渲染聊天模板。 +- 检查有效长度 1/17/255/256、右 padding、INT64 IDs、FP32 加性 mask、最后有效位置和 greedy 平局。mask 仅当 `key <= query` 且 `key < valid_length` 时为 0，其余为 FP32 最小有限值。 +- 验证生成 EOS、256 容量和 token 预算停止。空 prompt、超长输入、非法 ID、非有限 logits 等反例在单测中必须失败。 + +## 生产接口 + +```python +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer +from scratchv.runtime.llm_inputs import prepare_inputs, greedy_next_token + +tokenizer = Qwen3Tokenizer.from_directory("output/qwen3-tokenizer") +ids = tokenizer.encode("The capital of France is", add_special_tokens=False) +inputs = prepare_inputs(ids, pad_token_id=tokenizer.pad_token_id, + vocab_size=tokenizer.model_vocab_size) +# outputs = your_verified_model_runner(inputs.as_feed()) +# next_id = greedy_next_token(outputs, inputs.valid_length, +# vocab_size=tokenizer.model_vocab_size) +# tokenizer.validate_token_ids([next_id]) +``` + +`generation_stop_reason` 供后续生成循环在前向前判断 EOS/容量/预算。模型输出仍按原 ABI 完整校验，最后有效位置选择是 host 操作。这里未实现聊天模板、多轮历史、完整模型前向或端到端文本生成；也不能将官方 tokenizer 的 IDs 输入词表只有 128 的随机小模型。 + +## 报告与 CI + +`report.json/md/html` 保留逐语料结果、对照次数、词表/特殊 ID、资产与执行源码 SHA256、checkout、环境、阶段及错误。报告目录不含模型或 tokenizer 大文件；失败保留已完成的记录，不把未执行项计作通过。 + +LLM workflow 的 `unit:runtime` 在固定环境运行，下载固定的七个资产，随后全程离线调用官方参考；`w2-unit-gates-output` artifact 保存报告和单测 XML。`tests/test_llm_inputs.py`、`tests/test_qwen3_tokenizer.py` 和 `tests/test_w2_runtime_*.py` 覆盖输入、适配层和门禁的反例。具体运行结果见 [W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md)。 diff --git a/probes/w2_runtime/corpus.json b/probes/w2_runtime/corpus.json new file mode 100644 index 0000000..dc33765 --- /dev/null +++ b/probes/w2_runtime/corpus.json @@ -0,0 +1,487 @@ +{ + "schema_version": 1, + "model_id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "reference": { + "transformers": "4.51.3", + "tokenizers": "0.21.4", + "classes": [ + "Qwen2Tokenizer", + "Qwen2TokenizerFast" + ], + "add_special_tokens": false, + "skip_special_tokens": false, + "clean_up_tokenization_spaces": false + }, + "cases": [ + { + "name": "empty", + "text": "", + "ids": [], + "decoded": "" + }, + { + "name": "english", + "text": "The capital of France is", + "ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "decoded": "The capital of France is" + }, + { + "name": "chinese", + "text": "你好，世界！ScratchV 编译器。", + "ids": [ + 108386, + 3837, + 99489, + 6313, + 65508, + 754, + 53, + 84238, + 244, + 102610, + 31548, + 1773 + ], + "decoded": "你好，世界！ScratchV 编译器。" + }, + { + "name": "mixed", + "text": "Qwen3 在 RISC-V 上运行 0.6B 模型。", + "ids": [ + 48, + 16948, + 18, + 73562, + 431, + 27629, + 19625, + 64118, + 104001, + 220, + 15, + 13, + 21, + 33, + 6567, + 44401, + 24300, + 1773 + ], + "decoded": "Qwen3 在 RISC-V 上运行 0.6B 模型。" + }, + { + "name": "whitespace", + "text": " leading spaces\tand tabs\n\ntrailing ", + "ids": [ + 220, + 6388, + 220, + 12621, + 52477, + 22398, + 271, + 376, + 14277, + 256 + ], + "decoded": " leading spaces\tand tabs\n\ntrailing " + }, + { + "name": "line_endings", + "text": "line1\r\nline2\rline3\n", + "ids": [ + 1056, + 16, + 319, + 1056, + 17, + 201, + 1056, + 18, + 198 + ], + "decoded": "line1\r\nline2\rline3\n" + }, + { + "name": "code", + "text": "def f(x):\n return x * 2 + 1\n", + "ids": [ + 750, + 282, + 2075, + 982, + 262, + 470, + 856, + 353, + 220, + 17, + 488, + 220, + 16, + 198 + ], + "decoded": "def f(x):\n return x * 2 + 1\n" + }, + { + "name": "math", + "text": "x² + y² = z²; ∑ₖ αₖ ≈ 3.1415926", + "ids": [ + 87, + 29456, + 488, + 379, + 29456, + 284, + 1147, + 29456, + 26, + 11995, + 239, + 15896, + 244, + 19043, + 15896, + 244, + 21103, + 230, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 17, + 21 + ], + "decoded": "x² + y² = z²; ∑ₖ αₖ ≈ 3.1415926" + }, + { + "name": "emoji", + "text": "👩🏽‍💻 🧪 🚀 🇨🇳 👨‍👩‍👧‍👦", + "ids": [ + 145233, + 145375, + 378, + 235, + 145851, + 11162, + 100, + 103, + 11162, + 248, + 222, + 11162, + 229, + 101, + 145754, + 61804, + 101, + 378, + 235, + 145233, + 378, + 235, + 145665, + 378, + 235, + 145988 + ], + "decoded": "👩🏽‍💻 🧪 🚀 🇨🇳 👨‍👩‍👧‍👦" + }, + { + "name": "combining", + "text": "café café Å Å", + "ids": [ + 924, + 58858, + 51950, + 79252, + 79252 + ], + "decoded": "café café Å Å" + }, + { + "name": "cjk", + "text": "日本語のテスト。한국어 테스트。繁體中文測試。", + "ids": [ + 101059, + 102819, + 15767, + 140592, + 1773, + 23573, + 124785, + 31079, + 10764, + 72509, + 53189, + 1773, + 100112, + 100480, + 104811, + 107636, + 102854, + 1773 + ], + "decoded": "日本語のテスト。한국어 테스트。繁體中文測試。" + }, + { + "name": "rtl", + "text": "مرحبا بالعالم שלום עולם", + "ids": [ + 124122, + 29825, + 124671, + 124476, + 129634, + 124756, + 123881, + 123855, + 124907 + ], + "decoded": "مرحبا بالعالم שלום עולם" + }, + { + "name": "punctuation", + "text": "Hello , world ! Isn't this fun ?", + "ids": [ + 9707, + 1154, + 1879, + 753, + 220, + 52196, + 944, + 419, + 2464, + 937 + ], + "decoded": "Hello , world ! Isn't this fun ?" + }, + { + "name": "numbers", + "text": "0 1 -2 1e-5 3.14159 1234567890 00123", + "ids": [ + 15, + 220, + 16, + 481, + 17, + 220, + 16, + 68, + 12, + 20, + 220, + 18, + 13, + 16, + 19, + 16, + 20, + 24, + 220, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 15, + 220, + 15, + 15, + 16, + 17, + 18 + ], + "decoded": "0 1 -2 1e-5 3.14159 1234567890 00123" + }, + { + "name": "json", + "text": "{\"name\":\"Qwen3\",\"enabled\":true,\"items\":[1,2,3]}", + "ids": [ + 4913, + 606, + 3252, + 48, + 16948, + 18, + 2198, + 15868, + 788, + 1866, + 1335, + 3615, + 8899, + 16, + 11, + 17, + 11, + 18, + 13989 + ], + "decoded": "{\"name\":\"Qwen3\",\"enabled\":true,\"items\":[1,2,3]}" + }, + { + "name": "controls", + "text": "a\u0000b\t\u000b\f\n", + "ids": [ + 64, + 188, + 65, + 197, + 199, + 68762 + ], + "decoded": "a\u0000b\t\u000b\f\n" + }, + { + "name": "special", + "text": "<|endoftext|><|im_start|>user\nhello<|im_end|>", + "ids": [ + 151643, + 151644, + 872, + 198, + 14990, + 151645 + ], + "decoded": "<|endoftext|><|im_start|>user\nhello<|im_end|>" + }, + { + "name": "thinking", + "text": "Let me reason.\n42", + "ids": [ + 151667, + 10061, + 752, + 2874, + 13, + 151668, + 198, + 19, + 17 + ], + "decoded": "Let me reason.\n42" + }, + { + "name": "repeated", + "text": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello ", + "ids": [ + 14990, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 23811, + 220 + ], + "decoded": "hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello hello " + }, + { + "name": "all_added", + "text": "<|endoftext|><|im_start|><|im_end|><|object_ref_start|><|object_ref_end|><|box_start|><|box_end|><|quad_start|><|quad_end|><|vision_start|><|vision_end|><|vision_pad|><|image_pad|><|video_pad|><|fim_prefix|><|fim_middle|><|fim_suffix|><|fim_pad|><|repo_name|><|file_sep|>", + "ids": [ + 151643, + 151644, + 151645, + 151646, + 151647, + 151648, + 151649, + 151650, + 151651, + 151652, + 151653, + 151654, + 151655, + 151656, + 151657, + 151658, + 151659, + 151660, + 151661, + 151662, + 151663, + 151664, + 151665, + 151666, + 151667, + 151668 + ], + "decoded": "<|endoftext|><|im_start|><|im_end|><|object_ref_start|><|object_ref_end|><|box_start|><|box_end|><|quad_start|><|quad_end|><|vision_start|><|vision_end|><|vision_pad|><|image_pad|><|video_pad|><|fim_prefix|><|fim_middle|><|fim_suffix|><|fim_pad|><|repo_name|><|file_sep|>" + } + ] +} diff --git a/probes/w2_runtime/run.py b/probes/w2_runtime/run.py new file mode 100644 index 0000000..7ec2e08 --- /dev/null +++ b/probes/w2_runtime/run.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +"""W2 host-runtime gate against pinned official tokenizer assets and two oracles.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import html +import importlib.metadata +import json +from pathlib import Path +import platform +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) +MANIFEST_PATH = ROOT / "probes/w1_qwen3_export/manifest.json" +MANIFEST = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) +CORPUS = Path(__file__).with_name("corpus.json") +PINNED = {"numpy": "2.2.6", "tokenizers": "0.21.4", "transformers": "4.51.3"} +MAX_FILE_BYTES = 16 * 1024 * 1024 + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def verify_assets(directory, manifest=MANIFEST): + directory = Path(directory).resolve() + rows, seen = [], set() + for item in manifest["source_files"]: + name = item["name"] + if (not name or name in seen or name in (".", "..") + or any(c in name for c in "/\\:")): + raise ValueError(f"Non-flat or duplicate asset name: {name}") + seen.add(name) + path = (directory / name).resolve() + if not path.is_relative_to(directory) or not path.is_file(): + raise FileNotFoundError(f"Missing tokenizer asset: {name}") + size = path.stat().st_size + if size > MAX_FILE_BYTES or sha256(path) != item["sha256"]: + raise ValueError(f"Tokenizer asset SHA256/size mismatch: {name}") + rows.append({**item, "bytes": size}) + required = {"tokenizer.json", "tokenizer_config.json", "config.json", + "generation_config.json", "vocab.json", "merges.txt", "LICENSE"} + if seen != required: + raise ValueError("Tokenizer gate requires exactly the seven pinned source files") + return rows + + +def download_file(url, destination): + """Bound downloads; retry transient server failures without trusting a cache.""" + for attempt in range(3): + try: + request = urllib.request.Request(url, headers={"User-Agent": "ScratchV-W2-runtime"}) + with urllib.request.urlopen(request, timeout=30) as incoming, Path(destination).open("wb") as out: + count = 0 + while block := incoming.read(1024 * 1024): + count += len(block) + if count > MAX_FILE_BYTES: + raise ValueError("Tokenizer download exceeds per-file limit") + out.write(block) + return + except urllib.error.HTTPError as exc: + if exc.code not in (429, 500, 502, 503, 504) or attempt == 2: + raise + time.sleep(attempt + 1) + + +def acquire_assets(directory, manifest=MANIFEST): + directory = Path(directory).resolve() + if directory.exists(): + return {"reused": True, "files": verify_assets(directory, manifest)} + directory.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="qwen3-tokenizer-", dir=directory.parent) as tmp: + stage = Path(tmp).resolve() + for item in manifest["source_files"]: + name = item["name"] + if not name or name in (".", "..") or any(c in name for c in "/\\:"): + raise ValueError(f"Invalid tokenizer asset name: {name}") + url = f"https://huggingface.co/{manifest['model_id']}/resolve/{manifest['revision']}/{name}" + download_file(url, stage / name) + rows = verify_assets(stage, manifest) + if not stage.is_relative_to(directory.parent) or directory.exists(): + raise ValueError("Tokenizer destination changed during download") + stage.rename(directory) + return {"reused": False, "files": rows} + + +def require_environment(): + versions = {name: importlib.metadata.version(name) for name in PINNED} + if sys.version_info[:2] != (3, 12) or versions != PINNED: + raise RuntimeError(f"Use Python 3.12 and the pinned CPU probe environment: {versions}") + return {"python": platform.python_version(), "platform": platform.platform(), + "packages": versions, "reference": "Official Transformers fast and slow tokenizers, offline"} + + +def checkout_evidence(): + """Source archives remain runnable; never attribute them to a parent repo.""" + result = {"root": str(ROOT), "head": None, "status": None, "clean": None} + if not (ROOT / ".git").exists(): + result["unavailable"] = "Source directory has no .git entry; use source_sha256 for identity" + return result + try: + for key, arguments in (("head", ["rev-parse", "HEAD"]), + ("status", ["status", "--porcelain=v1", "--untracked-files=all"])): + result[key] = subprocess.check_output( + ["git", *arguments], cwd=ROOT, text=True, encoding="utf-8", errors="replace", + stderr=subprocess.PIPE, timeout=10, + ).rstrip("\r\n") + result["clean"] = not bool(result["status"]) + except (OSError, subprocess.SubprocessError) as exc: + result["unavailable"] = f"{type(exc).__name__}: {exc}" + return result + + +def tokenizer_cases(adapter, fast, slow, corpus, rows): + if not corpus["cases"]: + raise ValueError("Tokenizer corpus must not be empty") + names = set() + for case in corpus["cases"]: + if case["name"] in names: + raise ValueError("Duplicate corpus case name") + names.add(case["name"]) + row = {"name": case["name"], "passed": False, "checks": 0} + rows.append(row) + # Exercise public defaults as well as explicit flags. A default-only + # regression would otherwise leave every explicit-option check green. + default_ids = adapter.encode(case["text"]) + default_text = adapter.decode(default_ids) + for reference in (fast, slow): + if default_ids != reference.encode(case["text"]) or default_ids != case["ids"]: + raise ValueError(f"{case['name']}: default encoding differs from official/golden IDs") + row["checks"] += 1 + if default_text != reference.decode(default_ids) or default_text != case["decoded"]: + raise ValueError(f"{case['name']}: default decode differs from official/golden text") + row["checks"] += 1 + for special in (False, True): + actual = adapter.encode(case["text"], add_special_tokens=special) + for reference in (fast, slow): + expected = reference.encode(case["text"], add_special_tokens=special) + if actual != expected or actual != case["ids"]: + raise ValueError(f"{case['name']}: encoding differs from official/golden IDs") + row["checks"] += 1 + for skip in (False, True): + for cleanup in (False, True): + result = adapter.decode(actual, skip_special_tokens=skip, + clean_up_tokenization_spaces=cleanup) + for reference in (fast, slow): + expected = reference.decode(actual, skip_special_tokens=skip, + clean_up_tokenization_spaces=cleanup) + if result != expected: + raise ValueError(f"{case['name']}: decode differs (skip={skip}, cleanup={cleanup})") + row["checks"] += 1 + if not skip and not cleanup and result != case["decoded"]: + raise ValueError(f"{case['name']}: decode differs from golden text") + row.update(passed=True, token_count=len(case["ids"]), + raw_text_preserved=case["decoded"] == case["text"]) + + +def tokenizer_metadata(adapter, fast, slow, directory): + """Check metadata independently: model rows are not tokenizer membership.""" + directory = Path(directory) + model = json.loads((directory / "config.json").read_text(encoding="utf-8")) + generation = json.loads((directory / "generation_config.json").read_text(encoding="utf-8")) + expected_eos = generation["eos_token_id"] + if isinstance(expected_eos, int): + expected_eos = [expected_eos] + if adapter.model_vocab_size != model["vocab_size"]: + raise ValueError("Tokenizer model_vocab_size differs from the official model configuration") + if adapter.generation_eos_token_ids != tuple(expected_eos): + raise ValueError("Tokenizer generation EOS IDs differ from the official generation configuration") + if adapter.pad_token_id != generation["pad_token_id"]: + raise ValueError("Tokenizer pad ID differs from the official generation configuration") + for reference in (fast, slow): + if (adapter.base_vocab_size != reference.vocab_size + or adapter.tokenizer_vocab_size != len(reference) + or adapter.valid_token_ids != frozenset(reference.get_vocab().values())): + raise ValueError("Tokenizer vocabulary metadata differs from the official reference") + for name in ("pad_token_id", "eos_token_id", "bos_token_id", "unk_token_id"): + if getattr(adapter, name) != getattr(reference, name): + raise ValueError(f"Tokenizer {name} differs from the official reference") + return {"model_vocab_size": adapter.model_vocab_size, + "base_vocab_size": adapter.base_vocab_size, + "tokenizer_vocab_size": adapter.tokenizer_vocab_size, + "pad_token_id": adapter.pad_token_id, + "eos_token_id": adapter.eos_token_id, + "bos_token_id": adapter.bos_token_id, + "unk_token_id": adapter.unk_token_id, + "generation_eos_token_ids": list(adapter.generation_eos_token_ids)} + + +def input_cases(adapter, rows): + import numpy as np + from scratchv.runtime.llm_inputs import prepare_inputs, greedy_next_token, generation_stop_reason + # Explicit loop oracle, independent of the vectorized production mask. + for length in (1, 17, 255, 256): + ids = [42] * length + ids[0] = adapter.pad_token_id # A pad-like ID inside the prompt remains a valid token. + prepared = prepare_inputs(ids, pad_token_id=adapter.pad_token_id, + vocab_size=adapter.model_vocab_size) + expected = np.full((1, 1, 256, 256), np.finfo(np.float32).min, dtype=np.float32) + for query in range(256): + for key in range(min(query + 1, length)): + expected[0, 0, query, key] = 0 + if (prepared.valid_length != length or prepared.input_ids.dtype != np.int64 + or prepared.input_ids.shape != (1, 256) + or not np.array_equal(prepared.input_ids[0, :length], ids) + or not np.all(prepared.input_ids[0, length:] == adapter.pad_token_id) + or prepared.attention_mask.dtype != np.float32 + or not np.array_equal(prepared.attention_mask, expected)): + raise ValueError(f"Input/mask mismatch for valid length {length}") + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, -1, 7] = 100 + logits[0, length - 1, 7] = 0 + logits[0, length - 1, [2, 5]] = 1 + if greedy_next_token(logits, length, vocab_size=8) != 2: + raise ValueError("Greedy used a padding row or wrong tie rule") + rows.append({"name": f"mask-and-greedy-{length}", "passed": True}) + for eos in adapter.generation_eos_token_ids: + reason = generation_stop_reason(valid_length=2, generated_tokens=1, max_new_tokens=3, + last_token_id=eos, eos_token_ids=adapter.generation_eos_token_ids) + if reason != "eos": + raise ValueError(f"Generation EOS not recognized: {eos}") + if generation_stop_reason(valid_length=256, generated_tokens=0, max_new_tokens=1) != "capacity": + raise ValueError("Full context must stop before a new forward call") + if generation_stop_reason(valid_length=1, generated_tokens=0, max_new_tokens=0) != "max_new_tokens": + raise ValueError("Zero generation budget must stop") + rows.append({"name": "eos-capacity-budget", "passed": True}) + + +def run_checks(args, report): + report["stage"] = "environment" + report["environment"] = require_environment() + report["stage"] = "assets" + if args.mode == "download": + report["acquisition"] = acquire_assets(args.tokenizer_dir) + report["files"] = verify_assets(args.tokenizer_dir) + report["stage"] = "tokenizer" + from transformers import AutoTokenizer + from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + adapter = Qwen3Tokenizer.from_directory(args.tokenizer_dir) + fast = AutoTokenizer.from_pretrained(args.tokenizer_dir, use_fast=True, local_files_only=True, + trust_remote_code=False) + slow = AutoTokenizer.from_pretrained(args.tokenizer_dir, use_fast=False, local_files_only=True, + trust_remote_code=False) + corpus = json.loads(CORPUS.read_text(encoding="utf-8")) + if corpus["revision"] != MANIFEST["revision"] or corpus["model_id"] != MANIFEST["model_id"]: + raise ValueError("Corpus/model identity mismatch") + if (type(fast).__name__, type(slow).__name__) != ("Qwen2TokenizerFast", "Qwen2Tokenizer"): + raise ValueError("Expected two distinct official tokenizer reference classes") + report["tokenizer"] = tokenizer_metadata(adapter, fast, slow, args.tokenizer_dir) + tokenizer_cases(adapter, fast, slow, corpus, report["tokenizer_cases"]) + report["stage"] = "inputs-sampling" + input_cases(adapter, report["runtime_cases"]) + report.update(passed=True, stage="complete") + + +def save_report(directory, report): + """Keep the primary failure and make incomplete evidence an explicit FAIL.""" + def write(name, data): + try: + (directory / name).write_text(data, encoding="utf-8") + except OSError as exc: + return f"{name}: {type(exc).__name__}: {exc}" + return None + + lines = ["# W2 unit:runtime", "", f"Result: **{'PASS' if report['passed'] else 'FAIL'}**", + f"Stage: {report['stage']}; seconds: {report['seconds']:.3f}", "", + "Fixed official tokenizer assets; encode/decode vs Transformers fast + slow and golden corpus.", + "NFC normalization is official behavior; equivalence does not mean arbitrary raw-text byte preservation.", + "No pretrained weights, full-model inference or text generation is executed.", "", + "| Case | Checks | Passed |", "|---|---:|---|"] + for row in report["tokenizer_cases"] + report["runtime_cases"]: + lines.append(f"| {row['name']} | {row.get('checks', 1)} | {row['passed']} |") + if "error" in report: + lines.extend(["", "Error: " + report["error"]]) + markdown = "\n".join(lines) + "\n" + failures = [] + documents = { + "report.md": markdown, + "report.html": 'W2 runtime gate
'
+        + html.escape(markdown) + "
", + } + for name, content in documents.items(): + if error := write(name, content): + failures.append(error) + def record_failure(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete runtime gate evidence") + + def recover_views(): + # Preserve the primary probe error. Try both human-readable formats even + # when one destination is unwritable. + failure_text = ( + "# W2 unit:runtime\n\nResult: **FAIL**\n\n" + + f"Stage: {report['stage']}\n\nError: {report['error']}\n\n" + + "Report write errors:\n" + "\n".join(f"- {failure}" for failure in failures) + "\n" + ) + recovery = { + "report.md": failure_text, + "report.html": '
' + html.escape(failure_text) + "
", + } + for name, content in recovery.items(): + write(name, content) + + if failures: + record_failure() + recover_views() + # Publish machine-readable status only after the view outcomes are known. + # In particular, never write PASS and rely on a later corrective write: that + # second write could fail and leave a misleading successful JSON artifact. + data = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + # A write can complete its payload and still fail while flushing/closing. + # Keep even a fully written PASS private until write_text has returned; + # same-directory replacement then publishes the completed evidence. + pending = directory / ".report.json.tmp" + error = None + try: + pending.write_text(data, encoding="utf-8") + pending.replace(directory / "report.json") + except OSError as exc: + error = f"report.json: {type(exc).__name__}: {exc}" + try: + pending.unlink(missing_ok=True) + except OSError as cleanup_exc: + error += f"; temporary cleanup: {type(cleanup_exc).__name__}: {cleanup_exc}" + if error: + failures.append(error) + record_failure() + recover_views() + if failures: + print(f"[unit:runtime] {report['error']}; report write errors: {'; '.join(failures)}", file=sys.stderr) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--tokenizer-dir", type=Path, required=True) + ap.add_argument("--mode", choices=("verify", "download"), default="verify") + ap.add_argument("--output-dir", type=Path, default=Path("output/w2-runtime")) + args = ap.parse_args(argv) + args.tokenizer_dir = args.tokenizer_dir.resolve() + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + ap.error("--output-dir must be absent or empty; preserve earlier evidence") + out.mkdir(parents=True, exist_ok=True) + report = {"gate": "unit:runtime", "passed": False, "stage": "initialize", + "created_at": datetime.now(timezone.utc).isoformat(), "mode": args.mode, + "model_id": MANIFEST["model_id"], "revision": MANIFEST["revision"], + "tokenizer_dir": str(args.tokenizer_dir), "tokenizer_cases": [], "runtime_cases": []} + started = time.perf_counter() + try: + report["source_sha256"] = {p: sha256(ROOT / p) for p in ( + "probes/w2_runtime/run.py", "probes/w2_runtime/corpus.json", + "probes/w1_qwen3_export/manifest.json", "scratchv/runtime/llm_inputs.py", + "scratchv/runtime/qwen3_tokenizer.py")} + report["checkout"] = checkout_evidence() + run_checks(args, report) + except Exception as exc: + report.update(passed=False, error=f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + save_report(out, report) + print(f"[unit:runtime] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; report={out / 'report.json'}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probes/w2_runtime_model/README.md b/probes/w2_runtime_model/README.md new file mode 100644 index 0000000..794fa0e --- /dev/null +++ b/probes/w2_runtime_model/README.md @@ -0,0 +1,42 @@ +# W2 Runtime → Model 单步接口探测 + +把已实现的 Tokenizer、输入准备、模型调用、最后有效位置选择及 greedy 解码真正串起来。模型使用 `transformers==4.51.3` 官方 `Qwen3ForCausalLM`，两层、hidden=32、FFN=96、Q/KV heads=4/2、head_dim=16、随机种子 0、绑定 embedding/LM head。保留**完整模型词表 151936**；官方 Token ID 直接进入模型，不取模、不重新编号、不裁剪词表。 + +这是 W2 的接口集成证据；权重随机，不代表预训练 Qwen3-0.6B 的语言能力。执行单步前向与一个 greedy token，不执行生成循环、KV cache 或 QEMU。 + +## 本地复现 + +使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 + +```bash +python probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime +python probes/w2_runtime_model/run.py --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime-model +``` + +输出目录必须不存在或为空。既有证据不会被覆盖；版本不匹配、资产缺失/损坏、数值或采样不一致、报告写入失败均返回非零。 + +## 验收内容 + +- 两个固定原始文本：中文混合文本，以及包含官方特殊 token 和 `endoftext` 的输入。后者包含 pad ID，仍必须按原始 token 数计算有效长度。 +- 生产 `Qwen3Tokenizer` 的编码与官方 fast、slow Tokenizer 相同。生产 `prepare_inputs` 的 ID、dtype、shape、因果 mask 和 padding 由独立标量循环核验。 +- 只导出普通 logits ONNX：INT64 `[1,256]` input_ids、FP32 `[1,1,256,256]` attention_mask，FP32 `[1,256,151936]` logits；FP32、opset 18、固定形状、无 KV cache、eager attention。 +- Torch、ORT、共享 IR `none` 和 `all` 真正执行相同输入。ORT 对 Torch、IR 对 Torch 和 ORT 均检查所有位置，含 padding，严格要求有限 FP32、shape 完全相同、`max_abs < 1e-5`、`rtol=0`。 +- 每条路径选择 `valid_length - 1` 的 logits，greedy ID 必须与独立索引的 Torch argmax 相同；解码必须与官方 fast、slow 相同。即使精度误差小于阈值，argmax 改变仍会失败；若抽中模型词表的未定义 Tokenizer 行，明确失败，不能修改候选 mask 来掩盖。 + +## 输出及资源 + +`report.json`、`report.md`、`report.html` 包含结果、阶段、首次失败、源码 SHA256、实际 checkout、环境版本、资产 SHA256、模型参数与权重 hash、输入 token IDs/hash、ONNX hash、全/有效/padding 误差、最差位置、greedy ID/解码/top-2 margin、构建/导出/解析/各引擎执行耗时。JSON 在完整写入并关闭后原子发布；报告生成失败不能留下 PASS。 + +无 `.git` 的源码快照也可复现：checkout HEAD 明确标为未知，身份由运行前记录的源码 SHA256 确定，不借用上层仓库的 Git 信息。报告同时记录实际导入的 parser、IR interpreter、Tokenizer 和输入模块的 `__file__`；若这些模块来自当前源码树之外则失败，防止误用旧安装。 + +报告中的引擎 `seconds` 仅计该次主机前向调用；总 `seconds` 还包含环境加载、资产校验、导出、报告前的数据保存与数值比较。都不是 QEMU 时间，也不是目标硬件性能数据。 + +`model.onnx`、`config.json`、每个文本的 `*.inputs.npz`、`*.torch.npy`、`*.ort.npy` 供独立复核。一个完整输出约 148.4 MiB；参考输出保存后通过 mmap 顺序读取，ORT 会话在 IR 前释放，各次完整输出按顺序释放，比较按 8 个 token 的块进行。两种参考、两个文本的持久化输出合计约 594 MiB，另加约 20 MiB 模型；预留至少 1 GiB 磁盘和数 GiB 空闲内存。这里没有声称限制了解释器内部内存峰值。 + +负例回归： + +```bash +python -m pytest tests/test_w2_runtime_model.py -q +``` + +负例覆盖 ID 映射、padding/因果 mask、错误行选择、argmax 翻转、非法采样 ID、非有限/错误 shape/dtype、只在 padding 超差、解码差异及报告失败。单测不冒充真实模型运行；必须同时执行上面的集成 CLI 才能得到接口集成 PASS。 diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py new file mode 100644 index 0000000..05f6f29 --- /dev/null +++ b/probes/w2_runtime_model/run.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Text -> official tokenizer -> static Qwen3 -> shared IR -> one greedy ID. + +This is an interface integration probe with seeded random weights, not a +pretrained 0.6B model or a generation loop. The complete vocabulary is retained. +""" +from __future__ import annotations + +import argparse +import copy +from datetime import datetime, timezone +import gc +import html +import json +from pathlib import Path +import sys +import time + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w2_runtime.run import MANIFEST, checkout_evidence, require_environment, sha256, verify_assets +from probes.w2_qwen3_small.run import arrays_sha256, environment +from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token, prepare_inputs + +SEQ, VOCAB, ATOL = 256, 151936, 1e-5 +LEVELS = ("none", "all") +CASES = (("chinese", "你好，ScratchV！"), + ("special_pad_in_prompt", "<|im_start|>user\nTest<|endoftext|>!<|im_end|>\n")) + + +def checked_inputs(ids, prepared, pad_token_id): + """Independent ABI oracle: never infer valid length from pad-ID values.""" + length = len(ids) + if prepared.valid_length != length or not 1 <= length <= SEQ: + raise ValueError("Input preparation changed the prompt length") + actual = prepared.as_feed() + expected_ids = np.full((1, SEQ), pad_token_id, dtype=np.int64) + expected_ids[0, :length] = ids + expected_mask = np.full((1, 1, SEQ, SEQ), np.finfo(np.float32).min, dtype=np.float32) + for query in range(SEQ): + for key in range(min(query + 1, length)): + expected_mask[0, 0, query, key] = 0 + for name, expected in (("input_ids", expected_ids), ("attention_mask", expected_mask)): + value = actual[name] + if value.shape != expected.shape or value.dtype != expected.dtype or not np.array_equal(value, expected): + raise ValueError(f"Input preparation changed {name} ABI, IDs, causal mask or padding") + return actual + + +def compare_logits(actual, expected, valid_length, *, vocab_size=VOCAB): + """Compare every element in bounded FP64 chunks, including padding rows.""" + shape = (1, SEQ, vocab_size) + for name, array in (("actual", actual), ("expected", expected)): + if not isinstance(array, np.ndarray) or np.ma.isMaskedArray(array): + raise TypeError(f"{name} logits must be an ordinary ndarray") + if array.shape != shape or array.dtype != np.dtype(np.float32): + raise ValueError(f"{name} logits require FP32 {shape}; got {array.dtype} {array.shape}") + if not 1 <= valid_length <= SEQ: + raise ValueError("Invalid prompt length") + maxima = {"max_abs": 0.0, "valid_max_abs": 0.0, "padding_max_abs": 0.0} + worst = None + for start in range(0, SEQ, 8): + stop = min(start + 8, SEQ) + left, right = actual[0, start:stop], expected[0, start:stop] + if not np.isfinite(left).all() or not np.isfinite(right).all(): + raise ValueError(f"Nonfinite logits in positions {start}:{stop}") + difference = np.abs(left.astype(np.float64) - right.astype(np.float64)) + maximum = float(difference.max()) + if maximum > maxima["max_abs"]: + position, token = np.unravel_index(int(difference.argmax()), difference.shape) + worst = [0, start + int(position), int(token)] + maxima["max_abs"] = max(maxima["max_abs"], maximum) + boundary = min(max(valid_length - start, 0), stop - start) + if boundary: + maxima["valid_max_abs"] = max(maxima["valid_max_abs"], float(difference[:boundary].max())) + if boundary < stop - start: + maxima["padding_max_abs"] = max(maxima["padding_max_abs"], float(difference[boundary:].max())) + return {**maxima, "worst_index": worst, "elements": int(actual.size), + "passed": maxima["max_abs"] < ATOL, "atol": ATOL, "rtol": 0} + + +def sample_checked(logits, reference, length, adapter, references, *, vocab_size=VOCAB): + vector = last_valid_logits(logits, length, vocab_size=vocab_size) + expected = reference[0, length - 1, :] + # Independently index the reference to catch an incorrect host row selector. + if vector.shape != expected.shape or not np.array_equal(vector, logits[0, length - 1, :]): + raise ValueError("Host selected the wrong logits row") + actual_id, expected_id = greedy_token(vector), int(np.argmax(expected)) + if actual_id != expected_id: + raise ValueError(f"Greedy ID differs: actual={actual_id}, reference={expected_id}") + adapter.validate_token_ids([actual_id]) # Never hide invalid padded model rows. + text = adapter.decode([actual_id], skip_special_tokens=False, clean_up_tokenization_spaces=False) + for tokenizer in references: + if text != tokenizer.decode([actual_id], skip_special_tokens=False, clean_up_tokenization_spaces=False): + raise ValueError("Sampled token decode differs from official tokenizer") + ordered = np.partition(expected, -2)[-2:] + return {"id": actual_id, "decoded": text, "position": length - 1, + "reference_top2_margin": float(ordered[-1] - ordered[-2]), "passed": True} + + +def build_model(adapter): + import torch + from transformers import Qwen3Config, Qwen3ForCausalLM + from probes.w2_qwen3_small.model import MODEL_CONFIG + + config_values = {**MODEL_CONFIG, "vocab_size": VOCAB, + "pad_token_id": adapter.pad_token_id, + "bos_token_id": None, "eos_token_id": list(adapter.generation_eos_token_ids)} + config = Qwen3Config(**config_values) + config._attn_implementation = "eager" + with torch.random.fork_rng(devices=[]): + torch.manual_seed(0) + model = Qwen3ForCausalLM(config).float().eval() + if model.lm_head.weight is not model.model.embed_tokens.weight: + raise ValueError("Expected official tied embedding and LM head") + + class NormalModel(torch.nn.Module): + def __init__(self, qwen): + super().__init__() + self.model = qwen + self.register_buffer("positions", torch.arange(SEQ).reshape(1, SEQ), persistent=False) + + def forward(self, input_ids, attention_mask): + return self.model(input_ids=input_ids, attention_mask=attention_mask, + position_ids=self.positions, use_cache=False, + output_attentions=False, output_hidden_states=False, + return_dict=False, logits_to_keep=0)[0] + + return NormalModel(model).eval(), config_values + + +def check_onnx_contract(model): + import onnx + onnx.checker.check_model(model, full_check=True) + standard_opsets = [item.version for item in model.opset_import if item.domain in ("", "ai.onnx")] + if standard_opsets != [18]: + raise ValueError(f"Expected exactly one standard ONNX opset 18: {standard_opsets}") + expected = {"input_ids": (onnx.TensorProto.INT64, [1, SEQ]), + "attention_mask": (onnx.TensorProto.FLOAT, [1, 1, SEQ, SEQ]), + "logits": (onnx.TensorProto.FLOAT, [1, SEQ, VOCAB])} + if [v.name for v in model.graph.input] != ["input_ids", "attention_mask"] or [v.name for v in model.graph.output] != ["logits"]: + raise ValueError("ONNX must have two ordinary inputs and only logits output") + for value in [*model.graph.input, *model.graph.output]: + element = value.type.tensor_type + actual = element.elem_type, [d.dim_value for d in element.shape.dim] + if actual != expected[value.name]: + raise ValueError(f"Unexpected static ONNX ABI: {value.name}: {actual}") + if any(t.data_location == onnx.TensorProto.EXTERNAL for t in model.graph.initializer): + raise ValueError("This small model must embed its weights") + + +def source_evidence(): + paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", + ROOT / "probes/w2_qwen3_small/model.py", ROOT / "probes/w2_qwen3_small/run.py", + ROOT / "probes/w1_qwen3_export/manifest.json", + ROOT / "scratchv/runtime/qwen3_tokenizer.py", ROOT / "scratchv/runtime/llm_inputs.py", + ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/verification/ir_interpreter.py", + ROOT / "scratchv/verification/ir_numpy_ops.py", ROOT / "scratchv/pass_manager.py", + ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/ir/types.py", ROOT / "scratchv/ir/builder.py", + ROOT / "requirements/qwen3-small-probe.txt", + *sorted((ROOT / "scratchv/optimizer").glob("*.py")), + *sorted((ROOT / "scratchv/analysis").glob("*.py"))] + return {str(p.relative_to(ROOT)).replace("\\", "/"): sha256(p) for p in paths} + + +def run_probe(directory, out, report): + report.update(stage="environment", environment=environment(), tokenizer_environment=require_environment()) + import onnx + import onnxruntime as ort + import torch + from transformers import AutoTokenizer + from scratchv.analysis.ir_verifier import verify_ir + from scratchv.frontend.onnx_parser import ONNXParser + from scratchv.pass_manager import create_optimization_pass_manager + from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + from scratchv.verification.ir_interpreter import IRInterpreter + + def require_valid(program): + valid, errors = verify_ir(program) + if not valid: + raise ValueError(f"Invalid shared IR: {errors}") + + from scratchv.runtime import llm_inputs + modules = {"onnx_parser": sys.modules[ONNXParser.__module__], + "ir_interpreter": sys.modules[IRInterpreter.__module__], + "llm_inputs": llm_inputs, + "qwen3_tokenizer": sys.modules[Qwen3Tokenizer.__module__]} + report["module_files"] = {name: str(Path(module.__file__).resolve()) for name, module in modules.items()} + for name, path in report["module_files"].items(): + if not Path(path).is_relative_to(ROOT): + raise ValueError(f"{name} imported outside the current source tree: {path}") + report.update(stage="tokenizer", assets=verify_assets(directory)) + adapter = Qwen3Tokenizer.from_directory(directory) + if adapter.model_vocab_size != VOCAB: + raise ValueError("The complete official model vocabulary is mandatory") + references = [AutoTokenizer.from_pretrained(directory, use_fast=fast, local_files_only=True, + trust_remote_code=False) for fast in (True, False)] + if [type(t).__name__ for t in references] != ["Qwen2TokenizerFast", "Qwen2Tokenizer"]: + raise ValueError("Require two distinct official tokenizer references") + report["cases"] = [] + feeds = [] + for name, text in CASES: + ids = adapter.encode(text) + if any(ids != reference.encode(text) for reference in references): + raise ValueError(f"Official prompt encoding differs: {name}") + prepared = prepare_inputs(ids, pad_token_id=adapter.pad_token_id) + feed = checked_inputs(ids, prepared, adapter.pad_token_id) + np.savez(out / f"{name}.inputs.npz", **feed) + feeds.append((prepared.valid_length, feed)) + report["cases"].append({"name": name, "text": text, "token_ids": ids, + "valid_length": prepared.valid_length, "input_sha256": arrays_sha256(feed), + "pad_id_inside_prompt": adapter.pad_token_id in ids, + "passed": False, "executions": []}) + if len(feeds) != 2 or not report["cases"][1]["pad_id_inside_prompt"]: + raise ValueError("Require two original-text cases including an in-prompt pad ID") + torch.set_num_threads(1) + torch.use_deterministic_algorithms(True) + report["stage"] = "model" + started = time.perf_counter() + wrapper, config = build_model(adapter) + report.update(config=config, model_seed=0, model_seconds=time.perf_counter() - started, + model_class=type(wrapper.model).__module__ + "." + type(wrapper.model).__name__, + weights="seed 0 random initialization; no pretrained weights", + weights_sha256=arrays_sha256({name: value.detach().numpy() + for name, value in wrapper.model.state_dict().items()})) + (out / "config.json").write_text(wrapper.model.config.to_json_string(use_diff=False), encoding="utf-8") + # Persist references and release their complete tensors before ORT/IR runs. + for case, (length, feed) in zip(report["cases"], feeds): + report.update(stage="torch", current_case=case["name"]) + started = time.perf_counter() + with torch.inference_mode(): + value = wrapper(*(torch.from_numpy(feed[n]) for n in ("input_ids", "attention_mask"))).numpy() + case["torch_seconds"] = time.perf_counter() - started + check = compare_logits(value, value, length) + case["torch"] = {"contract": check, + "sampling": sample_checked(value, value, length, adapter, references)} + np.save(out / f"{case['name']}.torch.npy", value) + del value + gc.collect() + report["stage"] = "export" + started = time.perf_counter() + with torch.inference_mode(): + exported = torch.onnx.export(wrapper, tuple(torch.from_numpy(feeds[0][1][n]) for n in ("input_ids", "attention_mask")), + dynamo=True, opset_version=18, input_names=["input_ids", "attention_mask"], + output_names=["logits"], dynamic_shapes=None, optimize=False, fallback=False) + exported.save(str(out / "model.onnx"), external_data=False) + report["export_seconds"] = time.perf_counter() - started + del exported, wrapper + gc.collect() + exported = onnx.load(out / "model.onnx") + check_onnx_contract(exported) + report["onnx"] = {"sha256": sha256(out / "model.onnx"), "bytes": (out / "model.onnx").stat().st_size, + "nodes": len(exported.graph.node), "opset": 18, + "exporter": "torch.onnx.export(dynamo=True, optimize=False, fallback=False)", + "inputs": {"input_ids": "INT64[1,256]", "attention_mask": "FP32[1,1,256,256]"}, + "output": "FP32[1,256,151936]", "output_bytes": SEQ * VOCAB * 4} + del exported + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + options.enable_cpu_mem_arena = False + report["stage"] = "ort" + started = time.perf_counter() + session = ort.InferenceSession(str(out / "model.onnx"), options, providers=["CPUExecutionProvider"]) + report["ort_load_seconds"] = time.perf_counter() - started + for case, (length, feed) in zip(report["cases"], feeds): + report["current_case"] = case["name"] + started = time.perf_counter() + actual = session.run(["logits"], feed)[0] + row = {"engine": "ort", "seconds": time.perf_counter() - started, "passed": False} + case["executions"].append(row) + expected = np.load(out / f"{case['name']}.torch.npy", mmap_mode="r") + row["vs_torch"] = compare_logits(actual, expected, length) + row["sampling"] = sample_checked(actual, expected, length, adapter, references) + row["passed"] = row["vs_torch"]["passed"] + if not row["passed"]: + raise ValueError(f"ORT numeric error for {case['name']}") + # IR is compared independently to both references without retaining a + # second live full-size output array from another execution engine. + np.save(out / f"{case['name']}.ort.npy", actual) + del actual, expected + del session + gc.collect() + report["stage"] = "parse" + started = time.perf_counter() + parser = ONNXParser() + program = parser.parse(str(out / "model.onnx")) + require_valid(program) + report["parse_seconds"] = time.perf_counter() - started + report["ir_optimization_seconds"] = {} + for level in LEVELS: + report.update(stage="ir", current_level=level) + started = time.perf_counter() + manager = create_optimization_pass_manager(level) + manager.before_pass = lambda p, value: require_valid(value) + manager.after_pass = lambda p, value: require_valid(value) + optimized = manager.run_pipeline(copy.deepcopy(program)).data + require_valid(optimized) + report["ir_optimization_seconds"][level] = time.perf_counter() - started + for case, (length, feed) in zip(report["cases"], feeds): + report["current_case"] = case["name"] + row = {"engine": f"ir-{level}", "passed": False} + case["executions"].append(row) + started = time.perf_counter() + result = IRInterpreter(optimized).run(feed, initializers=parser.initializers) + row.update(seconds=time.perf_counter() - started, executed_steps=result.executed_steps) + actual = result.return_value + expected = np.load(out / f"{case['name']}.torch.npy", mmap_mode="r") + row["vs_torch"] = compare_logits(actual, expected, length) + row["sampling"] = sample_checked(actual, expected, length, adapter, references) + del expected + expected = np.load(out / f"{case['name']}.ort.npy", mmap_mode="r") + row["vs_ort"] = compare_logits(actual, expected, length) + row["passed"] = row["vs_torch"]["passed"] and row["vs_ort"]["passed"] + del result, actual, expected + gc.collect() + if not row["passed"]: + raise ValueError(f"IR {level} numeric error for {case['name']}") + print(f"[{case['name']}/ir-{level}] PASS max_abs={row['vs_torch']['max_abs']:.9g}", flush=True) + del optimized + for case in report["cases"]: + case["passed"] = ([row["engine"] for row in case["executions"]] == ["ort", "ir-none", "ir-all"] + and all(row["passed"] for row in case["executions"])) + report.update(passed=bool(report["cases"]) and all(case["passed"] for case in report["cases"]), stage="complete") + report["completed_model_invocations"] = len(report["cases"]) + sum(len(case["executions"]) for case in report["cases"]) + report.pop("current_case", None) + report.pop("current_level", None) + + +def save_reports(out, report): + failures = [] + + def views(): + text = ("# W2 runtime/model integration\n\nResult: **" + ("PASS" if report["passed"] else "FAIL") + + "**\n\nOfficial two-layer Qwen3, random weights, full vocabulary; one forward and greedy token.\n" + + "Not pretrained inference or a generation loop. All positions use strict max_abs < 1e-5.\n\n```json\n" + + json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n```\n") + return {"report.md": text, "report.html": '
' + html.escape(text) + "
"} + + def failed(): + report["report_write_errors"] = failures + if report["passed"]: + report.update(passed=False, stage="report-write", error="Incomplete integration evidence") + + def write_views(): + for name, value in views().items(): + try: + (out / name).write_text(value, encoding="utf-8") + except OSError as exc: + failures.append(f"{name}: {type(exc).__name__}: {exc}") + + write_views() + if failures: + failed() + write_views() + pending = out / ".report.json.tmp" + try: + pending.write_text(json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8") + pending.replace(out / "report.json") + except OSError as exc: + failures.append(f"report.json: {type(exc).__name__}: {exc}") + failed() + write_views() + finally: + try: + pending.unlink(missing_ok=True) + except OSError: + pass + if failures: + print("; ".join(failures), file=sys.stderr) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokenizer-dir", required=True, type=Path) + parser.add_argument("--output-dir", default=Path("output/w2-runtime-model"), type=Path) + args = parser.parse_args(argv) + out = args.output_dir.resolve() + if out.exists() and (not out.is_dir() or any(out.iterdir())): + parser.error("--output-dir must be absent or empty; preserve earlier evidence") + out.mkdir(parents=True, exist_ok=True) + report = {"gate": "integration:runtime-model", "passed": False, "stage": "initialize", + "created_at": datetime.now(timezone.utc).isoformat(), "model_id": MANIFEST["model_id"], + "tokenizer_revision": MANIFEST["revision"], "cases": []} + started = time.perf_counter() + try: + report["source_sha256"] = source_evidence() + report["checkout"] = checkout_evidence() + run_probe(args.tokenizer_dir.resolve(), out, report) + except Exception as exc: + report.update(passed=False, error=f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + save_reports(out, report) + print(f"[integration:runtime-model] {'PASS' if report['passed'] else 'FAIL'}; stage={report['stage']}; report={out / 'report.json'}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index a588db5..670aa63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ classifiers = [ riscv = ["tinyfive"] llvm = ["llvmlite"] # optional: LLVM IR JIT execution verify = ["onnxruntime"] # optional: ONNX Runtime comparison +llm = ["tokenizers>=0.21,<0.22"] # optional: local Qwen3 text tokenizer; no Torch all = ["tinyfive", "llvmlite", "onnxruntime"] topic06 = ["tinyfive", "pytest"] topic06-report = ["tinyfive", "pytest", "jinja2", "matplotlib"] diff --git a/scratchv/backend/tensor_c_codegen.py b/scratchv/backend/tensor_c_codegen.py index f333946..5b45db4 100644 --- a/scratchv/backend/tensor_c_codegen.py +++ b/scratchv/backend/tensor_c_codegen.py @@ -304,6 +304,15 @@ def _constant(self, value, data): spec = self.specs[value.name] if not isinstance(data, np.ndarray) or data.shape != spec.shape or data.dtype != spec.numpy_dtype: raise TensorCCodegenError(f"Initializer {value.name} dtype/shape disagrees with IR") + if value.is_constant: + # A scalar literal is part of the IR's meaning (and may have been + # used by optimizations). A caller binding cannot override it. + try: + expected = np.asarray(value.const_value, dtype=spec.numpy_dtype) + except (TypeError, ValueError, OverflowError) as exc: + raise TensorCCodegenError(f"Invalid scalar constant {value.name}") from exc + if data.shape != () or expected.shape != () or not np.array_equal(data, expected): + raise TensorCCodegenError(f"Initializer {value.name} disagrees with scalar constant") self.constant_bytes += spec.nbytes if self.constant_bytes > self.max_constant_bytes: raise TensorCCodegenError("Constant tensor storage exceeds configured limit") diff --git a/scratchv/runtime/llm_inputs.py b/scratchv/runtime/llm_inputs.py new file mode 100644 index 0000000..c1fadb6 --- /dev/null +++ b/scratchv/runtime/llm_inputs.py @@ -0,0 +1,179 @@ +"""Host input preparation and greedy sampling for fixed-length Qwen3 graphs. + +These helpers do not tokenize text, invoke a model, or run a generation loop. +The model ABI remains batch 1, sequence length 256, INT64 IDs and FP32 tensors. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Literal + +import numpy as np + + +SEQ_LENGTH = 256 +QWEN3_VOCAB_SIZE = 151936 +StopReason = Literal["eos", "capacity", "max_new_tokens"] + + +def _integer(value: object, name: str, *, minimum: int = 0, + maximum: int | None = None) -> int: + if isinstance(value, (bool, np.bool_)) or not isinstance(value, (int, np.integer)): + raise TypeError(f"{name} must be an integer, not {type(value).__name__}") + result = int(value) + if result < minimum or (maximum is not None and result > maximum): + bound = f"[{minimum}, {maximum}]" if maximum is not None else f">= {minimum}" + raise ValueError(f"{name} must be in {bound}; got {result}") + return result + + +def _vocabulary_size(vocab_size: object) -> int: + # A vocabulary size is a count; its largest valid ID must fit INT64. + return _integer(vocab_size, "vocab_size", minimum=1, + maximum=int(np.iinfo(np.int64).max) + 1) + + +def _valid_length(valid_length: object) -> int: + return _integer(valid_length, "valid_length", minimum=1, maximum=SEQ_LENGTH) + + +def _token_sequence(token_ids: Sequence[int] | np.ndarray, *, + vocab_size: int, name: str) -> list[int]: + if isinstance(token_ids, np.ndarray): + if np.ma.isMaskedArray(token_ids): + raise TypeError(f"{name} must not be a masked array") + if token_ids.ndim != 1: + raise ValueError(f"{name} must be a one-dimensional sequence") + if token_ids.dtype.kind not in "iu": + raise TypeError(f"{name} array must have an integer dtype") + elif isinstance(token_ids, (str, bytes)) or not isinstance(token_ids, Sequence): + raise TypeError(f"{name} must be a sequence of integer token IDs") + return [_integer(value, f"{name}[{index}]", maximum=vocab_size - 1) + for index, value in enumerate(token_ids)] + + +@dataclass(frozen=True) +class PreparedInputs: + """Owned model input arrays plus the unpadded token count. + + Arrays are mutable for the model caller. ``as_feed`` returns references to + these arrays, so changing the feed also changes this instance's tensors. + Construct instances through :func:`prepare_inputs` for validated inputs. + """ + + input_ids: np.ndarray + attention_mask: np.ndarray + valid_length: int + + def as_feed(self) -> dict[str, np.ndarray]: + return {"input_ids": self.input_ids, "attention_mask": self.attention_mask} + + +def build_attention_mask(valid_length: int) -> np.ndarray: + """Return causal + key-padding additive mask ``[1, 1, 256, 256]``. + + Query row ``q`` may attend to key ``k`` exactly when ``k <= q`` and + ``k < valid_length``. Padded query rows still see earlier valid keys; fully + masking those rows is not the fixed Qwen3 export's mask contract. + """ + length = _valid_length(valid_length) + positions = np.arange(SEQ_LENGTH) + allowed = (positions[None, :] <= positions[:, None]) & (positions[None, :] < length) + mask = np.where(allowed, np.float32(0), np.finfo(np.float32).min) + return mask.reshape(1, 1, SEQ_LENGTH, SEQ_LENGTH) + + +def prepare_inputs(token_ids: Sequence[int] | np.ndarray, *, pad_token_id: int, + vocab_size: int = QWEN3_VOCAB_SIZE) -> PreparedInputs: + """Validate unpadded IDs and return right-padded, contiguous ABI inputs. + + The length comes from the supplied sequence, never from occurrences of the + pad ID: a valid prompt may itself contain that ID. Empty or over-capacity + prompts fail instead of being silently truncated. Integer arrays of other + widths are accepted only after every ID passes the INT64 range checks. + """ + vocab = _vocabulary_size(vocab_size) + pad = _integer(pad_token_id, "pad_token_id", maximum=vocab - 1) + tokens = _token_sequence(token_ids, vocab_size=vocab, name="token_ids") + length = _valid_length(len(tokens)) + ids = np.full((1, SEQ_LENGTH), pad, dtype=np.int64) + ids[0, :length] = tokens + return PreparedInputs(ids, build_attention_mask(length), length) + + +def _fp32_finite(array: object, name: str) -> np.ndarray: + if not isinstance(array, np.ndarray): + raise TypeError(f"{name} must be a NumPy ndarray") + if np.ma.isMaskedArray(array): + raise TypeError(f"{name} must not be a masked array") + if array.dtype != np.dtype(np.float32): + raise TypeError(f"{name} must have dtype float32; got {array.dtype}") + if not np.isfinite(array).all(): + raise ValueError(f"{name} must contain only finite values") + return array + + +def last_valid_logits(logits: np.ndarray, valid_length: int, *, + vocab_size: int = QWEN3_VOCAB_SIZE) -> np.ndarray: + """Validate the full model output and copy the last valid token's logits. + + All positions, including padding, must be finite. Returning an owned vector + lets a runner release its full output buffer after selecting this row. + """ + length = _valid_length(valid_length) + vocab = _vocabulary_size(vocab_size) + if not isinstance(logits, np.ndarray): + raise TypeError("logits must be a NumPy ndarray") + expected = (1, SEQ_LENGTH, vocab) + if logits.shape != expected: + raise ValueError(f"logits must have shape {expected}; got {logits.shape}") + checked = _fp32_finite(logits, "logits") + return checked[0, length - 1, :].copy() + + +def greedy_token(logits: np.ndarray) -> int: + """Argmax of one finite FP32 vocabulary vector; ties choose the lowest ID.""" + if not isinstance(logits, np.ndarray): + raise TypeError("logits must be a NumPy ndarray") + if logits.ndim != 1 or logits.size == 0: + raise ValueError("logits must be a nonempty one-dimensional vocabulary vector") + checked = _fp32_finite(logits, "logits") + return int(np.argmax(checked)) + + +def greedy_next_token(logits: np.ndarray, valid_length: int, *, + vocab_size: int = QWEN3_VOCAB_SIZE) -> int: + """Select the next ID from ``logits[0, valid_length - 1]``, without softmax.""" + return greedy_token(last_valid_logits(logits, valid_length, vocab_size=vocab_size)) + + +def generation_stop_reason(*, valid_length: int, generated_tokens: int, + max_new_tokens: int, last_token_id: int | None = None, + eos_token_ids: Sequence[int] | np.ndarray = (), + vocab_size: int = QWEN3_VOCAB_SIZE) -> StopReason | None: + """Report whether a host loop must stop before its next forward call. + + Call initially with ``generated_tokens=0`` and after appending each newly + sampled ID. ``valid_length`` includes the nonempty prompt and generated IDs. + EOS applies only to a generated token, not an EOS-like token in the prompt. + Simultaneous stop conditions resolve as EOS, capacity, then token budget. + No truncation, model execution or token append is performed here. + """ + length = _valid_length(valid_length) + generated = _integer(generated_tokens, "generated_tokens", maximum=length - 1) + budget = _integer(max_new_tokens, "max_new_tokens") + vocab = _vocabulary_size(vocab_size) + eos = _token_sequence(eos_token_ids, vocab_size=vocab, name="eos_token_ids") + if generated and last_token_id is None: + raise ValueError("last_token_id is required after generating a token") + last = None if last_token_id is None else _integer(last_token_id, "last_token_id", + maximum=vocab - 1) + if generated and last in eos: + return "eos" + if length == SEQ_LENGTH: + return "capacity" + if generated >= budget: + return "max_new_tokens" + return None diff --git a/scratchv/runtime/qwen3_tokenizer.py b/scratchv/runtime/qwen3_tokenizer.py new file mode 100644 index 0000000..70ae2d7 --- /dev/null +++ b/scratchv/runtime/qwen3_tokenizer.py @@ -0,0 +1,223 @@ +"""Local Qwen3 text tokenization, without a Torch or Transformers dependency. + +The caller supplies a verified local snapshot containing tokenizer.json, +tokenizer_config.json, config.json and generation_config.json. This adapter never +downloads files or renders chat templates. Encoding follows tokenizer.json, +including its NFC normalization; a text round trip need not preserve decomposed +Unicode byte-for-byte. Model output width and the set of decodable token IDs are +different contracts (151936 versus 151669 entries in Qwen3-0.6B). +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Set +from numbers import Integral +from pathlib import Path + +from tokenizers import Tokenizer + + +def _read_config(path: Path) -> dict: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path.name} must contain a JSON object") + return value + + +def _boolean(value: object, name: str) -> bool: + if not isinstance(value, bool): + raise TypeError(f"{name} must be bool") + return value + + +def _cleanup(text: str) -> str: + # Match PreTrainedTokenizerBase.clean_up_tokenization in Transformers 4.51.3. + for before, after in ( + (" .", "."), (" ?", "?"), (" !", "!"), (" ,", ","), (" ' ", "'"), + (" n't", "n't"), (" 'm", "'m"), (" 's", "'s"), (" 've", "'ve"), + (" 're", "'re"), + ): + text = text.replace(before, after) + return text + + +class Qwen3Tokenizer: + """Tokenizer for a local Qwen3 snapshot; use :meth:`from_directory`. + + ``eos_token_id`` is the tokenizer's EOS. ``generation_eos_token_ids`` comes + from generation_config.json and includes *all* generation stop IDs. The + model's BOS metadata does not cause encoding to insert a BOS token. + """ + + @classmethod + def from_directory(cls, directory: str | Path) -> Qwen3Tokenizer: + directory = Path(directory) + config = _read_config(directory / "tokenizer_config.json") + model = _read_config(directory / "config.json") + generation = _read_config(directory / "generation_config.json") + if config.get("tokenizer_class") not in ("Qwen2Tokenizer", "Qwen2TokenizerFast"): + raise ValueError("tokenizer_class must identify the Qwen2 tokenizer used by Qwen3") + if model.get("model_type") != "qwen3": + raise ValueError("config.json model_type must be qwen3") + model_vocab_size = model.get("vocab_size") + if type(model_vocab_size) is not int or model_vocab_size <= 0: + raise ValueError("model vocab_size must be a positive integer") + for name in ("add_bos_token", "add_eos_token", "split_special_tokens", "add_prefix_space"): + if _boolean(config.get(name, False), name): + raise ValueError(f"Qwen3 text adapter does not support {name}=True") + # Transformers can add/mark these tokens during initialization. Loading + # tokenizer.json alone would silently ignore those extra semantics. + for name in ("mask_token", "sep_token", "cls_token"): + if config.get(name) is not None: + raise ValueError(f"Qwen3 text adapter does not support {name}") + if config.get("extra_special_tokens") not in (None, {}): + raise ValueError("Qwen3 text adapter does not support extra_special_tokens") + + self = cls() + self._tokenizer = Tokenizer.from_file(str(directory / "tokenizer.json")) + if self._tokenizer.padding is not None or self._tokenizer.truncation is not None: + raise ValueError("tokenizer.json must not enable automatic padding or truncation") + self._validate_prefix_space() + self.model_vocab_size = model_vocab_size + self.base_vocab_size = self._tokenizer.get_vocab_size(with_added_tokens=False) + self._valid_token_ids = frozenset(self._tokenizer.get_vocab(with_added_tokens=True).values()) + self.tokenizer_vocab_size = len(self._valid_token_ids) + if not self._valid_token_ids or any( + token_id < 0 or token_id >= model_vocab_size for token_id in self._valid_token_ids + ): + raise ValueError("tokenizer vocabulary does not fit the model vocab_size") + self.clean_up_tokenization_spaces = _boolean( + config.get("clean_up_tokenization_spaces", False), "clean_up_tokenization_spaces" + ) + self._validate_added_tokens(config) + self.pad_token_id = self._special_id(config, "pad_token", required=True) + self.eos_token_id = self._special_id(config, "eos_token", required=True) + self.bos_token_id = self._special_id(config, "bos_token", required=False) + # The official Qwen2 fast class defaults a missing unk_token to + # endoftext; an explicit JSON null intentionally disables it instead. + self.unk_token_id = self._special_id( + {"unk_token": config.get("unk_token", "<|endoftext|>")}, "unk_token", required=False + ) + eos_ids = generation.get("eos_token_id") + if type(eos_ids) is int: + eos_ids = [eos_ids] + if not isinstance(eos_ids, list) or not eos_ids: + raise ValueError("generation eos_token_id must be an integer or nonempty list") + self.generation_eos_token_ids = tuple(self.validate_token_ids(eos_ids)) + if self.eos_token_id not in self.generation_eos_token_ids: + raise ValueError("generation EOS IDs must include the tokenizer EOS") + if ( + generation.get("pad_token_id") != self.pad_token_id + or type(generation.get("pad_token_id")) is not int + ): + raise ValueError("generation pad_token_id must match the tokenizer pad token") + # Qwen3's JSON post-processor is ByteLevel and inserts no token IDs. + if self._tokenizer.num_special_tokens_to_add(is_pair=False) != 0: + raise ValueError("Qwen3 text encoding must not automatically insert BOS/EOS tokens") + return self + + def _validate_prefix_space(self) -> None: + # The fixed snapshot uses a Sequence containing ByteLevel(False). + # Transformers may override a top-level ByteLevel setting from config; + # refusing prefix insertion keeps this narrow adapter unambiguous for + # both that layout and the official nested layout. + pre_tokenizer = self._tokenizer.pre_tokenizer + pending = [] if pre_tokenizer is None else [json.loads(pre_tokenizer.__getstate__())] + while pending: + state = pending.pop() + if state.get("type") == "ByteLevel" and state.get("add_prefix_space"): + raise ValueError("tokenizer.json must not enable add_prefix_space") + if state.get("type") == "Sequence": + pending.extend(state["pretokenizers"]) + + @property + def valid_token_ids(self) -> frozenset[int]: + """Actual vocabulary members, including added tokens; not model rows.""" + return self._valid_token_ids + + def is_valid_token_id(self, token_id: object) -> bool: + return ( + isinstance(token_id, Integral) + and not isinstance(token_id, bool) + and int(token_id) in self._valid_token_ids + ) + + def validate_token_ids(self, token_ids: Iterable[int]) -> list[int]: + """Copy an ordered iterable to ints, refusing unknown IDs or bools.""" + if ( + isinstance(token_ids, (str, bytes, bytearray, Mapping, Set)) + or not isinstance(token_ids, Iterable) + ): + raise TypeError("token_ids must be an ordered iterable of integer IDs") + result = [] + for index, token_id in enumerate(token_ids): + if not isinstance(token_id, Integral) or isinstance(token_id, bool): + raise TypeError(f"token_ids[{index}] must be an integer (not bool)") + if not self.is_valid_token_id(token_id): + raise ValueError(f"token_ids[{index}]={token_id} is not in the tokenizer vocabulary") + result.append(int(token_id)) + return result + + def encode(self, text: str, *, add_special_tokens: bool = True) -> list[int]: + """Encode one string, with no padding, truncation, BOS/EOS, or chat template.""" + if not isinstance(text, str): + raise TypeError("text must be str") + text.encode("utf-8", errors="strict") # Reject lone surrogates explicitly. + _boolean(add_special_tokens, "add_special_tokens") + return self._tokenizer.encode(text, add_special_tokens=add_special_tokens).ids + + def decode( + self, + token_ids: Iterable[int] | int, + *, + skip_special_tokens: bool = False, + clean_up_tokenization_spaces: bool | None = None, + ) -> str: + """Decode valid IDs; defaults match the official fast tokenizer API. + + Unlike raw tokenizers.decode, unknown IDs are errors, including unused + rows below model_vocab_size. A sampled unused row must not disappear. + """ + _boolean(skip_special_tokens, "skip_special_tokens") + cleanup = self.clean_up_tokenization_spaces + if clean_up_tokenization_spaces is not None: + cleanup = _boolean(clean_up_tokenization_spaces, "clean_up_tokenization_spaces") + if isinstance(token_ids, Integral): + token_ids = [token_ids] + ids = self.validate_token_ids(token_ids) + text = self._tokenizer.decode(ids, skip_special_tokens=skip_special_tokens) + return _cleanup(text) if cleanup else text + + def _validate_added_tokens(self, config: dict) -> None: + decoder = config.get("added_tokens_decoder") + if not isinstance(decoder, dict): + raise ValueError("added_tokens_decoder must be an object") + actual = self._tokenizer.get_added_tokens_decoder() + if set(decoder) != {str(token_id) for token_id in actual}: + raise ValueError("tokenizer.json and tokenizer_config.json added token IDs differ") + for token_id, expected in decoder.items(): + if not isinstance(expected, dict): + raise ValueError(f"added token {token_id} must be an object") + token = actual[int(token_id)] + for name in ("content", "lstrip", "rstrip", "single_word", "normalized", "special"): + value = expected.get(name) + if type(value) is not type(getattr(token, name)) or value != getattr(token, name): + raise ValueError(f"added token {token_id} has inconsistent {name}") + additional = config.get("additional_special_tokens", []) + if not isinstance(additional, list): + raise ValueError("additional_special_tokens must be a list") + for content in additional: + self._special_id({"token": content}, "token", required=True) + + def _special_id(self, config: dict, name: str, *, required: bool) -> int | None: + content = config.get(name) + if content is None and not required: + return None + if not isinstance(content, str) or not content: + raise ValueError(f"{name} must be a nonempty token string") + token_id = self._tokenizer.token_to_id(content) + token = self._tokenizer.get_added_tokens_decoder().get(token_id) + if token_id is None or token is None or not token.special: + raise ValueError(f"{name}={content!r} is not a configured special token") + return token_id diff --git a/scratchv/runtime/riscv_tensor.py b/scratchv/runtime/riscv_tensor.py index 82d56aa..d6a401f 100644 --- a/scratchv/runtime/riscv_tensor.py +++ b/scratchv/runtime/riscv_tensor.py @@ -17,6 +17,7 @@ import signal import struct import subprocess +import threading import time from typing import Any, Mapping, Sequence @@ -165,6 +166,13 @@ def _run_process(command, *, cwd, timeout, env=None) -> subprocess.CompletedProc with subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) as process: job = _windows_job(process) + # The W2 supervisor first sends SIGTERM to the probe. This invocation + # owns a different session, so killing the probe group alone would + # orphan its compiler/QEMU. Unwind through cleanup before exiting. + previous_term = None + if os.name != "nt" and threading.current_thread() is threading.main_thread(): + previous_term = signal.getsignal(signal.SIGTERM) + signal.signal(signal.SIGTERM, _terminate_invocation) try: stdout, stderr = process.communicate(timeout=timeout) except subprocess.TimeoutExpired as exc: @@ -182,12 +190,44 @@ def _run_process(command, *, cwd, timeout, env=None) -> subprocess.CompletedProc process.wait(timeout=5) stdout, stderr = drain.output or b"", drain.stderr or b"" raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from exc + except BaseException: + # Cancellation and Ctrl-C must also close the owned invocation. + # Do not replace the primary interruption with a cleanup failure. + try: + _stop_process_tree(process, job) + process.communicate(timeout=5) + except (OSError, subprocess.SubprocessError): + process.kill() + for stream in (process.stdout, process.stderr): + if stream: + stream.close() + raise finally: + if previous_term is not None: + signal.signal(signal.SIGTERM, previous_term) if job is not None: job[0].CloseHandle(job[1]) return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) +def _terminate_invocation(signum, frame): + raise SystemExit(128 + signum) + + +def _save_process_logs(directory: Path, prefix: str, stdout, stderr): + """Attempt both logs without allowing a secondary I/O error to hide execution.""" + errors = [] + for stream, content in (("stdout", stdout), ("stderr", stderr)): + path = directory / f"{prefix}.{stream}" + try: + path.write_bytes(content or b"") + except OSError as exc: + errors.append((path.name, exc)) + detail = "; ".join(f"{name}: {error}" for name, error in errors) + suffix = f"; failed to preserve {prefix} logs: {detail}" if errors else "" + return suffix, errors[0][1] if errors else None + + def toolchain_versions(toolchain: RiscVToolchain) -> dict[str, str]: compiler = ([toolchain.cc[0], "version"] if toolchain.is_zig else [*toolchain.cc, "--version"]) @@ -516,15 +556,15 @@ def build_riscv_tensor( try: process = _run_process(command, cwd=build_dir, timeout=timeout, env=environment) except subprocess.TimeoutExpired as exc: - (build_dir / "compiler.stdout").write_bytes(exc.stdout or b"") - (build_dir / "compiler.stderr").write_bytes(exc.stderr or b"") - raise TimeoutError(f"RISC-V compilation exceeded {timeout} seconds") from exc + log_message, _ = _save_process_logs(build_dir, "compiler", exc.stdout, exc.stderr) + raise TimeoutError(f"RISC-V compilation exceeded {timeout} seconds{log_message}") from exc elapsed = time.perf_counter() - started - (build_dir / "compiler.stdout").write_bytes(process.stdout) - (build_dir / "compiler.stderr").write_bytes(process.stderr) + log_message, log_error = _save_process_logs(build_dir, "compiler", process.stdout, process.stderr) if process.returncode: message = process.stderr.decode("utf-8", errors="replace")[-12000:] - raise RuntimeError(f"RISC-V tensor compilation failed ({process.returncode}):\n{message}") + raise RuntimeError(f"RISC-V tensor compilation failed ({process.returncode}):\n{message}{log_message}") + if log_error is not None: + raise RuntimeError(log_message.lstrip("; ")) from log_error validate_riscv_elf(elf) return RiscVTensorExecutable(elf, tuple(artifact.inputs), artifact.output, toolchain, tuple(command), elapsed, int(artifact.workspace_bytes), @@ -562,17 +602,12 @@ def run_riscv_tensor( except subprocess.TimeoutExpired as exc: elapsed = time.perf_counter() - started message = f"RISC-V QEMU execution exceeded {timeout} seconds" - try: - stdout_path.write_bytes(exc.stdout or b"") - stderr_path.write_bytes(exc.stderr or b"") - except OSError as log_error: - message += f"; failed to preserve QEMU logs: {log_error}" - raise RiscVTensorTimeoutError(message, elapsed_s=elapsed, + log_message, _ = _save_process_logs(run_dir, "qemu", exc.stdout, exc.stderr) + raise RiscVTensorTimeoutError(message + log_message, elapsed_s=elapsed, command=command, timeout_s=timeout) from exc elapsed = time.perf_counter() - started + log_message, log_error = _save_process_logs(run_dir, "qemu", process.stdout, process.stderr) try: - stdout_path.write_bytes(process.stdout) - stderr_path.write_bytes(process.stderr) if process.returncode: diagnostic = process.stderr.decode("utf-8", errors="replace")[-4000:] if uart_path.exists(): @@ -584,5 +619,8 @@ def run_riscv_tensor( raise RuntimeError("QEMU exited without producing UART tensor output") result = decode_tensor_frame(uart_path.read_bytes(), executable.output) except Exception as exc: - raise RiscVTensorExecutionError(str(exc), elapsed_s=elapsed, command=command) from exc + raise RiscVTensorExecutionError(str(exc) + log_message, elapsed_s=elapsed, command=command) from exc + if log_error is not None: + raise RiscVTensorExecutionError(log_message.lstrip("; "), elapsed_s=elapsed, + command=command) from log_error return RiscVTensorRun(result, elapsed, tuple(command), uart_path, stdout_path, stderr_path) diff --git a/scripts/check_w2_ci_results.py b/scripts/check_w2_ci_results.py new file mode 100644 index 0000000..935fa55 --- /dev/null +++ b/scripts/check_w2_ci_results.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Fail W2 CI acceptance if a required job failed, was skipped or is absent.""" +from __future__ import annotations + +import json +import os +import sys + +REQUIRED_JOBS = ("llm-deploy", "full-qwen3-frontend") + + +def validate_results(results): + if not isinstance(results, dict): + raise ValueError("W2_JOB_RESULTS must contain a JSON object") + failures = [] + for name in REQUIRED_JOBS: + job = results.get(name) + outcome = job.get("result") if isinstance(job, dict) else None + if outcome != "success": + failures.append(f"{name}: {outcome or 'missing result'}") + if failures: + raise ValueError("Required W2 jobs did not succeed: " + "; ".join(failures)) + + +def main(): + print("## W2 required acceptance\n") + try: + validate_results(json.loads(os.environ.get("W2_JOB_RESULTS", "{}"))) + except (ValueError, TypeError) as exc: + print(f"FAIL: {exc}") + return 1 + print("PASS: small numerical/runtime gates and complete 28-layer frontend audit succeeded.") + print("\nThis does not certify second-person reproduction or full-model IR/QEMU inference.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_w2_acceptance.py b/scripts/run_w2_acceptance.py new file mode 100644 index 0000000..60439dd --- /dev/null +++ b/scripts/run_w2_acceptance.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Run fresh W2 acceptance gates, without installing dependencies or publishing. + +Use an existing pinned Python environment and explicit local asset/tool paths. +All seven gates are required for PASS. A successful --gates subset is PARTIAL +(exit 2); failures exit 1. Reports never certify independent-person signoff, +full-model numerical execution, or a complete text-generation loop. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import datetime, timezone +import hashlib +from html import escape +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +ROOT = Path(__file__).resolve().parents[1] +GATES = ("frontend", "backend", "runtime", "runtime-model", "small-ir", "small-qemu", "full-parse") +SCRIPTS = { + "backend": "probes/w2_backend_ops/run.py", "runtime": "probes/w2_runtime/run.py", + "runtime-model": "probes/w2_runtime_model/run.py", "small-ir": "probes/w2_qwen3_small/run.py", + "small-qemu": "probes/w2_qwen3_small/riscv.py", "full-parse": "probes/w2_qwen3_parse/run.py", +} +SMALL_CASES = ("full_seed_0", "full_seed_42", "one_token", "short_17", "short_255", + "changed_future", "changed_padding") +FAMILIES = {"matmul", "elementwise", "softmax", "rmsnorm", "rope", "swiglu", "gqa"} + + +def sha256(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def source_fingerprints(root=ROOT): + files = {root / "pyproject.toml", root / "scripts/run_w2_acceptance.py", + root / "scripts/check_w2_ci_results.py"} + for directory in ("scratchv", "probes", "requirements"): + files.update(p for p in (root / directory).rglob("*") + if p.is_file() and p.suffix in {".py", ".json", ".txt"} + and not any(part in {"out", "output", "__pycache__"} for part in p.relative_to(root).parts)) + files.update((root / "tests").glob("test_qwen3*.py")) + files.update((root / "tests").glob("test_w2*.py")) + files.add(root / "tests/test_llm_inputs.py") + return {p.relative_to(root).as_posix(): sha256(p) for p in sorted(files) if p.is_file()} + + +def checkout_evidence(root=ROOT): + """A source archive is valid evidence too; Git absence is not a failure.""" + if not (root / ".git").exists(): + # A copied source tree may sit under another checkout's ignored output/. + # Git's parent-directory discovery must not attribute that archive to it. + return {"head": None, "status": "unknown", "reason": "Source directory has no .git metadata"} + try: + def git(*args): + return subprocess.run(["git", *args], cwd=root, capture_output=True, + check=True, timeout=15).stdout + return {"head": git("rev-parse", "HEAD").decode().strip(), + "status": git("status", "--porcelain=v1", "--untracked-files=all").decode("utf-8", "replace"), + "tracked_diff_sha256": hashlib.sha256(git("diff", "--binary", "HEAD", "--")).hexdigest(), + "diff_scope": "Tracked staged and unstaged changes; source_sha256 also covers new source files"} + except (OSError, subprocess.SubprocessError) as exc: + return {"head": None, "status": "unknown", "error": f"{type(exc).__name__}: {exc}"} + + +@dataclass(frozen=True) +class Gate: + name: str + command: tuple[str, ...] + output: Path + dependencies: tuple[str, ...] = () + + +def build_plan(args, root=ROOT): + selected = set(args.gates) + if "small-qemu" in selected: + selected.add("small-ir") + if "runtime-model" in selected: + selected.add("runtime") + plans = [] + for name in GATES: + if name not in selected: + continue + out = args.output_dir / name + command = [args.python, "-B", "-X", "utf8"] + if name == "frontend": + command += ["-m", "pytest", "tests/test_qwen3_frontend_patterns.py", "-q", "-p", "no:cacheprovider", + "--junit-xml", str(out / "tests.xml"), "--basetemp", str(args.output_dir / "frontend-pytest-temp")] + else: + command += [str(root / SCRIPTS[name]), "--output-dir", str(out)] + if name in {"backend", "small-qemu"}: + command += ["--timeout", str(args.qemu_timeout)] + for option in ("cc", "qemu"): + if getattr(args, option): + command += ["--" + option, getattr(args, option)] + if name in {"runtime", "runtime-model"}: + command += ["--tokenizer-dir", str(args.tokenizer_dir)] + if name == "runtime": + command += ["--mode", args.tokenizer_mode] + if name == "small-qemu": + command += ["--model-dir", str(args.output_dir / "small-ir")] + if name == "full-parse": + command += ["--model-dir", str(args.model_dir), "--mode", args.model_mode, + "--timeout", str(args.parse_timeout)] + dependencies = (("small-ir",) if name == "small-qemu" else + ("runtime",) if name == "runtime-model" else ()) + plans.append(Gate(name, tuple(command), out, dependencies)) + return plans + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def check_numeric_tree(value): + """Reject contradictory nested failures and NaN/Infinity numeric evidence.""" + if isinstance(value, dict): + if "passed" in value: + require(value["passed"] is True, "Nested validation did not pass") + if "max_abs" in value: + number = value["max_abs"] + require(type(number) in (int, float) and math.isfinite(number) and 0 <= number < 1e-5, + "Numeric error is missing, nonfinite or outside strict 1e-5 threshold") + for child in value.values(): + check_numeric_tree(child) + elif isinstance(value, list): + for child in value: + check_numeric_tree(child) + elif isinstance(value, float): + require(math.isfinite(value), "Nonfinite report value cannot be accepted as evidence") + + +def checked_rows(rows, count, names=None): + require(isinstance(rows, list) and len(rows) == count, f"Expected {count} complete cases") + require(all(isinstance(row, dict) and row.get("passed") is True for row in rows), "Incomplete case results") + if names is not None: + require([row.get("name") for row in rows] == list(names), "Missing, duplicated or reordered cases") + return rows + + +def validate_gate_report(gate, sources, root=ROOT): + if gate.name == "frontend": + path = gate.output / "tests.xml" + cases = ET.parse(path).getroot().findall(".//testcase") + require(len(cases) == 22, "Frontend must execute all 22 pattern tests") + require(len({(case.get("classname"), case.get("name")) for case in cases}) == 22, + "Duplicated frontend test results") + require(not any(case.find(tag) is not None for case in cases for tag in ("failure", "error", "skipped")), + "Frontend failures, errors and skips are not passing acceptance") + return {"tests": len(cases), "report_sha256": sha256(path)} + path = gate.output / "report.json" + data = json.loads(path.read_text(encoding="utf-8")) + require(data.get("passed") is True and data.get("stage") == "complete", + str(data.get("error", "Child gate did not complete successfully"))) + for name in ("report.md", "report.html"): + require((gate.output / name).is_file(), f"Missing child report {name}") + hashes = data.get("source_sha256", data.get("provenance", {}).get("source_sha256")) + require(isinstance(hashes, dict) and bool(hashes), "Missing child source fingerprints") + require(hashes.get(SCRIPTS[gate.name]) == sources.get(SCRIPTS[gate.name]), "Gate script fingerprint mismatch") + require(all(name in sources and sources[name] == value for name, value in hashes.items()), + "Child source fingerprints do not match this acceptance run") + check_numeric_tree(data) + summary = {"report_sha256": sha256(path), "child_seconds": data.get("seconds")} + if gate.name == "backend": + require(data.get("backend") == "ScratchV tensor-c -> RV64GC -> QEMU", "Unexpected backend gate identity") + rows = checked_rows(data.get("cases"), 16) + require(len({row.get("name") for row in rows}) == 16 and {row.get("family") for row in rows} == FAMILIES, + "Missing backend families or duplicated cases") + for row in rows: + runs = checked_rows(row.get("executions"), 2) + require([run.get("optimization") for run in runs] == ["none", "all"], "Incomplete optimization levels") + for run in runs: + require(all(run.get(key, {}).get("passed") is True for key in + ("ir_vs_ort", "ir_vs_numpy", "qemu_vs_ort", "qemu_vs_numpy")), "Missing backend comparisons") + require(bool(run.get("command")) and bool(run.get("elf_sha256")), "Missing real QEMU evidence") + require((gate.output / row["name"] / run["optimization"] / "qemu.npy").is_file(), "Missing QEMU output") + require(data.get("completed_executions") == data.get("passed_executions") == 32, "Incomplete backend executions") + summary.update(executions=32, qemu_seconds=data.get("qemu_process_wall_seconds", {}).get("total")) + elif gate.name == "runtime": + require(data.get("gate") == "unit:runtime", "Unexpected runtime gate identity") + corpus = json.loads((root / "probes/w2_runtime/corpus.json").read_text(encoding="utf-8")) + rows = checked_rows(data.get("tokenizer_cases"), 20, [row["name"] for row in corpus["cases"]]) + require(all(row.get("checks") == 24 for row in rows), "Incomplete tokenizer comparisons") + checked_rows(data.get("runtime_cases"), 5) + summary.update(corpus_cases=20, tokenizer_comparisons=480, runtime_cases=5) + elif gate.name == "runtime-model": + require(data.get("gate") == "integration:runtime-model", "Unexpected runtime-model gate identity") + require(data.get("config", {}).get("vocab_size") == 151936 + and data.get("config", {}).get("num_hidden_layers") == 2, "Missing real tokenizer/model vocabulary contract") + require(data.get("completed_model_invocations") == 8, "Incomplete runtime-model forward invocations") + rows = checked_rows(data.get("cases"), 2, ("chinese", "special_pad_in_prompt")) + require(rows[1].get("pad_id_inside_prompt") is True, "Missing in-prompt PAD test") + for row in rows: + require(row.get("torch", {}).get("contract", {}).get("passed") is True + and row.get("torch", {}).get("sampling", {}).get("passed") is True, + "Missing independent model and sampling reference") + executions = checked_rows(row.get("executions"), 3) + require([run.get("engine") for run in executions] == ["ort", "ir-none", "ir-all"], + "Missing runtime-model execution engines") + for run in executions: + require(run.get("vs_torch", {}).get("passed") is True + and run.get("sampling", {}).get("passed") is True, "Missing runtime-model comparisons") + if run["engine"].startswith("ir-"): + require(run.get("vs_ort", {}).get("passed") is True, "Missing IR/ORT comparison") + summary.update(cases=2, executions=6) + elif gate.name == "small-ir": + require(data.get("config", {}).get("num_hidden_layers") == 2, "Expected real two-layer configuration") + checked_rows(data.get("cases"), 7, SMALL_CASES) + checked_rows(data.get("invariants"), 6) + for filename, field in (("model.onnx", "model_sha256"), ("diagnostics.onnx", "diagnostics_sha256")): + require(sha256(gate.output / filename) == data.get("onnx", {}).get(field), "Export artifact hash mismatch") + require((gate.output / "checkpoints.json").is_file(), "Missing diagnostic checkpoint schema") + summary["cases"] = 7 + elif gate.name == "small-qemu": + rows = checked_rows(data.get("cases"), 7, SMALL_CASES) + groups = [(graph, level) for graph in ("normal", "diagnostic") for level in ("none", "all")] + for row in rows: + runs = checked_rows(row.get("qemu"), 4) + require([(run.get("graph"), run.get("optimization")) for run in runs] == groups, "Missing QEMU graph/level") + require(all(run.get("status") == "success" and run.get("command") for run in runs), "Missing QEMU execution") + for run in runs: + require((gate.output / "runs" / row["name"] / f"{run['graph']}-{run['optimization']}" / "qemu.npy").is_file(), + "Missing two-layer QEMU output") + timing = data.get("timing", {}) + require(timing.get("completed_count") == timing.get("status_counts", {}).get("success") == 28, + "Incomplete two-layer QEMU executions") + require(data.get("export_evidence", {}).get("report_sha256") == sha256(gate.output.parent / "small-ir/report.json"), + "QEMU did not use the fresh export from this run") + summary.update(executions=28, qemu_seconds=timing.get("qemu_process_wall_seconds_total")) + elif gate.name == "full-parse": + require(data.get("gate") == "frontend:parse-qwen3" and data.get("schema_version") == 1, + "Unexpected full parse gate identity") + require(data.get("graph", {}).get("config", {}).get("num_hidden_layers") == 28, "Missing complete model configuration") + layers = checked_rows(data.get("graph", {}).get("layers"), 28) + require([row.get("layer") for row in layers] == list(range(28)), "Incomplete 28-layer chain") + require(data.get("audit", {}).get("passed") is True and data.get("ir_verifier", {}).get("passed") is True, + "Missing binding audit or IR verification") + for name in ("nodes.json", "bindings.json", "ir.txt"): + require(sha256(gate.output / name) == data.get("evidence", {}).get(name, {}).get("sha256"), + f"Missing or altered parse evidence: {name}") + summary["layers"] = 28 + return summary + + +def terminate_process_tree(process): + """The gates spawn compilers/QEMU/workers; stop owned descendants on timeout.""" + try: + if os.name == "nt": + subprocess.run(["taskkill", "/PID", str(process.pid), "/T", "/F"], capture_output=True, timeout=20, check=True) + else: + try: + # Let the probe unwind _run_process and stop its separate + # compiler/QEMU/worker session before the final group kill. + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=20) + + +def child_failure(row, gate, log): + try: + child = json.loads((gate.output / "report.json").read_text(encoding="utf-8")) + row.update(child_stage=child.get("stage"), child_error=child.get("error")) + except (OSError, ValueError): + pass + return row.get("child_error") or f"Child exited {row['returncode']}; inspect {log.name}" + + +def run_gate(gate, sources, *, timeout, root=ROOT): + started = time.perf_counter() + row = {"name": gate.name, "status": "FAIL", "passed": False, "command": list(gate.command), + "output": str(gate.output), "timeout_seconds": timeout} + log = gate.output.parent / "logs" / (gate.name + ".log") + row["log"] = str(log) + try: + require(not gate.output.exists(), "Gate output already exists; cannot reuse previous evidence") + log.parent.mkdir(exist_ok=True) + env = os.environ.copy() + env.update(PYTHONHASHSEED="0", OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1", MKL_NUM_THREADS="1") + options = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP} if os.name == "nt" else {"start_new_session": True} + with log.open("xb") as stream: + process = subprocess.Popen(gate.command, cwd=root, stdout=stream, stderr=subprocess.STDOUT, + env=env, **options) + try: + row["returncode"] = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + row["error"] = f"Gate exceeded {timeout} seconds" + try: + terminate_process_tree(process) + except (OSError, subprocess.SubprocessError) as exc: + row["cleanup_error"] = f"{type(exc).__name__}: {exc}" + raise + if row["returncode"] != 0: + # Save the primary failure before closing the log; close itself + # can fail and must not replace a child model/tool error. + row["error"] = child_failure(row, gate, log) + raise ValueError(row["error"]) + row["validation"] = validate_gate_report(gate, sources, root) + row.update(status="PASS", passed=True) + except Exception as exc: + row.setdefault("error", f"{type(exc).__name__}: {exc}") + row["seconds"] = time.perf_counter() - started + return row + + +def finalize_status(report): + if report.get("error") or any(row["status"] in {"FAIL", "BLOCKED"} for row in report["gates"]): + report.update(status="FAIL", passed=False) + elif all(row["status"] == "PASS" for row in report["gates"]) and len(report["gates"]) == len(GATES): + report.update(status="PASS", passed=True) + else: + report.update(status="PARTIAL", passed=False) + + +def write_reports(out, report): + errors = [] + + def fail(name, exc): + errors.append(f"{name}: {type(exc).__name__}: {exc}") + report.setdefault("error", "Could not preserve complete W2 acceptance evidence") + report.update(passed=False, status="FAIL", report_write_errors=list(errors)) + + def write(name, text): + temporary = out / ("." + name + ".tmp") + try: + temporary.write_text(text, encoding="utf-8") + temporary.replace(out / name) + return True + except OSError as exc: + fail(name, exc) + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + return False + + def views(): + lines = ["# W2 local acceptance", "", f"Result: **{report['status']}**", "", + "Fresh executions only. Subsets and blocked gates cannot certify all W2 technical gates.", "", + "| Gate | Result | Wall seconds | Reason |", "|---|---|---:|---|"] + for row in report["gates"]: + reason = str(row.get("error", "")).replace("|", "/").replace("\n", " ") + lines.append(f"| {row['name']} | {row['status']} | {row.get('seconds', 0):.3f} | {reason} |") + if report.get("error"): + lines += ["", "Error: " + report["error"]] + lines += ["", "Source hashes and checkout metadata: report.json. Per-gate logs: logs/.", + "This does not establish team signoff, full-model IR/QEMU correctness or text generation."] + markdown = "\n".join(lines) + "\n" + return {"report.md": markdown, "report.html": 'W2 acceptance' + '
'
+                + escape(markdown) + "
"} + + for name, value in views().items(): + write(name, value) + if errors: + for name, value in views().items(): + write(name, value) + if not write("report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n"): + for name, value in views().items(): + write(name, value) + write("report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + if errors: + print("; ".join(errors), file=sys.stderr) + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--python", default=sys.executable, help="Existing pinned Python interpreter; no installation") + cli.add_argument("--output-dir", type=Path, required=True, help="Must not exist, even if empty") + cli.add_argument("--tokenizer-dir", type=Path, help="Verified local official tokenizer assets") + cli.add_argument("--tokenizer-mode", choices=("verify", "download"), default="verify") + cli.add_argument("--model-dir", type=Path, help="Pinned full Qwen3 ONNX release directory") + cli.add_argument("--model-mode", choices=("verify", "download"), default="verify") + cli.add_argument("--cc", help="RISC-V compiler / Zig executable") + cli.add_argument("--qemu", help="qemu-system-riscv64 executable") + cli.add_argument("--gate-timeout", type=float, default=1800, help="Wall seconds per gate including child workers") + cli.add_argument("--qemu-timeout", type=float, default=180, help="Seconds per QEMU execution") + cli.add_argument("--parse-timeout", type=float, default=300, help="Full frontend worker timeout") + cli.add_argument("--gates", nargs="+", choices=GATES, default=list(GATES), + help="Optional subset: dependencies auto-added; successful subset exits 2 (PARTIAL)") + args = cli.parse_args(argv) + for field in ("gate_timeout", "qemu_timeout", "parse_timeout"): + if not math.isfinite(getattr(args, field)) or getattr(args, field) <= 0: + cli.error("Timeouts must be finite and positive") + if len(set(args.gates)) != len(args.gates): + cli.error("Duplicate gates are not allowed") + args.output_dir = args.output_dir.resolve() + for field in ("tokenizer_dir", "model_dir"): + if getattr(args, field) is not None: + setattr(args, field, getattr(args, field).resolve()) + if args.output_dir.exists(): + cli.error("--output-dir must not exist; preserve previous evidence") + args.output_dir.mkdir(parents=True) + report = {"schema_version": 1, "gate": "acceptance:w2", "status": "FAIL", "passed": False, + "created_at": datetime.now(timezone.utc).isoformat(), "gates": [], "requested_gates": args.gates, + "python": args.python, "checkout": checkout_evidence()} + started = time.perf_counter() + try: + report["source_sha256"] = source_fingerprints() + plan = {gate.name: gate for gate in build_plan(args)} + states = {} + for name in GATES: + gate = plan.get(name) + missing = ("--tokenizer-dir" if name in {"runtime", "runtime-model"} and args.tokenizer_dir is None else + "--model-dir" if name == "full-parse" and args.model_dir is None else None) + if gate is None: + row = {"name": name, "status": "NOT_RUN", "passed": False, "error": "Not selected"} + elif missing or any(states.get(dependency) != "PASS" for dependency in gate.dependencies): + row = {"name": name, "status": "BLOCKED", "passed": False, + "error": "Missing " + missing if missing else "Required preceding gate did not pass"} + else: + print(f"[W2] Running {name}", flush=True) + row = run_gate(gate, report["source_sha256"], timeout=args.gate_timeout) + report["gates"].append(row) + states[name] = row["status"] + print(f"[W2] {name}: {row['status']}", flush=True) + final_sources = source_fingerprints() + if final_sources != report["source_sha256"]: + report["error"] = "Source files changed while acceptance was running; rerun after edits finish" + report["final_source_sha256"] = final_sources + except Exception as exc: + report.setdefault("error", f"{type(exc).__name__}: {exc}") + report["seconds"] = time.perf_counter() - started + finalize_status(report) + write_reports(args.output_dir, report) + print(f"[W2] {report['status']}: {args.output_dir / 'report.json'}", flush=True) + return 0 if report["passed"] else 2 if report["status"] == "PARTIAL" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_llm_inputs.py b/tests/test_llm_inputs.py new file mode 100644 index 0000000..8aebbc5 --- /dev/null +++ b/tests/test_llm_inputs.py @@ -0,0 +1,228 @@ +"""Fixed Qwen3 host ABI, padding, finite greedy selection and stop boundaries.""" + +import numpy as np +import pytest + +from scratchv.runtime.llm_inputs import ( + QWEN3_VOCAB_SIZE, SEQ_LENGTH, build_attention_mask, generation_stop_reason, + greedy_next_token, greedy_token, last_valid_logits, prepare_inputs, +) + + +def test_prepare_inputs_preserves_prompt_and_does_not_infer_length_from_pad_id(): + original = np.array([7, 0, 9], dtype=np.int32) + prepared = prepare_inputs(original, pad_token_id=0, vocab_size=10) + original[:] = 1 + assert prepared.valid_length == 3 + assert prepared.input_ids.shape == (1, 256) + assert prepared.input_ids.dtype == np.int64 + np.testing.assert_array_equal(prepared.input_ids[0, :3], [7, 0, 9]) + assert np.all(prepared.input_ids[0, 3:] == 0) + assert prepared.input_ids.flags.c_contiguous + assert prepared.attention_mask.flags.c_contiguous + assert prepared.as_feed() == { + "input_ids": prepared.input_ids, "attention_mask": prepared.attention_mask, + } + assert set(prepared.as_feed()) == {"input_ids", "attention_mask"} + + +def test_prepare_inputs_accepts_maximum_int64_without_rounding_through_float(): + maximum = int(np.iinfo(np.int64).max) + prepared = prepare_inputs([maximum - 1, maximum], pad_token_id=maximum, + vocab_size=maximum + 1) + assert int(prepared.input_ids[0, 0]) == maximum - 1 + assert int(prepared.input_ids[0, 1]) == maximum + assert int(prepared.input_ids[0, -1]) == maximum + + +def test_prepare_inputs_defaults_to_full_qwen_vocabulary(): + prepared = prepare_inputs([QWEN3_VOCAB_SIZE - 1], pad_token_id=151643) + assert prepared.input_ids[0, 0] == 151935 + assert prepared.input_ids[0, 1] == 151643 + + +def test_prepare_inputs_accepts_full_capacity_without_truncation(): + prepared = prepare_inputs(tuple(range(256)), pad_token_id=0, vocab_size=256) + assert prepared.valid_length == 256 + np.testing.assert_array_equal(prepared.input_ids[0], np.arange(256)) + + +@pytest.mark.parametrize("tokens", [[], [1] * 257]) +def test_prepare_inputs_rejects_empty_and_oversize_prompts(tokens): + with pytest.raises(ValueError, match="valid_length"): + prepare_inputs(tokens, pad_token_id=0, vocab_size=2) + + +@pytest.mark.parametrize("tokens,exception", [ + ([-1], ValueError), ([4], ValueError), ([2**64], ValueError), + ([True], TypeError), ([np.bool_(False)], TypeError), ([1.0], TypeError), + (["1"], TypeError), ([None], TypeError), ("123", TypeError), + (b"123", TypeError), (iter([1]), TypeError), (1, TypeError), + (np.array([[1]], dtype=np.int64), ValueError), + (np.array([1.0], dtype=np.float32), TypeError), + (np.array([1], dtype=object), TypeError), + (np.array([2**64 - 1], dtype=np.uint64), ValueError), +]) +def test_prepare_inputs_rejects_invalid_ids_without_casting_them(tokens, exception): + with pytest.raises(exception): + prepare_inputs(tokens, pad_token_id=0, vocab_size=4) + + +@pytest.mark.parametrize("kwargs,exception", [ + ({"pad_token_id": -1}, ValueError), ({"pad_token_id": 4}, ValueError), + ({"pad_token_id": True}, TypeError), ({"pad_token_id": 0.0}, TypeError), + ({"vocab_size": 0}, ValueError), ({"vocab_size": -1}, ValueError), + ({"vocab_size": True}, TypeError), ({"vocab_size": 4.0}, TypeError), + ({"vocab_size": 2**63 + 1}, ValueError), +]) +def test_prepare_inputs_rejects_invalid_metadata(kwargs, exception): + options = {"pad_token_id": 0, "vocab_size": 4} + options.update(kwargs) + with pytest.raises(exception): + prepare_inputs([1], **options) + + +@pytest.mark.parametrize("length", [1, 17, 255, 256]) +def test_attention_mask_blocks_future_and_padding_keys_but_not_all_padded_queries(length): + mask = build_attention_mask(length) + blocked = np.finfo(np.float32).min + assert mask.shape == (1, 1, 256, 256) + assert mask.dtype == np.float32 + assert np.isfinite(mask).all() + assert set(np.unique(mask)) == {np.float32(0), blocked} + # Counts independently characterize every row, including padded queries. + counts = np.count_nonzero(mask[0, 0] == 0, axis=1) + np.testing.assert_array_equal(counts, [min(q + 1, length) for q in range(256)]) + assert mask[0, 0, 0, 0] == 0 + assert np.all(mask[0, 0, 0, 1:] == blocked) + assert np.all(mask[0, 0, 255, :length] == 0) + if length < SEQ_LENGTH: + assert np.all(mask[0, 0, :, length:] == blocked) + assert mask[0, 0, length, length - 1] == 0 + + +def test_mask_suppresses_large_padding_scores_and_never_hides_an_earlier_valid_key(): + mask = build_attention_mask(3)[0, 0] + scores = np.full((256, 256), 1000, dtype=np.float32) + scores[:, :3] = 0 + masked = scores + mask + weights = np.exp(masked - masked.max(axis=-1, keepdims=True)) + weights /= weights.sum(axis=-1, keepdims=True) + np.testing.assert_array_equal(weights[:, 3:], 0) + np.testing.assert_array_equal(weights[0, :3], [1, 0, 0]) + np.testing.assert_array_equal(weights[1, :3], [0.5, 0.5, 0]) + np.testing.assert_allclose(weights[2:, :3], np.float32(1 / 3), rtol=0, atol=0) + + +@pytest.mark.parametrize("length,exception", [ + (0, ValueError), (-1, ValueError), (257, ValueError), + (True, TypeError), (np.bool_(True), TypeError), (1.0, TypeError), (None, TypeError), +]) +def test_mask_rejects_invalid_lengths(length, exception): + with pytest.raises(exception, match="valid_length"): + build_attention_mask(length) + + +@pytest.mark.parametrize("length", [1, 17, 255, 256]) +def test_greedy_uses_last_valid_position_and_lowest_id_tie(length): + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, :, 7] = 100 + logits[0, length - 1, :] = [-9, 2, -1, 2, -5, 0, -3, 1] + selected = last_valid_logits(logits, length, vocab_size=8) + assert selected.shape == (8,) + assert selected.dtype == np.float32 + assert not np.shares_memory(selected, logits) + assert greedy_next_token(logits, length, vocab_size=8) == 1 + selected[:] = 0 + assert logits[0, length - 1, 1] == 2 + + +def test_greedy_supports_noncontiguous_fp32_view_and_negative_logits(): + values = np.array([-9, 0, -1, 0, -3, 0, -1, 0], dtype=np.float32)[::2] + assert not values.flags.c_contiguous + assert greedy_token(values) == 1 + assert greedy_token(np.array([-0.0, 0.0], dtype=np.float32)) == 0 + + +@pytest.mark.parametrize("logits,exception", [ + ([0, 1], TypeError), (np.zeros(3, dtype=np.float64), TypeError), + (np.zeros(3, dtype=np.float16), TypeError), (np.zeros(3, dtype=np.int64), TypeError), + (np.zeros(0, dtype=np.float32), ValueError), (np.zeros((), dtype=np.float32), ValueError), + (np.zeros((1, 3), dtype=np.float32), ValueError), + (np.array([np.nan, 0], dtype=np.float32), ValueError), + (np.array([0, np.inf], dtype=np.float32), ValueError), + (np.array([-np.inf, 0], dtype=np.float32), ValueError), +]) +def test_greedy_rejects_malformed_or_nonfinite_vocabulary(logits, exception): + with pytest.raises(exception): + greedy_token(logits) + + +def test_masked_arrays_cannot_hide_invalid_ids_or_nan_logits(): + ids = np.ma.array([1, -1], mask=[False, True], dtype=np.int64) + with pytest.raises(TypeError, match="masked array"): + prepare_inputs(ids, pad_token_id=0, vocab_size=8) + scores = np.ma.array([0, np.nan], mask=[False, True], dtype=np.float32) + with pytest.raises(TypeError, match="masked array"): + greedy_token(scores) + + +@pytest.mark.parametrize("shape", [(256, 8), (1, 255, 8), (2, 256, 8), (1, 256, 9)]) +def test_last_valid_logits_rejects_invalid_model_output_shape(shape): + with pytest.raises(ValueError, match="shape"): + last_valid_logits(np.zeros(shape, dtype=np.float32), 3, vocab_size=8) + + +def test_last_valid_logits_rejects_implicit_list_and_dtype_conversion(): + logits = np.zeros((1, 256, 8), dtype=np.float64) + with pytest.raises(TypeError, match="float32"): + last_valid_logits(logits, 3, vocab_size=8) + with pytest.raises(TypeError, match="ndarray"): + last_valid_logits(logits.tolist(), 3, vocab_size=8) + + +@pytest.mark.parametrize("position", [0, 2, 255]) +@pytest.mark.parametrize("invalid", [np.nan, np.inf, -np.inf]) +def test_full_output_must_be_finite_including_unused_padding(position, invalid): + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, position, 7] = invalid + with pytest.raises(ValueError, match="finite"): + greedy_next_token(logits, 3, vocab_size=8) + + +def stop_reason(**kwargs): + options = {"valid_length": 3, "generated_tokens": 0, "max_new_tokens": 10, + "vocab_size": 8, "eos_token_ids": [6, 7]} + options.update(kwargs) + return generation_stop_reason(**options) + + +def test_stop_reason_keeps_prompt_eos_and_stops_on_either_generated_eos(): + assert stop_reason(last_token_id=7) is None + assert stop_reason(generated_tokens=1, last_token_id=6) == "eos" + assert stop_reason(generated_tokens=1, last_token_id=7) == "eos" + assert stop_reason(generated_tokens=1, last_token_id=5) is None + + +def test_stop_reason_respects_zero_budget_exact_budget_and_capacity(): + assert stop_reason(max_new_tokens=0) == "max_new_tokens" + assert stop_reason(generated_tokens=2, max_new_tokens=2, last_token_id=1) == "max_new_tokens" + assert stop_reason(valid_length=255) is None + assert stop_reason(valid_length=256) == "capacity" + assert stop_reason(valid_length=256, generated_tokens=1, last_token_id=7, + max_new_tokens=1) == "eos" + assert stop_reason(valid_length=256, generated_tokens=1, last_token_id=1, + max_new_tokens=1) == "capacity" + + +@pytest.mark.parametrize("kwargs,exception", [ + ({"valid_length": 257}, ValueError), ({"generated_tokens": -1}, ValueError), + ({"generated_tokens": 3}, ValueError), ({"generated_tokens": True}, TypeError), + ({"generated_tokens": 1}, ValueError), ({"max_new_tokens": -1}, ValueError), + ({"max_new_tokens": 1.5}, TypeError), ({"last_token_id": 8}, ValueError), + ({"eos_token_ids": [8]}, ValueError), ({"eos_token_ids": [False]}, TypeError), + ({"eos_token_ids": 7}, TypeError), +]) +def test_stop_reason_rejects_inconsistent_state_or_invalid_configuration(kwargs, exception): + with pytest.raises(exception): + stop_reason(**kwargs) diff --git a/tests/test_qwen3_frontend_patterns.py b/tests/test_qwen3_frontend_patterns.py new file mode 100644 index 0000000..5f11a6d --- /dev/null +++ b/tests/test_qwen3_frontend_patterns.py @@ -0,0 +1,56 @@ +"""unit:frontend-ops: four Qwen3 decomposed patterns, before/after optimization.""" +import copy + +import numpy as np +import onnx +from onnx import helper, numpy_helper +import pytest + +from probes.w2_backend_ops.cases import build_cases +from probes.w2_backend_ops.run import ATOL, FRONTEND_FAMILIES, interpret, reference_case +from probes.w2_qwen3_small.diagnostics import tensor_diff + +CASES = [case for case in build_cases() if case.family in FRONTEND_FAMILIES] + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.name) +@pytest.mark.parametrize("level", ["none", "all"]) +def test_four_qwen3_patterns_match_independent_formula_and_ort(tmp_path, case, level): + expected, parser, program = reference_case(case, tmp_path / "pattern.onnx") + assert tensor_diff(expected, case.expected, ATOL)["passed"] + actual = interpret(program, parser.initializers, case.feed, level) + assert tensor_diff(actual, expected, ATOL)["passed"] + assert tensor_diff(actual, case.expected, ATOL)["passed"] + + +@pytest.mark.parametrize("family", FRONTEND_FAMILIES) +def test_pattern_cases_detect_semantic_mutations(tmp_path, family): + case = next(copy.deepcopy(case) for case in CASES if case.family == family) + graph = case.model.graph + if family == "rmsnorm": + epsilon = next(value for value in graph.initializer if value.name == "epsilon") + epsilon.CopyFrom(numpy_helper.from_array(np.array(1e-3, np.float32), "epsilon")) + elif family == "rope": + # Wrong rotate_half sign; rotation still preserves shape/dtype. + next(node for node in graph.node if node.op_type == "Neg").op_type = "Identity" + elif family == "swiglu": + next(node for node in graph.node if node.op_type == "Sigmoid").input[0] = "up" + else: + # Wrong interleaved KV repetition still has the expected head count. + graph.initializer.append(numpy_helper.from_array(np.array([0, 2, 1, 3], np.int64), "wrong_heads")) + for name in ("k", "v"): + nodes = list(graph.node) + position = next(index for index, node in enumerate(nodes) if f"{name}_repeated" in node.output) + nodes.insert(position + 1, helper.make_node("Gather", [f"{name}_repeated", "wrong_heads"], + [f"{name}_wrong"], axis=1)) + for node in nodes[position + 2:]: + for index, value in enumerate(node.input): + if value == f"{name}_repeated": + node.input[index] = f"{name}_wrong" + del graph.node[:] + graph.node.extend(nodes) + onnx.checker.check_model(case.model) + actual, parser, program = reference_case(case, tmp_path / "mutated.onnx") + ir = interpret(program, parser.initializers, case.feed, "all") + assert tensor_diff(actual, ir, ATOL)["passed"] + assert not tensor_diff(actual, case.expected, ATOL)["passed"], family diff --git a/tests/test_qwen3_full_audit.py b/tests/test_qwen3_full_audit.py new file mode 100644 index 0000000..1674c1e --- /dev/null +++ b/tests/test_qwen3_full_audit.py @@ -0,0 +1,375 @@ +"""The full-model acceptance audit must reject plausible but incomplete parses.""" + +import json + +import numpy as np +import onnx +import pytest +from onnx import TensorProto as T, helper, numpy_helper + +from probes.w2_qwen3_parse.audit import ( + AuditError, AuditedONNXParser, audit_parsed_model, +) +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.ir.types import DataType, OpCode, Value + + +def save(tmp_path, nodes, arrays=None, outputs=None, external=False): + graph = helper.make_graph( + nodes, "audit_fixture", [helper.make_tensor_value_info("x", T.FLOAT, [2, 2])], + outputs or [helper.make_tensor_value_info("logits", T.FLOAT, [2, 2])], + [numpy_helper.from_array(array, name) for name, array in (arrays or {}).items()], + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 18)], ir_version=9) + onnx.checker.check_model(model) + path = tmp_path / "model.onnx" + if external: + onnx.save_model(model, path, save_as_external_data=True, + all_tensors_to_one_file=True, location="weights.data", size_threshold=0) + else: + onnx.save_model(model, path) + return path + + +@pytest.fixture +def parsed(tmp_path): + nodes = [helper.make_node("MatMul", ["x", "w1"], ["a"], name="project"), + helper.make_node("Add", ["a", "w2"], ["b"], name="add_weight"), + helper.make_node("Identity", ["b"], ["alias"], name="alias"), + helper.make_node("Constant", [], ["bias"], name="bias", + value=numpy_helper.from_array(np.full((2, 2), .5, np.float32))), + helper.make_node("Add", ["alias", "bias"], ["logits"], name="final")] + path = save(tmp_path, nodes, {"w1": np.arange(4, dtype=np.float32).reshape(2, 2), + "w2": np.full((2, 2), 8, dtype=np.float32)}, external=True) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + return onnx.load(path, load_external_data=False), parser, program, tmp_path + + +def test_complete_parse_has_serializable_mapping_and_binding_evidence(parsed): + result = audit_parsed_model(*parsed) + json.dumps(result, allow_nan=False) + assert result["passed"] and result["node_count"] == 5 + assert result["value_count"] == 8 + assert result["initializer_count"] == 2 + assert result["constant_count"] == 1 + assert result["global_count"] == result["binding_count"] == 3 + assert result["binding_bytes"] == 48 + assert result["alias_count"] == 1 + assert result["return"]["onnx_name"] == "logits" + assert all(len(row["sha256"]) == 64 for row in result["tensor_bindings"]) + assert parsed[0].graph.initializer[0].raw_data == b"" + + +def test_parser_reuse_resets_observations_and_progress(parsed): + _, parser, _, directory = parsed + events = [] + parser.progress = events.append + parser.parse(str(directory / "model.onnx")) + assert len(parser.node_records) == 5 + assert events[0]["stage"] == "parse" and events[-1]["stage"] == "parsed" + assert [row["node_index"] for row in events if row["stage"] == "translate_node"] == list(range(5)) + + +@pytest.mark.parametrize("fault", ["missing_record", "missing_map", "missing_instruction", + "shape", "dtype", "alias", "return", "global", + "duplicate_global", "swapped_weights", "constant_content", + "operand", "opcode", "attrs", "parameter"]) +def test_incomplete_or_corrupt_translation_fails_closed(parsed, fault): + _, parser, program, _ = parsed + instructions = program.functions[0].blocks[0].instructions + if fault == "missing_record": + parser.node_records.pop() + elif fault == "missing_map": + del parser._value_map["a"] + elif fault == "missing_instruction": + instructions.pop(0) + elif fault == "shape": + parser._value_map["a"].shape = (4,) + elif fault == "dtype": + parser._value_map["a"].dtype = DataType.FLOAT64 + elif fault == "alias": + parser._value_map["alias"] = parser._value_map["a"] + elif fault == "return": + instructions[-1].operands = [parser._value_map["a"]] + elif fault == "global": + program.global_values.pop() + elif fault == "duplicate_global": + program.global_values.append(program.global_values[0]) + elif fault == "swapped_weights": + parser.initializers["w1"], parser.initializers["w2"] = ( + parser.initializers["w2"], parser.initializers["w1"]) + elif fault == "constant_content": + parser.initializers["bias"] = np.zeros((2, 2), dtype=np.float32) + elif fault == "operand": + instructions[0].operands[1] = parser._value_map["w2"] + elif fault == "opcode": + instructions[0].opcode = OpCode.ADD + elif fault == "attrs": + instructions[0].attrs["k"] = 1 + elif fault == "parameter": + program.functions[0].params = [] + with pytest.raises(AuditError): + audit_parsed_model(*parsed) + + +def test_skip_handler_is_not_counted_as_success(parsed, monkeypatch): + model, _, _, directory = parsed + parser = AuditedONNXParser() + monkeypatch.setattr(parser, "_handle_add", lambda node, inputs, outputs: + parser._define_outputs(outputs, inputs[0])) + program = parser.parse(str(directory / "model.onnx")) + with pytest.raises(AuditError, match="exactly one instruction"): + audit_parsed_model(model, parser, program, directory) + + +def test_equal_shape_value_copy_does_not_prove_identity_alias(parsed): + _, parser, _, _ = parsed + source = parser._value_map["alias"] + parser._value_map["alias"] = Value(source.name, source.dtype, shape=source.shape) + with pytest.raises(AuditError, match="alias binding"): + audit_parsed_model(*parsed) + + +@pytest.mark.parametrize("fault", ["missing", "truncated", "escape", "negative_offset", + "bad_length", "duplicate_key", "changed_content"]) +def test_external_data_errors_are_rejected_without_loading_second_weights(parsed, fault): + model, _, _, directory = parsed + tensor = model.graph.initializer[0] + fields = {row.key: row for row in tensor.external_data} + path = directory / "weights.data" + if fault == "missing": + path.unlink() + elif fault == "truncated": + path.write_bytes(b"\0" * 3) + elif fault == "escape": + fields["location"].value = "../outside.data" + elif fault == "negative_offset": + fields["offset"].value = "-1" + elif fault == "bad_length": + fields["length"].value = "15" + elif fault == "duplicate_key": + tensor.external_data.add(key="location", value="weights.data") + elif fault == "changed_content": + with path.open("r+b") as stream: + stream.write(b"\xFF" * 4) + with pytest.raises(AuditError): + audit_parsed_model(*parsed) + + +def test_noncontiguous_equal_binding_hashes_logical_c_order(parsed): + _, parser, _, _ = parsed + parser.initializers["w1"] = np.asfortranarray(parser.initializers["w1"]) + assert not parser.initializers["w1"].flags.c_contiguous + assert audit_parsed_model(*parsed)["passed"] + + +def test_multiple_graph_outputs_are_explicitly_rejected(tmp_path): + outputs = [helper.make_tensor_value_info(name, T.FLOAT, [2, 2]) for name in ("x", "logits")] + path = save(tmp_path, [helper.make_node("Identity", ["x"], ["logits"])], outputs=outputs) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + with pytest.raises(AuditError, match="exactly one graph output"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +def test_scalar_constants_and_static_shape_cast_chain(tmp_path): + nodes = [helper.make_node("Constant", [], ["dim"], value_int=2), + helper.make_node("Cast", ["dim"], ["cast"], to=T.INT64), + helper.make_node("Constant", [], ["axis"], value_ints=[0]), + helper.make_node("Unsqueeze", ["cast", "axis"], ["vec"]), + helper.make_node("Concat", ["vec", "vec"], ["shape"], axis=0), + helper.make_node("Reshape", ["x", "shape"], ["reshaped"]), + helper.make_node("Add", ["reshaped", "scalar"], ["logits"])] + path = save(tmp_path, nodes, {"scalar": np.array(1, dtype=np.float32)}) + model = onnx.load(path) + model.graph.value_info.append(helper.make_tensor_value_info("reshaped", T.FLOAT, [2, 2])) + onnx.save(model, path) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + result = audit_parsed_model(onnx.load(path), parser, program, tmp_path) + assert result["binding_count"] == 3 + assert result["node_count"] == 7 + + +def test_failed_translation_preserves_node_context(tmp_path, monkeypatch): + path = save(tmp_path, [helper.make_node("Identity", ["x"], ["logits"], name="failed_node")]) + parser = AuditedONNXParser() + + def broken(*args): + raise RuntimeError("injected") + + monkeypatch.setattr(parser, "_handle_identity", broken) + with pytest.raises(RuntimeError, match="injected"): + parser.parse(str(path)) + assert parser.current_context["node_name"] == "failed_node" + assert parser.node_records[-1]["status"] == "failed" + + +def test_scalar_constant_signed_zero_is_preserved(tmp_path): + path = save(tmp_path, [helper.make_node("Constant", [], ["zero"], value_float=-0.0), + helper.make_node("Add", ["x", "zero"], ["logits"])]) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + model = onnx.load(path) + assert audit_parsed_model(model, parser, program, tmp_path)["passed"] + program.functions[0].blocks[0].instructions[0].attrs["value"] = 0.0 + with pytest.raises(AuditError, match="Constant.*scalar instruction"): + audit_parsed_model(model, parser, program, tmp_path) + + +def test_complete_decoder_fixture_uses_production_parser_and_audit(tmp_path): + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + path = tmp_path / "decoder.onnx" + onnx.save_model(model, path, save_as_external_data=True, + all_tensors_to_one_file=True, location="decoder.data", size_threshold=0) + original = onnx.load(path, load_external_data=False) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + result = audit_parsed_model(original, parser, program, tmp_path) + assert result["passed"] and result["initializer_count"] == 26 + assert result["node_count"] == len(original.graph.node) + assert result["return"]["shape"] == [1, 3, 9] + shared = parser._value_map["model.lm_head.weight"] + consumers = [inst for inst in program.functions[0].blocks[0].instructions + if any(operand is shared for operand in inst.operands)] + assert sorted(inst.opcode.value for inst in consumers) == ["gather", "transpose"] + + +@pytest.mark.parametrize("operation", ["Reshape", "Slice"]) +def test_parser_control_evaluation_cannot_certify_its_own_wrong_results(tmp_path, operation): + from scratchv.analysis.ir_verifier import verify_ir + + if operation == "Reshape": + nodes = [helper.make_node("Reshape", ["x", "shape"], ["logits"])] + arrays = {"shape": np.array([2, 2], np.int64)} + shape = [2, 2] + wrong = {"shape": (1, 4)} + else: + nodes = [helper.make_node("Slice", ["x", "starts", "ends", "axes"], ["logits"])] + arrays = {name: np.array(value, np.int64) for name, value in + {"starts": [0], "ends": [1], "axes": [0]}.items()} + shape = [1, 2] + # The wrong row has the SAME output shape: metadata cannot catch this. + wrong = {"starts": (1,), "ends": (2,)} + path = save(tmp_path, nodes, arrays, + outputs=[helper.make_tensor_value_info("logits", T.FLOAT, shape)]) + parser = AuditedONNXParser() + correct = parser._constant_ints + parser._constant_ints = lambda value: wrong[value.name] if value.name in wrong else correct(value) + program = parser.parse(str(path)) + assert verify_ir(program) == (True, []) + with pytest.raises(AuditError, match="IR attributes mismatch"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +@pytest.fixture +def qwen_parsed(tmp_path): + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + path = tmp_path / "qwen.onnx" + onnx.save_model(model, path) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + return model, parser, program, tmp_path + + +@pytest.mark.parametrize("fault", ["transpose", "epsilon"]) +def test_parser_attribute_reader_cannot_certify_its_own_wrong_results(tmp_path, monkeypatch, fault): + from scratchv.analysis.ir_verifier import verify_ir + from tests.test_qwen3_full_structure import qwen3_fixture + + model, _ = qwen3_fixture() + original_bytes = model.SerializeToString() + epsilon_name = next(node.input[1] for node in model.graph.node + if list(node.output) == ["layer_0_input_norm_eps"]) + path = tmp_path / "qwen.onnx" + onnx.save(model, path) + correct = ONNXParser._attributes + + def wrong_reader(node): + attrs = correct(node) + if fault == "transpose" and list(node.output) == ["layer_0_q_weight"]: + attrs["perm"] = [0, 1] # Square Q projection: output shape stays unchanged. + if fault == "epsilon" and list(node.output) == [epsilon_name]: + attrs["value"] = numpy_helper.from_array(np.array(1e-3, dtype=np.float32)) + return attrs + + monkeypatch.setattr(ONNXParser, "_attributes", staticmethod(wrong_reader)) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + assert verify_ir(program) == (True, []) + expected = "IR attributes mismatch" if fault == "transpose" else "content SHA256 mismatch" + with pytest.raises(AuditError, match=expected): + audit_parsed_model(model, parser, program, tmp_path) + assert model.SerializeToString() == original_bytes + + +@pytest.mark.parametrize("parameter", [0, 1]) +@pytest.mark.parametrize("fault", ["name", "constant"]) +def test_qwen_parameter_binding_contract_is_checked(qwen_parsed, parameter, fault): + from scratchv.analysis.ir_verifier import verify_ir + + _, _, program, _ = qwen_parsed + value = program.functions[0].params[parameter] + if fault == "name": + value.name = "renamed_" + value.name + else: + value.is_constant = True + value.const_value = 0 + assert verify_ir(program) == (True, []) + expected = "parameter names" if fault == "name" else "must not be constants" + with pytest.raises(AuditError, match=expected): + audit_parsed_model(*qwen_parsed) + + +@pytest.mark.parametrize("source", ["initializer", "Constant"]) +@pytest.mark.parametrize("fault", ["shape", "constant_flag", "constant_value"]) +def test_auxiliary_scalar_load_result_metadata_is_checked(tmp_path, source, fault): + nodes = [helper.make_node("Add", ["x", "scalar"], ["logits"])] + arrays = {} + if source == "Constant": + nodes.insert(0, helper.make_node("Constant", [], ["scalar"], value_float=1.0)) + else: + arrays["scalar"] = np.array(1.0, dtype=np.float32) + path = save(tmp_path, nodes, arrays) + parser = AuditedONNXParser() + program = parser.parse(str(path)) + scalar_load = program.functions[0].blocks[0].instructions[0] + assert scalar_load.opcode is OpCode.LOAD_CONST + assert scalar_load.dest is not parser._value_map["scalar"] + if fault == "shape": + scalar_load.dest.shape = (2,) + elif fault == "constant_flag": + scalar_load.dest.is_constant = False + else: + scalar_load.dest.const_value = 2.0 + with pytest.raises(AuditError, match="scalar instruction metadata mismatch"): + audit_parsed_model(onnx.load(path), parser, program, tmp_path) + + +@pytest.mark.parametrize("binding", ["initializer", "Constant"]) +def test_qwen_bindings_are_not_silently_coerced_to_numpy(qwen_parsed, binding): + _, parser, _, _ = qwen_parsed + name = "positions" if binding == "initializer" else next( + name for name, value in parser.initializers.items() + if name.startswith("constant_") and value.dtype == np.dtype("int64") and value.ndim == 1) + original = parser.initializers[name] + parser.initializers[name] = original.tolist() + # Coercion would give the exact same bytes, shape and dtype, but execution + # requires the original binding object to already be a NumPy ndarray. + converted = np.asarray(parser.initializers[name]) + assert converted.dtype == original.dtype and converted.shape == original.shape + np.testing.assert_array_equal(converted, original) + with pytest.raises(AuditError, match="binding must be a NumPy array"): + audit_parsed_model(*qwen_parsed) + + +def test_qwen_read_only_ndarray_bindings_remain_valid(qwen_parsed): + _, parser, _, _ = qwen_parsed + for array in parser.initializers.values(): + array.setflags(write=False) + assert audit_parsed_model(*qwen_parsed)["passed"] diff --git a/tests/test_qwen3_full_parse.py b/tests/test_qwen3_full_parse.py new file mode 100644 index 0000000..970abec --- /dev/null +++ b/tests/test_qwen3_full_parse.py @@ -0,0 +1,422 @@ +"""Full-model gate orchestration and failure evidence; no large checkpoint needed.""" +import json +from pathlib import Path +import subprocess +from types import SimpleNamespace + +import pytest + +from probes.w2_qwen3_parse import run as gate + + +def completed_report(out, *, passed=True, stage="complete"): + report = gate.initial_report("verify") + report.update(passed=passed, stage=stage, validation_seconds=1.0, + graph={"passed": True, "layers": [{"layer": index} for index in range(28)]}, + stages_seconds={"scratchv-parse": 0.25}, + ir={"instructions": 12, "globals": 3}, ir_verifier={"passed": True}, + audit={"passed": True, "binding_bytes": 16}, + peak_memory={"bytes": 1024, "method": "test"}) + for name, content in (("nodes.json", "[]"), ("bindings.json", "[]"), ("ir.txt", "fixture IR")): + (out / name).write_text(content, encoding="utf-8") + gate.write_json(out / "worker-report.json", report) + return report + + +def invoke(tmp_path): + return ["--model-dir", str(tmp_path / "model"), "--output-dir", str(tmp_path / "report")] + + +def test_success_keeps_separate_parse_time_and_scope(tmp_path, monkeypatch): + def launch(args): + completed_report(args.output_dir) + return subprocess.CompletedProcess([], 0, b"parsed", b"") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 0 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["passed"] and report["stage"] == "complete" + assert report["stages_seconds"]["scratchv-parse"] == 0.25 + assert report["seconds"] >= 0 and report["validation_seconds"] == 1 + assert report["timeout_seconds"] == 300 + for name in ("report.md", "report.html"): + content = (out / name).read_text(encoding="utf-8") + assert "PASS" in content and "28 层" in content + assert "不证明完整 IR 数值或 QEMU 前向通过" in content + assert (out / "worker.stdout").read_bytes() == b"parsed" + + +@pytest.mark.parametrize("failure", ["nonzero", "failed", "unfinished", "missing_report", + "bad_json", "wrong_gate", "missing_evidence", "spawn", "killed_missing_report"]) +def test_worker_failure_cannot_be_success(tmp_path, monkeypatch, failure): + def launch(args): + if failure == "spawn": + raise OSError("cannot start worker ") + report = completed_report(args.output_dir) + path = args.output_dir / "worker-report.json" + if failure == "failed": + report.update(passed=False, stage="ir-verifier", error="invalid IR") + elif failure == "unfinished": + report["stage"] = "scratchv-parse" + elif failure == "wrong_gate": + report["gate"] = "ORT-only" + gate.write_json(path, report) + if failure in ("missing_report", "killed_missing_report"): + path.unlink() + elif failure == "bad_json": + path.write_text("{broken", encoding="utf-8") + elif failure == "missing_evidence": + (args.output_dir / "bindings.json").unlink() + code = -9 if failure == "killed_missing_report" else (4 if failure == "nonzero" else 0) + return subprocess.CompletedProcess([], code, b"partial", b"failure") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["passed"] is False and report["error"] + assert report["attempt"]["command"][0] == gate.sys.executable + assert report["attempt"]["model_dir"] == str((tmp_path / "model").resolve()) + assert "probes/w2_qwen3_parse/run.py" in report["attempt"]["source_sha256"] + if failure == "killed_missing_report": + assert report["worker_returncode"] == -9 + assert "FAIL" in (out / "report.md").read_text(encoding="utf-8") + html = (out / "report.html").read_text(encoding="utf-8") + assert "" not in html + + +def test_timeout_keeps_partial_progress_and_actual_elapsed_time(tmp_path, monkeypatch): + clock = iter((10.0, 13.75)) + monkeypatch.setattr(gate.time, "perf_counter", lambda: next(clock)) + def launch(args): + gate.write_json(args.output_dir / "worker-progress.json", + {"context": {"node_index": 17, "op": "MatMul"}, "elapsed_seconds": 2.0}) + raise subprocess.TimeoutExpired(["python"], 2, output=b"partial stdout", stderr=b"partial stderr") + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main([*invoke(tmp_path), "--timeout", "2"]) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["stage"] == "timeout" and report["timeout_seconds"] == 2 + assert report["attempt"]["source_sha256"] + assert report["seconds"] == 3.75 + assert report["last_progress"]["context"]["node_index"] == 17 + assert (out / "worker.stderr").read_bytes() == b"partial stderr" + + +def test_timeout_log_write_failure_still_produces_failure_report(tmp_path, monkeypatch): + original = Path.write_bytes + def locked_stderr(path, content): + if path.name == "worker.stderr": + raise PermissionError("log locked") + return original(path, content) + def timeout(args): + raise subprocess.TimeoutExpired(["python"], 1, output=b"partial") + monkeypatch.setattr(Path, "write_bytes", locked_stderr) + monkeypatch.setattr(gate, "launch_worker", timeout) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["stage"] == "timeout" and not report["passed"] + assert "log locked" in report["log_errors"][0] + assert "log locked" in (out / "report.md").read_text(encoding="utf-8") + assert (out / "worker.stdout").read_bytes() == b"partial" + + +@pytest.mark.parametrize("worker_passed", [False, True]) +@pytest.mark.parametrize("locked", ["worker.stdout", "worker.stderr"]) +def test_normal_log_failure_preserves_worker_result(tmp_path, monkeypatch, worker_passed, locked): + original = Path.write_bytes + def write(path, content): + if path.name == locked: + raise PermissionError("log locked") + return original(path, content) + def launch(args): + report = completed_report(args.output_dir, passed=worker_passed, + stage="complete" if worker_passed else "ir-and-bindings") + report.update(checkout={"head": "review-sha"}, failure_context={"tensor_name": "positions"}) + if not worker_passed: + report["error"] = "positions: bound tensor content SHA256 mismatch" + gate.write_json(args.output_dir / "worker-report.json", report) + return subprocess.CompletedProcess([], 0 if worker_passed else 1, b"out", b"err") + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + report = json.loads((out / "report.json").read_text(encoding="utf-8")) + assert report["checkout"]["head"] == "review-sha" + assert report["failure_context"] == {"tensor_name": "positions"} + assert len(report["log_errors"]) == 1 and locked in report["log_errors"][0] + assert report["worker_returncode"] == (0 if worker_passed else 1) + if worker_passed: + assert report["stage"] == "log-write" and "logs" in report["error"] + else: + assert report["stage"] == "ir-and-bindings" and "SHA256 mismatch" in report["error"] + assert "SHA256 mismatch" in (out / "report.md").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("worker_passed", [False, True]) +@pytest.mark.parametrize("locked", ["report.md", "report.html", "report.json"]) +def test_report_write_failure_cannot_publish_pass(tmp_path, monkeypatch, worker_passed, locked, capsys): + write_text, replace = Path.write_text, Path.replace + json_publications = [] + + def write(path, content, **kwargs): + if path.name == locked: + raise PermissionError("injected report lock") + return write_text(path, content, **kwargs) + + def publish(path, target): + if Path(target).name == "report.json": + if locked == "report.json": + raise PermissionError("injected report lock") + json_publications.append(json.loads(path.read_text(encoding="utf-8"))) + return replace(path, target) + + def launch(args): + report = completed_report(args.output_dir, passed=worker_passed, + stage="complete" if worker_passed else "ir-and-bindings") + if not worker_passed: + report["error"] = "positions: bound tensor content SHA256 mismatch" + report["failure_context"] = {"tensor_name": "positions"} + gate.write_json(args.output_dir / "worker-report.json", report) + return subprocess.CompletedProcess([], 0 if worker_passed else 1, b"out", b"err") + + monkeypatch.setattr(Path, "write_text", write) + monkeypatch.setattr(Path, "replace", publish) + monkeypatch.setattr(gate, "launch_worker", launch) + assert gate.main(invoke(tmp_path)) == 1 + out = tmp_path / "report" + assert "injected report lock" in capsys.readouterr().err + if locked == "report.json": + assert not (out / "report.json").exists() + assert not json_publications + else: + # JSON must never have been published as PASS, even temporarily. + assert len(json_publications) == 1 + report = json_publications[0] + assert report["passed"] is False + assert locked in report["report_write_errors"][0] + if worker_passed: + assert report["stage"] == "report-write" + else: + assert report["stage"] == "ir-and-bindings" + assert "SHA256 mismatch" in report["error"] + assert report["failure_context"] == {"tensor_name": "positions"} + for name in ("report.md", "report.html"): + if name != locked: + content = (out / name).read_text(encoding="utf-8") + assert "FAIL" in content and "PASS" not in content + assert "injected report lock" in content + if not worker_passed: + assert "SHA256 mismatch" in content + + +def test_w2_docs_links_resolve_without_local_output(monkeypatch): + from scripts import check_docs_links + exists = Path.exists + def clean_checkout(path): + if path.resolve().is_relative_to(gate.ROOT / "output"): + return False + return exists(path) + monkeypatch.setattr(Path, "exists", clean_checkout) + _, broken = check_docs_links.check_links([ + "docs/llm-deploy-v1.0/W2/README.md", "probes/w2_qwen3_parse/README.md"]) + assert not broken, broken + + +def test_checkout_evidence_preserves_porcelain_columns(monkeypatch, tmp_path): + # A linked worktree stores .git as a file, not a directory. + (tmp_path / ".git").write_text("gitdir: unused-test-path\n", encoding="utf-8") + monkeypatch.setattr(gate, "ROOT", tmp_path) + def run(command, **kwargs): + assert kwargs["timeout"] == 10 and kwargs["cwd"] == tmp_path + return SimpleNamespace(stdout="abcdef\n" if command[1] == "rev-parse" else " M file.py\n") + monkeypatch.setattr(gate.subprocess, "run", run) + report = gate.checkout_evidence() + assert report["head"] == "abcdef" and report["status"] == " M file.py" + assert report["clean"] is False + + +def test_source_archive_does_not_query_parent_checkout(monkeypatch, tmp_path): + (tmp_path / ".git").mkdir() + archive = tmp_path / "source-archive" + archive.mkdir() + monkeypatch.setattr(gate, "ROOT", archive) + monkeypatch.setattr(gate.subprocess, "run", lambda *a, **kw: pytest.fail("must not search parent Git")) + report = gate.checkout_evidence() + assert report["head"] is None and report["status"] == "unknown" and report["clean"] is None + assert report["reason"] == "Source directory has no .git metadata" + + +@pytest.mark.parametrize("failure_at", ["rev-parse", "status"]) +def test_git_timeout_is_unknown_and_does_not_leave_partial_identity(monkeypatch, tmp_path, failure_at): + (tmp_path / ".git").mkdir() + monkeypatch.setattr(gate, "ROOT", tmp_path) + def query(command, **kwargs): + assert kwargs["timeout"] == 10 + if command[1] == failure_at: + raise subprocess.TimeoutExpired(command, kwargs["timeout"]) + return SimpleNamespace(stdout="partial-sha\n") + monkeypatch.setattr(gate.subprocess, "run", query) + report = gate.checkout_evidence() + assert report["head"] is None and report["status"] == "unknown" and report["clean"] is None + assert report["reason"] == "Checkout metadata unavailable" + assert "TimeoutExpired" in report["error"] + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +def test_invalid_timeout_does_not_start_worker(tmp_path, monkeypatch, timeout): + monkeypatch.setattr(gate, "launch_worker", lambda args: pytest.fail("must not launch")) + with pytest.raises(SystemExit, match="2"): + gate.main([*invoke(tmp_path), "--timeout", timeout]) + assert not (tmp_path / "report").exists() + + +def test_existing_evidence_is_preserved(tmp_path): + out = tmp_path / "report" + out.mkdir() + evidence = out / "report.json" + evidence.write_text("prior evidence", encoding="utf-8") + with pytest.raises(SystemExit, match="2"): + gate.main(invoke(tmp_path)) + assert evidence.read_text(encoding="utf-8") == "prior evidence" + + +def test_worker_uses_same_python_explicit_paths_and_timeout(tmp_path, monkeypatch): + import scratchv.runtime.riscv_tensor as runtime + captured = {} + def process(command, **kwargs): + captured.update(command=command, **kwargs) + return subprocess.CompletedProcess(command, 0, b"", b"") + monkeypatch.setattr(runtime, "_run_process", process) + args = SimpleNamespace(mode="verify", model_dir=tmp_path / "model with spaces", + output_dir=tmp_path / "output with spaces", timeout=12.5) + gate.launch_worker(args) + assert captured["command"][0] == gate.sys.executable + assert str(args.model_dir.resolve()) in captured["command"] + assert "--worker" in captured["command"] and captured["timeout"] == 12.5 + + +def test_worker_dependency_failure_writes_report_before_parsing(tmp_path, monkeypatch): + def bad_environment(): + raise RuntimeError("pinned environment mismatch") + monkeypatch.setattr(gate.artifacts, "environment", bad_environment) + monkeypatch.setattr(gate, "run_validation", lambda *args, **kwargs: pytest.fail("must not parse")) + args = SimpleNamespace(mode="verify", model_dir=tmp_path / "model", output_dir=tmp_path) + assert gate.worker_main(args) == 1 + result = json.loads((tmp_path / "worker-report.json").read_text(encoding="utf-8")) + assert result["stage"] == "environment" and not result["passed"] + assert "mismatch" in result["error"] and result["validation_seconds"] >= 0 + + +def test_corrupt_hash_cannot_reach_scratchv_parser(tmp_path, monkeypatch): + from probes.w2_qwen3_parse import audit + def bad_files(*args): + raise ValueError("Artifact size/SHA-256 mismatch: weights.data") + monkeypatch.setattr(gate.artifacts, "verify_files", bad_files) + monkeypatch.setattr(audit.AuditedONNXParser, "parse", lambda *args: pytest.fail("must not parse")) + report = gate.initial_report("verify") + with pytest.raises(ValueError, match="SHA-256"): + gate.run_validation(tmp_path / "model", tmp_path, report) + assert report["stage"] == "hashes" and not report["passed"] + assert "hashes" in report["stages_seconds"] + + +@pytest.mark.parametrize("fault,evidence_locked", [ + (None, False), ("parse", False), ("verifier", False), ("binding", False), + ("parse", True), ("binding", True), (None, True), +]) +def test_real_decoder_pipeline_and_failure_context(tmp_path, monkeypatch, fault, evidence_locked): + """Exercise the stage wiring with real files, parser and audits, cheaply.""" + import onnx + from scratchv.analysis import ir_verifier + from probes.w2_qwen3_parse import audit, validation + from tests.test_qwen3_full_structure import qwen3_fixture + + model, config = qwen3_fixture() + model_dir, out = tmp_path / "model", tmp_path / "out" + model_dir.mkdir() + out.mkdir() + onnx.save_model(model, model_dir / "model.onnx", save_as_external_data=True, + all_tensors_to_one_file=True, location="weights.data", size_threshold=0) + manifest = {"files": [{"name": path.name, "bytes": path.stat().st_size, + "sha256": gate.artifacts.sha256(path)} for path in model_dir.iterdir()]} + # Only the full-model dimensions are adapted; loading, file integrity, + # lowering, verifier and content audits still run against real fixtures. + structure = validation.audit_graph_structure + monkeypatch.setattr(validation, "audit_graph_structure", lambda graph: structure(graph, config)) + def inspect(directory, files): + onnx.checker.check_model(str(directory / "model.onnx"), full_check=True) + return {"operators": {"fixture": len(model.graph.node)}} + monkeypatch.setattr(gate.artifacts, "inspect_model", inspect) + if fault == "parse": + def broken_gather(*args): + raise RuntimeError("injected Gather failure") + monkeypatch.setattr(audit.AuditedONNXParser, "_handle_gather", broken_gather) + elif fault == "verifier": + monkeypatch.setattr(ir_verifier, "verify_ir", lambda *args, **kwargs: (False, ["injected verifier failure"])) + elif fault == "binding": + original = audit.AuditedONNXParser.parse + def misbind(self, path): + program = original(self, path) + self.initializers["model.lm_head.weight"] = self.initializers["model.lm_head.weight"].copy() + self.initializers["model.lm_head.weight"][0, 0] += 1 + return program + monkeypatch.setattr(audit.AuditedONNXParser, "parse", misbind) + report = gate.initial_report("verify") + if evidence_locked: + original_write = gate.write_json + def write(path, value): + # The binding failure occurs after the first, successful nodes.json + # save; lock only its diagnostic retry, not the earlier operation. + locked = ((fault == "parse" and path.name == "nodes.partial.json") + or (fault == "binding" and path.name == "nodes.json" + and "failure_context" in report) + or (fault is None and path.name == "bindings.json")) + if locked: + raise PermissionError("injected evidence lock") + return original_write(path, value) + monkeypatch.setattr(gate, "write_json", write) + if fault or evidence_locked: + expected_error = ("injected Gather failure" if fault == "parse" else + "content SHA256 mismatch" if fault == "binding" else + "injected evidence lock" if evidence_locked else "IR verifier rejected") + with pytest.raises((RuntimeError, ValueError, PermissionError), match=expected_error) as failure: + gate.run_validation(model_dir, out, report, manifest=manifest) + assert report["passed"] is False + assert report["stage"] == {"parse": "scratchv-parse", "verifier": "ir-verifier", + "binding": "ir-and-bindings", None: "ir-and-bindings"}[fault] + if fault == "parse": + assert report["failure_context"]["op_type"] == "Gather" + if evidence_locked: + assert not (out / "nodes.partial.json").exists() + assert report["nodes_completed"] == 0 + else: + partial = json.loads((out / "nodes.partial.json").read_text(encoding="utf-8")) + assert partial[-1]["status"] == "failed" + assert report["nodes_completed"] == len(partial) - 1 + elif fault == "verifier": + assert report["ir_verifier"]["issues"] == ["injected verifier failure"] + assert (out / "nodes.json").is_file() + elif fault == "binding": + assert "model.lm_head.weight" in json.dumps(report["failure_context"]) + assert (out / "nodes.json").is_file() + if evidence_locked and fault: + assert len(report["evidence_errors"]) == 1 + assert "injected evidence lock" in report["evidence_errors"][0] + report["error"] = f"{type(failure.value).__name__}: {failure.value}" + gate.write_reports(out, report) + markdown = (out / "report.md").read_text(encoding="utf-8") + assert expected_error in markdown and "injected evidence lock" in markdown + elif evidence_locked: + # Successful validation cannot pass if required evidence is lost. + assert isinstance(failure.value, PermissionError) + assert "evidence_errors" not in report + assert not (out / "bindings.json").exists() + else: + gate.run_validation(model_dir, out, report, manifest=manifest) + assert report["passed"] and report["stage"] == "complete" + assert report["graph"]["layer_count"] == 2 + assert report["audit"]["node_count"] == len(model.graph.node) + assert report["audit"]["return"]["shape"] == [1, 3, 9] + for name, evidence in report["evidence"].items(): + assert evidence["sha256"] == gate.artifacts.sha256(out / name) diff --git a/tests/test_qwen3_full_structure.py b/tests/test_qwen3_full_structure.py new file mode 100644 index 0000000..e199933 --- /dev/null +++ b/tests/test_qwen3_full_structure.py @@ -0,0 +1,327 @@ +"""Small complete decoder fixtures and deliberately corrupted structure gates.""" + +from copy import deepcopy + +import numpy as np +import pytest + +onnx = pytest.importorskip("onnx") + +from probes.w2_qwen3_parse.validation import ( + Qwen3StructureError, audit_graph_structure, make_control_constant_reader, +) + + +def qwen3_fixture(layers=2): + """A tiny, valid ONNX decoder using every required architectural path.""" + config = {"num_hidden_layers": layers, "hidden_size": 4, "intermediate_size": 6, + "num_attention_heads": 2, "num_key_value_heads": 1, "head_dim": 2, + "vocab_size": 9, "sequence_length": 3, "rms_norm_eps": 1e-6} + nodes, weights, constants = [], [], {} + + def node(op, inputs, output, **attrs): + nodes.append(onnx.helper.make_node(op, inputs, [output], name=f"opaque_{len(nodes)}", **attrs)) + return output + + def constant(value, dtype=np.int64): + array = np.asarray(value, dtype=dtype) + key = (array.dtype.str, array.shape, array.tobytes()) + if key not in constants: + constants[key] = node("Constant", [], f"constant_{len(constants)}", + value=onnx.numpy_helper.from_array(array)) + return constants[key] + + def weight(name, shape): + data = np.arange(np.prod(shape), dtype=np.float32).reshape(shape) / 100 + len(weights) / 1000 + weights.append(onnx.numpy_helper.from_array(data, name)) + + shapes = {"self_attn.q_proj.weight": (4, 4), "self_attn.k_proj.weight": (2, 4), + "self_attn.v_proj.weight": (2, 4), "self_attn.o_proj.weight": (4, 4), + "self_attn.q_norm.weight": (2,), "self_attn.k_norm.weight": (2,), + "mlp.gate_proj.weight": (6, 4), "mlp.up_proj.weight": (6, 4), + "mlp.down_proj.weight": (4, 6), "input_layernorm.weight": (4,), + "post_attention_layernorm.weight": (4,)} + for layer in range(layers): + for name, shape in shapes.items(): + weight(f"model.model.layers.{layer}.{name}", shape) + weight("model.model.norm.weight", (4,)) + weight("model.lm_head.weight", (9, 4)) + weights.append(onnx.numpy_helper.from_array(np.arange(3, dtype=np.int64).reshape(1, 3), "positions")) + weights.append(onnx.numpy_helper.from_array(np.array([1.0], dtype=np.float32), "model.model.rotary_emb.inv_freq")) + + def norm(raw, name, weight_name): + square = node("Pow", [raw, constant(2, np.float32)], name + "_square") + mean = node("ReduceMean", [square, constant([-1])], name + "_mean", keepdims=1) + plus = node("Add", [mean, constant(config["rms_norm_eps"], np.float32)], name + "_eps") + sqrt = node("Sqrt", [plus], name + "_sqrt") + inv = node("Reciprocal", [sqrt], name + "_inv") + scaled = node("Mul", [raw, inv], name + "_scaled") + return node("Mul", [weight_name, scaled], name) + + def projection(raw, name, weight_name): + transpose = node("Transpose", [weight_name], name + "_weight", perm=[1, 0]) + return node("MatMul", [raw, transpose], name) + + def rope(raw, name, cosine, sine): + first = node("Slice", [raw, constant([0]), constant([1]), constant([-1])], name + "_first") + second = node("Slice", [raw, constant([1]), constant([2]), constant([-1])], name + "_second") + negative = node("Neg", [second], name + "_negative") + rotated = node("Concat", [negative, first], name + "_rotated", axis=-1) + original = node("Mul", [raw, cosine], name + "_cos") + rotated = node("Mul", [rotated, sine], name + "_sin") + return node("Add", [original, rotated], name) + + def repeat(raw, name): + expanded = node("Unsqueeze", [raw, constant([2])], name + "_unsqueezed") + expanded = node("Expand", [expanded, constant([1, 1, 2, 3, 2])], name + "_expanded") + return node("Reshape", [expanded, constant([1, 2, 3, 2])], name) + + previous = node("Gather", ["model.lm_head.weight", "input_ids"], "embedding", axis=0) + positions = node("Cast", ["positions"], "float_positions", to=onnx.TensorProto.FLOAT) + positions = node("Unsqueeze", [positions, constant([1])], "positions_3d") + freq = node("Unsqueeze", ["model.model.rotary_emb.inv_freq", constant([0, 2])], "freq_3d") + freq = node("MatMul", [freq, positions], "freq_matmul") + freq = node("Transpose", [freq], "freq_transpose", perm=[0, 2, 1]) + freq = node("Concat", [freq, freq], "freq_full_head", axis=-1) + cosine = node("Cos", [freq], "cosine") + sine = node("Sin", [freq], "sine") + cosine = node("Unsqueeze", [cosine, constant([1])], "cosine_4d") + sine = node("Unsqueeze", [sine, constant([1])], "sine_4d") + for layer in range(layers): + prefix = f"layer_{layer}" + path = f"model.model.layers.{layer}." + normalized = norm(previous, prefix + "_input_norm", path + "input_layernorm.weight") + q = projection(normalized, prefix + "_q", path + "self_attn.q_proj.weight") + k = projection(normalized, prefix + "_k", path + "self_attn.k_proj.weight") + v = projection(normalized, prefix + "_v", path + "self_attn.v_proj.weight") + q = node("Reshape", [q, constant([1, 3, 2, 2])], prefix + "_q_view") + k = node("Reshape", [k, constant([1, 3, 1, 2])], prefix + "_k_view") + v = node("Reshape", [v, constant([1, 3, 1, 2])], prefix + "_v_view") + q = norm(q, prefix + "_q_norm", path + "self_attn.q_norm.weight") + k = norm(k, prefix + "_k_norm", path + "self_attn.k_norm.weight") + q = node("Transpose", [q], prefix + "_q_transpose", perm=[0, 2, 1, 3]) + k = node("Transpose", [k], prefix + "_k_transpose", perm=[0, 2, 1, 3]) + v = node("Transpose", [v], prefix + "_v_transpose", perm=[0, 2, 1, 3]) + q = rope(q, prefix + "_q_rope", cosine, sine) + k = rope(k, prefix + "_k_rope", cosine, sine) + k = repeat(k, prefix + "_k_repeat") + v = repeat(v, prefix + "_v_repeat") + k = node("Transpose", [k], prefix + "_k_scores", perm=[0, 1, 3, 2]) + scores = node("MatMul", [q, k], prefix + "_scores") + scores = node("Mul", [scores, constant(2 ** -0.5, np.float32)], prefix + "_scaled_scores") + scores = node("Add", [scores, "attention_mask"], prefix + "_masked_scores") + probabilities = node("Softmax", [scores], prefix + "_probabilities", axis=-1) + values = node("MatMul", [probabilities, v], prefix + "_weighted_values") + values = node("Transpose", [values], prefix + "_attention_transpose", perm=[0, 2, 1, 3]) + values = node("Reshape", [values, constant([1, 3, 4])], prefix + "_attention_view") + output = projection(values, prefix + "_o", path + "self_attn.o_proj.weight") + residual = node("Add", [previous, output], prefix + "_attention_residual") + normalized = norm(residual, prefix + "_post_norm", path + "post_attention_layernorm.weight") + gate = projection(normalized, prefix + "_gate", path + "mlp.gate_proj.weight") + up = projection(normalized, prefix + "_up", path + "mlp.up_proj.weight") + sigmoid = node("Sigmoid", [gate], prefix + "_sigmoid") + activated = node("Mul", [gate, sigmoid], prefix + "_silu") + swiglu = node("Mul", [activated, up], prefix + "_swiglu") + down = projection(swiglu, prefix + "_down", path + "mlp.down_proj.weight") + previous = node("Add", [residual, down], prefix + "_mlp_residual") + normalized = norm(previous, "final_norm", "model.model.norm.weight") + projection(normalized, "logits", "model.lm_head.weight") + helper = onnx.helper.make_tensor_value_info + graph = onnx.helper.make_graph(nodes, "small_full_structure", + [helper("input_ids", onnx.TensorProto.INT64, [1, 3]), + helper("attention_mask", onnx.TensorProto.FLOAT, [1, 1, 3, 3])], + [helper("logits", onnx.TensorProto.FLOAT, [1, 3, 9])], weights) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 18)], ir_version=10) + return model, config + + +def output_node(model, name): + return next(node for node in model.graph.node if name in node.output) + + +def replace_control(model, output, input_index, value, dtype=np.int64): + target_index = next(i for i, node in enumerate(model.graph.node) if output in node.output) + constant_name = output + "_corrupt_control" + constant = onnx.helper.make_node("Constant", [], [constant_name], + value=onnx.numpy_helper.from_array(np.asarray(value, dtype=dtype))) + model.graph.node.insert(target_index, constant) + output_node(model, output).input[input_index] = constant_name + + +def test_complete_small_graph_validates_actual_dataflow(): + model, config = qwen3_fixture() + onnx.checker.check_model(model, full_check=True) + result = audit_graph_structure(model, config) + assert result["passed"] and result["tied_embeddings"] + assert result["layer_count"] == 2 + assert result["graph"]["initializer_count"] == 26 + assert result["live_node_count"] == len(model.graph.node) + assert result["layers"][1]["input"] == "layer_0_mlp_residual" + + +def test_layer_detection_does_not_depend_on_node_names_or_exporter_metadata(): + model, config = qwen3_fixture() + for node in model.graph.node: + node.name = "" + del node.metadata_props[:] + assert audit_graph_structure(model, config)["layer_count"] == 2 + + +def test_external_weight_bytes_are_not_loaded(monkeypatch): + model, config = qwen3_fixture() + for tensor in model.graph.initializer: + onnx.external_data_helper.set_external_data(tensor, "does-not-exist.data", offset=0, + length=len(tensor.raw_data)) + tensor.ClearField("raw_data") + tensor.data_location = onnx.TensorProto.EXTERNAL + def forbidden(*args, **kwargs): + raise AssertionError("structure validation must not read external weight data") + monkeypatch.setattr(onnx.external_data_helper, "load_external_data_for_tensor", forbidden) + assert audit_graph_structure(model, config)["passed"] + + +def test_control_reader_uses_original_onnx_graph_without_parser_state(): + model, _ = qwen3_fixture() + reader = make_control_constant_reader(model) + shape_name = output_node(model, "layer_0_k_repeat_expanded").input[1] + np.testing.assert_array_equal(reader(shape_name), [1, 1, 2, 3, 2]) + assert reader(shape_name).dtype == np.int64 + + +def test_control_reader_rejects_external_and_large_tensors_without_loading(): + model, _ = qwen3_fixture() + tensor = model.graph.initializer[0] + onnx.external_data_helper.set_external_data(tensor, "missing.data", offset=0, length=len(tensor.raw_data)) + tensor.ClearField("raw_data") + tensor.data_location = onnx.TensorProto.EXTERNAL + reader = make_control_constant_reader(model) + with pytest.raises(Qwen3StructureError, match="external or large tensor"): + reader(tensor.name) + big = onnx.numpy_helper.from_array(np.zeros(1025, dtype=np.int64), "big_control") + model.graph.initializer.append(big) + with pytest.raises(Qwen3StructureError, match="external or large tensor"): + make_control_constant_reader(model)("big_control") + + +@pytest.mark.parametrize("change,expected", [("shape", "expected shape"), ("dtype", "dtype"), + ("missing", "missing="), ("extra", "unexpected="), + ("duplicate", "duplicate weight")]) +def test_weight_inventory_rejects_incorrect_metadata(change, expected): + model, config = qwen3_fixture() + tensor = model.graph.initializer[0] + if change == "shape": + tensor.dims[0] += 1 + elif change == "dtype": + tensor.data_type = onnx.TensorProto.FLOAT16 + elif change == "missing": + del model.graph.initializer[0] + elif change == "extra": + extra = deepcopy(tensor) + extra.name = "model.model.layers.99.self_attn.q_proj.weight" + model.graph.initializer.append(extra) + else: + model.graph.initializer.append(deepcopy(tensor)) + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("target", ["layer_1_input_norm_square", "layer_1_input_norm_scaled"]) +def test_cross_layer_norm_must_use_same_previous_residual(target): + model, config = qwen3_fixture() + output_node(model, target).input[0] = "embedding" + with pytest.raises(Qwen3StructureError, match="layer 1.*input mismatch"): + audit_graph_structure(model, config) + + +def test_bypassed_layer_is_rejected_even_when_all_weights_remain(): + model, config = qwen3_fixture() + for name in ("final_norm_square", "final_norm_scaled"): + output_node(model, name).input[0] = "layer_0_mlp_residual" + with pytest.raises(Qwen3StructureError, match="final norm must consume the last layer"): + audit_graph_structure(model, config) + + +def test_wrong_residual_bypass_is_rejected(): + model, config = qwen3_fixture() + output_node(model, "layer_1_attention_residual").input[0] = "embedding" + with pytest.raises(Qwen3StructureError, match="layer 1: attention residual"): + audit_graph_structure(model, config) + + +def test_dead_final_norm_is_rejected(): + model, config = qwen3_fixture() + output_node(model, "logits").input[0] = "layer_1_mlp_residual" + with pytest.raises(Qwen3StructureError, match="LM Head must consume final norm"): + audit_graph_structure(model, config) + + +def test_shared_embedding_weight_cannot_be_replaced_by_another_matrix(): + model, config = qwen3_fixture() + output_node(model, "embedding").input[0] = "model.model.layers.0.self_attn.q_proj.weight" + with pytest.raises(Qwen3StructureError, match="embedding: tied LM Head"): + audit_graph_structure(model, config) + + +def test_same_shape_layer_weights_cannot_be_substituted(): + model, config = qwen3_fixture() + output_node(model, "layer_0_q_weight").input[0] = "model.model.layers.1.self_attn.q_proj.weight" + with pytest.raises(Qwen3StructureError, match="layer 0 self_attn.q_proj"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("node_name,new_input,expected", [ + ("layer_0_q_norm_square", "layer_0_k_view", "input mismatch"), + ("layer_0_sigmoid", "layer_0_up", "sigmoid input must be gate"), + ("layer_0_masked_scores", "layer_0_scores", "additive attention mask"), +]) +def test_key_computations_cannot_be_rewired(node_name, new_input, expected): + model, config = qwen3_fixture() + node = output_node(model, node_name) + node.input[1 if node_name == "layer_0_masked_scores" else 0] = new_input + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +def test_disconnected_extra_compute_node_cannot_hide_in_a_complete_graph(): + model, config = qwen3_fixture() + model.graph.node.append(onnx.helper.make_node("Identity", ["logits"], ["dead_output"], name="dead_node")) + with pytest.raises(Qwen3StructureError, match="disconnected from logits.*dead_node"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("output,input_index,value,dtype,expected", [ + ("layer_0_input_norm_square", 1, 3, np.float32, "must be scalar 2"), + ("layer_0_input_norm_mean", 1, [0], np.int64, "axes=\\[-1\\]"), + ("layer_0_input_norm_eps", 1, 0.001, np.float32, "must be scalar 1e-06"), + ("layer_0_scaled_scores", 1, 1, np.float32, "attention scale"), + ("layer_0_k_repeat_expanded", 1, [1, 2, 1, 3, 2], np.int64, "GQA K expansion shape"), + ("layer_0_v_repeat", 1, [1, 1, 3, 4], np.int64, "GQA V head reshape"), + ("layer_0_v_repeat_unsqueezed", 1, [1], np.int64, "GQA V repeat group must be axis 2"), +]) +def test_numerical_structure_constants_and_group_layout_are_checked(output, input_index, value, dtype, expected): + model, config = qwen3_fixture() + replace_control(model, output, input_index, value, dtype) + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) + + +def test_norm_weight_used_as_both_operands_has_a_clear_validation_error(): + model, config = qwen3_fixture() + node = output_node(model, "layer_0_input_norm") + node.input[1] = node.input[0] + with pytest.raises(Qwen3StructureError, match="normalized activation"): + audit_graph_structure(model, config) + + +@pytest.mark.parametrize("change,expected", [("output", "graph output"), ("shape", "incorrect static shape"), + ("dtype", "incorrect static shape")]) +def test_static_interface_is_required(change, expected): + model, config = qwen3_fixture() + if change == "output": + model.graph.output[0].name = "layer_1_mlp_residual" + elif change == "shape": + model.graph.output[0].type.tensor_type.shape.dim[1].dim_value = 2 + else: + model.graph.input[0].type.tensor_type.elem_type = onnx.TensorProto.INT32 + with pytest.raises(Qwen3StructureError, match=expected): + audit_graph_structure(model, config) diff --git a/tests/test_qwen3_tokenizer.py b/tests/test_qwen3_tokenizer.py new file mode 100644 index 0000000..81eb29d --- /dev/null +++ b/tests/test_qwen3_tokenizer.py @@ -0,0 +1,221 @@ +"""Offline contract tests; official assets are checked by the runtime probe.""" + +import json +from pathlib import Path + +import numpy as np +import pytest + +pytest.importorskip("tokenizers") +from tokenizers import AddedToken, Tokenizer, decoders, models, normalizers, pre_tokenizers, processors + +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + + +@pytest.fixture +def source(tmp_path): + alphabet = sorted(pre_tokenizers.ByteLevel.alphabet()) + engine = Tokenizer(models.BPE(vocab={s: i for i, s in enumerate(alphabet)}, merges=[])) + engine.normalizer = normalizers.NFC() + engine.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + engine.decoder = decoders.ByteLevel() + engine.post_processor = processors.ByteLevel(trim_offsets=False) + engine.add_special_tokens([ + AddedToken("<|endoftext|>", special=True, normalized=False), + AddedToken("<|im_start|>", special=True, normalized=False), + AddedToken("<|im_end|>", special=True, normalized=False), + ]) + engine.add_tokens([AddedToken("", special=False, normalized=False)]) + engine.save(str(tmp_path / "tokenizer.json")) + added = { + str(token_id): {name: getattr(token, name) for name in ( + "content", "lstrip", "rstrip", "single_word", "normalized", "special" + )} + for token_id, token in engine.get_added_tokens_decoder().items() + } + _write(tmp_path, "tokenizer_config.json", { + "tokenizer_class": "Qwen2Tokenizer", "add_bos_token": False, + "bos_token": None, "unk_token": None, "eos_token": "<|im_end|>", + "pad_token": "<|endoftext|>", "clean_up_tokenization_spaces": False, + "added_tokens_decoder": added, "additional_special_tokens": ["<|im_start|>", "<|im_end|>"], + }) + _write(tmp_path, "config.json", {"model_type": "qwen3", "vocab_size": 288, "bos_token_id": 256}) + _write(tmp_path, "generation_config.json", {"eos_token_id": [258, 256], "pad_token_id": 256}) + return tmp_path + + +def _write(directory: Path, filename: str, value: dict): + (directory / filename).write_text(json.dumps(value), encoding="utf-8") + + +def _change(directory, filename, **changes): + data = json.loads((directory / filename).read_text(encoding="utf-8")) + data.update(changes) + _write(directory, filename, data) + + +def test_metadata_distinguishes_vocab_and_generation_eos(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + assert tokenizer.base_vocab_size == 256 + assert tokenizer.tokenizer_vocab_size == 260 + assert tokenizer.model_vocab_size == 288 + assert tokenizer.pad_token_id == 256 + assert tokenizer.eos_token_id == 258 + assert tokenizer.generation_eos_token_ids == (258, 256) + assert tokenizer.bos_token_id is None + assert tokenizer.unk_token_id is None + assert tokenizer.valid_token_ids == frozenset(range(260)) + assert tokenizer.is_valid_token_id(np.int64(259)) + assert not tokenizer.is_valid_token_id(260) + assert not tokenizer.is_valid_token_id(True) + + +def test_missing_unk_uses_official_default_but_explicit_null_disables_it(source): + config = json.loads((source / "tokenizer_config.json").read_text(encoding="utf-8")) + assert Qwen3Tokenizer.from_directory(source).unk_token_id is None + del config["unk_token"] + _write(source, "tokenizer_config.json", config) + assert Qwen3Tokenizer.from_directory(source).unk_token_id == 256 + + +@pytest.mark.parametrize("text", ["", "Hello, world!", " a\t\r\nb ", "中文日本語", "😀👩‍💻", "a\x00b"]) +def test_text_round_trip_and_no_implicit_special_tokens(source, text): + tokenizer = Qwen3Tokenizer.from_directory(source) + ids = tokenizer.encode(text) + assert ids == tokenizer.encode(text, add_special_tokens=False) + assert not set(ids) & {256, 257, 258} + assert tokenizer.decode(ids) == text + + +def test_normalization_is_official_json_behavior(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + assert tokenizer.encode("e\u0301") == tokenizer.encode("é") + assert tokenizer.decode(tokenizer.encode("e\u0301")) == "é" + + +def test_special_tokens_and_ordinary_added_tokens(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + text = "a<|im_start|><|im_end|><|endoftext|>b" + ids = tokenizer.encode(text) + assert all(token_id in ids for token_id in (256, 257, 258, 259)) + assert tokenizer.decode(ids) == text + assert tokenizer.decode(ids, skip_special_tokens=True) == "ab" + assert tokenizer.decode(np.int64(258)) == "<|im_end|>" + assert tokenizer.decode(np.array([257, 258], dtype=np.int64)) == "<|im_start|><|im_end|>" + + +def test_cleanup_is_opt_in_and_matches_transformers_rules(source): + tokenizer = Qwen3Tokenizer.from_directory(source) + text = "I 'm sure , you 're here . We 've seen it ! He 's fine ? I do n't know ." + ids = tokenizer.encode(text) + assert tokenizer.decode(ids) == text + assert tokenizer.decode(ids, clean_up_tokenization_spaces=True) == "I'm sure, you're here. We've seen it! He's fine? I don't know." + _change(source, "tokenizer_config.json", clean_up_tokenization_spaces=True) + configured = Qwen3Tokenizer.from_directory(source) + assert configured.decode(ids) == tokenizer.decode(ids, clean_up_tokenization_spaces=True) + assert configured.decode(ids, clean_up_tokenization_spaces=False) == text + + +@pytest.mark.parametrize("text", [None, 12, b"text", ["text"]]) +def test_encode_rejects_non_text(source, text): + with pytest.raises(TypeError, match="text must be str"): + Qwen3Tokenizer.from_directory(source).encode(text) + + +def test_encode_rejects_invalid_unicode(source): + with pytest.raises(UnicodeEncodeError): + Qwen3Tokenizer.from_directory(source).encode("\ud800") + + +@pytest.mark.parametrize("ids", [ + [True], [1.0], ["1"], [[1]], [None], "123", b"123", bytearray(b"123"), + None, True, {1, 2}, {1: "one"}, +]) +def test_decode_rejects_noninteger_ids(source, ids): + with pytest.raises(TypeError): + Qwen3Tokenizer.from_directory(source).decode(ids) + + +@pytest.mark.parametrize("ids", [[-1], [260], [287], [288], [2**80]]) +def test_decode_refuses_unknown_or_unused_model_rows(source, ids): + with pytest.raises(ValueError, match="not in the tokenizer vocabulary"): + Qwen3Tokenizer.from_directory(source).decode(ids) + + +@pytest.mark.parametrize("option", ["add_special_tokens", "skip_special_tokens", "clean_up_tokenization_spaces"]) +def test_flags_are_boolean(source, option): + tokenizer = Qwen3Tokenizer.from_directory(source) + with pytest.raises(TypeError, match="must be bool"): + if option == "add_special_tokens": + tokenizer.encode("text", **{option: 1}) + else: + tokenizer.decode([], **{option: 1}) + + +@pytest.mark.parametrize(("filename", "changes", "message"), [ + ("config.json", {"model_type": "other"}, "model_type"), + ("config.json", {"vocab_size": True}, "vocab_size"), + ("config.json", {"vocab_size": 259}, "vocab_size"), + ("tokenizer_config.json", {"tokenizer_class": "OtherTokenizer"}, "tokenizer_class"), + ("tokenizer_config.json", {"add_bos_token": True}, "add_bos_token"), + ("tokenizer_config.json", {"add_prefix_space": True}, "add_prefix_space"), + ("tokenizer_config.json", {"split_special_tokens": True}, "split_special_tokens"), + ("tokenizer_config.json", {"mask_token": ""}, "mask_token"), + ("tokenizer_config.json", {"sep_token": ""}, "sep_token"), + ("tokenizer_config.json", {"cls_token": ""}, "cls_token"), + ("tokenizer_config.json", {"extra_special_tokens": {"image_token": ""}}, "extra_special_tokens"), + ("tokenizer_config.json", {"pad_token": "absent"}, "pad_token"), + ("tokenizer_config.json", {"additional_special_tokens": [""]}, "special token"), + ("tokenizer_config.json", {"added_tokens_decoder": {}}, "added token IDs differ"), + ("generation_config.json", {"eos_token_id": []}, "eos_token_id"), + ("generation_config.json", {"eos_token_id": [256]}, "include the tokenizer EOS"), + ("generation_config.json", {"eos_token_id": [258, 280]}, "not in the tokenizer vocabulary"), + ("generation_config.json", {"pad_token_id": 257}, "pad_token_id"), +]) +def test_configuration_mismatch_is_not_silently_ignored(source, filename, changes, message): + _change(source, filename, **changes) + with pytest.raises(ValueError, match=message): + Qwen3Tokenizer.from_directory(source) + + +def test_inconsistent_added_token_flags_rejected(source): + data = json.loads((source / "tokenizer_config.json").read_text(encoding="utf-8")) + data["added_tokens_decoder"]["259"]["special"] = True + _write(source, "tokenizer_config.json", data) + with pytest.raises(ValueError, match="inconsistent special"): + Qwen3Tokenizer.from_directory(source) + + +@pytest.mark.parametrize("setting", ["padding", "truncation", "post_processor"]) +def test_no_implicit_padding_truncation_or_bos(source, setting): + engine = Tokenizer.from_file(str(source / "tokenizer.json")) + if setting == "padding": + engine.enable_padding(length=10, pad_id=256, pad_token="<|endoftext|>") + elif setting == "truncation": + engine.enable_truncation(max_length=2) + else: + engine.post_processor = processors.TemplateProcessing(single="<|im_start|> $A", special_tokens=[("<|im_start|>", 257)]) + engine.save(str(source / "tokenizer.json")) + with pytest.raises(ValueError, match="automatic"): + Qwen3Tokenizer.from_directory(source) + + +@pytest.mark.parametrize("nested", [False, True]) +def test_json_cannot_insert_prefix_space_even_when_config_disables_it(source, nested): + engine = Tokenizer.from_file(str(source / "tokenizer.json")) + prefix = pre_tokenizers.ByteLevel(add_prefix_space=True) + engine.pre_tokenizer = pre_tokenizers.Sequence([prefix]) if nested else prefix + engine.save(str(source / "tokenizer.json")) + with pytest.raises(ValueError, match="add_prefix_space"): + Qwen3Tokenizer.from_directory(source) + + +def test_generation_single_eos_supported(source): + _change(source, "generation_config.json", eos_token_id=258) + assert Qwen3Tokenizer.from_directory(source).generation_eos_token_ids == (258,) + + +def test_files_are_local_and_required(source): + (source / "generation_config.json").unlink() + with pytest.raises(FileNotFoundError): + Qwen3Tokenizer.from_directory(source) diff --git a/tests/test_riscv_tensor_runtime.py b/tests/test_riscv_tensor_runtime.py index de1c8de..517a9c3 100644 --- a/tests/test_riscv_tensor_runtime.py +++ b/tests/test_riscv_tensor_runtime.py @@ -217,6 +217,32 @@ def _assert_timed_child_terminated(pid, started): _assert_posix_process_terminated(pid, starttime=started) +@pytest.mark.parametrize("interruption", [KeyboardInterrupt, SystemExit]) +def test_interruption_stops_actual_owned_process(monkeypatch, tmp_path, interruption): + communicate = subprocess.Popen.communicate + interrupted = [] + + def interrupt_once(process, *args, **kwargs): + if not interrupted: + interrupted.append(process) + raise interruption(143) + return communicate(process, *args, **kwargs) + + monkeypatch.setattr(subprocess.Popen, "communicate", interrupt_once) + with pytest.raises(interruption): + _run_process([sys.executable, "-B", "-c", "import time;time.sleep(60)"], + cwd=tmp_path, timeout=30) + assert len(interrupted) == 1 + assert interrupted[0].poll() is not None + + +def test_termination_signal_requests_normal_cleanup(): + from scratchv.runtime.riscv_tensor import _terminate_invocation + with pytest.raises(SystemExit) as caught: + _terminate_invocation(15, None) + assert caught.value.code == 143 + + @pytest.mark.parametrize("field,value", [ ("class", 1), ("endian", 2), ("machine", 62), ("entry", INPUT_BASE), ("abi", 0), ]) @@ -347,6 +373,68 @@ def process(command, *, cwd, timeout): assert error.__cause__ is not None +@pytest.mark.parametrize("outcome,message", [ + ("guest", "failure status 3"), ("nonzero", r"QEMU failed \(7\): rejected"), + ("bad_frame", "truncated or missing"), +]) +def test_qemu_log_failure_keeps_primary_error_and_other_log( + tmp_path, monkeypatch, timed_guest, outcome, message): + runtime, executable, feed, clock = timed_guest + original_write = Path.write_bytes + + def write(path, data): + if path.name == "qemu.stdout": + raise PermissionError("stdout is locked") + return original_write(path, data) + + def process(command, *, cwd, timeout): + clock.now += 2.5 + if outcome == "guest": + (cwd / "uart.bin").write_bytes(frame(b"", status=3)) + elif outcome == "bad_frame": + (cwd / "uart.bin").write_bytes(b"broken") + return subprocess.CompletedProcess(command, 7 if outcome != "bad_frame" else 0, + b"partial stdout", b"rejected") + + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(runtime, "_run_process", process) + run_dir = tmp_path / "run" + with pytest.raises(RiscVTensorExecutionError, match=message) as captured: + run_riscv_tensor(executable, feed, run_dir) + assert "stdout is locked" in str(captured.value) + assert captured.value.elapsed_s == 2.5 and clock.reads == 2 + assert (run_dir / "qemu.stderr").read_bytes() == b"rejected" + + +@pytest.mark.parametrize("timeout", [False, True]) +def test_compiler_log_failure_keeps_primary_error_and_other_log(tmp_path, monkeypatch, timeout): + from scratchv.runtime import riscv_tensor as runtime + + original_write = Path.write_bytes + + def write(path, data): + if path.name == "compiler.stdout": + raise PermissionError("compiler stdout is locked") + return original_write(path, data) + + def process(command, **kwargs): + if timeout: + raise subprocess.TimeoutExpired(command, kwargs["timeout"], output=b"partial", stderr=b"rejected") + return subprocess.CompletedProcess(command, 11, b"partial", b"rejected") + + artifact = SimpleNamespace(source="int scratchv_run(void){return 0;}", inputs=(), + output=spec("result", (1,), "float32"), function_name="scratchv_run") + tools = SimpleNamespace(cc=("fixture-clang",), qemu="fixture-qemu", is_zig=False) + monkeypatch.setattr(Path, "write_bytes", write) + monkeypatch.setattr(runtime, "_run_process", process) + error = TimeoutError if timeout else RuntimeError + message = "compilation exceeded 2" if timeout else r"compilation failed \(11\)" + with pytest.raises(error, match=message) as captured: + build_riscv_tensor(artifact, tmp_path / "build", tools, timeout=2) + assert "compiler stdout is locked" in str(captured.value) + assert (tmp_path / "build/compiler.stderr").read_bytes() == b"rejected" + + @pytest.mark.parametrize("log_error", [False, True]) def test_timeout_reports_actual_time_including_cleanup(tmp_path, monkeypatch, timed_guest, log_error): runtime, executable, feed, clock = timed_guest @@ -380,7 +468,7 @@ def write(path, data): assert "cannot preserve timeout log" in str(error) else: assert (tmp_path / "run/qemu.stdout").read_bytes() == b"partial" - assert (tmp_path / "run/qemu.stderr").read_bytes() == b"timed out" + assert (tmp_path / "run/qemu.stderr").read_bytes() == b"timed out" @pytest.mark.parametrize("failure", ["elf", "input", "stale", "write_input", "spawn"]) diff --git a/tests/test_tensor_c_codegen.py b/tests/test_tensor_c_codegen.py index 6fccf71..23dd265 100644 --- a/tests/test_tensor_c_codegen.py +++ b/tests/test_tensor_c_codegen.py @@ -330,3 +330,45 @@ def test_same_generator_is_reusable_and_never_mutates_ir(): first, second = generator.generate(), generator.generate() assert first == second assert b.program.dump() == before + + +@pytest.mark.parametrize("dtype,numpy_dtype", [(D.FLOAT32, np.float32), (D.INT32, np.int32), + (D.INT64, np.int64)]) +def test_scalar_initializer_cannot_override_ir_literal(dtype, numpy_dtype): + from scratchv.verification.ir_interpreter import IRExecutionError, IRInterpreter + + scale = Value("scale", dtype, is_constant=True, const_value=1) + b = builder() + b.program.global_values.append(scale) + b.ret(scale) + bindings = {"scale": np.array(2, dtype=numpy_dtype)} + with pytest.raises(IRExecutionError, match="initializer disagrees with scalar constant"): + IRInterpreter(b.program).run({}, initializers=bindings) + with pytest.raises(TensorCCodegenError, match="scale disagrees with scalar constant"): + TensorCCodegen(b.program, bindings).generate() + + +def test_scalar_constant_binding_cannot_be_a_vector(): + scale = Value("scale", D.FLOAT32, is_constant=True, const_value=1.0, shape=(1,)) + b = builder() + b.program.global_values.append(scale) + b.ret(scale) + with pytest.raises(TensorCCodegenError, match="scale disagrees with scalar constant"): + TensorCCodegen(b.program, {"scale": np.array([1], np.float32)}).generate() + + +def test_matching_scalar_initializer_preserves_compiled_result(tmp_path, compiler): + from scratchv.verification.ir_interpreter import IRInterpreter + + x = Value("x", shape=(2,)) + scale = Value("scale", D.FLOAT32, is_constant=True, const_value=1.25) + b = builder(x) + b.program.global_values.append(scale) + b.ret(b.mul(x, scale)) + bindings = {"scale": np.array(1.25, np.float32)} + feed = {"x": np.array([2, -3], np.float32)} + expected = IRInterpreter(b.program).run(feed, initializers=bindings).return_value + with compiled(tmp_path, TensorCCodegen(b.program, bindings).generate(), compiler) as execute: + status, actual = execute(feed) + assert status == 0 + np.testing.assert_array_equal(actual, expected) diff --git a/tests/test_w2_acceptance.py b/tests/test_w2_acceptance.py new file mode 100644 index 0000000..44a8796 --- /dev/null +++ b/tests/test_w2_acceptance.py @@ -0,0 +1,296 @@ +"""Acceptance orchestration must not turn partial, stale or failed evidence into PASS.""" +import argparse +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +import pytest + +SPEC = importlib.util.spec_from_file_location("w2_acceptance", Path(__file__).parents[1] / "scripts/run_w2_acceptance.py") +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +def arguments(tmp_path, gates=None): + return argparse.Namespace(output_dir=tmp_path / "fresh", gates=list(runner.GATES) if gates is None else gates, + python=sys.executable, tokenizer_dir=tmp_path / "assets", tokenizer_mode="verify", + model_dir=tmp_path / "model", model_mode="verify", cc="compiler", qemu="emulator", + qemu_timeout=180, parse_timeout=300) + + +def runtime_fixture(tmp_path): + root = tmp_path / "source" + corpus = root / "probes/w2_runtime/corpus.json" + corpus.parent.mkdir(parents=True) + names = [f"case_{i}" for i in range(20)] + corpus.write_text(json.dumps({"cases": [{"name": name} for name in names]}), encoding="utf-8") + folder = tmp_path / "run/runtime" + folder.mkdir(parents=True) + for name in ("report.md", "report.html"): + (folder / name).write_text("PASS", encoding="utf-8") + sources = {runner.SCRIPTS["runtime"]: "a" * 64} + data = {"gate": "unit:runtime", "passed": True, "stage": "complete", "source_sha256": sources, + "tokenizer_cases": [{"name": name, "passed": True, "checks": 24} for name in names], + "runtime_cases": [{"name": str(i), "passed": True} for i in range(5)]} + return root, runner.Gate("runtime", (), folder), sources, data + + +def validate_fixture(fixture, mutate=lambda data: None): + root, gate, sources, data = fixture + mutate(data) + (gate.output / "report.json").write_text(json.dumps(data), encoding="utf-8") + return runner.validate_gate_report(gate, sources, root) + + +def test_plan_is_fresh_serial_and_explicit(tmp_path): + args = arguments(tmp_path) + plan = runner.build_plan(args) + assert [gate.name for gate in plan] == list(runner.GATES) + for gate in plan: + assert gate.command[:4] == (sys.executable, "-B", "-X", "utf8") + assert str(gate.output) in " ".join(gate.command) + assert plan[5].dependencies == ("small-ir",) + assert str(args.output_dir / "small-ir") in plan[5].command + assert "--mode" in plan[-1].command and "verify" in plan[-1].command + assert "compiler" in plan[1].command and "emulator" in plan[1].command + + +def test_frontend_real_pytest_tmp_path_in_fresh_gate_directory(tmp_path): + """Pytest's basetemp.mkdir has no parents=True; its parent must already exist.""" + source = tmp_path / "source" + tests = source / "tests" + tests.mkdir(parents=True) + (tests / "test_qwen3_frontend_patterns.py").write_text( + "import pytest\n@pytest.mark.parametrize('number', range(22))\n" + "def test_real_tmp_path(tmp_path, number):\n" + " path = tmp_path / str(number)\n" + " path.write_text('fresh')\n" + " assert path.read_text() == 'fresh'\n", encoding="utf-8") + args = arguments(tmp_path, ["frontend"]) + args.output_dir.mkdir() + gate = runner.build_plan(args, source)[0] + assert not gate.output.exists() + result = runner.run_gate(gate, {}, timeout=30, root=source) + assert result["status"] == "PASS", Path(result["log"]).read_text(encoding="utf-8") + assert result["validation"]["tests"] == 22 + + +@pytest.mark.parametrize("chosen,expected", [(["small-qemu"], ["small-ir", "small-qemu"]), + (["runtime-model"], ["runtime", "runtime-model"]), + (["frontend"], ["frontend"])]) +def test_subset_auto_adds_only_required_dependencies(tmp_path, chosen, expected): + assert [gate.name for gate in runner.build_plan(arguments(tmp_path, chosen))] == expected + + +@pytest.mark.parametrize("status,expected", [("PASS", "PASS"), ("NOT_RUN", "PARTIAL"), ("BLOCKED", "FAIL"), ("FAIL", "FAIL")]) +def test_overall_requires_every_gate(status, expected): + report = {"gates": [{"name": name, "status": "PASS"} for name in runner.GATES]} + report["gates"][-1]["status"] = status + runner.finalize_status(report) + assert report["status"] == expected + assert report["passed"] is (expected == "PASS") + + +def test_top_level_source_error_blocks_pass(): + report = {"error": "source changed", "gates": [{"status": "PASS"} for _ in runner.GATES]} + runner.finalize_status(report) + assert report["status"] == "FAIL" + + +def test_git_unavailable_archive_is_not_failed(monkeypatch, tmp_path): + monkeypatch.setattr(runner.subprocess, "run", lambda *a, **k: (_ for _ in ()).throw(FileNotFoundError("git"))) + result = runner.checkout_evidence(tmp_path) + assert result["head"] is None and result["status"] == "unknown" + + +def test_new_source_files_have_fingerprints(tmp_path): + path = tmp_path / "scratchv/runtime/new.py" + path.parent.mkdir(parents=True) + path.write_text("x = 1\n") + before = runner.source_fingerprints(tmp_path) + path.write_text("x = 2\n") + assert before != runner.source_fingerprints(tmp_path) + + +def test_complete_runtime_evidence(tmp_path): + assert validate_fixture(runtime_fixture(tmp_path))["tokenizer_comparisons"] == 480 + + +@pytest.mark.parametrize("mutation", [ + lambda d: d.update(gate="different"), lambda d: d.update(passed=False), + lambda d: d.update(stage="tokenizer"), lambda d: d["tokenizer_cases"].pop(), + lambda d: d["runtime_cases"].pop(), lambda d: d["tokenizer_cases"][0].update(checks=23), + lambda d: d["tokenizer_cases"][1].update(name="case_0"), + lambda d: d["runtime_cases"][0].update(passed=False), + lambda d: d.update(source_sha256={}), + lambda d: d.update(source_sha256={runner.SCRIPTS["runtime"]: "wrong"}), +]) +def test_runtime_missing_or_inconsistent_evidence_fails(tmp_path, mutation): + with pytest.raises(ValueError): + validate_fixture(runtime_fixture(tmp_path), mutation) + + +@pytest.mark.parametrize("number", [None, float("nan"), float("inf"), -1, 1e-5, 0.1, True]) +def test_invalid_numeric_evidence_fails(number): + with pytest.raises(ValueError): + runner.check_numeric_tree({"passed": True, "nested": [{"passed": True, "max_abs": number}]}) + + +@pytest.mark.parametrize("field", ["seconds", "qemu_process_wall_seconds"]) +def test_nonfinite_timing_cannot_break_machine_report(field): + with pytest.raises(ValueError): + runner.check_numeric_tree({"passed": True, field: float("nan")}) + + +@pytest.mark.parametrize("tag", ["skipped", "failure", "error"]) +def test_frontend_skips_are_failures(tmp_path, tag): + cases = [f'{"<" + tag + "/>" if i == 0 else ""}' for i in range(22)] + (tmp_path / "tests.xml").write_text("" + "".join(cases) + "") + with pytest.raises(ValueError): + runner.validate_gate_report(runner.Gate("frontend", (), tmp_path), {}) + + +def test_existing_output_is_not_read_as_pass(tmp_path): + folder = tmp_path / "old" + folder.mkdir() + (folder / "report.json").write_text('{"passed":true}') + result = runner.run_gate(runner.Gate("runtime", (), folder), {}, timeout=1) + assert result["status"] == "FAIL" and "already exists" in result["error"] + + +def test_zero_exit_without_report_is_failure(tmp_path): + gate = runner.Gate("runtime", (sys.executable, "-c", "print('PASS')"), tmp_path / "gate") + result = runner.run_gate(gate, {}, timeout=10) + assert result["returncode"] == 0 and result["status"] == "FAIL" + assert "PASS" in Path(result["log"]).read_text() + + +def test_child_error_is_preserved(tmp_path): + folder = tmp_path / "gate" + code = "import pathlib,json; p=pathlib.Path(" + repr(str(folder)) + ");p.mkdir();(p/'report.json').write_text(json.dumps({'passed':False,'stage':'assets','error':'Checksum mismatch'}));raise SystemExit(1)" + result = runner.run_gate(runner.Gate("runtime", (sys.executable, "-c", code), folder), {}, timeout=10) + assert result["status"] == "FAIL" and "Checksum mismatch" in result["error"] + assert result["child_stage"] == "assets" + + +def test_timeout_is_failure(tmp_path): + result = runner.run_gate(runner.Gate("runtime", (sys.executable, "-c", "import time;time.sleep(30)"), tmp_path / "gate"), {}, timeout=0.2) + assert result["status"] == "FAIL" and "exceeded" in result["error"] + + +@pytest.mark.skipif(sys.platform != "linux", reason="Linux nested-session SIGTERM integration") +def test_supervisor_cancels_nested_worker_session(tmp_path): + """Real probe -> _run_process -> different session, not a mocked killpg.""" + pidfile = tmp_path / "ready.json" + worker = ("import os,time,json;from pathlib import Path;" + "pid=os.getpid();stat=Path(f'/proc/{pid}/stat').read_text();" + f"Path({str(pidfile)!r}).write_text(json.dumps([pid,stat.rsplit(')',1)[1].split()[19]]));" + "time.sleep(60)") + probe = ("import sys;from scratchv.runtime.riscv_tensor import _run_process;" + f"_run_process([sys.executable,'-B','-c',{worker!r}],cwd=None,timeout=50)") + unrelated = subprocess.Popen([sys.executable, "-c", "import time;time.sleep(60)"], + start_new_session=True) + process = subprocess.Popen([sys.executable, "-B", "-c", probe], cwd=runner.ROOT, + start_new_session=True) + pid = None + try: + deadline = time.monotonic() + 15 + while not pidfile.exists(): + assert process.poll() is None, "Probe exited before worker readiness" + assert time.monotonic() < deadline, "Worker did not become ready" + time.sleep(0.01) + # Existence can precede completion of the tiny JSON write. + while True: + try: + pid, started = json.loads(pidfile.read_text()) + break + except ValueError: + assert time.monotonic() < deadline + time.sleep(0.01) + assert os.getpgid(pid) != process.pid + runner.terminate_process_tree(process) + assert process.returncode == 143 + assert unrelated.poll() is None + deadline = time.monotonic() + 5 + while True: + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): + break + if fields[0] in {"Z", "X"} or fields[19] != started: + break + assert time.monotonic() < deadline, "Detached worker survived supervisor cleanup" + time.sleep(0.01) + finally: + for child in (process, unrelated): + if child.poll() is None: + child.kill() + child.wait(timeout=5) + if pid is not None: + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + if fields[19] == started and fields[0] not in {"Z", "X"}: + os.kill(pid, 9) + except (FileNotFoundError, ProcessLookupError): + pass + + +@pytest.mark.parametrize("filename", ["report.md", "report.html", "report.json"]) +def test_report_write_failure_cannot_publish_pass(tmp_path, monkeypatch, filename): + original = Path.replace + def replace(path, target): + if Path(target).name == filename: + raise OSError("disk failed") + return original(path, target) + monkeypatch.setattr(Path, "replace", replace) + report = {"status": "PASS", "passed": True, "gates": []} + runner.write_reports(tmp_path, report) + assert report["status"] == "FAIL" and not report["passed"] + if (tmp_path / "report.json").exists(): + assert json.loads((tmp_path / "report.json").read_text())["passed"] is False + + +def test_report_secondary_error_preserves_primary(tmp_path, monkeypatch): + monkeypatch.setattr(Path, "replace", lambda *a: (_ for _ in ()).throw(OSError("disk failed"))) + report = {"status": "FAIL", "passed": False, "gates": [], "error": "original numerical divergence"} + runner.write_reports(tmp_path, report) + assert report["error"] == "original numerical divergence" + + +def test_subset_main_is_partial_and_exit_two(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + monkeypatch.setattr(runner, "source_fingerprints", lambda: {"source": "hash"}) + monkeypatch.setattr(runner, "run_gate", lambda gate, *a, **k: {"name": gate.name, "status": "PASS", "passed": True}) + out = tmp_path / "new" + assert runner.main(["--output-dir", str(out), "--gates", "frontend"]) == 2 + report = json.loads((out / "report.json").read_text()) + assert report["status"] == "PARTIAL" and not report["passed"] + + +def test_source_mutation_during_run_fails(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + values = iter([{"source": "old"}, {"source": "new"}]) + monkeypatch.setattr(runner, "source_fingerprints", lambda: next(values)) + monkeypatch.setattr(runner, "run_gate", lambda gate, *a, **k: {"name": gate.name, "status": "PASS", "passed": True}) + assert runner.main(["--output-dir", str(tmp_path / "new"), "--gates", "frontend"]) == 1 + + +def test_missing_assets_block_and_do_not_launch_dependent_gates(tmp_path, monkeypatch): + monkeypatch.setattr(runner, "checkout_evidence", lambda: {"head": None}) + monkeypatch.setattr(runner, "source_fingerprints", lambda: {}) + monkeypatch.setattr(runner, "run_gate", lambda *a, **k: pytest.fail("must not run without required assets")) + out = tmp_path / "new" + assert runner.main(["--output-dir", str(out), "--gates", "runtime-model"]) == 1 + data = json.loads((out / "report.json").read_text()) + assert [row["status"] for row in data["gates"] if row["name"] in {"runtime", "runtime-model"}] == ["BLOCKED", "BLOCKED"] + + +def test_help_has_no_heavy_imports(tmp_path): + code = "import runpy,sys;sys.argv=['run_w2_acceptance.py','--help'];\ntry: runpy.run_path(" + repr(str(Path(runner.__file__))) + ",run_name='__main__')\nexcept SystemExit: pass\nassert not ({'torch','numpy','onnx','transformers'} & sys.modules.keys())" + result = subprocess.run([sys.executable, "-S", "-c", code], capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr diff --git a/tests/test_w2_backend_ops_gate.py b/tests/test_w2_backend_ops_gate.py new file mode 100644 index 0000000..cf295fb --- /dev/null +++ b/tests/test_w2_backend_ops_gate.py @@ -0,0 +1,220 @@ +"""Mandatory QEMU gate failure/report handling; target execution is a separate job.""" +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from probes.w2_backend_ops import run as probe +from scratchv.runtime.riscv_tensor import RiscVTensorTimeoutError + + +@pytest.fixture +def fake_target(monkeypatch): + case = probe.build_cases()[0] + monkeypatch.setattr(probe, "build_cases", lambda: [case]) + monkeypatch.setattr(probe, "FAMILIES", (case.family,)) + monkeypatch.setattr(probe, "discover_toolchain", lambda **kwargs: SimpleNamespace(cc=("fixture-cc",), qemu="fixture-qemu")) + monkeypatch.setattr(probe, "build_riscv_tensor", lambda *args: SimpleNamespace( + elf_sha256="fixture", compile_command=("fixture-cc",), compile_seconds=0.1, + workspace_bytes=16, tool_versions={"fixture": "unit-test-only"})) + monkeypatch.setattr(probe, "run_riscv_tensor", lambda *args, **kwargs: SimpleNamespace( + output=case.expected.copy(), elapsed_s=0.25, command=("fixture-qemu",))) + return case + + +def result(tmp_path): + directory = tmp_path / "report" + code = probe.main(["--output-dir", str(directory)]) + report = json.loads((directory / "report.json").read_text(encoding="utf-8")) + assert (directory / "report.md").is_file() and (directory / "report.html").is_file() + return code, report + + +def test_missing_tools_fail_with_all_report_formats(tmp_path, monkeypatch): + def unavailable(**kwargs): + raise FileNotFoundError("QEMU unavailable") + monkeypatch.setattr(probe, "discover_toolchain", unavailable) + code, report = result(tmp_path) + assert code == 1 and report["passed"] is False and report["stage"] == "toolchain" + assert "QEMU unavailable" in report["error"] + assert report["planned_executions"] == 32 + + +def test_success_requires_both_optimization_levels(tmp_path, monkeypatch, fake_target): + code, report = result(tmp_path) + assert code == 0 and report["passed"] is True + assert report["completed_executions"] == report["planned_executions"] == 2 + assert report["passed_executions"] == 2 + assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.5} + assert report["atol"] == 1e-5 and report["rtol"] == 0 + + +@pytest.mark.parametrize("fault", ["numeric", "nan", "shape", "dtype", "second_level"]) +def test_incorrect_qemu_output_is_not_a_pass(tmp_path, monkeypatch, fake_target, fault): + calls = [] + def execute(*args, **kwargs): + actual = fake_target.expected.copy() + calls.append(1) + if fault == "numeric" or (fault == "second_level" and len(calls) == 2): + actual.flat[0] += np.float32(0.01) + elif fault == "nan": + actual.flat[0] = np.nan + elif fault == "shape": + actual = actual.flatten() + elif fault == "dtype": + actual = actual.astype(np.float64) + return SimpleNamespace(output=actual, elapsed_s=0.25, command=("fixture-qemu",)) + monkeypatch.setattr(probe, "run_riscv_tensor", execute) + code, report = result(tmp_path) + assert code == 1 and report["passed"] is False + assert report["completed_executions"] == 2 + assert report["passed_executions"] == (1 if fault == "second_level" else 0) + + +def test_qemu_timeout_preserves_measured_time_and_failures(tmp_path, monkeypatch, fake_target): + def timeout(*args, **kwargs): + raise RiscVTensorTimeoutError("QEMU test timeout", elapsed_s=0.125, + command=["fixture-qemu"], timeout_s=0.1) + monkeypatch.setattr(probe, "run_riscv_tensor", timeout) + code, report = result(tmp_path) + assert code == 1 + assert report["completed_executions"] == 0 + assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.25} + assert all(item["status"] == "timeout" for item in report["cases"][0]["executions"]) + + +def test_formula_disagreement_cannot_enter_qemu(tmp_path, monkeypatch, fake_target): + fake_target.expected.flat[0] += np.float32(1) + def unexpected(*args, **kwargs): + pytest.fail("Invalid reference must stop before QEMU") + monkeypatch.setattr(probe, "run_riscv_tensor", unexpected) + code, report = result(tmp_path) + assert code == 1 and report["cases"][0]["executions"] == [] + assert "NumPy formula" in report["cases"][0]["error"] + + +def test_compiler_failure_cannot_become_qemu_success(tmp_path, monkeypatch, fake_target): + monkeypatch.setattr(probe.CompilerDriver, "compile", lambda *args: SimpleNamespace( + success=False, errors=["fixture unsupported operator"])) + def unexpected(*args, **kwargs): + pytest.fail("Compiler failure must stop before QEMU") + monkeypatch.setattr(probe, "run_riscv_tensor", unexpected) + code, report = result(tmp_path) + assert code == 1 and report["completed_executions"] == 0 + assert all(item["stage"] == "compile" and "unsupported operator" in item["error"] + for item in report["cases"][0]["executions"]) + + +@pytest.mark.parametrize("fault", ["empty", "missing_family", "duplicate_name"]) +def test_invalid_coverage_fails_before_tools(tmp_path, monkeypatch, fault): + cases = probe.build_cases() + selected = [] if fault == "empty" else (cases[:-2] if fault == "missing_family" else cases + [cases[0]]) + monkeypatch.setattr(probe, "build_cases", lambda: selected) + def unexpected(**kwargs): + pytest.fail("Invalid coverage must stop before toolchain discovery") + monkeypatch.setattr(probe, "discover_toolchain", unexpected) + code, report = result(tmp_path) + assert code == 1 and "cover every required operator family" in report["error"] + + +@pytest.mark.parametrize("timeout", ["0", "-1", "nan", "inf"]) +def test_invalid_timeout_is_rejected(tmp_path, timeout): + with pytest.raises(SystemExit, match="2"): + probe.main(["--output-dir", str(tmp_path / "report"), "--timeout", timeout]) + + +def test_existing_report_is_preserved(tmp_path): + sentinel = tmp_path / "existing.txt" + sentinel.write_text("keep", encoding="utf-8") + with pytest.raises(SystemExit, match="2"): + probe.main(["--output-dir", str(tmp_path)]) + assert sentinel.read_text(encoding="utf-8") == "keep" + + +@pytest.mark.parametrize("filename", ["report.json", "report.md", "report.html"]) +def test_report_failure_invalidates_cli_success(tmp_path, monkeypatch, capsys, filename): + def successful_probe(out, report, **kwargs): + report.update(passed=True, stage="complete", cases=[]) + + original = Path.write_text + + def blocked(path, data, *args, **kwargs): + if path.name.removesuffix(".tmp") == filename: + raise PermissionError("blocked evidence") + return original(path, data, *args, **kwargs) + + monkeypatch.setattr(probe, "run_probe", successful_probe) + monkeypatch.setattr(Path, "write_text", blocked) + directory = tmp_path / "report" + assert probe.main(["--output-dir", str(directory)]) == 1 + for name in ("report.json", "report.md", "report.html"): + if name == filename: + continue + content = (directory / name).read_text(encoding="utf-8") + if name == "report.json": + saved = json.loads(content) + assert saved["passed"] is False and saved["stage"] == "report-write" + assert "blocked evidence" in saved["report_write_errors"][0] + else: + assert "Result: FAIL" in content + assert filename in capsys.readouterr().err + + +def test_report_failure_does_not_replace_primary_error(tmp_path, monkeypatch): + def failed_probe(out, report, **kwargs): + report.update(stage="toolchain", cases=[]) + raise RuntimeError("original missing compiler") + + original = Path.write_text + + def blocked(path, data, *args, **kwargs): + if path.name == "report.md": + raise OSError("disk full") + return original(path, data, *args, **kwargs) + + monkeypatch.setattr(probe, "run_probe", failed_probe) + monkeypatch.setattr(Path, "write_text", blocked) + directory = tmp_path / "report" + assert probe.main(["--output-dir", str(directory)]) == 1 + saved = json.loads((directory / "report.json").read_text(encoding="utf-8")) + assert saved["error"] == "RuntimeError: original missing compiler" + assert saved["stage"] == "toolchain" + assert "disk full" in saved["report_write_errors"][0] + + +def test_json_write_error_after_bytes_written_cannot_publish_pass(tmp_path, monkeypatch): + original = Path.write_text + + def written_then_failed(path, data, *args, **kwargs): + result = original(path, data, *args, **kwargs) + if path.name.removesuffix(".tmp") == "report.json": + raise OSError("late flush failure") + return result + + monkeypatch.setattr(Path, "write_text", written_then_failed) + report = {"passed": True, "stage": "complete", "cases": [], "seconds": 0.1} + probe.write_reports(tmp_path, report) + assert report["passed"] is False + assert not (tmp_path / "report.json").exists() + assert "Result: FAIL" in (tmp_path / "report.md").read_text(encoding="utf-8") + assert "late flush failure" in report["report_write_errors"][0] + + +def test_json_publish_failure_is_not_success(tmp_path, monkeypatch): + original = Path.replace + + def fail_publish(path, target): + if Path(target).name == "report.json": + raise PermissionError("JSON destination locked") + return original(path, target) + + monkeypatch.setattr(Path, "replace", fail_publish) + report = {"passed": True, "stage": "complete", "cases": [], "seconds": 0.1} + probe.write_reports(tmp_path, report) + assert report["passed"] is False + assert report["stage"] == "report-write" + assert not (tmp_path / "report.json").exists() + assert not (tmp_path / "report.json.tmp").exists() + assert "Result: FAIL" in (tmp_path / "report.md").read_text(encoding="utf-8") diff --git a/tests/test_w2_ci_results.py b/tests/test_w2_ci_results.py new file mode 100644 index 0000000..b411e65 --- /dev/null +++ b/tests/test_w2_ci_results.py @@ -0,0 +1,49 @@ +"""A skipped heavy frontend job must never yield green W2 acceptance.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +SCRIPT = Path(__file__).parents[1] / "scripts/check_w2_ci_results.py" +SPEC = importlib.util.spec_from_file_location("w2_ci_results", SCRIPT) +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +def successful_jobs(): + return {name: {"result": "success"} for name in gate.REQUIRED_JOBS} + + +@pytest.mark.parametrize("name", gate.REQUIRED_JOBS) +@pytest.mark.parametrize("outcome", ["skipped", "failure", "cancelled", "pending", None, True]) +def test_required_job_must_actually_succeed(name, outcome): + results = successful_jobs() + results[name]["result"] = outcome + with pytest.raises(ValueError, match=name): + gate.validate_results(results) + + +@pytest.mark.parametrize("payload", [{}, [], None, {"llm-deploy": {"result": "success"}}, + {"llm-deploy": "success", "full-qwen3-frontend": {"result": "success"}}]) +def test_absent_or_malformed_job_results_fail(payload): + with pytest.raises(ValueError): + gate.validate_results(payload) + + +@pytest.mark.parametrize("payload,expected", [("invalid-json", 1), ("{}", 1), + (json.dumps(successful_jobs()), 0)]) +def test_cli_exit_matches_gate_status(monkeypatch, payload, expected): + monkeypatch.setenv("W2_JOB_RESULTS", payload) + result = subprocess.run([sys.executable, "-B", str(SCRIPT)], capture_output=True, + text=True, timeout=10) + assert result.returncode == expected + assert ("PASS:" if expected == 0 else "FAIL:") in result.stdout + + +def test_optional_full_ort_skip_does_not_replace_required_frontend(): + results = successful_jobs() + results["full-qwen3-onnx"] = {"result": "skipped"} + gate.validate_results(results) diff --git a/tests/test_w2_runtime_gate.py b/tests/test_w2_runtime_gate.py new file mode 100644 index 0000000..b33f478 --- /dev/null +++ b/tests/test_w2_runtime_gate.py @@ -0,0 +1,192 @@ +"""Runtime acceptance must reject missing/corrupt assets and reference drift.""" +import copy +import hashlib +import io +import json +from pathlib import Path +import urllib.error + +import pytest + +from probes.w2_runtime import run as gate + + +@pytest.fixture +def assets(tmp_path): + data = {item["name"]: ("fixture:" + item["name"]).encode() for item in gate.MANIFEST["source_files"]} + manifest = copy.deepcopy(gate.MANIFEST) + for item in manifest["source_files"]: + item["sha256"] = hashlib.sha256(data[item["name"]]).hexdigest() + source = tmp_path / "source" + source.mkdir() + for name, payload in data.items(): + (source / name).write_bytes(payload) + return source, data, manifest + + +@pytest.mark.parametrize("problem", ["missing", "corrupt", "lfs", "duplicate", "traversal", "missing_manifest_entry"]) +def test_asset_verification_rejects_incomplete_or_altered_source(assets, problem): + source, _, manifest = assets + file = source / "tokenizer.json" + if problem == "missing": + file.unlink() + elif problem == "corrupt": + file.write_bytes(b"corrupted") + elif problem == "lfs": + file.write_bytes(b"version https://git-lfs.github.com/spec/v1\n") + elif problem == "duplicate": + manifest["source_files"].append(manifest["source_files"][0]) + elif problem == "traversal": + manifest["source_files"][0]["name"] = "../config.json" + else: + manifest["source_files"].pop() + with pytest.raises((ValueError, FileNotFoundError)): + gate.verify_assets(source, manifest) + + +def test_download_only_pinned_tokenizer_files_and_reuses_verified_assets(assets, tmp_path, monkeypatch): + _, data, manifest = assets + urls = [] + def download(url, path): + urls.append(url) + Path(path).write_bytes(data[Path(path).name]) + monkeypatch.setattr(gate, "download_file", download) + out = tmp_path / "download" + result = gate.acquire_assets(out, manifest) + assert not result["reused"] and len(result["files"]) == 7 + assert len(urls) == 7 and all(manifest["revision"] in url for url in urls) + assert not any("safetensors" in url or ".onnx" in url for url in urls) + assert gate.acquire_assets(out, manifest)["reused"] + assert len(urls) == 7 + + +def test_download_hash_failure_never_installs_partial_directory(assets, tmp_path, monkeypatch): + _, _, manifest = assets + monkeypatch.setattr(gate, "download_file", lambda url, path: Path(path).write_bytes(b"wrong")) + out = tmp_path / "download" + with pytest.raises(ValueError, match="SHA256"): + gate.acquire_assets(out, manifest) + assert not out.exists() + assert not list(tmp_path.glob("qwen3-tokenizer-*")) + + +def test_corrupt_existing_cache_is_preserved_not_silently_replaced(assets, monkeypatch): + source, _, manifest = assets + (source / "tokenizer.json").write_bytes(b"keep this evidence") + monkeypatch.setattr(gate, "download_file", lambda *a: pytest.fail("must not redownload")) + with pytest.raises(ValueError, match="SHA256"): + gate.acquire_assets(source, manifest) + assert (source / "tokenizer.json").read_bytes() == b"keep this evidence" + + +def test_download_retries_only_transient_http_errors(tmp_path, monkeypatch): + attempts = [] + def open_url(*args, **kwargs): + attempts.append(1) + if len(attempts) == 1: + raise urllib.error.HTTPError("https://example.invalid/", 503, "busy", {}, None) + return io.BytesIO(b"payload") + monkeypatch.setattr(gate.urllib.request, "urlopen", open_url) + monkeypatch.setattr(gate.time, "sleep", lambda _: None) + gate.download_file("https://example.invalid/test", tmp_path / "asset") + assert len(attempts) == 2 and (tmp_path / "asset").read_bytes() == b"payload" + + +def test_download_rejects_oversize_data(tmp_path, monkeypatch): + monkeypatch.setattr(gate, "MAX_FILE_BYTES", 4) + monkeypatch.setattr(gate.urllib.request, "urlopen", lambda *a, **k: io.BytesIO(b"12345")) + with pytest.raises(ValueError, match="exceeds"): + gate.download_file("https://example.invalid/test", tmp_path / "asset") + + +def test_missing_assets_produces_failed_report_with_source_identity(tmp_path, monkeypatch): + monkeypatch.setattr(gate, "require_environment", lambda: {"fixture": True}) + out = tmp_path / "report" + assert gate.main(["--tokenizer-dir", str(tmp_path / "absent"), "--output-dir", str(out)]) == 1 + r = json.loads((out / "report.json").read_text()) + assert not r["passed"] and r["stage"] == "assets" + assert "Missing tokenizer asset" in r["error"] + assert r["source_sha256"]["scratchv/runtime/llm_inputs.py"] + assert "FAIL" in (out / "report.md").read_text() + + +def test_validation_failure_preserves_completed_cases_and_escapes_html(tmp_path, monkeypatch): + def fail(args, report): + report["stage"] = "tokenizer" + report["tokenizer_cases"].append({"name": "first", "passed": True}) + raise ValueError("bad ") + monkeypatch.setattr(gate, "run_checks", fail) + out = tmp_path / "report" + assert gate.main(["--tokenizer-dir", str(tmp_path / "fixture"), "--output-dir", str(out)]) == 1 + r = json.loads((out / "report.json").read_text()) + assert r["stage"] == "tokenizer" and len(r["tokenizer_cases"]) == 1 and not r["passed"] + assert " + +ONNX 算子 benchmark Linux 复跑入口 +

ONNX 算子 benchmark：Linux 复跑入口

+

正式环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。本页是入口，不是新测量报告。

+

请按环境与单测指南准备依赖、固定 Git 基线和 LFS 模型，再按Linux benchmark 命令生成自己的 JSON、Markdown 和 HTML。

+

当前同名 JSON 为 historical_benchmark_index，不能作为 benchmark 结果。历史个人测量不改名为 Linux 数据，也不能与不同环境的耗时直接计算加速比。

+

查看固定提交中的原始 HTML 测量

+ diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" index 9b68647..1ce68a3 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" @@ -1,2388 +1,14 @@ { + "document_kind": "historical_benchmark_index", "schema_version": 1, - "baseline": { - "commit": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "source_sha256": "a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c" + "measurement_available": false, + "reproduction_target": "Ubuntu 24.04 x86_64 / Bash / Python 3.12", + "guide": "算子补齐与单测-benchmark.md", + "original_commit": "2d07ccc42936dcfde19d552bca02a324cd33821c", + "historical_reports": { + "md": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md", + "json": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json", + "html": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html" }, - "current": { - "head": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "dirty": true, - "source_sha256": "57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf" - }, - "environment": { - "python": "3.12.13", - "platform": "Windows-11-10.0.26200-SP0", - "numpy": "2.5.3", - "onnx": "1.23.0", - "onnxruntime": "1.29.0", - "threads": { - "OMP_NUM_THREADS": "1", - "OPENBLAS_NUM_THREADS": "1", - "MKL_NUM_THREADS": "1", - "NUMEXPR_NUM_THREADS": "1", - "VECLIB_MAXIMUM_THREADS": "1" - }, - "blas": { - "Compilers": { - "c": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - }, - "cython": { - "name": "cython", - "linker": "cython", - "version": "3.3.0", - "commands": "cython" - }, - "c++": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - } - }, - "Machine Information": { - "host": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - }, - "build": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - } - }, - "Build Dependencies": { - "blas": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - }, - "lapack": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - } - }, - "Python Information": { - "version": "3.12" - }, - "SIMD Extensions": { - "baseline": [ - "X86_V2" - ], - "found": [ - "X86_V3" - ] - } - }, - "ort": "CPUExecutionProvider, sequential, one thread, optimizations disabled" - }, - "warmup": 1, - "repeats": 10, - "repeats_per_round": 5, - "rounds_per_variant": 2, - "execution_order": [ - "before", - "after", - "after", - "before" - ], - "timing_scope": "parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU.", - "binding_policy": "Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change.", - "results": [ - { - "name": "add_float32", - "group": "control", - "description": "FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs.", - "operators": [ - "Add" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba", - "inputs_sha256": "8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036", - "expected_sha256": "9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012459990102797747, - 0.00012260000221431255, - 0.00012610002886503935, - 0.00012240000069141388, - 0.00011799996718764305, - 0.00017060001846402884, - 0.00017589994240552187, - 0.0001556000206619501, - 0.00013669999316334724, - 0.0001281999284401536 - ], - "run_samples_s": [ - 9.820004925131798e-05, - 9.109999518841505e-05, - 9.620003402233124e-05, - 9.129999671131372e-05, - 8.999998681247234e-05, - 0.0001316999550908804, - 0.0001227000029757619, - 0.00012240000069141388, - 9.77999297901988e-05, - 9.400001727044582e-05 - ], - "parse_median_s": 0.00012714997865259647, - "run_median_s": 9.699998190626502e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00011799996718764305, - "parse_max_s": 0.00017589994240552187, - "run_min_s": 8.999998681247234e-05, - "run_max_s": 0.0001316999550908804 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004192000487819314, - 0.00039460009429603815, - 0.00038620003033429384, - 0.0003771999618038535, - 0.00038330000825226307, - 0.00037699996028095484, - 0.00035340001340955496, - 0.00035879993811249733, - 0.00034790008794516325, - 0.0003451999509707093 - ], - "run_samples_s": [ - 0.00011930009350180626, - 0.00010579999070614576, - 0.00010310008656233549, - 0.00010159995872527361, - 0.0001004999503493309, - 0.00010600010864436626, - 0.00010289996862411499, - 0.0001020999625325203, - 0.00010160007514059544, - 0.00010059995111078024 - ], - "parse_median_s": 0.0003770999610424042, - "run_median_s": 0.00010249996557831764, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003451999509707093, - "parse_max_s": 0.0004192000487819314, - "run_min_s": 0.0001004999503493309, - "run_max_s": 0.00011930009350180626 - }, - "fixture_files": { - "model": "add_float32.onnx", - "inputs": "add_float32.inputs.npz", - "expected": "add_float32.expected.npy", - "initializers": "add_float32.initializers.npz" - } - }, - { - "name": "matmul_32", - "group": "control", - "description": "FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs.", - "operators": [ - "MatMul" - ], - "atol": 1e-05, - "rtol": 1e-05, - "model_sha256": "c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203", - "inputs_sha256": "3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0", - "expected_sha256": "d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012470001820474863, - 0.00012370001059025526, - 0.0001233000075444579, - 0.0001382000045850873, - 0.00012390001211315393, - 0.00015219999477267265, - 0.00013090006541460752, - 0.0001327999634668231, - 0.00012919993605464697, - 0.00012370001059025526 - ], - "run_samples_s": [ - 0.00010169995948672295, - 9.799993131309748e-05, - 9.8800053820014e-05, - 9.50000248849392e-05, - 9.69000393524766e-05, - 0.00011190003715455532, - 0.00010029994882643223, - 0.00010699999984353781, - 0.00010060006752610207, - 9.679992217570543e-05 - ], - "parse_median_s": 0.0001269499771296978, - "run_median_s": 9.955000132322311e-05, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0001233000075444579, - "parse_max_s": 0.00015219999477267265, - "run_min_s": 9.50000248849392e-05, - "run_max_s": 0.00011190003715455532 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000394199974834919, - 0.00036610011011362076, - 0.00036529998760670424, - 0.0003619999624788761, - 0.00037210003938525915, - 0.00037340004928410053, - 0.00036669999826699495, - 0.0003709999145939946, - 0.00040339992847293615, - 0.000367900007404387 - ], - "run_samples_s": [ - 0.00011040002573281527, - 0.00010629999451339245, - 0.00010939990170300007, - 0.00010389997623860836, - 0.00011880008969455957, - 0.00010979990474879742, - 0.00010659999679774046, - 0.00010810000821948051, - 0.00010469998233020306, - 0.0001050999853760004 - ], - "parse_median_s": 0.0003694499609991908, - "run_median_s": 0.00010735000250861049, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003619999624788761, - "parse_max_s": 0.00040339992847293615, - "run_min_s": 0.00010389997623860836, - "run_max_s": 0.00011880008969455957 - }, - "fixture_files": { - "model": "matmul_32.onnx", - "inputs": "matmul_32.inputs.npz", - "expected": "matmul_32.expected.npy", - "initializers": "matmul_32.initializers.npz" - } - }, - { - "name": "gather", - "group": "operators", - "description": "FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4].", - "operators": [ - "Gather" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e", - "inputs_sha256": "dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf", - "expected_sha256": "4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000382099999114871, - 0.0003637999761849642, - 0.00036070006899535656, - 0.00037329993210732937, - 0.0003640999784693122, - 0.0003535000141710043, - 0.0003445000620558858, - 0.00034570007119327784, - 0.00036100007127970457, - 0.000354600022546947 - ], - "run_samples_s": [ - 0.00010419997852295637, - 0.00010269996710121632, - 0.00010080006904900074, - 0.0001020999625325203, - 0.00010219996329396963, - 0.00010059995111078024, - 0.00010079995263367891, - 0.00010109995491802692, - 0.00010179996024817228, - 0.00010019994806498289 - ], - "parse_median_s": 0.00036085007013753057, - "run_median_s": 0.0001014499575830996, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003445000620558858, - "parse_max_s": 0.000382099999114871, - "run_min_s": 0.00010019994806498289, - "run_max_s": 0.00010419997852295637 - }, - "fixture_files": { - "model": "gather.onnx", - "inputs": "gather.inputs.npz", - "expected": "gather.expected.npy", - "initializers": "gather.initializers.npz" - } - }, - { - "name": "sqrt", - "group": "operators", - "description": "Positive FP32 x[2,4] -> Sqrt [2,4].", - "operators": [ - "Sqrt" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97", - "inputs_sha256": "dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6", - "expected_sha256": "2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sqrt", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003466999623924494, - 0.0003605000674724579, - 0.0003635999746620655, - 0.00034799997229129076, - 0.00034380005672574043, - 0.00040040002204477787, - 0.0003725000424310565, - 0.0003456000704318285, - 0.0003480000887066126, - 0.0003408000338822603 - ], - "run_samples_s": [ - 0.00010339997243136168, - 9.490002412348986e-05, - 9.290000889450312e-05, - 9.370001498609781e-05, - 9.099999442696571e-05, - 9.990006219595671e-05, - 9.940005838871002e-05, - 9.69000393524766e-05, - 9.69000393524766e-05, - 9.26000066101551e-05 - ], - "parse_median_s": 0.0003480000304989517, - "run_median_s": 9.590003173798323e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003408000338822603, - "parse_max_s": 0.00040040002204477787, - "run_min_s": 9.099999442696571e-05, - "run_max_s": 0.00010339997243136168 - }, - "fixture_files": { - "model": "sqrt.onnx", - "inputs": "sqrt.inputs.npz", - "expected": "sqrt.expected.npy", - "initializers": "sqrt.initializers.npz" - } - }, - { - "name": "reduce_mean", - "group": "operators", - "description": "FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1].", - "operators": [ - "ReduceMean" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132", - "inputs_sha256": "037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17", - "expected_sha256": "43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7", - "initializers_sha256": "31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: ReduceMean", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 9.933077874436559e-08, - "parse_samples_s": [ - 0.0003934999695047736, - 0.00037949997931718826, - 0.00037489994429051876, - 0.000372299924492836, - 0.00037619995418936014, - 0.0003952999832108617, - 0.00040259992238134146, - 0.0003915999550372362, - 0.0003774999640882015, - 0.0003698000218719244 - ], - "run_samples_s": [ - 0.00011030002497136593, - 0.00010720000136643648, - 0.0001078000059351325, - 0.00010830000974237919, - 0.00010569998994469643, - 0.00010750000365078449, - 0.00010809989180415869, - 0.00010909989941865206, - 0.00010730000212788582, - 0.00010609999299049377 - ], - "parse_median_s": 0.0003784999717026949, - "run_median_s": 0.0001076500047929585, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003698000218719244, - "parse_max_s": 0.00040259992238134146, - "run_min_s": 0.00010569998994469643, - "run_max_s": 0.00011030002497136593 - }, - "fixture_files": { - "model": "reduce_mean.onnx", - "inputs": "reduce_mean.inputs.npz", - "expected": "reduce_mean.expected.npy", - "initializers": "reduce_mean.initializers.npz" - } - }, - { - "name": "transpose", - "group": "operators", - "description": "FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4].", - "operators": [ - "Transpose" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6", - "inputs_sha256": "c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a", - "expected_sha256": "b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Transpose", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003711999161168933, - 0.00037619995418936014, - 0.0003662999952211976, - 0.00041259999852627516, - 0.0003730000462383032, - 0.00035260000731796026, - 0.0003722000401467085, - 0.0003524000057950616, - 0.0003497001016512513, - 0.0003529998939484358 - ], - "run_samples_s": [ - 0.00010429997928440571, - 0.00010159995872527361, - 9.989994578063488e-05, - 0.00010140007361769676, - 0.00010170007590204477, - 0.00010349997319281101, - 9.93999419733882e-05, - 9.840005077421665e-05, - 9.790004696696997e-05, - 9.710004087537527e-05 - ], - "parse_median_s": 0.00036874995566904545, - "run_median_s": 0.00010065000969916582, - "executed_steps": 2, - "actual_shape": [ - 1, - 3, - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003497001016512513, - "parse_max_s": 0.00041259999852627516, - "run_min_s": 9.710004087537527e-05, - "run_max_s": 0.00010429997928440571 - }, - "fixture_files": { - "model": "transpose.onnx", - "inputs": "transpose.inputs.npz", - "expected": "transpose.expected.npy", - "initializers": "transpose.initializers.npz" - } - }, - { - "name": "concat", - "group": "operators", - "description": "FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4].", - "operators": [ - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977", - "inputs_sha256": "09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9", - "expected_sha256": "c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Concat", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003693000180646777, - 0.00038620003033429384, - 0.0003693000180646777, - 0.0003645999822765589, - 0.00036700000055134296, - 0.00041009997949004173, - 0.0003698000218719244, - 0.00037619995418936014, - 0.00035749992821365595, - 0.00035550002939999104 - ], - "run_samples_s": [ - 9.770004544407129e-05, - 0.00010369997471570969, - 9.69000393524766e-05, - 9.790004696696997e-05, - 9.599991608411074e-05, - 0.00010299996938556433, - 9.889993816614151e-05, - 9.699992369860411e-05, - 9.910005610436201e-05, - 9.550002869218588e-05 - ], - "parse_median_s": 0.0003693000180646777, - "run_median_s": 9.780004620552063e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00035550002939999104, - "parse_max_s": 0.00041009997949004173, - "run_min_s": 9.550002869218588e-05, - "run_max_s": 0.00010369997471570969 - }, - "fixture_files": { - "model": "concat.onnx", - "inputs": "concat.inputs.npz", - "expected": "concat.expected.npy", - "initializers": "concat.initializers.npz" - } - }, - { - "name": "slice", - "group": "operators", - "description": "FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4].", - "operators": [ - "Slice" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30", - "inputs_sha256": "04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa", - "expected_sha256": "15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df", - "initializers_sha256": "6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Slice", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00040770007763057947, - 0.0004149000160396099, - 0.00039679999463260174, - 0.000395399983972311, - 0.000394199974834919, - 0.0004005999071523547, - 0.0003872000379487872, - 0.0003947000950574875, - 0.00039820000529289246, - 0.0003963999915868044 - ], - "run_samples_s": [ - 0.00010409997776150703, - 0.00010469998233020306, - 0.00010289996862411499, - 0.00010169995948672295, - 0.00010259996633976698, - 0.00010469998233020306, - 0.00010290008503943682, - 0.0001039999770000577, - 0.00010349997319281101, - 0.00010409997776150703 - ], - "parse_median_s": 0.00039659999310970306, - "run_median_s": 0.00010374997509643435, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003872000379487872, - "parse_max_s": 0.0004149000160396099, - "run_min_s": 0.00010169995948672295, - "run_max_s": 0.00010469998233020306 - }, - "fixture_files": { - "model": "slice.onnx", - "inputs": "slice.inputs.npz", - "expected": "slice.expected.npy", - "initializers": "slice.initializers.npz" - } - }, - { - "name": "unsqueeze", - "group": "operators", - "description": "INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1].", - "operators": [ - "Unsqueeze" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd", - "inputs_sha256": "4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50", - "expected_sha256": "d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193", - "initializers_sha256": "a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Unsqueeze", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004278999986127019, - 0.00037889997474849224, - 0.0003894000547006726, - 0.0004752000095322728, - 0.00046479993034154177, - 0.00037020002491772175, - 0.00035470002330839634, - 0.0003568999236449599, - 0.0003622000804170966, - 0.0003958999877795577 - ], - "run_samples_s": [ - 8.939998224377632e-05, - 8.649996016174555e-05, - 8.329993579536676e-05, - 8.98001017048955e-05, - 8.560006972402334e-05, - 8.510006591677666e-05, - 8.279993198812008e-05, - 8.359993807971478e-05, - 8.44000605866313e-05, - 8.329993579536676e-05 - ], - "parse_median_s": 0.00038415001472458243, - "run_median_s": 8.475006325170398e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 2, - 3, - 1 - ], - "actual_dtype": "int64", - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00035470002330839634, - "parse_max_s": 0.0004752000095322728, - "run_min_s": 8.279993198812008e-05, - "run_max_s": 8.98001017048955e-05 - }, - "fixture_files": { - "model": "unsqueeze.onnx", - "inputs": "unsqueeze.inputs.npz", - "expected": "unsqueeze.expected.npy", - "initializers": "unsqueeze.initializers.npz" - } - }, - { - "name": "expand", - "group": "operators", - "description": "FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4].", - "operators": [ - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab", - "inputs_sha256": "4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6", - "expected_sha256": "eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6", - "initializers_sha256": "7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Expand", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004062999505549669, - 0.0003779999678954482, - 0.0004357000580057502, - 0.00038139999378472567, - 0.0004078999627381563, - 0.0003739000530913472, - 0.0003749999450519681, - 0.0003727000439539552, - 0.00036589999217540026, - 0.000359599944204092 - ], - "run_samples_s": [ - 0.00011890009045600891, - 0.00010760000441223383, - 0.00010830000974237919, - 0.00010639999527484179, - 0.0001023000804707408, - 0.00010429997928440571, - 0.00010469998233020306, - 0.00010419997852295637, - 0.00010289996862411499, - 0.00010410009417682886 - ], - "parse_median_s": 0.00037649995647370815, - "run_median_s": 0.00010449998080730438, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000359599944204092, - "parse_max_s": 0.0004357000580057502, - "run_min_s": 0.0001023000804707408, - "run_max_s": 0.00011890009045600891 - }, - "fixture_files": { - "model": "expand.onnx", - "inputs": "expand.inputs.npz", - "expected": "expand.expected.npy", - "initializers": "expand.initializers.npz" - } - }, - { - "name": "abs", - "group": "operators", - "description": "Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype.", - "operators": [ - "Abs" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf", - "inputs_sha256": "898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a", - "expected_sha256": "7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Abs", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00036099995486438274, - 0.0003568999236449599, - 0.0003541000187397003, - 0.0003519000019878149, - 0.0003469999646767974, - 0.00033569999504834414, - 0.00035450002178549767, - 0.00034719996619969606, - 0.0003350999904796481, - 0.000327100045979023 - ], - "run_samples_s": [ - 8.889997843652964e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.420005906373262e-05, - 8.320005144923925e-05, - 8.759996853768826e-05, - 8.549995254725218e-05, - 8.499994874000549e-05, - 8.68999632075429e-05, - 8.470006287097931e-05 - ], - "parse_median_s": 0.0003495499840937555, - "run_median_s": 8.585001341998577e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.000327100045979023, - "parse_max_s": 0.00036099995486438274, - "run_min_s": 8.320005144923925e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "abs.onnx", - "inputs": "abs.inputs.npz", - "expected": "abs.expected.npy", - "initializers": "abs.initializers.npz" - } - }, - { - "name": "cast", - "group": "operators", - "description": "INT64 position IDs[1,8] -> FP32 [1,8].", - "operators": [ - "Cast" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5", - "inputs_sha256": "93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f", - "expected_sha256": "5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.00036179996095597744, - 0.0004943999228999019, - 0.0005131000652909279, - 0.00037509994581341743, - 0.00034440006129443645, - 0.00033980002626776695, - 0.00034899997990578413, - 0.00033760000951588154, - 0.00035889993887394667 - ], - "run_samples_s": [ - 7.98999099060893e-05, - 0.00017800007481127977, - 0.00011080002877861261, - 8.82000895217061e-05, - 8.230004459619522e-05, - 7.829989772289991e-05, - 7.860001642256975e-05, - 7.840001489967108e-05, - 8.060003165155649e-05, - 7.970002479851246e-05 - ], - "parse_median_s": 0.00036034994991496205, - "run_median_s": 8.02499707788229e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033760000951588154, - "parse_max_s": 0.0005131000652909279, - "run_min_s": 7.829989772289991e-05, - "run_max_s": 0.00017800007481127977 - }, - "fixture_files": { - "model": "cast.onnx", - "inputs": "cast.inputs.npz", - "expected": "cast.expected.npy", - "initializers": "cast.initializers.npz" - } - }, - { - "name": "constant", - "group": "operators", - "description": "Constant tensor attribute FP32 [2,3]; no runtime inputs.", - "operators": [ - "Constant" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.000381799996830523, - 0.00039549998473376036, - 0.000381799996830523, - 0.0003598000621423125, - 0.00036259996704757214, - 0.0003591999411582947, - 0.0003554000286385417, - 0.0003502999898046255, - 0.0003415999235585332 - ], - "run_samples_s": [ - 6.250001024454832e-05, - 6.170000415295362e-05, - 6.10999995842576e-05, - 6.200000643730164e-05, - 6.0399994254112244e-05, - 6.150000263005495e-05, - 6.219989154487848e-05, - 6.0399994254112244e-05, - 6.049999501556158e-05, - 5.909998435527086e-05 - ], - "parse_median_s": 0.00036120001459494233, - "run_median_s": 6.130000110715628e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003415999235585332, - "parse_max_s": 0.00039549998473376036, - "run_min_s": 5.909998435527086e-05, - "run_max_s": 6.250001024454832e-05 - }, - "fixture_files": { - "model": "constant.onnx", - "inputs": "constant.inputs.npz", - "expected": "constant.expected.npy", - "initializers": "constant.initializers.npz" - } - }, - { - "name": "cos", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4].", - "operators": [ - "Cos" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298", - "inputs_sha256": "8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998", - "expected_sha256": "7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cos", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 5.960464477539063e-08, - "max_rel_error": 8.023689783612319e-08, - "parse_samples_s": [ - 0.0003530000103637576, - 0.00034549995325505733, - 0.00033980002626776695, - 0.000340400030836463, - 0.0003451999509707093, - 0.00034739996772259474, - 0.000332699972204864, - 0.0003412999212741852, - 0.00033599999733269215, - 0.00033199996687471867 - ], - "run_samples_s": [ - 8.690007962286472e-05, - 8.670007809996605e-05, - 8.889997843652964e-05, - 8.739996701478958e-05, - 8.559995330870152e-05, - 8.60999571159482e-05, - 8.629995863884687e-05, - 8.690007962286472e-05, - 8.579995483160019e-05, - 8.400005754083395e-05 - ], - "parse_median_s": 0.0003408499760553241, - "run_median_s": 8.650001836940646e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033199996687471867, - "parse_max_s": 0.0003530000103637576, - "run_min_s": 8.400005754083395e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "cos.onnx", - "inputs": "cos.inputs.npz", - "expected": "cos.expected.npy", - "initializers": "cos.initializers.npz" - } - }, - { - "name": "identity", - "group": "operators", - "description": "INT64 indices[2,3] -> Identity [2,3]; frontend alias.", - "operators": [ - "Identity" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b", - "inputs_sha256": "19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413", - "expected_sha256": "a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Identity", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000341000035405159, - 0.00034380005672574043, - 0.0003355999942868948, - 0.0003361001145094633, - 0.0003378000110387802, - 0.00033820001408457756, - 0.0003301999531686306, - 0.00032760004978626966, - 0.0003744999412447214, - 0.0003630999708548188 - ], - "run_samples_s": [ - 5.170004442334175e-05, - 5.280005279928446e-05, - 5.219993181526661e-05, - 5.2200048230588436e-05, - 5.080003757029772e-05, - 5.179992876946926e-05, - 5.079992115497589e-05, - 5.139992572367191e-05, - 6.809993647038937e-05, - 5.4600066505372524e-05 - ], - "parse_median_s": 0.0003380000125616789, - "run_median_s": 5.1999930292367935e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00032760004978626966, - "parse_max_s": 0.0003744999412447214, - "run_min_s": 5.079992115497589e-05, - "run_max_s": 6.809993647038937e-05 - }, - "fixture_files": { - "model": "identity.onnx", - "inputs": "identity.inputs.npz", - "expected": "identity.expected.npy", - "initializers": "identity.initializers.npz" - } - }, - { - "name": "pow", - "group": "operators", - "description": "Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3].", - "operators": [ - "Pow" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771", - "inputs_sha256": "b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1", - "expected_sha256": "dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004491000436246395, - 0.00043999997433274984, - 0.00040019990410655737, - 0.00037569995038211346, - 0.00038789992686361074, - 0.0003453000681474805, - 0.00048369995784014463, - 0.00044920004438608885, - 0.00047619990073144436, - 0.0004214000655338168 - ], - "run_samples_s": [ - 0.00014999997802078724, - 0.00014180003199726343, - 0.00011959997937083244, - 0.00011729996185749769, - 0.00011809996794909239, - 0.00011589995119720697, - 0.00015820004045963287, - 0.00015029998030513525, - 0.00015840004198253155, - 0.0001241000136360526 - ], - "parse_median_s": 0.00043070001993328333, - "run_median_s": 0.00013295002281665802, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003453000681474805, - "parse_max_s": 0.00048369995784014463, - "run_min_s": 0.00011589995119720697, - "run_max_s": 0.00015840004198253155 - }, - "fixture_files": { - "model": "pow.onnx", - "inputs": "pow.inputs.npz", - "expected": "pow.expected.npy", - "initializers": "pow.initializers.npz" - } - }, - { - "name": "reciprocal", - "group": "operators", - "description": "Nonzero signed FP32 x[2,4] -> Reciprocal [2,4].", - "operators": [ - "Reciprocal" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3", - "inputs_sha256": "fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f", - "expected_sha256": "c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Reciprocal", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00035170000046491623, - 0.0003496999852359295, - 0.00034989998675882816, - 0.00034769997000694275, - 0.00033870001789182425, - 0.00037759996484965086, - 0.00035119999665766954, - 0.0003469000803306699, - 0.00033660000190138817, - 0.00033329997677356005 - ], - "run_samples_s": [ - 9.190000128000975e-05, - 9.019998833537102e-05, - 9.029998909682035e-05, - 9.14999982342124e-05, - 8.899997919797897e-05, - 9.609991684556007e-05, - 9.510002564638853e-05, - 9.06999921426177e-05, - 9.109999518841505e-05, - 9.059999138116837e-05 - ], - "parse_median_s": 0.0003486999776214361, - "run_median_s": 9.089999366551638e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033329997677356005, - "parse_max_s": 0.00037759996484965086, - "run_min_s": 8.899997919797897e-05, - "run_max_s": 9.609991684556007e-05 - }, - "fixture_files": { - "model": "reciprocal.onnx", - "inputs": "reciprocal.inputs.npz", - "expected": "reciprocal.expected.npy", - "initializers": "reciprocal.initializers.npz" - } - }, - { - "name": "sin", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4].", - "operators": [ - "Sin" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a", - "inputs_sha256": "efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e", - "expected_sha256": "e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sin", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00034879997838288546, - 0.0003532000118866563, - 0.00035470002330839634, - 0.0003448999486863613, - 0.00033730000723153353, - 0.00033710000570863485, - 0.0003510999958962202, - 0.0003322999691590667, - 0.00032819993793964386, - 0.000327700050547719 - ], - "run_samples_s": [ - 8.570007048547268e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.480006363242865e-05, - 8.730008266866207e-05, - 8.71000811457634e-05, - 8.659996092319489e-05, - 8.569995407015085e-05, - 8.390005677938461e-05, - 8.750008419156075e-05 - ], - "parse_median_s": 0.0003410999779589474, - "run_median_s": 8.640001760795712e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000327700050547719, - "parse_max_s": 0.00035470002330839634, - "run_min_s": 8.390005677938461e-05, - "run_max_s": 8.750008419156075e-05 - }, - "fixture_files": { - "model": "sin.onnx", - "inputs": "sin.inputs.npz", - "expected": "sin.expected.npy", - "initializers": "sin.initializers.npz" - } - }, - { - "name": "scalar_fp32_add_chain", - "group": "regressions", - "description": "FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output.", - "operators": [ - "Add" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a", - "initializers_sha256": "32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a", - "before": { - "status": "NUMERIC_ERROR", - "phase": "compare", - "error": "NumericMismatch: output differs from ORT: max_abs_error=2", - "max_abs_error": 2.0, - "max_rel_error": 1.1920928955078125e-07, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0005631999811157584, - 0.0004637000383809209, - 0.0004089000867679715, - 0.00039579998701810837, - 0.0003945999778807163, - 0.000381799996830523, - 0.00041089998558163643, - 0.00040959997568279505, - 0.0003945999778807163, - 0.0003902000607922673 - ], - "run_samples_s": [ - 0.00019599997904151678, - 0.00015169999096542597, - 0.00014780007768422365, - 0.0001476999605074525, - 0.0001452999422326684, - 0.00015129998791962862, - 0.00014889996964484453, - 0.0001441999338567257, - 0.00014589994680136442, - 0.0001444999361410737 - ], - "parse_median_s": 0.00040235003689303994, - "run_median_s": 0.00014775001909583807, - "executed_steps": 5, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": 0.000381799996830523, - "parse_max_s": 0.0005631999811157584, - "run_min_s": 0.0001441999338567257, - "run_max_s": 0.00019599997904151678 - }, - "fixture_files": { - "model": "scalar_fp32_add_chain.onnx", - "inputs": "scalar_fp32_add_chain.inputs.npz", - "expected": "scalar_fp32_add_chain.expected.npy", - "initializers": "scalar_fp32_add_chain.initializers.npz" - } - }, - { - "name": "singleton_initializer", - "group": "regressions", - "description": "FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar.", - "operators": [], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8", - "initializers_sha256": "7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df", - "before": { - "status": "EXECUTION_ERROR", - "phase": "execute", - "error": "IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003521000035107136, - 0.00034039991442114115, - 0.00033650000113993883, - 0.0003445999464020133, - 0.0003336999798193574, - 0.0003494999837130308, - 0.0003339999821037054, - 0.00035889993887394667, - 0.00032849994022399187, - 0.00033849989995360374 - ], - "run_samples_s": [ - 6.290001329034567e-05, - 6.280001252889633e-05, - 6.140000186860561e-05, - 6.009999196976423e-05, - 6.079999729990959e-05, - 6.370001938194036e-05, - 6.200000643730164e-05, - 6.330001633614302e-05, - 6.260001100599766e-05, - 6.0399994254112244e-05 - ], - "parse_median_s": 0.00033944990718737245, - "run_median_s": 6.230000872164965e-05, - "executed_steps": 1, - "actual_shape": [ - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00032849994022399187, - "parse_max_s": 0.00035889993887394667, - "run_min_s": 6.009999196976423e-05, - "run_max_s": 6.370001938194036e-05 - }, - "fixture_files": { - "model": "singleton_initializer.onnx", - "inputs": "singleton_initializer.inputs.npz", - "expected": "singleton_initializer.expected.npy", - "initializers": "singleton_initializer.initializers.npz" - } - }, - { - "name": "constant_cast_abs_expand", - "group": "composite", - "description": "Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3].", - "operators": [ - "Constant", - "Cast", - "Abs", - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3", - "inputs_sha256": "9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822", - "expected_sha256": "eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00043970008846372366, - 0.00044329999946057796, - 0.0004450000124052167, - 0.0004384000785648823, - 0.0006199999479576945, - 0.00047199998516589403, - 0.00045169994700700045, - 0.00044700002763420343, - 0.00044139998499304056, - 0.00043539993930608034 - ], - "run_samples_s": [ - 0.00013560010120272636, - 0.0001338999718427658, - 0.00013189995661377907, - 0.0002367999404668808, - 0.00017560005653649569, - 0.00013229995965957642, - 0.0001343999756500125, - 0.000130399945192039, - 0.00013109995052218437, - 0.00013479997869580984 - ], - "parse_median_s": 0.00044415000593289733, - "run_median_s": 0.00013414997374638915, - "executed_steps": 4, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00043539993930608034, - "parse_max_s": 0.0006199999479576945, - "run_min_s": 0.000130399945192039, - "run_max_s": 0.0002367999404668808 - }, - "fixture_files": { - "model": "constant_cast_abs_expand.onnx", - "inputs": "constant_cast_abs_expand.inputs.npz", - "expected": "constant_cast_abs_expand.expected.npy", - "initializers": "constant_cast_abs_expand.initializers.npz" - } - }, - { - "name": "qwen_rmsnorm", - "group": "composite", - "description": "FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8].", - "operators": [ - "Pow", - "ReduceMean", - "Add", - "Sqrt", - "Reciprocal", - "Mul" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237", - "inputs_sha256": "f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48", - "expected_sha256": "5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec", - "initializers_sha256": "6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000506900018081069, - 0.0005015999777242541, - 0.0005096999229863286, - 0.0005014999769628048, - 0.0004960000514984131, - 0.0005388999124988914, - 0.0005028999876230955, - 0.000512199942022562, - 0.000493900035507977, - 0.0004851999692618847 - ], - "run_samples_s": [ - 0.0002813000464811921, - 0.0002788000274449587, - 0.0002808000426739454, - 0.00028170004952698946, - 0.00027820002287626266, - 0.0002930000191554427, - 0.00028490007389336824, - 0.00028249993920326233, - 0.0002837000647559762, - 0.0002848999574780464 - ], - "parse_median_s": 0.0005022499826736748, - "run_median_s": 0.0002820999943651259, - "executed_steps": 10, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0004851999692618847, - "parse_max_s": 0.0005388999124988914, - "run_min_s": 0.00027820002287626266, - "run_max_s": 0.0002930000191554427 - }, - "fixture_files": { - "model": "qwen_rmsnorm.onnx", - "inputs": "qwen_rmsnorm.inputs.npz", - "expected": "qwen_rmsnorm.expected.npy", - "initializers": "qwen_rmsnorm.initializers.npz" - } - }, - { - "name": "rope_numeric", - "group": "composite", - "description": "INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8].", - "operators": [ - "Cast", - "Mul", - "Cos", - "Sin", - "Sub", - "Add", - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa", - "inputs_sha256": "823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a", - "expected_sha256": "eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 1.698927025647682e-07, - "parse_samples_s": [ - 0.0005266999360173941, - 0.000520200002938509, - 0.0005088000325486064, - 0.0006041000597178936, - 0.0006553001003339887, - 0.0005080000264570117, - 0.0005031999899074435, - 0.0005010999739170074, - 0.0005029999883845448, - 0.00049919995944947 - ], - "run_samples_s": [ - 0.0002498000394552946, - 0.00024560000747442245, - 0.0002465000143274665, - 0.0003208999987691641, - 0.00031370006036013365, - 0.00025370006915181875, - 0.0002449000021442771, - 0.00024610001128166914, - 0.0002482000272721052, - 0.0002460000105202198 - ], - "parse_median_s": 0.000508400029502809, - "run_median_s": 0.00024735002079978585, - "executed_steps": 12, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00049919995944947, - "parse_max_s": 0.0006553001003339887, - "run_min_s": 0.0002449000021442771, - "run_max_s": 0.0003208999987691641 - }, - "fixture_files": { - "model": "rope_numeric.onnx", - "inputs": "rope_numeric.inputs.npz", - "expected": "rope_numeric.expected.npy", - "initializers": "rope_numeric.initializers.npz" - } - }, - { - "name": "pr89_transformer_2l", - "group": "composite", - "operators": [ - "Add", - "Concat", - "Div", - "Expand", - "Gather", - "MatMul", - "Mul", - "Neg", - "ReduceMean", - "Reshape", - "Sigmoid", - "Slice", - "Softmax", - "Sqrt", - "Transpose", - "Unsqueeze" - ], - "description": "PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3", - "atol": 1e-05, - "rtol": 0, - "model_sha256": "17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1", - "inputs_sha256": "bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457", - "expected_sha256": "413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c", - "initializers_sha256": "955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 7.152557373046875e-07, - "max_rel_error": 0.0005722416551695097, - "parse_samples_s": [ - 0.004760700045153499, - 0.004594899946823716, - 0.004717700066976249, - 0.004990800051018596, - 0.004586199996992946, - 0.004221900017000735, - 0.004203699994832277, - 0.004572000005282462, - 0.004674999974668026, - 0.004224899923428893 - ], - "run_samples_s": [ - 0.010629900032654405, - 0.011460799956694245, - 0.009812200092710555, - 0.009985200013034046, - 0.010765800019726157, - 0.009604100021533668, - 0.010238799965009093, - 0.009660700103268027, - 0.010715999989770353, - 0.009805700043216348 - ], - "parse_median_s": 0.004590549971908331, - "run_median_s": 0.01011199998902157, - "executed_steps": 159, - "actual_shape": [ - 1, - 256, - 128 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": 0.004203699994832277, - "parse_max_s": 0.004990800051018596, - "run_min_s": 0.009604100021533668, - "run_max_s": 0.011460799956694245 - }, - "fixture_files": { - "model": "pr89_transformer_2l.onnx", - "inputs": "pr89_transformer_2l.inputs.npz", - "expected": "pr89_transformer_2l.expected.npy", - "initializers": "pr89_transformer_2l.initializers.npz" - } - } - ], - "passed": true, - "archive_note": "归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。" + "note": "Index only; not a benchmark measurement or a Linux execution result. Generate a new report with benchmarks.bench_onnx_operators." } diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" index 87d5a06..2428b17 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" @@ -1,1228 +1,25 @@ -# ONNX 算子支持与正确性：修改前后对照 +# ONNX 算子补齐 benchmark：Linux 复跑入口 -> 归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。 +正式交付与复现使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。本页是入口，**不是新测量报告**。环境、Git 历史和 LFS 准备见 [算子补齐与单测报告](算子补齐与单测报告.txt)，通用交付约定见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。 -总验收：通过。修改后 24/24 个用例通过；原先不支持、现通过的用例 20 个，原先错误、现通过的用例 2 个，出现回归的用例 0 个。用例数不等于新增算子种类数。 +在仓库根目录激活 `.venv-operators` 后执行： -本次改动扩展算子支持并修正计算语义；计时用于本地观测，不代表性能优化结论。只有修改前后均 PASS 的用例才计算执行耗时比。不支持或缺少有效测量时显示“—”，不能解释为零耗时或无限加速。 - -## 基线兼容适配与输入绑定 - -输入绑定兼容适配不属于基线编译器实现；以下为 runner 的实际策略，需与数值结果一起审阅。 - -> Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change. - -## 按组通过情况 - -| 分组 | 用例总数 | 修改前 PASS | 修改后 PASS | -| --- | --- | --- | --- | -| 独立算子 | 16 | 0/16 | 16/16 | -| 已有支持对照 | 2 | 2/2 | 2/2 | -| 错误回归 | 2 | 0/2 | 2/2 | -| 组合链路 | 4 | 0/4 | 4/4 | - -## 版本与测量条件 - -| 项目 | 值 | -| --- | --- | -| 报告 schema | 1 | -| 基线提交 | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 基线源码 SHA-256 | a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c | -| 当前 HEAD | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 当前包含未提交改动 | 是 | -| 当前源码 SHA-256 | 57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf | -| 执行顺序 | ABBA：修改前 → 修改后 → 修改后 → 修改前 | -| 每版本轮数 | 2 | -| 每轮预热次数 | 1 | -| 每轮正式计时次数 | 5 | -| 每版本正式样本总数 | 10 | -| 计时统计 | 合并各轮正式样本后计算中位数（ms），预热不计入；原始秒数见用例详情 | -| runner 计时说明 | parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU. | - -执行计时覆盖完整 IRInterpreter.run，包括 IR 校验与输入/权重复制；不含 ONNX 解析和 ONNX Runtime 参考计算。ONNX → IR 解析单独计时。 - -环境： - -| 项目 | 值 | -| --- | --- | -| blas | {"Build Dependencies": {"blas": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}, "lapack": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}}, "Compilers": {"c": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "c++": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "cython": {"commands": "cython", "linker": "cython", "name": "cython", "version": "3.3.0"}}, "Machine Information": {"build": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}, "host": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}}, "Python Information": {"version": "3.12"}, "SIMD Extensions": {"baseline": ["X86_V2"], "found": ["X86_V3"]}} | -| numpy | 2.5.3 | -| onnx | 1.23.0 | -| onnxruntime | 1.29.0 | -| ort | CPUExecutionProvider, sequential, one thread, optimizations disabled | -| platform | Windows-11-10.0.26200-SP0 | -| python | 3.12.13 | -| threads | {"MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1"} | - -## 全部用例 - -耗时比 = 修改前 / 修改后；大于 1 表示该次观测中修改后较快，小于 1 表示较慢。 - -| 用例 | 分组 | 修改前 | 修改后 | 前最大绝对误差 | 后最大绝对误差 | 前解析 ms | 后解析 ms | 前执行 ms | 后执行 ms | 执行耗时比 前/后 | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| add_float32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1271 | 0.3771 | 0.0970 | 0.1025 | 0.946× | -| matmul_32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1269 | 0.3694 | 0.0996 | 0.1074 | 0.927× | -| gather | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3609 | — | 0.1014 | — | -| sqrt | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3480 | — | 0.0959 | — | -| reduce_mean | 独立算子 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.3785 | — | 0.1077 | — | -| transpose | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3687 | — | 0.1007 | — | -| concat | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3693 | — | 0.0978 | — | -| slice | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3966 | — | 0.1037 | — | -| unsqueeze | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3842 | — | 0.0848 | — | -| expand | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3765 | — | 0.1045 | — | -| abs | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3495 | — | 0.0859 | — | -| cast | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3603 | — | 0.0802 | — | -| constant | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3612 | — | 0.0613 | — | -| cos | 独立算子 | UNSUPPORTED | PASS | — | 5.960e-08 | — | 0.3408 | — | 0.0865 | — | -| identity | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3380 | — | 0.0520 | — | -| pow | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4307 | — | 0.1330 | — | -| reciprocal | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3487 | — | 0.0909 | — | -| sin | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3411 | — | 0.0864 | — | -| scalar_fp32_add_chain | 错误回归 | NUMERIC_ERROR | PASS | 2.000e+00 | 0.000e+00 | — | 0.4024 | — | 0.1478 | — | -| singleton_initializer | 错误回归 | EXECUTION_ERROR | PASS | — | 0.000e+00 | — | 0.3394 | — | 0.0623 | — | -| constant_cast_abs_expand | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4442 | — | 0.1341 | — | -| qwen_rmsnorm | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.5022 | — | 0.2821 | — | -| rope_numeric | 组合链路 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.5084 | — | 0.2474 | — | -| pr89_transformer_2l | 组合链路 | UNSUPPORTED | PASS | — | 7.153e-07 | — | 4.5905 | — | 10.1120 | — | - -## 用例与错误详情 - -
add_float32 · 前后均通过 - -FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba | -| 输入 SHA-256 | 8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036 | -| 参考输出 SHA-256 (expected_sha256) | 9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012459990102797747, 0.00012260000221431255, 0.00012610002886503935, 0.00012240000069141388, 0.00011799996718764305, 0.00017060001846402884, 0.00017589994240552187, 0.0001556000206619501, 0.00013669999316334724, 0.0001281999284401536] | -| 执行原始样本 s | [9.820004925131798e-05, 9.109999518841505e-05, 9.620003402233124e-05, 9.129999671131372e-05, 8.999998681247234e-05, 0.0001316999550908804, 0.0001227000029757619, 0.00012240000069141388, 9.77999297901988e-05, 9.400001727044582e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004192000487819314, 0.00039460009429603815, 0.00038620003033429384, 0.0003771999618038535, 0.00038330000825226307, 0.00037699996028095484, 0.00035340001340955496, 0.00035879993811249733, 0.00034790008794516325, 0.0003451999509707093] | -| 执行原始样本 s | [0.00011930009350180626, 0.00010579999070614576, 0.00010310008656233549, 0.00010159995872527361, 0.0001004999503493309, 0.00010600010864436626, 0.00010289996862411499, 0.0001020999625325203, 0.00010160007514059544, 0.00010059995111078024] | - -
- -
matmul_32 · 前后均通过 - -FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["MatMul"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 1e-05 | -| 模型 SHA-256 | c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203 | -| 输入 SHA-256 | 3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0 | -| 参考输出 SHA-256 (expected_sha256) | d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012470001820474863, 0.00012370001059025526, 0.0001233000075444579, 0.0001382000045850873, 0.00012390001211315393, 0.00015219999477267265, 0.00013090006541460752, 0.0001327999634668231, 0.00012919993605464697, 0.00012370001059025526] | -| 执行原始样本 s | [0.00010169995948672295, 9.799993131309748e-05, 9.8800053820014e-05, 9.50000248849392e-05, 9.69000393524766e-05, 0.00011190003715455532, 0.00010029994882643223, 0.00010699999984353781, 0.00010060006752610207, 9.679992217570543e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000394199974834919, 0.00036610011011362076, 0.00036529998760670424, 0.0003619999624788761, 0.00037210003938525915, 0.00037340004928410053, 0.00036669999826699495, 0.0003709999145939946, 0.00040339992847293615, 0.000367900007404387] | -| 执行原始样本 s | [0.00011040002573281527, 0.00010629999451339245, 0.00010939990170300007, 0.00010389997623860836, 0.00011880008969455957, 0.00010979990474879742, 0.00010659999679774046, 0.00010810000821948051, 0.00010469998233020306, 0.0001050999853760004] | - -
- -
gather · 原先不支持 → 现通过 - -FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Gather"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e | -| 输入 SHA-256 | dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf | -| 参考输出 SHA-256 (expected_sha256) | 4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000382099999114871, 0.0003637999761849642, 0.00036070006899535656, 0.00037329993210732937, 0.0003640999784693122, 0.0003535000141710043, 0.0003445000620558858, 0.00034570007119327784, 0.00036100007127970457, 0.000354600022546947] | -| 执行原始样本 s | [0.00010419997852295637, 0.00010269996710121632, 0.00010080006904900074, 0.0001020999625325203, 0.00010219996329396963, 0.00010059995111078024, 0.00010079995263367891, 0.00010109995491802692, 0.00010179996024817228, 0.00010019994806498289] | - -
- -
sqrt · 原先不支持 → 现通过 - -Positive FP32 x[2,4] -> Sqrt [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sqrt"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97 | -| 输入 SHA-256 | dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6 | -| 参考输出 SHA-256 (expected_sha256) | 2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sqrt -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003466999623924494, 0.0003605000674724579, 0.0003635999746620655, 0.00034799997229129076, 0.00034380005672574043, 0.00040040002204477787, 0.0003725000424310565, 0.0003456000704318285, 0.0003480000887066126, 0.0003408000338822603] | -| 执行原始样本 s | [0.00010339997243136168, 9.490002412348986e-05, 9.290000889450312e-05, 9.370001498609781e-05, 9.099999442696571e-05, 9.990006219595671e-05, 9.940005838871002e-05, 9.69000393524766e-05, 9.69000393524766e-05, 9.26000066101551e-05] | - -
- -
reduce_mean · 原先不支持 → 现通过 - -FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["ReduceMean"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132 | -| 输入 SHA-256 | 037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17 | -| 参考输出 SHA-256 (expected_sha256) | 43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7 | -| 初始化器 SHA-256 (initializers_sha256) | 31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: ReduceMean -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 1] / float32 | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 9.933e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003934999695047736, 0.00037949997931718826, 0.00037489994429051876, 0.000372299924492836, 0.00037619995418936014, 0.0003952999832108617, 0.00040259992238134146, 0.0003915999550372362, 0.0003774999640882015, 0.0003698000218719244] | -| 执行原始样本 s | [0.00011030002497136593, 0.00010720000136643648, 0.0001078000059351325, 0.00010830000974237919, 0.00010569998994469643, 0.00010750000365078449, 0.00010809989180415869, 0.00010909989941865206, 0.00010730000212788582, 0.00010609999299049377] | - -
- -
transpose · 原先不支持 → 现通过 - -FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Transpose"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6 | -| 输入 SHA-256 | c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a | -| 参考输出 SHA-256 (expected_sha256) | b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Transpose -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 3, 2, 4] / float32 | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003711999161168933, 0.00037619995418936014, 0.0003662999952211976, 0.00041259999852627516, 0.0003730000462383032, 0.00035260000731796026, 0.0003722000401467085, 0.0003524000057950616, 0.0003497001016512513, 0.0003529998939484358] | -| 执行原始样本 s | [0.00010429997928440571, 0.00010159995872527361, 9.989994578063488e-05, 0.00010140007361769676, 0.00010170007590204477, 0.00010349997319281101, 9.93999419733882e-05, 9.840005077421665e-05, 9.790004696696997e-05, 9.710004087537527e-05] | - -
- -
concat · 原先不支持 → 现通过 - -FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977 | -| 输入 SHA-256 | 09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9 | -| 参考输出 SHA-256 (expected_sha256) | c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Concat -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003693000180646777, 0.00038620003033429384, 0.0003693000180646777, 0.0003645999822765589, 0.00036700000055134296, 0.00041009997949004173, 0.0003698000218719244, 0.00037619995418936014, 0.00035749992821365595, 0.00035550002939999104] | -| 执行原始样本 s | [9.770004544407129e-05, 0.00010369997471570969, 9.69000393524766e-05, 9.790004696696997e-05, 9.599991608411074e-05, 0.00010299996938556433, 9.889993816614151e-05, 9.699992369860411e-05, 9.910005610436201e-05, 9.550002869218588e-05] | - -
- -
slice · 原先不支持 → 现通过 - -FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Slice"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30 | -| 输入 SHA-256 | 04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa | -| 参考输出 SHA-256 (expected_sha256) | 15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df | -| 初始化器 SHA-256 (initializers_sha256) | 6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Slice +```bash +set -euo pipefail +source .venv-operators/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 ``` -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00040770007763057947, 0.0004149000160396099, 0.00039679999463260174, 0.000395399983972311, 0.000394199974834919, 0.0004005999071523547, 0.0003872000379487872, 0.0003947000950574875, 0.00039820000529289246, 0.0003963999915868044] | -| 执行原始样本 s | [0.00010409997776150703, 0.00010469998233020306, 0.00010289996862411499, 0.00010169995948672295, 0.00010259996633976698, 0.00010469998233020306, 0.00010290008503943682, 0.0001039999770000577, 0.00010349997319281101, 0.00010409997776150703] | - -
- -
unsqueeze · 原先不支持 → 现通过 - -INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Unsqueeze"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd | -| 输入 SHA-256 | 4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50 | -| 参考输出 SHA-256 (expected_sha256) | d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193 | -| 初始化器 SHA-256 (initializers_sha256) | a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Unsqueeze -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 2, 3, 1] / int64 | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004278999986127019, 0.00037889997474849224, 0.0003894000547006726, 0.0004752000095322728, 0.00046479993034154177, 0.00037020002491772175, 0.00035470002330839634, 0.0003568999236449599, 0.0003622000804170966, 0.0003958999877795577] | -| 执行原始样本 s | [8.939998224377632e-05, 8.649996016174555e-05, 8.329993579536676e-05, 8.98001017048955e-05, 8.560006972402334e-05, 8.510006591677666e-05, 8.279993198812008e-05, 8.359993807971478e-05, 8.44000605866313e-05, 8.329993579536676e-05] | - -
- -
expand · 原先不支持 → 现通过 - -FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab | -| 输入 SHA-256 | 4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6 | -| 参考输出 SHA-256 (expected_sha256) | eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6 | -| 初始化器 SHA-256 (initializers_sha256) | 7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Expand -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004062999505549669, 0.0003779999678954482, 0.0004357000580057502, 0.00038139999378472567, 0.0004078999627381563, 0.0003739000530913472, 0.0003749999450519681, 0.0003727000439539552, 0.00036589999217540026, 0.000359599944204092] | -| 执行原始样本 s | [0.00011890009045600891, 0.00010760000441223383, 0.00010830000974237919, 0.00010639999527484179, 0.0001023000804707408, 0.00010429997928440571, 0.00010469998233020306, 0.00010419997852295637, 0.00010289996862411499, 0.00010410009417682886] | - -
- -
abs · 原先不支持 → 现通过 - -Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Abs"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf | -| 输入 SHA-256 | 898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a | -| 参考输出 SHA-256 (expected_sha256) | 7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Abs -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00036099995486438274, 0.0003568999236449599, 0.0003541000187397003, 0.0003519000019878149, 0.0003469999646767974, 0.00033569999504834414, 0.00035450002178549767, 0.00034719996619969606, 0.0003350999904796481, 0.000327100045979023] | -| 执行原始样本 s | [8.889997843652964e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.420005906373262e-05, 8.320005144923925e-05, 8.759996853768826e-05, 8.549995254725218e-05, 8.499994874000549e-05, 8.68999632075429e-05, 8.470006287097931e-05] | - -
- -
cast · 原先不支持 → 现通过 - -INT64 position IDs[1,8] -> FP32 [1,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5 | -| 输入 SHA-256 | 93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f | -| 参考输出 SHA-256 (expected_sha256) | 5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 8] / float32 | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003632999723777175, 0.00036179996095597744, 0.0004943999228999019, 0.0005131000652909279, 0.00037509994581341743, 0.00034440006129443645, 0.00033980002626776695, 0.00034899997990578413, 0.00033760000951588154, 0.00035889993887394667] | -| 执行原始样本 s | [7.98999099060893e-05, 0.00017800007481127977, 0.00011080002877861261, 8.82000895217061e-05, 8.230004459619522e-05, 7.829989772289991e-05, 7.860001642256975e-05, 7.840001489967108e-05, 8.060003165155649e-05, 7.970002479851246e-05] | - -
- -
constant · 原先不支持 → 现通过 - -Constant tensor attribute FP32 [2,3]; no runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003632999723777175, 0.000381799996830523, 0.00039549998473376036, 0.000381799996830523, 0.0003598000621423125, 0.00036259996704757214, 0.0003591999411582947, 0.0003554000286385417, 0.0003502999898046255, 0.0003415999235585332] | -| 执行原始样本 s | [6.250001024454832e-05, 6.170000415295362e-05, 6.10999995842576e-05, 6.200000643730164e-05, 6.0399994254112244e-05, 6.150000263005495e-05, 6.219989154487848e-05, 6.0399994254112244e-05, 6.049999501556158e-05, 5.909998435527086e-05] | - -
- -
cos · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cos"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298 | -| 输入 SHA-256 | 8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998 | -| 参考输出 SHA-256 (expected_sha256) | 7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cos -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 5.960e-08 | -| 最大相对误差 | 8.024e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003530000103637576, 0.00034549995325505733, 0.00033980002626776695, 0.000340400030836463, 0.0003451999509707093, 0.00034739996772259474, 0.000332699972204864, 0.0003412999212741852, 0.00033599999733269215, 0.00033199996687471867] | -| 执行原始样本 s | [8.690007962286472e-05, 8.670007809996605e-05, 8.889997843652964e-05, 8.739996701478958e-05, 8.559995330870152e-05, 8.60999571159482e-05, 8.629995863884687e-05, 8.690007962286472e-05, 8.579995483160019e-05, 8.400005754083395e-05] | - -
- -
identity · 原先不支持 → 现通过 - -INT64 indices[2,3] -> Identity [2,3]; frontend alias. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Identity"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b | -| 输入 SHA-256 | 19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413 | -| 参考输出 SHA-256 (expected_sha256) | a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Identity -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.000341000035405159, 0.00034380005672574043, 0.0003355999942868948, 0.0003361001145094633, 0.0003378000110387802, 0.00033820001408457756, 0.0003301999531686306, 0.00032760004978626966, 0.0003744999412447214, 0.0003630999708548188] | -| 执行原始样本 s | [5.170004442334175e-05, 5.280005279928446e-05, 5.219993181526661e-05, 5.2200048230588436e-05, 5.080003757029772e-05, 5.179992876946926e-05, 5.079992115497589e-05, 5.139992572367191e-05, 6.809993647038937e-05, 5.4600066505372524e-05] | - -
- -
pow · 原先不支持 → 现通过 - -Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771 | -| 输入 SHA-256 | b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1 | -| 参考输出 SHA-256 (expected_sha256) | dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004491000436246395, 0.00043999997433274984, 0.00040019990410655737, 0.00037569995038211346, 0.00038789992686361074, 0.0003453000681474805, 0.00048369995784014463, 0.00044920004438608885, 0.00047619990073144436, 0.0004214000655338168] | -| 执行原始样本 s | [0.00014999997802078724, 0.00014180003199726343, 0.00011959997937083244, 0.00011729996185749769, 0.00011809996794909239, 0.00011589995119720697, 0.00015820004045963287, 0.00015029998030513525, 0.00015840004198253155, 0.0001241000136360526] | - -
- -
reciprocal · 原先不支持 → 现通过 - -Nonzero signed FP32 x[2,4] -> Reciprocal [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Reciprocal"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3 | -| 输入 SHA-256 | fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f | -| 参考输出 SHA-256 (expected_sha256) | c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Reciprocal -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00035170000046491623, 0.0003496999852359295, 0.00034989998675882816, 0.00034769997000694275, 0.00033870001789182425, 0.00037759996484965086, 0.00035119999665766954, 0.0003469000803306699, 0.00033660000190138817, 0.00033329997677356005] | -| 执行原始样本 s | [9.190000128000975e-05, 9.019998833537102e-05, 9.029998909682035e-05, 9.14999982342124e-05, 8.899997919797897e-05, 9.609991684556007e-05, 9.510002564638853e-05, 9.06999921426177e-05, 9.109999518841505e-05, 9.059999138116837e-05] | - -
- -
sin · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sin"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a | -| 输入 SHA-256 | efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e | -| 参考输出 SHA-256 (expected_sha256) | e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sin -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00034879997838288546, 0.0003532000118866563, 0.00035470002330839634, 0.0003448999486863613, 0.00033730000723153353, 0.00033710000570863485, 0.0003510999958962202, 0.0003322999691590667, 0.00032819993793964386, 0.000327700050547719] | -| 执行原始样本 s | [8.570007048547268e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.480006363242865e-05, 8.730008266866207e-05, 8.71000811457634e-05, 8.659996092319489e-05, 8.569995407015085e-05, 8.390005677938461e-05, 8.750008419156075e-05] | - -
- -
scalar_fp32_add_chain · 原先错误 → 现正确 - -FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a | -| 初始化器 SHA-256 (initializers_sha256) | 32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | NUMERIC_ERROR / compare | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 2.000e+00 | -| 最大相对误差 | 1.192e-07 | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -NumericMismatch: output differs from ORT: max_abs_error=2 -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 5 | -| 解析原始样本 s | [0.0005631999811157584, 0.0004637000383809209, 0.0004089000867679715, 0.00039579998701810837, 0.0003945999778807163, 0.000381799996830523, 0.00041089998558163643, 0.00040959997568279505, 0.0003945999778807163, 0.0003902000607922673] | -| 执行原始样本 s | [0.00019599997904151678, 0.00015169999096542597, 0.00014780007768422365, 0.0001476999605074525, 0.0001452999422326684, 0.00015129998791962862, 0.00014889996964484453, 0.0001441999338567257, 0.00014589994680136442, 0.0001444999361410737] | - -
- -
singleton_initializer · 原先错误 → 现正确 - -FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar. - -| 项目 | 值 | -| --- | --- | -| 算子 | [] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8 | -| 初始化器 SHA-256 (initializers_sha256) | 7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | EXECUTION_ERROR / execute | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1] / float32 | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003521000035107136, 0.00034039991442114115, 0.00033650000113993883, 0.0003445999464020133, 0.0003336999798193574, 0.0003494999837130308, 0.0003339999821037054, 0.00035889993887394667, 0.00032849994022399187, 0.00033849989995360374] | -| 执行原始样本 s | [6.290001329034567e-05, 6.280001252889633e-05, 6.140000186860561e-05, 6.009999196976423e-05, 6.079999729990959e-05, 6.370001938194036e-05, 6.200000643730164e-05, 6.330001633614302e-05, 6.260001100599766e-05, 6.0399994254112244e-05] | - -
- -
constant_cast_abs_expand · 原先不支持 → 现通过 - -Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant", "Cast", "Abs", "Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3 | -| 输入 SHA-256 | 9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822 | -| 参考输出 SHA-256 (expected_sha256) | eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 4 | -| 解析原始样本 s | [0.00043970008846372366, 0.00044329999946057796, 0.0004450000124052167, 0.0004384000785648823, 0.0006199999479576945, 0.00047199998516589403, 0.00045169994700700045, 0.00044700002763420343, 0.00044139998499304056, 0.00043539993930608034] | -| 执行原始样本 s | [0.00013560010120272636, 0.0001338999718427658, 0.00013189995661377907, 0.0002367999404668808, 0.00017560005653649569, 0.00013229995965957642, 0.0001343999756500125, 0.000130399945192039, 0.00013109995052218437, 0.00013479997869580984] | - -
- -
qwen_rmsnorm · 原先不支持 → 现通过 - -FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow", "ReduceMean", "Add", "Sqrt", "Reciprocal", "Mul"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237 | -| 输入 SHA-256 | f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48 | -| 参考输出 SHA-256 (expected_sha256) | 5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec | -| 初始化器 SHA-256 (initializers_sha256) | 6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 10 | -| 解析原始样本 s | [0.000506900018081069, 0.0005015999777242541, 0.0005096999229863286, 0.0005014999769628048, 0.0004960000514984131, 0.0005388999124988914, 0.0005028999876230955, 0.000512199942022562, 0.000493900035507977, 0.0004851999692618847] | -| 执行原始样本 s | [0.0002813000464811921, 0.0002788000274449587, 0.0002808000426739454, 0.00028170004952698946, 0.00027820002287626266, 0.0002930000191554427, 0.00028490007389336824, 0.00028249993920326233, 0.0002837000647559762, 0.0002848999574780464] | - -
- -
rope_numeric · 原先不支持 → 现通过 - -INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast", "Mul", "Cos", "Sin", "Sub", "Add", "Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa | -| 输入 SHA-256 | 823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a | -| 参考输出 SHA-256 (expected_sha256) | eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 1.699e-07 | -| 执行 IR 步数 | 12 | -| 解析原始样本 s | [0.0005266999360173941, 0.000520200002938509, 0.0005088000325486064, 0.0006041000597178936, 0.0006553001003339887, 0.0005080000264570117, 0.0005031999899074435, 0.0005010999739170074, 0.0005029999883845448, 0.00049919995944947] | -| 执行原始样本 s | [0.0002498000394552946, 0.00024560000747442245, 0.0002465000143274665, 0.0003208999987691641, 0.00031370006036013365, 0.00025370006915181875, 0.0002449000021442771, 0.00024610001128166914, 0.0002482000272721052, 0.0002460000105202198] | - -
- -
pr89_transformer_2l · 原先不支持 → 现通过 - -PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3 - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add", "Concat", "Div", "Expand", "Gather", "MatMul", "Mul", "Neg", "ReduceMean", "Reshape", "Sigmoid", "Slice", "Softmax", "Sqrt", "Transpose", "Unsqueeze"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 0 | -| 模型 SHA-256 | 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 | -| 输入 SHA-256 | bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457 | -| 参考输出 SHA-256 (expected_sha256) | 413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c | -| 初始化器 SHA-256 (initializers_sha256) | 955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` +输出目录每次换新名字，生成本次 JSON、Markdown、HTML 及原始样本。仅前后均 PASS 的用例比较耗时；缺失、不支持或失败不等于零耗时。结果用于支持/正确性对比，IR 解释器计时不等于 QEMU 或目标硬件性能。 -**修改后** +原始个人测量保持原平台身份和数据，分别保存于固定提交： -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 256, 128] / float32 | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | 7.153e-07 | -| 最大相对误差 | 5.722e-04 | -| 执行 IR 步数 | 159 | -| 解析原始样本 s | [0.004760700045153499, 0.004594899946823716, 0.004717700066976249, 0.004990800051018596, 0.004586199996992946, 0.004221900017000735, 0.004203699994832277, 0.004572000005282462, 0.004674999974668026, 0.004224899923428893] | -| 执行原始样本 s | [0.010629900032654405, 0.011460799956694245, 0.009812200092710555, 0.009985200013034046, 0.010765800019726157, 0.009604100021533668, 0.010238799965009093, 0.009660700103268027, 0.010715999989770353, 0.009805700043216348] | +- [原始 MD](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md) +- [原始 JSON](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json) +- [原始 HTML](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html) -
+当前同名 JSON 的 `document_kind=historical_benchmark_index`，只保存归档链接，不含 PASS、环境测量或计时字段，不能作为 benchmark 结果输入。新 Linux 测量必须从 CLI 输出目录读取。 diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" index 13b7df2..4d516da 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" @@ -1,148 +1,92 @@ -Qwen3 ONNX 算子补齐与单测报告 -日期：2026-10-01 - -工作目录：仓库根目录；本文使用仓库相对路径。 -主干基线：20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 -历史测量状态：原始算子探测和 benchmark 在实现尚未提交时执行；归档报告保留测量时的 HEAD、dirty 和源码指纹。 -本文用于随算子实现及测试 PR 归档，不以历史 dirty 标记描述当前提交或远程仓库状态。 -本报告更新上一阶段“真实 Qwen3 尚缺八类前端算子”的临时结论。 - -一、完成范围 - -完成 ONNX 前端 → 共享 IR → NumPy 解释器路径所需的两组算子： -1. PR #89 探测图八类：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 -2. 真实 Qwen3 导出图额外八类：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 -第一组复用已有 IR 算子与内核，补齐前端和边界测试；第二组新增六个 IR opcode，Constant 绑定为全局常量，Identity 复用输入值。 - -二、本轮代码改动 - -scratchv/frontend/onnx_parser.py -- 补齐第二组前端映射；支持 Constant 的 tensor、value_int/value_ints、value_float/value_floats。 -- 保持零维标量、单元素向量与 INT64 值的区别，支持外部权重及 Constant 外部数据。 -- 求值 Constant → Cast/Abs/Reshape 等静态 shape/axis 链，适配真实 Qwen3 导出形式。 -- 不将中间结果冒充 initializer；限制静态参数求值规模，避免超大 Expand 与尺寸乘积溢出。 -- Add/Mul 统一发出 IR，修复 Python 标量折叠造成的 FP32 舍入与整数宽度错误。 -- 预留全图 v_N 名字，修复 ONNX 原始名字和临时结果名碰撞。 -- 明确拒绝当前 IR 不支持的数据类型。 - -scratchv/ir/types.py、scratchv/ir/builder.py -- 新增 ABS、CAST、COS、SIN、POW、RECIPROCAL 及 builder 方法。 -- Cast 的结果 dtype 表示目标类型；Pow 允许底数与指数类型不同，结果保持底数类型。 - -scratchv/analysis/ir_verifier.py -- 增加新算子签名、浮点算子类型约束和 Pow 输出类型校验。 - -scratchv/verification/ir_numpy_ops.py -- 增加新算子的实际 NumPy 求值和数值错误诊断。 -- Cast 整数缩窄保留低位；浮点转整数按截断后范围检查，避免 INT64 边界错误。 -- Pow 支持广播、混合类型以及有界的精确整数计算。 -- Slice 使用 ONNX 的正反向钳制规则，修复过小负 start 的反向切片，并处理空维和 INT64 哨兵。 - -三、测试 - -本轮新增四个测试文件，共 282 个用例： -- tests/test_onnx_shape_ops.py：116 个，第一组八类算子与 ORT 对照、边界和非法参数。 -- tests/test_onnx_qwen_numeric_ops.py：78 个，第二组算子、外部常量、实际静态参数链及 RMSNorm 组合。 -- tests/test_ir_qwen_numeric_ops.py：74 个，IR builder、类型签名、广播、整数精度、数值边界与错误定位。 -- tests/test_onnx_constant_regressions.py：14 个，FP32 舍入、整数宽度、名字碰撞和静态求值规模限制。 -更新 tests/test_ir_verifier.py 的穷举签名表，覆盖六个新增 opcode。 -延续上一阶段的 tests/test_onnx_transformer_execution.py 和 initializer 测试修复。 - -算子及原有相关回归共 715 项，加上 22 项 benchmark 自检；提交前合并重跑结果为 737 passed，无失败。 -该次合并运行有 7 条 NumPy 配置提示 warnings，未影响测试通过。 -其中 715 项算子及原有相关回归的分组记录如下： -- IR、解释器、校验器、优化器、PassManager 相关：439 passed。 -- 前端、ONNX 算子、Transformer 组合、诊断和相关 benchmark：262 passed。 -- Constant 专项回归：14 passed（另与已计入的 85 项用例联合运行，结果 99 passed）。 -这是相关回归集合，未声称运行整个项目测试全集。 -单测测量环境：Python 3.12.13，NumPy 2.5.3、ONNX 1.23.0、ORT 1.29.0，Python -X utf8 -B。 - -四、模型验证 - -1. PR #89 原始 LFS 模型未修改： - probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx - SHA-256：17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 - 本轮重新运行 seed=1：ORT vs 独立 NumPy 参考最大绝对误差 1.09718e-6；IR vs ORT 为 7.1525574e-7。 - 均通过最大绝对误差 <1e-5；IR 执行 159 步。 - 另有 seed=0/1/42 的组合回归通过。 - 原始探测 JSON 保留在本机，未随本 PR 归档；此处保留模型指纹和数值结论。 - -2. 本地真实 Qwen3-0.6B 导出模型： - models/qwen3-0.6b-onnx/model.onnx（额外交付模型，不包含在本次算子 PR 中） - 7847 个 ONNX 节点、23 类算子；未识别算子为 0。 - 完整载入外部权重，得到 2148 个全局定义、5910 条 IR 指令。 - verify_ir 通过，问题数为 0。 - 原始结构审计 JSON 保留在本机，未随本 PR 归档。 - 此项是完整图解析及结构/类型校验，不是全模型 IR 推理的数值对照。 -模型探测使用独立导出依赖环境（NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1）。 - -五、支持边界与下一步 - -本次范围是部署方案所需的静态图及 FP32 路径，同时覆盖 IR 已有的 FLOAT64、INT32、INT64 类型。 -没有实现 ONNX 的全部 dtype/opset 扩展；动态 shape/axis 参数及 Reshape allowzero=1 仍明确拒绝。 -解释器沿用严格数值策略：非法定义域、非有限结果以及新增 Abs/整数 Pow 的溢出会报错。 -整数 Pow 对大整数采用精确计算，ORT CPU 在超 2**53 的整数幂上可能存在浮点精度差异；实际 Qwen 使用 FP32 底数和 INT64 指数 2。 - -下一步应运行真实 Qwen3 结构的小模型，包含 Q/K RMSNorm、完整 RoPE、GQA、SwiGLU，逐层对照 PyTorch/ORT/IR。 -之后再验证优化流水线、完整 0.6B 推理的数值/内存，以及 RISC-V 后端和 QEMU。 -本轮没有补 RISC-V 降低和执行支持，不能据此前端/IR 结果判断 QEMU 已可运行。 - -六、修改前后 benchmark 对比 - -归档文件（同目录）： -- 算子补齐与单测-benchmark.json：移除本机路径后的测量数据，保留全部样本、误差及指纹。 -- 算子补齐与单测-benchmark.md：GitHub 可直接阅读的结果报告。 -- 算子补齐与单测-benchmark.html：独立 HTML 报告，下载后用浏览器打开，可筛选并展开错误详情。 - -共 24 个固定用例，修改前 2/24 PASS，修改后 24/24 PASS： -- 16 个独立算子用例：0/16 → 16/16。 -- 2 个已有支持的对照用例：2/2 → 2/2。 -- 2 个错误回归用例：0/2 → 2/2。 -- 4 个组合链路用例：0/4 → 4/4，包含 PR #89 的原始两层 Transformer。 -因此，“原先不支持、现通过”是 20 个用例，不是新增 20 种算子。 - -使用同一批模型、输入、ORT 参考及初始化器，并校验指纹；基线通过解释器已有 initializers 参数接受原始 ONNX 数组，属于基准适配，不改基线实现，也不重塑权重以掩盖旧错误。 -ABBA 顺序为修改前、修改后、修改后、修改前；每版本 2 轮，每轮预热 1 次、正式计时 5 次，合计每版本 10 个正式样本。 -IR.run 完整计时包括校验、执行规划、绑定复制、NumPy 计算及返回复制；ONNX 解析单独计时，ORT 参考计算不计入。 -只在前后均 PASS 时显示执行耗时比；本报告证明支持和正确性改善，不宣称性能优化或 QEMU 性能。 -新增 tests/test_onnx_operator_benchmark.py 的 22 项自检与原 715 项回归已合并运行，共 737 passed。 -CI 的测试 checkout 获取完整历史，以便历史基线集成测试实际运行。 - -七、复跑准备与命令（在仓库根目录 PowerShell 执行） - -建议 Python 3.12，并使用独立虚拟环境。依赖安装： -python -m pip install -e ".[all]" pytest -其中项目 all 依赖包含 onnxruntime；基准和测试直接依赖 NumPy、ONNX、onnxruntime、pytest。 -本次单测/benchmark 版本为 NumPy 2.5.3、ONNX 1.23.0、onnxruntime 1.29.0；解释器路径不是复跑前提。 - -确保 Git 历史中存在固定基线提交，浅克隆需补充该历史；origin 应指向包含该提交的项目仓库或 fork： +Qwen3 ONNX 算子补齐与单测：Linux 复现说明 +更新：2026-10-06 + +正式交付与他人复现环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。 +所有命令在仓库根目录执行。通用环境约定见 docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md。 +个人本机测量的原始版本保留在固定提交： +https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测报告.txt +原始平台、计数、计时和数值不改名为 Linux 数据；新结果必须由下面的 Linux 命令重新生成。 + +一、实现范围 + +ONNX 前端 → 共享 IR → NumPy 解释器覆盖两组算子： +1. PR #89 探测图：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 +2. 真实 Qwen3 导出额外需要：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 +第一组复用已有 IR 与内核，补前端/边界；第二组新增 ABS、CAST、COS、SIN、POW、RECIPROCAL，Constant 使用绑定，Identity 复用输入值。 + +语义修复包括零维标量与向量区别、INT64 精度、静态 shape/axis 参数链、外部权重和 Constant、FP32 常量舍入、临时值命名冲突、静态求值规模限制、Cast 截断/溢出、Pow 广播与整数精度，以及 Slice 的反向边界和 INT64 哨兵。前端不将中间值伪装为 initializer。 + +二、验收用例 + +tests/test_onnx_shape_ops.py：八类形状/索引算子，ORT 对照、边界和非法参数。 +tests/test_onnx_qwen_numeric_ops.py：数值算子、常量、静态参数链及 RMSNorm 组合。 +tests/test_ir_qwen_numeric_ops.py：IR 类型、广播、整数精度与错误定位。 +tests/test_onnx_constant_regressions.py：FP32 舍入、整数宽度、名字碰撞、求值规模。 +tests/test_onnx_operator_benchmark.py：基准自身的身份、正确性及失败传播。 +实际 passed/skipped/failed 以本次 Linux 输出为准，不沿用历史计数。 + +三、Linux 环境与资产准备 + +以下为 Bash，已存在环境时可复用，但必须核对依赖版本。benchmark 使用单独环境，以保持历史前后对照依赖；它不替代完整模型门禁的固定导出环境。 + +set -euo pipefail +sudo apt-get update +sudo apt-get install --no-install-recommends python3.12-venv git-lfs +python3.12 -m venv .venv-operators +source .venv-operators/bin/activate +python -m pip install "numpy==2.5.3" "onnx==1.23.0" "onnxruntime==1.29.0" "pytest==9.1.1" +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 + +git rev-parse HEAD +git status --short git fetch origin 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 - -安装 Git LFS 后，下载已由 PR #89 提交的模型实体，不能把 LFS 指针作为 ONNX 运行： -git lfs install +git lfs install --local git lfs pull --include="probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx" -以下 24 用例基准不需要完整 Qwen3-0.6B 权重；完整模型结构审计需要另外准备本文第四节所述模型。 -生成新的基准报告及临时 fixtures，不覆盖 docs 中的历史归档： -python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5 - -复跑相关测试： - -$operatorTests = @( - 'tests/test_ir.py', 'tests/test_ir_qwen_numeric_ops.py', - 'tests/test_ir_interpreter.py', 'tests/test_ir_interpreter_ops.py', - 'tests/test_ir_interpreter_control_flow.py', 'tests/test_ir_verifier.py', - 'tests/test_ir_verifier_integration.py', 'tests/test_optimizer.py', - 'tests/test_optimizer_advanced.py', 'tests/test_pass_manager.py', - 'tests/test_pass_registry.py', 'tests/test_parser.py', - 'tests/test_onnx_initializers.py', 'tests/test_onnx_shape_ops.py', - 'tests/test_onnx_qwen_numeric_ops.py', 'tests/test_onnx_transformer_execution.py', - 'tests/test_ir_interpreter_benchmark.py', 'tests/test_ir_verifier_benchmark.py', - 'tests/test_ir_diagnostics.py', 'tests/test_onnx_constant_regressions.py', - 'tests/test_onnx_operator_benchmark.py' +基线为固定提交 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6，不能用不同分支 HEAD 代替。浅克隆必须补齐历史。原模型 SHA256 应为 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1，不可把 LFS pointer 当 ONNX。 + +四、生成修改前后 benchmark + +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 + +每轮选择新的输出目录。CLI 生成自己的 fixtures、参考和 JSON/Markdown/HTML 报告，不读取 docs 目录中的历史索引作为测量。源码、模型、输入、环境身份与错误详情由新报告保存。 + +24 个固定用例包含 16 独立算子、2 已支持的对照、2 错误回归和 4 组合链路，含原 PR #89 两层 Transformer。ABBA 顺序比较修改前/后，每版本两轮；前后都通过的用例才计算耗时比。不支持、失败或缺测不能解释为零耗时或无限加速。 + +IR.run 计时包含验证、执行规划、绑定复制、NumPy 计算与返回复制；ONNX 解析单独计时，ORT 参考不计入。此基准验证支持与正确性变化，不证明 QEMU 性能改善。重新生成的 Linux 样本不可与不同平台的历史耗时直接算加速比。 + +五、运行相关回归与合成图 + +operator_tests=( + tests/test_ir.py tests/test_ir_qwen_numeric_ops.py + tests/test_ir_interpreter.py tests/test_ir_interpreter_ops.py + tests/test_ir_interpreter_control_flow.py tests/test_ir_verifier.py + tests/test_ir_verifier_integration.py tests/test_optimizer.py + tests/test_optimizer_advanced.py tests/test_pass_manager.py + tests/test_pass_registry.py tests/test_parser.py + tests/test_onnx_initializers.py tests/test_onnx_shape_ops.py + tests/test_onnx_qwen_numeric_ops.py tests/test_onnx_transformer_execution.py + tests/test_ir_interpreter_benchmark.py tests/test_ir_verifier_benchmark.py + tests/test_ir_diagnostics.py tests/test_onnx_constant_regressions.py + tests/test_onnx_operator_benchmark.py ) -python -X utf8 -B -m pytest @operatorTests -q --basetemp=output/pytest-operator-completion +mkdir -p output +python -X utf8 -B -m pytest "${operator_tests[@]}" -q \ + --junit-xml=output/operator-tests-linux-01.xml +python -X utf8 -B probes/w1_tiny_transformer/run.py \ + --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx \ + --output-dir output/operator-completion-linux-01 --seed 1 + +六、范围与后续门禁 + +这是静态图和 FP32 所需前端/IR 能力，同时覆盖现有 FLOAT64、INT32、INT64；不宣称全部 ONNX dtype/opset 或动态 shape/axis。Reshape allowzero=1 等不支持语义明确拒绝；非法定义域、非有限值及部分整数溢出明确报错。当前固定 Qwen 图使用 FP32 底数与整数指数 2；通用 Pow 范围需要分别验证。 -python -X utf8 -B probes/w1_tiny_transformer/run.py --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx --output-dir output/operator-completion-probe --seed 1 +完整 28 层解析、真实两层 Qwen3 数值与 RV64/QEMU 分别从 W1/README.md、W2/README.md 和 probes/w2_qwen3_small/README.md 进入；基础算子 benchmark 通过不能代替这些门禁。 -上述探测命令默认使用已下载的 ONNX，不要求重新导出；切换导出依赖环境时请遵循模型探测的已记录版本。 +同目录 benchmark.md 为 Linux 复跑入口，benchmark.html 为可打开的指引，benchmark.json 为明确标识的历史索引，不包含新测量。原始完整测量保留在固定提交历史中。 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index e233014..0db79de 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -1,262 +1,85 @@ -# W2：完整前端解析、基础后端与 Host 运行时 +# W2 Linux 独立复现与验收 -本目录记录开发计划中的 W2 验收：完整 28 层 Qwen3-0.6B 的前端解析与审计、四类模型模式回归、基础算子 QEMU，以及 Tokenizer/输入/mask/greedy。已有真实两层模型的 IR/ORT/QEMU 门禁继续保留。 +正式交付、CI 和第二人复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。完整环境准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。Linux 为宿主环境；QEMU 内的 RV64GC/LP64D 程序仍使用当前裸机 ABI。 -完整图的范围仍止于前端验收。完整 IR 与 ORT 的数值对照、真实维度预训练子图、权重独立装载、完整 QEMU 前向和生成循环仍是后续工作。本轮也不自动关闭 W1 的第二人复现、接口确认和出口评审。 +W2 的范围是完整前端解析、小模型 IR 数值、基础后端和 Host 单步接口。完整 28 层模型的 IR 数值前向属于 W3；完整模型 QEMU 与生成循环属于后续阶段。W1 的人工接口决议继续见 [W1 记录](../W1/README.md) 和 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92)。 -**当前交付状态（2026-10-03）**：本次按 W1/W2 拆分为独立分支。W2 直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含另一 W1 分支的两层报告/参考测试修复。当前分支的范围、重新执行的证据及 PR 归属见 [分支验收与发布说明](W2-分支验收与发布说明.md)。下方原本地整合记录及历史 CI 按各自源码归属，不代表本分支已通过远端 CI。 +## 独立复现版本 -## W2 独立技术验收入口 +[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 的既有 Linux 记录对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。后续文档或源码更新不自动继承该提交的执行结果。开始前记录约定 SHA、`git rev-parse HEAD` 和 `git status --short`；有未提交增量则保留完整 diff 和新增文件，不能只报告 HEAD。 -使用已有的 Python 3.12 固定 CPU 环境（[依赖](../../../requirements/qwen3-small-probe.txt)）、Zig 0.14.1、QEMU RV64，以及固定完整 ONNX 和官方 Tokenizer 资产。在仓库或完整源码副本根目录执行： +发布前个人工作树的详细测量按原始身份保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/README.md)。当前页面只提供 Linux 交付入口与可关联到具体提交的 Linux CI 记录，不把历史个人测量改名为 Linux 结果。 -```powershell -$w2Python = 'D:/path/to/qwen3-venv/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-new ` - --tokenizer-dir D:/path/to/qwen3-tokenizer ` - --model-dir D:/path/to/qwen3-0.6b-onnx ` - --cc D:/path/to/zig.exe ` - --qemu D:/path/to/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -Linux 使用相同参数，将解释器和工具替换为本机路径即可。程序不安装依赖；默认两个资产目录均使用 `verify` 离线校验。无资产时可显式传 `--tokenizer-mode download` 或 `--model-mode download`，后者需要下载约 1.24 GB 的完整模型发布包。所有结果使用本次全新的输出目录，不能复用已有目录，即使为空。 - -| 执行顺序 | 独立门禁 | 要求 | -|---|---|---| -| 1 | `frontend` | 22 项 RMSNorm/RoPE/SwiGLU/GQA 模式单测，不接受 skip | -| 2 | `backend` | 16 个小图 × none/all，32 次实际 RV64 QEMU | -| 3 | `runtime` | 官方 Tokenizer 20 组/480 对照，以及 5 组输入、mask、采样与停止条件 | -| 4 | `runtime-model` | 官方文本 ID 直入完整词表的真实两层小配置，Torch/ORT/IR none/all 单步接口集成 | -| 5 | `small-ir` | 从固定种子重新导出原两层普通/诊断图，7 组输入、29 检查点与不变性验证 | -| 6 | `small-qemu` | 使用本轮新导出的模型与报告，28 次两层实际 QEMU | -| 7 | `full-parse` | 固定 28 层完整图解析、逐节点/权重绑定审计与 IR verifier | - -只有七项全部实际执行并通过才返回 **PASS / 退出 0**。`--gates frontend backend` 等子集支持开发期间定位，但最多返回 **PARTIAL / 退出 2**；失败或阻断返回 **FAIL / 退出 1**。small-qemu 自动包含 small-ir，runtime-model 自动包含 runtime。默认每门禁截止 1800 秒、每次 QEMU 截止 180 秒、完整解析 worker 截止 300 秒，可用 `--gate-timeout`、`--qemu-timeout`、`--parse-timeout` 明确调整。 - -总报告在 `output/w2-acceptance-new/report.json/md/html`，每项有独立目录和 `logs/.log`。总入口核对退出码、报告身份、必要计数、实际输出和源码指纹；运行期间源文件变化也会失败。报告记录 HEAD/dirty diff（可用时）及本次源码 SHA256；没有 `.git` 的源码包允许运行，但必须明确标为未知 Git 身份，不能借用父目录仓库的提交。 - -新 [单步模型集成探测](../../../probes/w2_runtime_model/README.md) 保留完整词表 151936，hidden32/FFN96/两层随机权重。原文本 ID 不裁剪、不取模、不重新映射。逐元素检查完整 logits（含 padding），校验最后有效位置、原始 greedy ID 和官方解码；不要求随机权重生成有意义的文本。本地会保存较大的参考数组（约 614 MiB），CI 仅上传摘要报告，不上传这些参考数组或权重。该探测不执行完整 0.6B、完整词表 QEMU 或多步生成。 - -当前本地 workflow 已加入单步集成、完整 28 层解析必跑任务和汇总检查。`W2 / required acceptance` 要求轻量/数值任务与完整解析任务均为 success；失败、取消、跳过、缺失结果均不通过。完整 ONNX 的 ORT 前向仍是独立的手动/夜间 W1 任务。配置随本 W2 分支发布，远端 CI 以对应提交的实际执行为准；分支保护规则未变更。源码隔离复现共用本机 Python/工具链时，只能证明不依赖旧源码目录与旧产物，不能代替第二人或另一台机器复现。 - -统一验收的 Linux 超时路径会先发 SIGTERM，让运行器清理独立会话内的编译器、QEMU 或解析 worker，再执行进程组兜底清理。Windows 继续使用进程树/Job 清理。Linux 专属进程集成回归在 Windows 上明确跳过，发布前须在 Linux 实跑；不能用 Windows 结果替代它。 - -## 本轮继续交付与最终实测（2026-10-02，本地未提交） - -本轮新增完整词表的单步模型接口集成、七项统一验收入口及回归，并补齐原两层 IR 门禁的报告原子写入：MD/HTML 或 JSON 发布失败时必须返回失败，保留原数值错误，不遗留成功 JSON。此前四条报告路径的修复继续保留。 - -从当前工作树复制 **553 个源码文件**到全新的 `output/w2-independent-source-v2`，包含未提交改动，不复制 `.git`、历史 output、Python 环境或模型产物；从该目录真实运行统一入口，最终 **7/7 PASS，退出 0**。固定 ONNX/Tokenizer 资产重新验证哈希，Python 环境与 Zig/QEMU 工具仍共用本机现有安装。导入路径确认来自新副本，模型重新导出、ELF 重新编译、所有结果重新执行；总报告的 **134 项源码指纹与最终工作树一致**。 - -总证据：`output/w2-independent-source-v2/output/w2-acceptance/report.json/md/html`；每项明细、误差位置、日志和实际命令位于同一目录下对应子目录。快照来源及逐文件哈希在 `output/w2-independent-source-v2-manifest.json`，外层日志在 `output/w2-independent-acceptance-v2.log`。这些本机产物不进入版本库。 - -| 门禁 | 最终实测 | -|---|---| -| 前端四模式 | **22/22** 通过，未跳过 | -| 基础后端 | **32/32 QEMU**；最大绝对误差 **2.384185791015625e-7** | -| 运行时 | **20/20** 官方语料、**480/480** 编码/解码对照、**5/5** 输入与采样场景 | -| 单步模型集成 | **2 个原始文本、8 次模型调用**（2 Torch + 2 ORT + 4 IR）；完整输出 `[1,256,151936]`；最大绝对误差 **2.8312206268310547e-7**，采样 ID/解码一致 | -| 两层 IR | **7 组输入、29 检查点、6 项不变性**全部通过；所有数值比较最大绝对误差 **1.9073486328125e-6** | -| 两层 QEMU | **28/28**，普通/诊断图及 none/all 全通过；普通 logits 最大误差 **3.129243850708008e-7**，诊断检查点最大误差 **1.6689300537109375e-6** | -| 完整前端 | **28/28 层、7,847 节点、8,161 value、2,148 绑定**全部通过；5,910 条 IR 指令、verifier 无错误；绑定数据 **2,384,216,772 bytes** | - -统一流程 **264.340 秒**；两层 QEMU 进程累计 **157.642 秒**，其门禁全流程 **172.264 秒**；完整解析门禁 **9.538 秒**，其中 parser **2.031 秒**，worker 峰值 RSS **5,064,372,224 bytes（约 4.72 GiB）**。QEMU 计时包含进程启动、guest 运算和输出传输，诊断图包含检查点工作；这些是单次主机观察，不是纯推理性能，也不能据此声称提速。 - -首次源码副本实跑保存在 `output/w2-independent-source`，总结果正确为 FAIL：前端 pytest 临时目录的父目录未创建，导致 22 个 fixture setup 错误，其余六项通过。修复为使用统一输出根下的临时目录，新增真实 pytest 子进程回归后从新副本完整重跑，没有复用第一次的 PASS。另修复源码包向上查找 Git 的身份错误：运行时、两层 IR 和完整解析入口检查自身 `.git`；缺失时保留源码指纹并明确未知提交。最终实跑不依赖 `GIT_CEILING_DIRECTORIES` 等外部补救设置。 - -专项回归先执行 **631 passed / 0 skipped**（`output/w2-closeout-combined-tests.xml`）。最终修正上述入口后，对受影响两组分别复验 **72 passed / 0 skipped**（`output/w2-source-identity-all-tests.xml`）与 **47 passed / 0 skipped**（`output/w2-acceptance-tests-d.xml`）；批次重叠，不相加。工作流 YAML、25 段 Bash 语法、文档链接和 diff 格式检查通过。本轮未执行全仓测试或远端 CI，未提交、未推送、未更新 PR #91。 - -本轮补齐的是 W2 的独立技术验收和单步接口集成。第二人独立复现、团队接口确认以及发布后对应源码的远端 CI 仍需完成；完整预训练模型的 IR 数值、真实维度后端、独立权重装载和多步生成仍属于后续阶段。 - -## 前一轮：拆分后的本地 Review（2026-10-02） - -本轮审查完整解析、四模式前端、基础后端、Tokenizer、输入/mask/greedy，以及本地分支携带的两层 QEMU 修复。发现一类 P2 验收漏洞并修复四条报告路径：必需的报告写入失败时，可能留下可解析的 `passed=true` JSON。它会使仅消费 artifact 的脚本误判，不能以数值计算正确为理由忽略。 - -| 入口 | 复现与修复 | -|---|---| -| `probes/w2_backend_ops/run.py` | 原来先写 PASS JSON，随后 MD/HTML 失败；改为保存视图结果后发布 JSON，保存失败使门禁非零退出，保留原错误 | -| `probes/w2_qwen3_parse/run.py` | 同类先发布成功状态问题；保留验证阶段、failure_context 和主错误，报告保存异常单列，JSON 最后原子发布 | -| `probes/w2_runtime/run.py` | 虽然原来已最后写 JSON，完整内容写入后关闭/flush 抛异常仍可留下 PASS；改为同目录临时文件成功关闭后原子替换，失败不发布成功 JSON | -| `probes/w2_qwen3_small/riscv.py` | 既有两层 QEMU 门禁存在同类问题；视图与 JSON 原子写入，JSON 最后发布，保留原执行错误与时间证据。此修复只在本地，未同步 PR #91 | - -新增 **30 项**失败注入回归，覆盖 MD/HTML/JSON 无法保存、完整写入后报错、JSON 发布失败、原探测已失败及 CLI 状态。没有修改数值算法、精度阈值或模型配置。在本轮实际核对的路径和样本中，未发现新的前端/MatMul/Tokenizer 计算错误；这不构成完整模型数值正确性的证明。 - -| 本轮验证 | 结果与证据 | -|---|---| -| W2 与两层报告相关专项 | **396 passed / 0 skipped**；`output/w2-review-combined-tests.xml` | -| 前端模式、C 后端与 RISC-V 运行时相关回归 | **138 passed / 0 skipped**；`output/w2-review-compiler-regression.xml`。与上一行有重叠，不相加 | -| 基础后端实际 RV64 QEMU | **32/32**；对 ORT/独立 NumPy 最大绝对误差均 **2.384185791015625e-7**；流程 **23.588 秒**，QEMU 累计 **5.800 秒**；`output/w2-review-backend-real/report.json` | -| 官方 Tokenizer 离线探测 | **20/20** 语料、**480/480** 对照、**5/5** 输入与采样场景；流程 **3.739 秒**；`output/w2-review-runtime-offline-fixed/report.json` | - -已核对上述后端 30 项、运行时 5 项源码 SHA256 与修复后代码一致。额外 500 组随机 Unicode/特殊 token 文本编码和 4,000 次解码对照也通过；它们是本轮探索性检查，不算新增 CI 用例。本轮未重跑完整 28 层模型、整两层 QEMU 或全仓测试，也没有远端 CI；历史结果仍按原 SHA 归属。计时为单次观察，不用于声称性能改善。 - -### 可以继续的工作，按优先级 - -1. **W2 交付收尾**：第二人用自己的环境执行统一验收入口；未来按用户指示整理独立 W2 PR，明确哪些两层门禁修复应单独回补 W1。当前 PR #91 的绿色状态不是本地 W2 的验收；同机源码副本复现也不能代替第二人验收。 -2. **真实维度与预训练子图**：先验证投影 MatMul、RMSNorm、RoPE、Attention、SwiGLU，保留 Torch/ORT/IR 和适当小子图的 QEMU 误差。固定源配置为 hidden=1024、head_dim=128、16 个 Q head/8 个 KV head、FFN=3072。现有两层缩小配置和 3-token GQA case 不能证明这些长累加维度满足误差要求。优先建立基线，再决定 MatMul 分块/布局优化。 -3. **参数化中等模型与逐层定位（W3）**：将两层配置和固定 29 检查点推广成配置驱动，跑计划中的六层中等模型，记录首个偏差、最大绝对误差、耗时与峰值内存。Attention 可先沿用现有分解 IR，融合不是正确性前置。 -4. **完整 IR 内存准备（W3）**：当前解释器会复制 initializer，并在 `values` 中保留中间结果，没有最后使用后回收。先测权重、活跃张量与 RSS，设计只读权重复用和按生命周期回收；再执行完整 IR/ORT，沿用计划中的 `<1e-4` 门槛。 -5. **完整后端与生成集成（W4/W5）**：当前常量/工作区默认各 256 MiB，QEMU 为 512 MiB，权重展开为 C 字面量；仅真实 embedding 就约 593.5 MiB。需要代码与权重分离、装载 ABI、地址布局和容量验证，再接完整前向与生成循环。Host helper 与完整词表两层随机模型的单步调用现已集成；真实预训练模型调用、逐 token 循环、chat template 与流式解码仍未完成。 - -建议下一轮从第 2 项的真实维度数值基线开始，同时准备第 3 项的可配置探测，避免直接以完整模型试跑替代可定位的小范围验证。 - -## W2 基础能力补齐（2026-10-02，历史提交 ff193c7） - -该次在已通过完整解析和两层数值探测的 `dfc9437` 上补齐三个独立入口，曾更新 PR #91，现已按 W1/W2 拆分要求撤出该 PR 并保留在本地。表中“通过”必须读取对应提交的报告或 CI；本页不把旧提交或跳过项计作当前本地修改通过。 - -| W2 门禁 | 当前入口与范围 | -|---|---| -| `unit:frontend-ops` | `tests/test_qwen3_frontend_patterns.py`：RMSNorm、RoPE、SwiGLU、GQA 四类模式，含错误 epsilon、旋转符号、门控分支与 KV 重复顺序反例 | -| `unit:backend-ops` | `probes/w2_backend_ops/run.py`：16 小图，MatMul/逐元素/Softmax/RMSNorm/RoPE/SwiGLU/GQA 七类；none/all 共 32 次实际 RV64 QEMU；独立 NumPy 公式和 ORT 双参考 | -| `unit:runtime` | `probes/w2_runtime/run.py`：固定官方七份 tokenizer/配置文件，20 组语料、fast/slow 双参考与 golden IDs/解码文本；mask/最后有效位置/greedy/EOS/容量/预算 | -| `numeric:ir-small` | 既有 `probes/w2_qwen3_small/run.py`：真实两层缩小配置随机权重，继续验收 IR 对照；另保留两层 QEMU 门禁 | -| `frontend:parse-qwen3` | 既有 `probes/w2_qwen3_parse/run.py`：完整 28 层固定产物解析与绑定审计；手动/定时完整模型 CI 实际执行 | - -具体命令见 [基础后端门禁](../../../probes/w2_backend_ops/README.md) 和 [运行时门禁](../../../probes/w2_runtime/README.md)。生产新增 `scratchv/runtime/llm_inputs.py` 与 `scratchv/runtime/qwen3_tokenizer.py`；Tokenizer 只依赖可选 `tokenizers`，不在生产模块引入 Torch/Transformers。纯文本编码、完整模型输入准备与单步采样为可复用函数，尚未组合成端到端生成循环。 +## 七项统一验收 -交叉 review 修复了非默认 prefix/special-token 配置被静默接受而与官方语义不同、缺省 unk token 偏离参考的问题；门禁也增加词表/特殊 ID 元数据与默认参数对照，并修正报告写入失败遗留 PASS 的路径。官方 NFC 规范化是保留行为，不把组合 Unicode 归一化误判成编译器错误。未知或空余 logits 行解码时失败，不静默丢弃或改变 greedy 候选。 - -本次 CI 增加明确的三个门禁、固定环境单测、独立 Summary 与 `w2-unit-gates-output` artifact。运行时下载约 15.2 MiB 的 tokenizer/config/许可证，按固定 revision 和既有 manifest SHA256 校验，不下载模型权重。源码和测试文件进入 Git，模型、下载缓存、ELF 与本地 output 不进入 Git。 - -### 本轮本地验收 - -| 验证 | 结果 | 本地证据 | -|---|---|---| -| 新增专项测试（固定 CPU 环境） | **228 passed / 0 skipped**，含四类前端模式 22 项、后端门禁 18 项、运行时 188 项 | `output/w2-final-unit-tests.xml` | -| 全仓回归（通用开发环境） | **2,732 passed / 41 skipped / 0 failed** | `output/w2-final-full-ready-tests.xml` | -| 基础后端真实 RV64 QEMU | **32 / 32** 通过；对 ORT 与独立 NumPy 参考的最大绝对误差均为 **2.384185791015625e-7** | `output/w2-backend-ops-final/report.json/md/html` | -| 官方 Tokenizer 下载与运行时门禁 | **20 / 20** 语料、**480 / 480** 比较、**5 / 5** 输入与采样场景通过 | `output/w2-runtime-final-download/report.json/md/html` | -| 同一执行身份离线复验 | 禁用 Hugging Face 在线访问，复用经过哈希校验的本地资源；上述 **20 / 480 / 5** 全部通过 | `output/w2-runtime-final-offline-same-user/report.json/md/html` | - -41 个跳过项包括本机缺少 `qemu-riscv32` 的 38 项，以及通用环境没有 Torch/Tokenizers 的 3 个模块；新 Tokenizer 模块已在上述固定 CPU 环境执行。两个既有 Torch 模块在本轮未本地重跑，PR CI 的固定环境继续执行原两层数值探测。各批次测试有重叠，不能相加成通过总数。 - -首次全仓运行因子进程未获得 LLVM 工具路径失败，修正测试进程的 PATH 后得到上表结果；首次跨执行身份的离线复验因 Windows 目录读取权限失败，同一身份复验通过。失败报告仍保留，未通过修改数值门槛或跳过检查消除失败。 - -后端探测全流程 **22.194 秒**，32 次 QEMU 进程墙钟累计 **5.660 秒**；Tokenizer 下载门禁 **14.824 秒**，同一身份离线复验 **4.198 秒**。均为本机单次观察，不是性能改善结论。已核对后端 30 项、运行时 5 项源码指纹与本次源码一致。仓库指定的本地 L2 harness 文件缺失，未宣称执行；本轮以实际专项、全仓和真实探测结果交付。 - -## 完整前端解析的本地复现 - -详细环境、Windows/Linux 命令、下载模式和报告说明见 [探测入口 README](../../../probes/w2_qwen3_parse/README.md)。所有命令在仓库根目录执行；使用 Python 3.12 和固定 NumPy/ONNX/ORT/protobuf 环境，无需 torch、Zig 或 QEMU。 - -已有完整模型时： +先完成环境准备，激活 `.venv-linux`，并设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU`。模型及官方 Tokenizer 目录必须包含固定 manifest 对应的真实资产。在仓库根目录执行： ```bash -python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 +set -euo pipefail +source .venv-linux/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export PYTHONIOENCODING=utf-8 +export SCRATCHV_PYTHON="$(pwd)/.venv-linux/bin/python" +export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" +export SCRATCHV_QEMU="$(command -v qemu-system-riscv64)" +W2_TOKENIZER=/absolute/path/to/fixed-qwen3-tokenizer +W2_MODEL=/absolute/path/to/fixed-qwen3-onnx + +git rev-parse HEAD +git status --short +"$SCRATCHV_PYTHON" -B -X utf8 scripts/run_w2_acceptance.py \ + --python "$SCRATCHV_PYTHON" \ + --output-dir output/w2-acceptance-linux-01 \ + --tokenizer-dir "$W2_TOKENIZER" \ + --model-dir "$W2_MODEL" \ + --cc "$SCRATCHV_CC" \ + --qemu "$SCRATCHV_QEMU" ``` -PowerShell 可使用实际虚拟环境解释器： +把两项资产路径改成 Linux 实际目录。默认使用 `verify` 离线校验；无资产时可显式增加 `--tokenizer-mode download` 与 `--model-mode download`。后者下载约 1.24 GB 的固定完整发布包，解包及解析需要额外磁盘与数 GiB 空闲内存。入口不安装依赖。`--output-dir` 必须是本次全新的路径，连已存在的空目录也会被拒绝。 -```powershell -& .\output\qwen3-parse-venv\Scripts\python.exe -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE -``` - -没有完整模型时，将 `verify` 改成 `download`，仍使用同一固定 W1 manifest。模型必须包含真实 external-data 分片。每次使用新的报告目录，失败时保留已有文件与日志。 - -## 如何判断本阶段完成 - -| 项目 | 应保留的证据 | +| 门禁 | 实际执行与判断 | |---|---| -| 源码身份 | checkout SHA、是否有本地改动、实际 diff 与新增文件 | -| 固定模型身份 | 发布产物和 external-data 文件校验结果 | -| 完整结构与层覆盖 | 本次结构审计摘要及 28 层检查结果 | -| 实际节点转换 | `nodes.json` 与审计摘要 | -| initializer 绑定 | `bindings.json` 与审计摘要 | -| 共享 IR | `ir.txt` 与 IR 验证器结果 | -| 本次执行 | 命令、退出码、`report.json/md/html`、实测时间及内存（如平台可测） | -| CI | 对应提交和实际 run URL；未执行时明确写未执行 | +| `frontend` | RMSNorm、RoPE、SwiGLU、GQA 四种基础算子分解图；独立公式与反例 | +| `backend` | 七类基础算子的 16 个小图 × none/all，共 32 次 QEMU；误差 `<1e-5` | +| `runtime` | 官方 Tokenizer 的 20 组语料、480 次对照，以及输入、掩码、greedy、停止边界 | +| `runtime-model` | 完整词表两层随机小配置；2 文本、8 次 Torch/ORT/IR 调用及单步 greedy/解码 | +| `small-ir` | 官方 Qwen3 类两层小配置，7 输入、29 检查点、none/basic/all，误差 `<1e-5` | +| `small-qemu` | 同一本轮普通/诊断图 × none/all × 7 输入，共 28 次真实 QEMU，误差 `<1e-5` | +| `full-parse` | 固定完整 28 层 ONNX 的结构、逐节点转换、真实权重绑定与 IR verifier 审计 | -退出 0 和 `passed=true` 必须来自本次执行。轻量 fixture 测试、历史 ORT 通过或一个可打开的报告文件都不足以代替完整图审计。失败发生较早时，某些明细可能尚未产生；按实际阶段和错误定位，不将缺失记录补成成功。 +所有数值门禁同时检查 shape、dtype、有限值；上述阈值使用最大绝对误差、`rtol=0`，包括 padding 输出。Attention 使用基础算子组合，不要求新增融合 opcode。Tokenizer 的“往返 100%”指与官方 NFC/特殊 Token 规则一致，不是任意字节可逆。 -本地 W2 workflow 现在让 PR、手动和定时任务均执行独立的 `full-qwen3-frontend` 完整解析任务，无须等待完整 ONNX 的 ORT 前向。该任务下载约 1.24 GB 的固定发布包，解析时需约 5 GB RSS；手动/夜间同时运行 ORT 任务时，两台 runner 会各自下载。汇总任务拒绝失败或跳过的依赖任务。PR #91 已合并；未来 W2 远端结果须关联其独立提交和实际 run，不能继承旧提交的 PASS。 +仅七项全部执行并通过才会退出 **0 / PASS**；`--gates` 选择部分门禁即使成功，也只会退出 **2 / PARTIAL**。失败退出 1，中断按实际退出状态记录。`runtime-model` 自动带上 `runtime`，`small-qemu` 自动带上 `small-ir`。缺失、跳过、非有限误差、复用旧输出或执行中源码变化均不能冒充完整通过。 -## 完整解析 Review 与实测（2026-10-02，历史提交 dfc9437） +## 报告与独立复现记录 -该次同时复核本地完整解析门禁和 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91)。核对时 PR head 为 `df18b02f152878b1ea838bbf6073def01d367456`，base 为 `d1c2b5f04cad817acf0b2f14d3bfa4ff56dbfa11`，当时差异涉及 52 个文件。审查覆盖编译入口、IR 优化、tensor-c 后端、RISC-V 运行时、数值探测、CI 和复现说明。下述完整解析与 Review 修复曾在 `dfc9437` 纳入 PR #91，现已撤出该 PR 并保留于本地；当时实际执行源码由报告中的 SHA256 确定。 +总报告位于本次 `--output-dir` 下的 `report.json`、`report.md`、`report.html`；子目录保存各门禁原始报告、命令、日志、数组和模型/ELF 身份。源码指纹核对的是实际执行文件，不能把别处安装的模块误当当前 checkout。 -### 完整解析门禁修复 +- 保存 checkout SHA、dirty/diff、Linux 发行版、CPU、Python/NumPy/ORT/Torch、Zig/QEMU 版本。 +- 保存每项实际退出码、PASS/FAIL/PARTIAL/未执行、原始误差、首次失败阶段及报告位置。 +- QEMU 时长是进程墙钟，包含启动、guest 执行、UART 和退出；不是目标硬件性能或纯 forward 时间。 +- 重新生成自己的模型、参考与输出，不能下载第一人的数组后称为独立执行。 +- 独立复现和团队接口/出口确认分开记录，作者再次执行或 CI 自动运行不替代第二人。 -| 问题 | 修复与验证 | -|---|---| -| 属性审计复用生产 parser 的属性解码，可能让相同错误在转换和审计两端同时通过 | 直接从原 ONNX protobuf 独立读取属性，拒绝重复属性；增加错误 Transpose 轴和错误 epsilon 的反例 | -| 未检查函数参数名称与常量标记 | 对照原 ONNX 输入校验名称、顺序、shape/dtype，并拒绝被标记为常量的输入 | -| 标量辅助 `LOAD_CONST` 校验不完整 | 对 initializer 和 Constant 两条路径核对 opcode、目标 Value、空操作数、shape/dtype、常量标记、常量值及指令属性 | -| 审计时隐式 `np.asarray` 可能掩盖错误的绑定类型 | 要求实际绑定是 `numpy.ndarray`；list 等错误类型直接失败，合法只读 ndarray 仍接受 | -| 日志写入失败可能覆盖 worker 原始错误 | stdout/stderr 独立尝试保存；保留原错误、阶段及退出码，附加 `log_errors`；成功路径缺少日志也不能报 PASS | -| 文档链接指向被忽略的本地输出，干净 checkout 中无法访问 | 改为明确的本地输出路径说明，并增加模拟缺少 output 目录的文档链接回归测试 | - -进一步修正失败明细写入的同类问题：保存 `nodes.partial.json` 等失败证据时，写入异常记录到 `evidence_errors`，不覆盖原始解析或绑定错误；成功路径缺少必需证据仍失败。supervisor 在启动 worker 前保存 `attempt`，记录命令、模型路径、解释器、checkout 和源码指纹，便于定位过早超时、worker 未能启动等情况；该字段不代表验证已经执行。 +## 已有 Linux CI 证据 -### PR #91 修复 +以下全部归属历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c`，不作为这次文档更新的新执行结果： -| 问题 | 修复与验证 | +| 工作流 | 已记录结果 | |---|---| -| tensor-c 接受与 IR 标量常量值不同的外部绑定，可能与解释器行为不一致 | 校验绑定与常量声明一致；增加 FP32/INT32/INT64 错值、错误向量形状和合法标量执行的回归测试 | -| 编译或 QEMU 失败时，保存日志的异常可能掩盖编译错误、超时或 guest 错误 | 独立保存两路日志；保留主错误及原耗时，并附加日志错误。覆盖编译失败/成功与 QEMU 超时/失败/成功等组合 | -| RISC-V 报告不足以确定本次 ORT/IR 对照的实际执行环境和完整源码身份 | 顶层 `environment` 记录当前进程，`export_evidence.environment` 保留导出时环境；源码指纹覆盖 parser、IR、优化、后端和运行时，共 30 项 | - -未发现本次审查范围内需要额外修改的 MatMul 实现问题；下列结果只证明已测试输入和路径。未调整精度门槛、模型权重或 D8 等团队待确认决策。 - -### 修复后的实测 - -| 验证 | 当前结果 | 本地证据 | -|---|---|---| -| 全仓回归 | 2,563 passed / 40 skipped / 0 failed | `output/review-fix-full-tests.xml`、同名 `.log` | -| 固定 CPU 环境最终相关测试 | 286 passed / 0 skipped，包含全部 114 项完整解析测试 | `output/review-fix-pinned-tests.xml`、同名 `.log` | -| 真实 Qwen3 结构两层小模型重新导出与 PyTorch/ORT/IR 对照 | 7 / 7 组输入通过；全部数值比较最大绝对误差 `1.9073486328125e-6` | `output/review-fix-qwen3-small/report.json` | -| RISC-V QEMU 两层探测 | 7 组输入 × 普通/诊断图 × none/all 优化，共 28 / 28 执行通过；QEMU 对照最大绝对误差 `1.6689300537109375e-6`，门槛 `atol=1e-5, rtol=0` | `output/review-fix-qwen3-riscv/report.json`、`report.md` | -| QEMU MatMul | 7 / 7 通过，最大绝对误差 `2.384185791015625e-7` | `output/review-fix-matmul/report.json` | -| 完整 Qwen3 前端解析 | 28 / 28 层；7,847 节点、23 类算子；5,910 条 IR；2,148 项绑定；IR verifier 0 问题 | `output/review-fix-full-qwen3-final/report.json`、`report.md`、`report.html` | - -全仓回归中的 38 个 RV32 执行测试因缺少 `qemu-riscv32` 跳过，另有 2 个依赖 torch 的模块因通用开发环境不含 torch 而跳过；最终固定 CPU 环境相关测试已覆盖这两个模块。全仓测试收集后补充了 3 项失败证据回归测试，均包含在最后的 286 项测试内；没有把不同批次或重叠测试相加为一个通过总数。RV64 tensor 真实运行时集成测试和上述实际 QEMU 探测已执行。 - -本次 QEMU 进程墙钟时间累计 **160.450 秒**，探测全流程 **174.634 秒**，4 个成功构建累计 **6.586 秒**。普通图每次平均约 **1.143 / 1.147 秒**（none/all），诊断图约 **10.138 / 10.494 秒**。QEMU 时间包含进程启动、guest 计算、输出打包/UART 和退出；诊断图还包含中间张量输出，不能当作纯前向耗时或真实硬件性能。 - -完整模型本次纯解析 **3.636 秒**，入口至验证及日志保存结束 **12.772 秒**，worker 峰值 RSS **5,064,331,264 bytes，约 4.72 GiB**。完整模型输出静态契约为 FP32 `[1,256,151936]`，绑定共 **2,384,216,772 bytes**；这些是解析与绑定审计结果，本次未执行完整模型数值前向。以上耗时均为本机单次运行，期间存在并行测试，不用于声称性能改善。 - -已核对完整解析报告的 20 项、RISC-V 报告的 30 项源码 SHA256 与当前源码一致；完整解析的 `nodes.json`、`bindings.json`、`ir.txt` 的文件大小和 SHA256 均与报告一致。测试后仅补充说明文档，执行源码未再修改。输出目录被 Git 忽略，仅在本机保存；他人复现需运行入口生成自己的报告。 - -最终还实跑了 0.1 秒截止和模型缺失两个失败路径：均正确退出 1，分别记录 `timeout`、`hashes` 阶段，且保留启动前的 20 项源码指纹；报告在 `output/review-fix-final-expected-timeout/` 和 `output/review-fix-final-expected-missing/`。文档检查包含尚未跟踪的新文档，共检查 97 个文件、196 个相对链接，0 个失效；LLM workflow YAML 与 19 段 shell 命令语法检查通过，`git diff --check` 通过。 - -当前没有已确认且尚未修复的本轮阻断性问题。完整 IR/ORT 数值与容量验收、完整模型 QEMU 前向、权重独立装载、W1 第二人复现和接口签字仍未由本次工作完成。本页实测均为本地证据，远端 CI 状态及 run 链接以 PR 描述和 Checks 为准。 - -## 首次交付结果（2026-10-02，历史记录） - -首次交付工作在独立本地分支 `codex/w2-full-qwen3-parse`，基于 PR #91 的 `df18b02f152878b1ea838bbf6073def01d367456`。实际工作目录为 `D:/cyq/code/ScratchV/output/w1-pr89-review`；当时修改未暂存、未提交、未推送，尚未更新 PR #91。上层原工作目录中的已有改动未纳入本轮。 - -首次交付的完整实物执行退出 0，结果 **PASS**。本地证据保存在 `output/w2-full-qwen3-parse-complete/`：`report.md`、`report.json`、`nodes.json`、`bindings.json` 和 `ir.txt`。这些是该次 checkout 的本地输出，`output/` 不进入版本库；其他机器需执行入口重新生成，不能依赖这些历史路径存在。 - -| 验收项 | 首次交付实测 | -|---|---| -| 固定模型 | Qwen3-0.6B，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 / L=256 / 无 KV Cache | -| 完整层链 | 28 / 28 层，含最终归一化与共享 embedding / LM Head | -| ONNX 节点 | 7,847 / 7,847；23 类基础算子；全部可达 logits | -| ONNX value | 8,161 项静态 shape / dtype 审计通过 | -| 实际 IR | 5,910 条指令；IR verifier 0 个问题 | -| 真实绑定 | 312 个 initializer + 1,836 个 Constant，共 2,148 项；逐项内容哈希通过 | -| 绑定字节数 | 2,384,216,772 bytes | -| 合法别名 | 112 个 Identity，均核对实际对象映射 | -| 返回值 | `logits`，FP32 `[1,256,151936]` | -| 纯解析耗时 | 2.493 秒 | -| 全入口耗时 | 10.533 秒，包含检查、解析、审计与子进程开销 | -| 验证进程峰值 RSS | 5,063,430,144 bytes，约 4.72 GiB，Windows PeakWorkingSetSize | - -耗时来自单次本地运行，不是平均值或性能承诺；本轮没有做解析性能优化。报告记录运行环境、实际源码 SHA256 和产物哈希。工作树有本地改动，不能将此结果仅归属于基准 HEAD。 - -首次交付的轻量测试 **91 / 91 通过**，在固定 CPU 环境和现有开发环境各执行一次，分别记录于 `output/w2-gates-pinned-delivery.xml` 和 `output/w2-gates-general-delivery.xml`。测试含真实两层 fixture 经文件哈希、结构检查、生产 parser、IR verifier 和绑定审计的贯通，以及漏节点、错权重、错 shape/dtype、外部数据损坏、解析异常、缺报告与超时等反例。当时全仓回归 **2,521 passed / 40 skipped**，记录于 `output/w2-regression-final.xml`；首次交付最后修正后重跑上述全部 91 项相关测试。当前修复后的验证以本页上方结果为准；跳过项不计为通过。 - -实际进程反例也已执行：模型缺失在 `hashes` 阶段退出 1；0.1 秒截止测试在 `timeout` 阶段退出 1，实测总耗时约 0.184 秒，均保存 FAIL 报告。记录分别位于 `output/w2-expected-missing-model/` 和 `output/w2-expected-timeout/`。 - -首次交付新增三个探测模块、三组测试与复现文档，并更新本地 LLM CI 配置。没有发现这份完整固定图需要新增前端算子，当时未修改通用 ONNX parser 或编译后端。首次互审修正了验收实现中的两个重点问题：控制参数必须从原 ONNX 独立求值，避免解析器错误自证通过；超时日志写入失败、进程异常退出及 verifier 失败时应尽量保留原始错误和节点证据。后续 Review 的更多修复见本页上方。 +| [通用 CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915411) | 成功；2927 passed / 7 skipped。Linux `test_w2_acceptance.py` 55/55、无跳过 | +| [LLM Deploy](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915420) | 成功；32/32 基础 QEMU、28/28 两层 QEMU、3/3 显式产物、20/20 超时清理；两层诊断最大误差 `1.7285346984863281e-6` | +| [Topic 06](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915428) | 成功；范围以该 run 的实际步骤与 artifact 为准 | -本阶段已具备完整前端解析的本地验收证据；远端 CI、第二人复现、完整模型 IR 数值及完整模型 QEMU 前向尚未由本轮验证。 +PR 中按条件未运行的完整 ONNX ORT 重型任务仍是未执行，不因上述绿色记录而算作完整 ORT 重验。新的提交需要新的 Checks；已有 Linux CI 不替代本次七项统一入口的第二人独立复现。 -## 后续衔接 +## 分项入口与后续边界 -1. 前端审计通过后，依据实际图和权重规模建立完整 IR 数值与峰值内存基线，逐步扩大预训练模型验证范围。 -2. 设计代码与权重分离、权重绑定 ABI 和内存布局；当前 C 常量展开及小模型 guest 配置不能直接承担完整 0.6B 模型。 -3. 完整 IR 数值验收通过后，再验证完整 QEMU 单次前向；随后将已有 Tokenizer、输入准备、最后有效 token logits、采样和停止条件组合成生成循环，并验证真实文本输出。 +- [完整前端解析](../../../probes/w2_qwen3_parse/README.md)：无须 Torch 或 QEMU，可单独准备固定解析环境。报告含 `nodes.json`、`bindings.json`、`ir.txt`。 +- [前端模式与基础后端](../../../probes/w2_backend_ops/README.md)：NumPy 独立公式、ORT、IR 和 QEMU 四路检查。 +- [Tokenizer 与输入](../../../probes/w2_runtime/README.md) 和 [单步模型接口](../../../probes/w2_runtime_model/README.md)：模型词表 151936 与实际可解码 ID 集合分开处理，不取模重映射。 +- [两层数值探测](../../../probes/w2_qwen3_small/README.md)：普通图、诊断图、产物绑定、仿真计时与精度边界。 +- [发布范围与修复](W2-分支验收与发布说明.md)：共享后端、日志和 Linux 取消清理的交付归属。 -前端解析通过不能跳过这些阶段，也不能据此宣布完整 Qwen3 已在 ScratchV 上运行。开发范围见 [总开发计划](../开发计划.md)，W1 人工验收状态见 [W1 独立复现记录](../W1/README.md)。 +完整解析通过不能替代完整 IR 数值、完整权重独立装载、完整模型 QEMU 前向或真实文本生成。阶段安排见 [开发计划](../开发计划.md)。 diff --git "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" index e08da37..1cde2ae 100644 --- "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -1,125 +1,9 @@ -# W1 修复与 W2 本地验收报告 +# W1 修复与 W2 历史验收记录索引 -日期：2026-10-03。**历史整合工作树报告**：记录的是发布前 W1+W2 合并在本地的验证；其中“未提交”等为当时状态。后来按 W1/W2 拆分独立发布，当前 W2 分支结果见 [分支验收与发布说明](W2-分支验收与发布说明.md)，不能将下述组合源码哈希当作当前分支哈希。 +本文件名保留以兼容已有链接。当前正式交付、CI 和独立复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，从 [Linux 复现约定](../LINUX_REPRODUCTION.md) 和 [W2 七项统一入口](README.md) 开始。 -## 1. 结论与源码位置 +原报告记录 2026-10-03 发布前的个人混合工作树，包含 W1 修复与 W2 增量，既不是独立 W2 分支的精确源码，也不是 Linux 实测。原始测量和当时状态完整保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W1修复与W2本地验收报告.md)。没有修改原始平台标签、计数、误差或耗时来生成新的通过结论。 -W1 的报告可靠性与源码归属修复已补入；W2 已在 PR #91 合并基线上整合，并重新执行七项技术验收，结果 **7/7 PASS**。完整图的通过范围是前端解析、结构和权重绑定审计；完整 0.6B 的 IR 数值、QEMU 前向与生成循环仍未完成。本报告不替代团队冻结接口、第二人复现或远端 CI。 +已交付的修复包括报告写入失败与源码归属、标量常量绑定一致性、Linux 嵌套会话退出清理、七项统一验收及 CI 缺失/跳过检测。具体代码归属与 Linux run 见 [W2 分支验收与发布说明](W2-分支验收与发布说明.md)；W1 报告修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。 -本轮源码在 `D:/cyq/code/ScratchV/output/w1-w2-local-closeout`，本地分支 `codex/w1-w2-local-closeout`。基线是 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，所有增量均未提交。原 `output/w1-pr89-review` 的 W2 修改、`output/w1-pr91-repro-fixes` 和最外层工作目录的代码保持原样。 - -下述 `output/...` 均相对于本轮源码目录。它们是本机证据，不是仓库发布资产；复现者应使用新的目录生成自己的报告。 - -## 2. W1 补完了什么 - -| 修复 | 原问题 | 当前行为与主要文件 | -|---|---|---| -| 两层报告完整发布 | MD/HTML 写入失败或 JSON 发布失败可能留下成功证据 | 视图先写、JSON 最后原子发布；失败返回非零，尽力保存 FAIL，保留原始数值错误。`probes/w2_qwen3_small/run.py`、`riscv.py` | -| 源码与环境归属 | 无 `.git` 的源码副本可能向父目录查找 Git，误认其他 checkout；QEMU 对照环境可能与导出环境混淆 | 无本目录元数据时明确未知 Git 身份，保留源码 SHA256；分别记录当前执行与导出环境，并补齐 parser/IR/优化/运行时指纹 | -| 回归覆盖 | 成功路径测试不足以验证失败时的证据 | 故障注入覆盖写入、关闭、替换失败与原始错误保留；补源码副本及当前环境归属回归 | -| 原有算子用例与固定 ORT 环境兼容 | ORT 1.22.1 无 DOUBLE Cos 内核，ReduceMean 的 omitted axes + noop=1 路径报内核错误 | 三项用例显式使用 ONNX 官方 ReferenceEvaluator，继续实际运行生产 parser/IR，ReduceMean 再检查与输入完全相等；其余用例继续 ORT 对照，不捕获或忽略数值错误 | -| 共享后端与运行时防错 | 外部绑定可能覆盖标量 IR 常量；日志 I/O 失败可能遮蔽原编译/QEMU 错误 | `tensor_c_codegen.py` 校验常量绑定一致；`riscv_tensor.py` 独立保存两路日志，保留主错误和耗时；一并整合之前 W2 分支暂存的修复 | -| 文档状态 | 仍使用 PR 未合并、旧提交或“所有第二人复现待做”的描述 | 固定已合并基线，保留 `df18b02f` 第二人两层复现事实，区分完整模型复现与接口签字待办 | - -在整合 W2 前，W1 四个源码/测试文件的专项测试已执行 **120 passed / 0 skipped**，证据为 `output/w1-fix-tests-v2.xml`。W1 独立检查点在 `output/w1-fix-checkpoint/`：包含补丁、四文件快照与哈希，便于后续按 W1/W2 分拆提交。这次没有重新打开或更新已合并的 PR #91。 - -最终共享修复的建议拆分补丁为 `output/w1-final-review/w1-fixes.patch`，含上述报告/归属、标量绑定、运行时清理/日志及 ORT 用例兼容的 10 个源码/测试文件，配有 `manifest.json`；它是供审阅的本地补丁，不是已暂存或已提交的改动。 - -最后还单独实跑原 W1 入口：MatMul **7/7**（4×4 小数最大误差 `1.4901161193847656e-8`，七例最大 `2.384185791015625e-7`）；PR #89 合成两层图的 IR vs ORT 最大误差约 `7.153e-7`，ORT vs 独立 NumPy 约 `1.097e-6`。证据在 `output/w1-matmul-final/`、`output/w1-transformer-final/`。完整 0.6B ONNX 的 ORT 前向本轮没有重跑，其历史验收仍按原记录归属。 - -W1 人工出口继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪：团队 D1–D8 接口决议、风险处置责任与 Plan B、完整模型第二人复现和最终出口评审。本轮没有代签或关闭这些事项。 - -## 3. W2 工作与开发计划逐项对照 - -| 开发计划要求 | 当前实现与本轮证据 | 状态边界 | -|---|---|---| -| `unit:frontend-ops`：RMSNorm/RoPE/SwiGLU/GQA | 22 项真实 ONNX 模式测试全部通过 | 通过基础算子组合解析，未新增同名融合 IR 指令 | -| `unit:backend-ops`：基础算子 QEMU | MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA，7 类/16 小图 × none/all，共 32 次通过 | 超过表中六类最低覆盖；仅证明这些 shape 与输入 | -| `numeric:ir-small`：两层 IR vs ORT | 真实 Qwen3 类的两层缩小配置，7 输入、29 检查点、6 项不变性通过；none/basic/all 优化对照 | 随机固定权重，不是预训练权重切片 | -| Attention IR 定义 | 候选接口文档明确基础算子组合、Q/K 独立 RMSNorm、全维 RoPE、GQA 分组与加性 mask 语义 | 当前不强制新增融合 ATTENTION opcode；这项接口决议仍需团队确认 | -| `unit:runtime`：Tokenizer、greedy、mask | 固定官方 Tokenizer 20 组语料/480 对照，5 组输入、mask、采样与停止条件通过 | 往返要求按官方 tokenizer 的特殊 token/规范化语义校验，不声称任意文本字节级恒等 | -| `frontend:parse-qwen3`：完整 28 层 | 固定 FP32/L=256 产物完整解析；7,847 节点、8,161 value、2,148 绑定、5,910 IR 指令通过审计，verifier 无问题 | 不代表完整 IR 数值或 QEMU 已运行 | -| 单步运行时与模型接口集成 | 2 个文本、完整词表 151936；2 Torch + 2 ORT + 4 IR，共 8 次调用；原始 token ID、最后有效位置、greedy ID 与解码一致 | hidden32/FFN96 两层随机小配置；未执行完整词表 QEMU 或多步生成 | -| W1 数值链路不退化 | 两层普通/诊断图 × none/all × 7 输入，28 次真实 QEMU 全通过 | 原 W1 的 20 次 Linux 超时回归与 3 项显式产物集成 CI 配置继续保留 | -| CI 全绿 | 本地配置增加完整 28 层解析 PR 必跑任务与汇总检查，拒绝跳过/失败/缺失 | **尚未提交，所以没有本轮远端 CI 通过结论** | - -本轮整合原先仅本地保存的完整解析、后端和运行时工作，新增修复 CI 验收缺口与嵌套进程超时清理。生产后端还保留审查所得的标量常量绑定一致性检查，拒绝调用方用不同绑定覆盖 IR 常量；日志写入失败保留原编译/QEMU 错误与耗时。这些此前 W2 分支的修复均已在本轮基线上重新验证。 - -## 4. 本轮重新执行的验收数据 - -总入口：`scripts/run_w2_acceptance.py`。总报告：`output/w2-acceptance-20261003/report.json`，另有同名 Markdown/HTML。每项包含自己的原始报告、日志、命令、输出数组、模型或 ELF 身份。总入口核对 **136 项源码指纹**，拒绝旧目录、缺失门禁、skip、非有限误差及运行中代码变化。 - -| 门禁 | 结果 | 最大绝对误差 | 顶层观测耗时 | -|---|---|---:|---:| -| frontend | 22/22 | 不适用 | 1.157 秒 | -| backend | 32/32 QEMU | `2.384185791015625e-7` | 23.769 秒 | -| runtime | 20/20、480/480、5/5 | 离散结果精确对照 | 4.905 秒 | -| runtime-model | 2 文本、8 次调用 | `2.8312206268310547e-7` | 14.889 秒 | -| small-ir | 7 输入/29 检查点/6 不变性 | `1.9073486328125e-6` | 6.971 秒 | -| small-qemu | 28/28 QEMU | 普通图 `3.129243850708008e-7`；诊断图 `1.6689300537109375e-6` | 153.658 秒 | -| full-parse | 28/28 层、逐节点/绑定审计通过 | 本项不执行数值前向 | 11.111 秒 | - -数值门槛保持 `max_abs < 1e-5`、`rtol=0`，没有放宽精度、裁剪 token ID 或用历史产物替代本次执行。small-qemu 报告也包含主机 IR 参考，其全报告最大误差 `1.9073486328125e-6` 不应误写成 QEMU 自身误差。 - -整轮 **216.597 秒（约 3 分 37 秒）**。基础后端 32 次 QEMU 进程墙钟累计 **5.545 秒**；两层 28 次累计 **140.312 秒**，4 次 C→RV64 编译累计 **6.189 秒**。两层普通图平均约 1.06/1.07 秒（none/all），诊断图约 8.94/8.97 秒；诊断输出多，不能当作纯前向速度。 - -完整前端 worker 的纯解析 **2.579 秒**，验证流程 **10.818 秒**，峰值 RSS **5,064,474,624 bytes，约 4.72 GiB**；绑定数据 **2,384,216,772 bytes**。这些是本机单次观察，不是优化前后基准对比，也不等同真实硬件性能。 - -## 5. Review、回归及 CI 检查 - -本轮 Review 先修复两个新发现的问题： - -1. 完整解析原来只在手动/夜间任务运行，PR 绿色无法证明 W2 全部门禁。现在 `full-qwen3-frontend` 在匹配路径的 PR 上必跑；`w2-acceptance` 对所有依赖要求 success。对应 21 项结果校验测试覆盖失败、跳过、取消、缺失和畸形 JSON。分支保护设置未修改。 -2. 顶层探测的 Linux 超时直接杀进程组，可能遗留 `_run_process` 创建的独立编译器/QEMU 会话。现在先 SIGTERM 触发运行器清理，再强制清理原组；增加实际进程中断回归和 Linux 嵌套会话回归。Windows 路径沿用 Job/进程树处理。 - -扩大回归时还修复了上文三项 ORT 参考内核兼容问题；相关四个测试文件最终 **251 passed / 0 skipped**，证据 `output/review-regression-fixes.xml`。没有修改编译器计算算法或数值门槛来适应参考执行器。 - -超时与 CI 专项结果 **128 passed / 1 skipped**，证据 `output/timeout-ci-tests.xml`；跳过的是 Linux 专属嵌套会话集成用例，本机 Windows 无法执行，不能计作通过。W2 集成/报告/CI 结果校验前一轮为 **111 passed / 0 skipped**，证据 `output/w2-integration-tests.xml`。各批次有重叠，不能相加。 - -最终 `tests/` 全量回归 **3020 passed / 39 skipped / 0 failed，177.66 秒**，证据为 `output/final-all-tests-v4.xml` 和同名 `.log`。39 个 skip 明确分为：本机无 `qemu-riscv32` 的 38 个旧 RV32 用户态执行测试，以及 1 个 Linux 专属嵌套会话清理测试。真实 RV64 tensor 运行时测试、三项显式产物优化集成及本轮所有可在 Windows 执行的 W1/W2 测试均已执行；不能把 39 个 skip 写成通过。 - -最终工作流 YAML、29 段 Bash 语法、文档相对链接、W1 四类接口/21 条风险及 Plan B 的结构检查、`git diff --check` 通过；详见 `output/final-static-review.json`。七项验收记录的 136 项源码指纹仍与最终执行代码一致。报告及状态文档在数值执行结束后补写，不改变该数值结果对应的执行代码。项目约定的 `.claude/harness/verify/run.py` 不存在，未宣称运行 L2 harness。 - -失败尝试也保留：W1 首次测试因自定义 pytest 临时目录的父目录尚未创建而产生 82 个 setup error，创建目录后 120 项通过；全仓第一次在收集阶段发现固定模型环境缺少可选 `llvmlite`。后者使用已有开发环境的 llvmlite 0.49.0 与 tinyfive 1.0.0 的本地副本补入仅本轮测试的路径，未改固定模型环境或全局安装。 - -中间全仓尝试 `final-all-tests-v2` 为 228 failed / 2791 passed / 40 skipped：224 项是 LLVM 工具未发现，3 项是上述 ORT 参考限制，1 项文档链接失败发生在报告文件尚未写完时。`v3` 显式指定 llvm-mca 后为 5 failed / 3014 passed / 40 skipped，剩余 5 项均因 clang/ld.lld 未从 PATH 找到。最终通过 `output/run_local_regressions.py` 在 Python 进程内配置路径，并确认三项 LLVM 工具实际可运行；不把这些中间失败隐藏或计算成通过。 - -## 6. 尚待完成与后续工作 - -- 本轮 Linux 嵌套会话清理集成回归、修改后的 GitHub workflow 和完整解析 runner 的实际容量，需要发布后取得对应源码的 Linux CI 证据。源码/配置本地通过不等于远端已通过。 -- W2 需要第二人用自己的环境和全新输出目录运行七项统一验收；W1 已确认的是旧 SHA 的两层链路，不能自动覆盖这次新增修复或 W2。 -- W1 接口冻结与风险责任决议保持候选，继续按 Issue #92 讨论；不要因技术门禁通过代替团队确认。 -- 后续按 W3 先做真实维度、预训练权重子图和更深层 IR 数值/内存基线，再做权重与代码分离、权重加载 ABI、内存复用和完整模型 QEMU 前向；最后接多步生成。原 `backend=riscv` 标量选择器并未因此支持大模型，本路线使用 `tensor-c`。 - -## 7. 本机复现命令 - -在本轮源码目录执行，输出目录必须是全新的。下面使用现有工具与固定模型资产，不自动安装或下载。 - -```powershell -$w2Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-review-new ` - --tokenizer-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` - --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` - --cc D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/zig-python/ziglang/zig.exe ` - --qemu D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/qemu/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -人工首先检查本报告、七项总报告和文件清单，再决定后续拆分提交方式。本轮没有创建、修改或关闭 GitHub PR/Issue。 - -## 8. 修改文件审阅顺序 - -| 范围 | 主要文件 | -|---|---| -| W1 两层证据可靠性 | `probes/w2_qwen3_small/run.py`、`riscv.py` 及对应 gate 单测 | -| 编译/运行时基础修复 | `scratchv/backend/tensor_c_codegen.py`、`scratchv/runtime/riscv_tensor.py` 及对应单测 | -| 固定 ORT 测试兼容 | `tests/test_onnx_qwen_numeric_ops.py`、`tests/test_onnx_shape_ops.py` | -| 完整图审计 | `probes/w2_qwen3_parse/{run,validation,audit}.py`、三组 `test_qwen3_full_*.py` | -| 基础后端与前端模式 | `probes/w2_backend_ops/`、`tests/test_qwen3_frontend_patterns.py`、后端 gate 测试 | -| Host 运行时与单步模型集成 | `scratchv/runtime/{qwen3_tokenizer,llm_inputs}.py`、`probes/w2_runtime/`、`probes/w2_runtime_model/` 及对应测试 | -| 验收入口与 CI | `scripts/run_w2_acceptance.py`、`scripts/check_w2_ci_results.py`、`.github/workflows/llm-deploy.yml` 及对应测试 | -| 文档与依赖声明 | W1/W2 指南、执行/探测计划、接口/风险候选、总开发计划、各 probe README、`pyproject.toml` 的可选 tokenizer 依赖 | - -最终全文件清单与哈希保存在 `output/final-review-manifest.json`；完整已跟踪文件差异在 `output/final-tracked.patch`，新增文件保留在上表目录。模型、参考数组、ELF、可选测试依赖副本均在被忽略目录，未纳入源码差异。 +新复现必须在指定 SHA 上自行生成报告，记录 Linux 环境、实际退出码、源码与模型身份以及误差。旧原始数据可用于调查历史问题，不能代替新提交执行或第二人确认。 diff --git "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" index 4e95d1e..fc5b57d 100644 --- "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" +++ "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" @@ -1,27 +1,27 @@ -# W2 独立分支验收与发布说明 +# W2 Linux 分支验收与发布说明 -W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。本分支直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含独立 W1 PR 中的六个两层报告/参考测试文件。两部分可分别审阅和合并。共享后端的标量绑定、运行时日志和取消清理加固随 W2 交付。 +正式复现环境为 Ubuntu 24.04 x86_64 / Bash / Python 3.12。环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)，七项统一入口和实际 CI 链接见 [W2 README](README.md)。 -## 范围 +W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。W2 初次独立交付基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`；[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 既有 Linux 执行对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。新修改按新 SHA 验收，不能继承旧 CI PASS。 -- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和权重绑定审计。 -- RMSNorm/RoPE/SwiGLU/GQA 前端模式，七类基础后端的 16 小图/32 次 QEMU。 -- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层小配置的单步模型集成。 -- 七项统一验收入口，完整前端 PR 必跑任务，以及拒绝失败、取消、跳过、缺失的 CI 汇总。 -- 标量 IR 常量外部绑定一致性；日志写入失败保留主错误；取消时清理独立编译器/QEMU 会话。 +## 交付范围 -本分支 Windows 固定 CPU 环境重新执行：**629 passed / 1 skipped**（仅 Linux 嵌套会话清理用例跳过）；**七项统一验收 7/7 PASS**，实际 32 次基础后端 + 28 次两层 QEMU 全通过。完整图为 28 层、7,847 节点、2,148 绑定、5,910 条 IR，verifier 与逐项审计通过。运行时为 20 组语料/480 对照/5 场景；完整词表单步集成为 2 文本、8 次调用。136 项源码指纹与发布前代码一致。 +- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和真实权重绑定审计。 +- RMSNorm/RoPE/SwiGLU/GQA 前端模式；七类基础后端的 16 小图、32 次 QEMU。 +- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层随机小配置的单步模型集成。 +- 七项统一验收、完整前端 PR 必跑任务，以及拒绝失败、取消、跳过和缺失的 CI 汇总。 +- 标量 IR 常量外部绑定一致性；保存日志失败时保留主错误；Linux 取消时清理独立编译器/QEMU 会话。 -总入口耗时 251.494 秒，两层 QEMU 进程累计 154.125 秒，基础后端 QEMU 累计 6.194 秒；并行验收时的本机单次观察，不用作性能对比。精度最大误差：基础后端 `2.384185791015625e-7`，单步集成 `2.8312206268310547e-7`，两层 IR `1.9073486328125e-6`，两层目标诊断 `1.6689300537109375e-6`，均小于 `1e-5`。 +## Linux 验证与源码归属 -另外，以本分支新生成的模型显式执行 none/basic/all 三项产物优化集成，**3/3 通过、0 跳过**，证据为 `output/w2-publish-artifact.xml`。 +历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c` 的通用 CI、LLM Deploy、Topic 06 均成功。通用 CI 为 2927 passed / 7 skipped；Linux 统一验收单测 55/55、无跳过；LLM 记录基础 QEMU 32/32、两层 QEMU 28/28、显式优化产物 3/3、退出清理 20/20。两层 QEMU 诊断最大绝对误差 `1.7285346984863281e-6`，小于 `1e-5`。原始 run 链接在 [W2 README](README.md)。 -证据在 `output/w2-publish-tests.xml`、`output/w2-publish-acceptance/report.json/md/html` 及各子目录。工作流 YAML、29 段 Bash、文档链接及 diff 格式检查通过。本分支没有重跑全仓 tests；原整合工作树 3020 项通过属于下述历史记录，不能用其代替本分支验证。固定 ORT 环境下三个原有算子参考用例的兼容修复归属 PR #93。项目的本地 L2 harness 文件缺失，未声称运行。此前 [W1 修复与 W2 本地验收报告](W1修复与W2本地验收报告.md) 记录的是两部分合在一起的历史工作树，不应冒充本分支的精确源码证据。 +跳过项不计为通过；按 PR 条件未运行的完整 ONNX ORT 重型任务不计为完整 ORT 重验。上述工作流集合与作者运行统一七项 CLI、第二人完整独立执行，是不同证据，分别记录。 -## 复现与限制 +发布前个人工作树的计数、数值和耗时保留于 [固定提交中的原报告](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W2-分支验收与发布说明.md)，不迁移成 Linux 测量值。早期 W1/W2 混合工作树记录见 [历史归档说明](W1修复与W2本地验收报告.md)，不代表独立 W2 分支的精确源码。 -运行 [统一入口](README.md) 的七项门禁；每次使用新的输出目录。源文件哈希、模型/ELF 身份、原始命令、误差和仿真墙钟时间均保存在生成的 JSON/Markdown/HTML 中。 +## 复现与限制 -完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合，无新增融合 opcode。第二人复现、接口冻结、完整模型权重独立装载和生成循环仍需后续完成。 +运行统一入口时每次使用不存在的新输出目录。保留 Linux 版本、CPU、依赖和工具版本、源文件哈希、模型/ELF 身份、原始命令、误差及仿真墙钟；七项完整通过才能记为 PASS。新的提交需要新的实际运行证据。 -Linux 嵌套会话清理用例在 Windows 上明确跳过；CI 配置已经提供入口，但只有对应提交的实际远端执行才能证明 Linux 通过。完整 ONNX ORT 前向继续由独立手动/夜间任务验证。 +完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合；完整词表单步模型使用随机小配置，不具备预训练模型语言能力。第二人独立复现和团队接口确认仍由参与者完成；完整权重装载、生成循环等后续阶段不能由本分支的小模型结果代替。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 030275d..8c6182e 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -11,6 +11,8 @@ --- +**交付环境约定（2026-10-06）**：W1 及后续阶段的公开指南、第二人复现和 CI 统一使用 Linux（Ubuntu 24.04 / Bash / Python 3.12）。作者个人环境不构成 Linux 验收；保留历史记录的真实平台和提交身份。环境安装、资产和后续提交规则见 [Linux 复现约定](LINUX_REPRODUCTION.md)。这只规定 Host 环境，目标仍为 RISC-V 裸机程序。 + ## 1. 项目概述 ### 1.1 背景 diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" index b39dab2..b1395f8 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -17,14 +17,22 @@ DSL -> DSLInterpreter -> 解释器结果 -> 期望值对比 ## 运行依赖 -测试命令需要在 ScratchV 仓库根目录运行。`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +复现环境统一为 Linux / Bash，使用 Python 3.12。测试命令需要在 ScratchV 仓库根目录运行。 +首次使用时创建并激活虚拟环境： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 调用真实 TinyFive 所需的配套接口位于 `scratchv/simulator/tinyfive.py`、 `scratchv/backend/register_alloc.py`、`scratchv/compiler.py` 和 `scratchv/main.py`。 安装基础测试依赖： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 基础模式需要以下依赖： @@ -34,8 +42,8 @@ pip install -e ".[topic06]" 如需生成 HTML 和 PNG 完整报告，再安装可选依赖： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` 其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 @@ -67,13 +75,13 @@ build/topic06/ # 运行时生成，不提交 在仓库根目录执行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` 默认同时执行解释器与 TinyFive，并分别记录状态。也可以显式选择： -```powershell +```bash python scripts/run_topic06_benchmarks.py --verification-backend both python scripts/run_topic06_benchmarks.py --verification-backend interpreter python scripts/run_topic06_benchmarks.py --verification-backend tinyfive @@ -102,13 +110,13 @@ python scripts/run_topic06_benchmarks.py --verification-backend tinyfive 更新性能基线： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` 性能退化阈值默认是 5%，可以按场景调整： -```powershell +```bash # 超过 2% 即判定为退化 python scripts/run_topic06_benchmarks.py --regression-threshold 2 @@ -122,19 +130,19 @@ python scripts/run_topic06_benchmarks.py --regression-threshold 10 按类别运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按用例名称进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 组合筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -160,7 +168,7 @@ benchmarks/topic06/baseline.json 测试和报告生成采用两阶段流程。先运行测试并生成与展示格式无关的元数据： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -172,7 +180,7 @@ benchmark_reports/topic06/metadata.json 随后由独立报告工具读取元数据并生成轻量报告： -```powershell +```bash python scripts/generate_topic06_report.py ``` @@ -187,7 +195,7 @@ benchmark_reports/topic06/cases/*.json 需要 HTML 和 PNG 时运行： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" index b5a687c..a95e6b0 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -366,7 +366,7 @@ Markdown 和 HTML 报告固定展示相同核心指标。CI 推荐读取 `benchm 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -374,7 +374,7 @@ python scripts/run_topic06_benchmarks.py 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` @@ -386,7 +386,7 @@ benchmarks/topic06/baseline.json 后续再次运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -398,7 +398,7 @@ python scripts/run_topic06_benchmarks.py 如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： -```powershell +```bash python scripts/run_topic06_benchmarks.py --regression-threshold 2 ``` @@ -478,48 +478,57 @@ benchmark_reports/topic06/course_report_instructions.png 测试运行时仍使用绝对路径调用编译器、读取用例和写入文件，避免工作目录变化影响执行。写入报告前，`_report_path()` 会将仓库内路径转换为相对于 ScratchV 仓库根目录的 POSIX 格式。 -例如，原报告中的： +例如，Linux 工作区中的绝对路径： ```text -D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl +/home/reviewer/ScratchV/tests/topic06/cases/activation/relu_only.dsl ``` -现在写为： +在报告中写为： ```text tests/topic06/cases/activation/relu_only.dsl ``` -该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖本机盘符与工作区位置，可以在 Windows、Linux、macOS 和 CI artifact 中查看。 +该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖作者机器的工作区位置，可在其他 Linux 工作区和 CI artifact 中查看。 --- ## 六、运行方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行，使用 Python 3.12： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +各节保留的历史测试结果不因命令格式调整而变成新的 Linux 执行证据；新结果以本次生成的报告为准。 + ### 6.1 安装依赖 基础测试： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 完整报告： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` ### 6.2 普通测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py python scripts/generate_topic06_report.py ``` 第一条命令生成 `metadata.json`，第二条命令读取元数据并生成 `report.md`、`report.json` 和单用例报告。生成 HTML 和 PNG 完整报告： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` @@ -527,19 +536,19 @@ python scripts/generate_topic06_report.py --full-report 按测试目录类别筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按 DSL 文件名进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 两个条件可以组合，组合时必须同时满足： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -549,7 +558,7 @@ python scripts/run_topic06_benchmarks.py --category tensor --filter relu ### 6.4 性能回归测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -557,7 +566,7 @@ python scripts/run_topic06_benchmarks.py ### 6.5 更新性能基线 -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" index 9cd0e8c..bf93541 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" @@ -8,7 +8,11 @@ push 使用 self-hosted）、Python 3.12 ## 专项测试 -```powershell +以下复现命令统一使用 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境： + +```bash +python3.12 -m venv .venv-dsl-diagnostics +source .venv-dsl-diagnostics/bin/activate python -m pip install -e . "pytest>=7,<10" python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_diagnostics_benchmark.py -v --tb=short ``` @@ -23,7 +27,7 @@ python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test 先为基线建立独立 checkout。以下 `997d2aa` 是本 PR 加入诊断实现之前的历史版本； rebase 后与最新上游比较时，应改用对应的 PR base SHA。 -```powershell +```bash git worktree add --detach ../ScratchV-dsl-baseline 997d2aa python -m benchmarks.bench_dsl_diagnostics --baseline-root ../ScratchV-dsl-baseline --json-output benchmark_reports/dsl_diagnostics.json --markdown benchmark_reports/dsl_diagnostics.md --html benchmark_reports/dsl_diagnostics.html ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" index dd5e243..2346026 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -507,6 +507,9 @@ scratchv bad.dsl -o output.s 2> error.txt ## 16. 测试计划 +本节复现命令统一使用 Linux / Bash，在仓库根目录和 Python 3.12 虚拟环境中执行； +依赖安装见 [CI 与 Benchmark](09-DSL错误提示美化器-CI与Benchmark.md#专项测试)。 + ### 16.1 快速测试 开发过程中每个小步骤运行： @@ -526,12 +529,6 @@ python -m pytest \ tests/test_dsl_diagnostics_cli.py -q ``` -Windows PowerShell 可把路径放在同一行执行。 - -```powershell -python -m pytest tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py -q -``` - ### 16.3 项目验证 ```bash diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" index 9fed5ea..bd89c20 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -286,20 +286,32 @@ convert_onnx_to_llvm(model) → LLVM IR (866K lines, 183MB) ## 7. 使用方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境和依赖： + +```bash +python3.12 -m venv .venv-regalloc +source .venv-regalloc/bin/activate +python -m pip install -e ".[all]" +``` + +CNN 用例还需要 `models/graph/cnn.onnx`；请先确认模型文件存在。LLVM 对比依赖 +`llvmlite`，报告中的 `llvm_available=false` 表示该项未执行，不能作为 LLVM 对比通过。 + ```bash -# 运行全部 4 项 benchmark，生成三格式报告（Windows PowerShell） -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json report.json ` - --output-html report.html ` - --output-md report.md +# 运行全部 4 项 benchmark，生成三格式报告 +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ + --output-md benchmark_reports/regalloc_bench.md # 单独运行某项 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_simple --repeats 100 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_dense --repeats 50 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_cnn ` +python -m benchmarks.test_regalloc.bench_simple --repeats 100 +python -m benchmarks.test_regalloc.bench_dense --repeats 50 +python -m benchmarks.test_regalloc.bench_cnn \ --cnn-path models/graph/cnn.onnx --repeats 30 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_pseudo --repeats 30 +python -m benchmarks.test_regalloc.bench_pseudo --repeats 30 ``` --- diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" index 8f5e0f3..a4c2dda 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" @@ -137,13 +137,16 @@ ### 5.2 30 次运行结果 -命令： - -```powershell -python -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json benchmark_reports/regalloc_bench.json ` - --output-html benchmark_reports/regalloc_bench.html ` +以下是原验证命令的 Linux / Bash 等价写法，在仓库根目录和已安装依赖的 Python 3.12 +虚拟环境中执行。环境准备见 [Benchmark 设计文档](17-寄存器分配-Benchmark设计文档.md#7-使用方式)。 +下表保留原历史测量，不代表使用这份 Linux 命令重新运行过。 + +```bash +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ --output-md benchmark_reports/regalloc_bench.md ``` diff --git a/probes/w1_qwen3_export/README.md b/probes/w1_qwen3_export/README.md index 02d5da2..f72c019 100644 --- a/probes/w1_qwen3_export/README.md +++ b/probes/w1_qwen3_export/README.md @@ -15,24 +15,19 @@ for blocked positions. Short-input predictions use the last valid token. ## Environment -Use Python 3.12. The existing `requirements/qwen3-small-probe.txt` CPU +The published reproduction target is Ubuntu 24.04 x86_64 with Bash and Python 3.12. See the [Linux reproduction contract](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md). The existing `requirements/qwen3-small-probe.txt` CPU environment also satisfies this gate. For a separate reproducible export environment: -```sh -python -m venv output/qwen3-export-venv -source output/qwen3-export-venv/bin/activate +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate python -m pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cpu python -m pip install -r requirements/qwen3-export.txt ``` -Keep this environment activated for all `python` commands below. On Windows -PowerShell, create it with `py -3.12 -m venv output/qwen3-export-venv`, then -activate it with `& .\output\qwen3-export-venv\Scripts\Activate.ps1` and run -the two installation commands above. If activation is unavailable, replace -every later `python` with `& .\output\qwen3-export-venv\Scripts\python.exe` -(or `output/qwen3-export-venv/bin/python` on Linux); put multiline commands on -one line in PowerShell rather than using shell backslash continuations. +Keep this environment activated for all `python` commands below; the interpreter is `.venv-linux/bin/python`. Record the checkout SHA and actual dependency versions with each new run. Verify/download mode enforces NumPy 2.2.6, ONNX 1.18.0, ONNX Runtime 1.22.1 and protobuf 5.29.5; @@ -41,7 +36,7 @@ export mode additionally enforces the complete fixed export stack, including ## Verify the fixed release -```sh +```bash python probes/w1_qwen3_export/run.py --mode download \ --model-dir output/qwen3-full-model \ --output-dir output/qwen3-full-probe --threads 2 @@ -59,7 +54,7 @@ uses a previously downloaded ZIP with the same size/hash checks. For an already available complete model directory, skip network access: -```sh +```bash python probes/w1_qwen3_export/run.py --mode verify \ --model-dir models/qwen3-0.6b-onnx \ --output-dir output/qwen3-full-probe --threads 2 @@ -78,7 +73,7 @@ ScratchV's full-model execution or language quality. Obtain the official `Qwen/Qwen3-0.6B` Hugging Face snapshot at revision `c1899de289a04d12100db370d81485cdf75e47ca`. For example, in the pinned environment: -```sh +```bash python -c "from huggingface_hub import snapshot_download; snapshot_download('Qwen/Qwen3-0.6B', revision='c1899de289a04d12100db370d81485cdf75e47ca', local_dir='output/qwen3-source')" python probes/w1_qwen3_export/run.py --mode export \ --source-dir output/qwen3-source \ @@ -110,7 +105,7 @@ measurement. Export mode also saves `export.log` and its reference artifacts. Source fingerprints are included in failed reports as well as successful ones. Upload the report directory as CI evidence, not the multi-gigabyte model. -```sh +```bash python -m pytest tests/test_w1_qwen3_export.py -q ``` diff --git a/probes/w2_backend_ops/README.md b/probes/w2_backend_ops/README.md index a535d14..dd26184 100644 --- a/probes/w2_backend_ops/README.md +++ b/probes/w2_backend_ops/README.md @@ -1,9 +1,11 @@ # W2 前端模式与基础后端算子验收 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 准备并激活环境。下文 `python` 为 `.venv-linux/bin/python`。 + `unit:frontend-ops` 明确检查 RMSNorm、RoPE、SwiGLU、GQA 四种分解图模式。 这些模式由现有 ONNX 基础算子表达，不声明已有独立融合 Attention/RMSNorm IR。 -```powershell +```bash python -m pytest tests/test_qwen3_frontend_patterns.py -q ``` @@ -11,12 +13,12 @@ python -m pytest tests/test_qwen3_frontend_patterns.py -q 交叉编译为 RV64GC ELF，然后在 QEMU `virt` 裸机执行。16 个用例分别运行 `none`、`all` 两个优化级别，共 32 次目标执行；不下载模型权重。 -```powershell -python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc path/to/zig --qemu path/to/qemu-system-riscv64 +```bash +python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" python -m pytest tests/test_w2_backend_ops_gate.py -q ``` -Linux 可省略工具路径，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 +也可省略工具参数，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 使用现有 `requirements/qwen3-small-probe.txt` 中的 NumPy/ONNX/ORT 环境即可， 本探测不需要 PyTorch。输出目录须不存在或为空；依赖、工具、编译、执行或 比较失败均非零退出，不将跳过当通过。 diff --git a/probes/w2_qwen3_parse/README.md b/probes/w2_qwen3_parse/README.md index 23a69e7..d303efe 100644 --- a/probes/w2_qwen3_parse/README.md +++ b/probes/w2_qwen3_parse/README.md @@ -6,29 +6,18 @@ ## 准备环境 -使用 Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 -Windows PowerShell： - -```powershell -py -3.12 -m venv output/qwen3-parse-venv -$parsePython = ".\output\qwen3-parse-venv\Scripts\python.exe" -& $parsePython -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" -& $parsePython -m pip check -& $parsePython -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE -``` - -Linux shell： +Linux Bash： ```bash +set -euo pipefail python3.12 -m venv output/qwen3-parse-venv . output/qwen3-parse-venv/bin/activate python -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" python -m pip check python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify \ --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -echo $? ``` 所有命令在仓库根目录执行。`--model-dir` 必填，应指向同目录保存 `model.onnx` 和三个真实 external-data 分片的完整产物，不能只有 ONNX 文件或 LFS pointer。`verify` 是默认模式，不访问网络；不存在完整模型时，将命令中的 `--mode verify` 改成 `--mode download`，由入口获取并校验 W1 固定发布产物。不要用新导出或其他 revision 的文件替换固定产物。 @@ -78,10 +67,10 @@ python -m pip install "pytest==9.1.1" python -m pytest tests/test_qwen3_full_structure.py tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py -q ``` -PowerShell 将上面命令中的 `python` 替换为 `& $parsePython`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 +命令中的 `python` 指已激活的 `output/qwen3-parse-venv/bin/python`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 [LLM 工作流](../../.github/workflows/llm-deploy.yml) 的 PR 路径接入上述三组测试，并在独立 `full-qwen3-frontend` job 以 `--mode download` 实跑完整固定图。手动/定时同样运行该解析任务。解析有独立 Summary，并以 `always()` 上传 `output/qwen3-parse/`，artifact 名为 `qwen3-full-frontend-parse-output`。下载约 1.24 GB，解析 worker 峰值 RSS 约 5 GB；runner 的实际容量与耗时仍须远端验证。 `w2-acceptance` 汇总任务要求数值任务和完整解析任务均实际成功，失败、取消、跳过或缺失都不能通过。手动/定时的 `full-qwen3-onnx` job 独立保留 W1 完整 ONNX ORT 验证；解析不依赖它成功。并行的两个完整模型任务会各自下载资产。 -PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存本地实跑与 Review 修复结果，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 +PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存 Linux 复现入口与已记录的 Linux CI 证据，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 diff --git a/probes/w2_qwen3_small/README.md b/probes/w2_qwen3_small/README.md index 66ccc2b..bda658d 100644 --- a/probes/w2_qwen3_small/README.md +++ b/probes/w2_qwen3_small/README.md @@ -27,20 +27,22 @@ RV64GC ELF，在 QEMU `virt` 裸机执行。下文“不是 RISC-V 验证”的 ONNX 输入为 `input_ids: INT64 [1,256]` 与 `attention_mask: FP32 [1,1,256,256]`。mask 是加性因果和 key-padding 掩码（允许位置为 0，屏蔽位置为 FP32 最小有限值），不是 tokenizer 常用的二维 0/1 mask；位置编号固定为 `0..255`。普通图输出为 `logits: FP32 [1,256,128]`。 -## 本地运行 - -先确认用于建环境的 `python` 是 Python 3.12。下面是 PowerShell 命令；Linux/macOS 可将 `& $probePython` 换成 `output/qwen3-probe-venv/bin/python`。 - -```powershell -python -m venv output/qwen3-probe-venv -$probePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $probePython -m pip install --upgrade pip -& $probePython -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu -& $probePython -m pip install -r requirements/qwen3-small-probe.txt -& $probePython -m pip install --no-deps -e . -& $probePython -m pip check -& $probePython probes/w2_qwen3_small/run.py --output-dir output/qwen3-small -& $probePython -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q +## Linux 复现 + +正式交付使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12。通用工具准备见 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)；在仓库根目录执行： + +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate +python -m pip install --upgrade pip +python -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu +python -m pip install -r requirements/qwen3-small-probe.txt +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +python probes/w2_qwen3_small/run.py --output-dir output/qwen3-small +python -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q ``` CPU PyTorch 必须在安装依赖文件前单独安装；随后 `torch==2.7.1` 会保留已安装的 `2.7.1+cpu`。不要只执行依赖文件安装而让 pip 从默认索引另选 CUDA 依赖。导出相关包采用已验证的固定组合，避免环境升级改变导出图或数值基线。 @@ -121,15 +123,13 @@ ScratchV 承担 ONNX/IR、IR 优化、张量循环生成、内存规划与输入 Linux 工具准备（Python 导出依赖仍按前面的固定版本安装）： ```bash +sudo apt-get update sudo apt-get install --no-install-recommends qemu-system-misc python -m pip install ziglang==0.14.1 export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" ``` -Windows 可使用便携 Zig 和 QEMU，设置 `SCRATCHV_CC` 为 `zig.exe` 的完整路径、 -`SCRATCHV_QEMU` 为 `qemu-system-riscv64.exe` 的完整路径。也会自动搜索仓库的 -`output/tools/`。不需要 WSL、Linux 镜像或系统安装；QEMU Windows 分发来源见 -[QEMU 下载页](https://www.qemu.org/download/#windows)。工具仅在本机准备，脚本不会隐式联网安装。 +设置 `SCRATCHV_QEMU="$(command -v qemu-system-riscv64)"`；所有工具均为 Linux 可执行文件。脚本不会隐式联网安装工具。 先验证矩阵乘，再导出模型并运行完整验收；输出目录必须为空，避免覆盖旧证据： diff --git a/probes/w2_runtime/README.md b/probes/w2_runtime/README.md index 462324a..3ed2242 100644 --- a/probes/w2_runtime/README.md +++ b/probes/w2_runtime/README.md @@ -8,18 +8,10 @@ 使用 [两层模型指南](../w2_qwen3_small/README.md) 中的 Python 3.12 固定 CPU 环境；其中已包含 `numpy==2.2.6`、`tokenizers==0.21.4`、`transformers==4.51.3`。仅使用生产 tokenizer 时可安装 `pip install -e ".[llm]"`，无需 Torch；复现 gate 仍使用固定版本和官方参考依赖。 -在仓库根目录运行，PowerShell： - -```powershell -$runtimePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $runtimePython -B -X utf8 probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime -$LASTEXITCODE -``` - -Linux： +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活环境，在仓库根目录运行： ```bash -output/qwen3-probe-venv/bin/python -B -X utf8 probes/w2_runtime/run.py \ +.venv-linux/bin/python -B -X utf8 probes/w2_runtime/run.py \ --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime ``` diff --git a/probes/w2_runtime_model/README.md b/probes/w2_runtime_model/README.md index 794fa0e..d58cd4f 100644 --- a/probes/w2_runtime_model/README.md +++ b/probes/w2_runtime_model/README.md @@ -4,9 +4,9 @@ 这是 W2 的接口集成证据；权重随机，不代表预训练 Qwen3-0.6B 的语言能力。执行单步前向与一个 greedy token，不执行生成循环、KV cache 或 QEMU。 -## 本地复现 +## Linux 复现 -使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活 `.venv-linux`。使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 ```bash python probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime diff --git a/scripts/check_linux_repro_docs.py b/scripts/check_linux_repro_docs.py new file mode 100644 index 0000000..dfe8b6f --- /dev/null +++ b/scripts/check_linux_repro_docs.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Reject host-specific reproduction instructions; optionally parse Bash blocks. + +This checks documentation, not execution or numerical acceptance. Historical +platform descriptions and portable implementation details remain valid. +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import shutil +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] +FENCE = re.compile(r"^\s*(`{3,}|~{3,})\s*([\w+-]*)\s*$") +HOST_PATTERNS = ( + (re.compile(r"(?= len(active) and not match.group(2): + if language in ("bash", "sh", "shell"): + blocks.append((start, "\n".join(body) + "\n")) + active = None + elif active: + body.append(line) + if language in ("bash", "sh", "shell") and re.search(r"\b[\w.-]+\.exe\b", line, re.I): + problems.append(f"{path}:{number}: non-Linux executable in shell example") + if active: + problems.append(f"{path}:{start}: unterminated fenced block") + if bash: + for start, content in blocks: + try: + result = subprocess.run([bash, "--noprofile", "--norc", "-n"], input=content, + capture_output=True, text=True, encoding="utf-8", timeout=10) + except (OSError, subprocess.TimeoutExpired) as exc: + problems.append(f"{path}:{start}: Bash validation could not run: {exc}") + continue + if result.returncode: + problems.append(f"{path}:{start}: invalid Bash: {result.stderr.strip()}") + return problems, len(blocks) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--check-bash", action="store_true", help="Require Bash and syntax-check every shell block") + parser.add_argument("--bash", help="Explicit Bash executable; defaults to PATH") + args = parser.parse_args(argv) + bash = (args.bash or shutil.which("bash")) if args.check_bash else None + if args.check_bash and not bash: + parser.exit(1, "FAIL: Bash is required for --check-bash\n") + paths = document_paths(args.root.resolve()) + if not paths: + parser.exit(1, "FAIL: no reproduction documents found\n") + problems, blocks = [], 0 + for path in paths: + findings, count = inspect_document(path, bash=bash) + problems.extend(findings) + blocks += count + for problem in problems: + print(problem) + print(f"{'FAIL' if problems else 'PASS'}: {len(paths)} documents, {blocks} shell blocks; " + f"Bash syntax {'checked' if bash else 'not requested'}; no model execution") + return int(bool(problems)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_linux_repro_docs.py b/tests/test_linux_repro_docs.py new file mode 100644 index 0000000..b528a79 --- /dev/null +++ b/tests/test_linux_repro_docs.py @@ -0,0 +1,67 @@ +"""Check rejection of unusable public reproduction commands, not doc wording.""" +import shutil + +import pytest + +from scripts import check_linux_repro_docs as docs + + +@pytest.mark.parametrize("content", [ + "```powershell\npython run.py\n```\n", + "~~~pwsh\npython run.py\n~~~\n", + "Use D:/private/models for the checkpoint\n", + "python C:\\private\\run.py\n", + "$env:OMP_NUM_THREADS = '1'\n", + "$LASTEXITCODE\n", + "venv/Scripts/python.exe\n", + "```bash\nzig.exe version\n```\n", + "```bash\necho unfinished\n", +]) +def test_rejects_non_linux_or_broken_examples(tmp_path, content): + path = tmp_path / "README.md" + path.write_text(content, encoding="utf-8") + assert docs.inspect_document(path)[0] + + +def test_linux_examples_and_truthful_history_are_allowed(tmp_path): + path = tmp_path / "README.md" + path.write_text('Historical Windows results are not Linux evidence.\n' + 'https://example.com/source\n' + '```bash\nset -euo pipefail\n' + '"$SCRATCHV_PYTHON" run.py --model-dir /srv/models/qwen3\n```\n', encoding="utf-8") + assert docs.inspect_document(path) == ([], 1) + + +def test_required_bash_cannot_silently_skip(monkeypatch): + monkeypatch.setattr(docs.shutil, "which", lambda _: None) + with pytest.raises(SystemExit) as exc: + docs.main(["--check-bash"]) + assert exc.value.code == 1 + + +def test_empty_input_is_not_success(tmp_path): + with pytest.raises(SystemExit) as exc: + docs.main(["--root", str(tmp_path)]) + assert exc.value.code == 1 + + +def test_inventory_covers_shared_guides_and_benchmarks(tmp_path): + names = ("README.md", "CONTRIBUTING.md", "docs/guide/setup.md", + "docs/topics/topic/reproduce.md", "probes/w4/example.md", "benchmarks/sample/README.md") + for name in names: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("Linux reproduction\n", encoding="utf-8") + assert {p.relative_to(tmp_path).as_posix() for p in docs.document_paths(tmp_path)} == set(names) + + +def test_actual_bash_rejects_bad_syntax_without_executing(tmp_path): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is required; Linux CI exercises this case") + path = tmp_path / "README.md" + marker = tmp_path / "must-not-exist" + path.write_text(f'```bash\ntouch "{marker.as_posix()}"\nif then\n```\n', encoding="utf-8") + problems, count = docs.inspect_document(path, bash=bash) + assert count == 1 and any("invalid Bash" in item for item in problems) + assert not marker.exists() diff --git a/tests/test_tensor_c_codegen.py b/tests/test_tensor_c_codegen.py index 23dd265..06b037f 100644 --- a/tests/test_tensor_c_codegen.py +++ b/tests/test_tensor_c_codegen.py @@ -1,7 +1,7 @@ """Execute generated tensor C against ORT, plus static/numeric rejection cases. -Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Repository -local output/tools/zig-python/ziglang/zig.exe is also recognized on Windows. +Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Optional +repository-local tools are resolved for the actual host platform. Validation tests do not require a C toolchain; execution tests skip explicitly when no compiler is available rather than reporting unexecuted cases as passes. """ @@ -35,7 +35,7 @@ def builder(*params): @pytest.fixture(scope="module") def compiler(): zig = os.environ.get("SCRATCHV_ZIG") - local = ROOT / "output/tools/zig-python/ziglang/zig.exe" + local = ROOT / "output/tools/zig-python/ziglang" / ("zig.exe" if os.name == "nt" else "zig") if zig or local.is_file(): return [zig or str(local), "cc"] for name in ("clang", "gcc", "cc"): From a17ba5fb419b965ecd94c227638c3d3044d5776c Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Tue, 6 Oct 2026 17:19:40 +0800 Subject: [PATCH 13/30] docs: standardize deployment reproduction on Linux --- .github/workflows/linux-reproduction-docs.yml | 38 + .github/workflows/llm-deploy.yml | 13 +- .gitignore | 2 + CONTRIBUTING.md | 5 + benchmarks/test_regalloc/regalloc.md | 32 +- ...351\232\234\346\216\222\351\231\244FAQ.md" | 2 +- docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md | 104 + ...77\347\224\250\350\257\264\346\230\216.md" | 2 + ...00\345\217\221\346\226\207\346\241\243.md" | 2 + ...76\350\256\241\346\226\207\346\241\243.md" | 2 + docs/llm-deploy-v1.0/W1/README.md | 72 +- ...47\350\241\214\350\256\241\345\210\222.md" | 65 +- ...42\346\265\213\346\226\271\346\241\210.md" | 70 +- ...41\346\237\245\346\212\245\345\221\212.md" | 45 +- docs/llm-deploy-v1.0/W1/interfaces.md | 2 + docs/llm-deploy-v1.0/W1/risks.md | 2 +- ...16\345\215\225\346\265\213-benchmark.html" | 682 +---- ...16\345\215\225\346\265\213-benchmark.json" | 2394 +---------------- ...\216\345\215\225\346\265\213-benchmark.md" | 1237 +-------- ...5\346\265\213\346\212\245\345\221\212.txt" | 228 +- docs/llm-deploy-v1.0/W2/README.md | 297 +- ...14\346\224\266\346\212\245\345\221\212.md" | 126 +- ...21\345\270\203\350\257\264\346\230\216.md" | 32 +- ...00\345\217\221\350\256\241\345\210\222.md" | 2 + ...77\347\224\250\350\257\264\346\230\216.md" | 38 +- ...76\350\256\241\346\226\207\346\241\243.md" | 47 +- ...26\345\231\250-CI\344\270\216Benchmark.md" | 8 +- ...00\345\217\221\346\226\207\346\241\243.md" | 9 +- ...76\350\256\241\346\226\207\346\241\243.md" | 32 +- ...52\346\237\245\346\212\245\345\221\212.md" | 17 +- probes/w1_qwen3_export/README.md | 25 +- probes/w2_backend_ops/README.md | 10 +- probes/w2_qwen3_parse/README.md | 21 +- probes/w2_qwen3_small/README.md | 36 +- probes/w2_runtime/README.md | 12 +- probes/w2_runtime_model/README.md | 4 +- scripts/check_linux_repro_docs.py | 95 + tests/test_linux_repro_docs.py | 67 + tests/test_tensor_c_codegen.py | 6 +- 39 files changed, 763 insertions(+), 5120 deletions(-) create mode 100644 .github/workflows/linux-reproduction-docs.yml create mode 100644 docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md create mode 100644 scripts/check_linux_repro_docs.py create mode 100644 tests/test_linux_repro_docs.py diff --git a/.github/workflows/linux-reproduction-docs.yml b/.github/workflows/linux-reproduction-docs.yml new file mode 100644 index 0000000..68eccb2 --- /dev/null +++ b/.github/workflows/linux-reproduction-docs.yml @@ -0,0 +1,38 @@ +name: Linux reproduction docs + +on: + pull_request: + paths: + - 'docs/**' + - 'probes/**/*.md' + - 'probes/**/*.txt' + - 'benchmarks/**/*.md' + - 'benchmarks/**/*.txt' + - 'README.md' + - 'CONTRIBUTING.md' + - 'scripts/check_linux_repro_docs.py' + - 'tests/test_linux_repro_docs.py' + - '.github/workflows/linux-reproduction-docs.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux-docs: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: '3.12' + - name: Check public instructions and Bash syntax + run: python scripts/check_linux_repro_docs.py --check-bash + - name: Exercise invalid-input rejection + run: | + set -euo pipefail + python -m pip install pytest==9.1.1 + python -m pytest tests/test_linux_repro_docs.py -q diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index adaedac..6dc4a42 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -24,6 +24,8 @@ on: - 'docs/llm-deploy-v1.0/**' - 'probes/**' - 'scripts/check_llm_deploy_docs.py' + - 'scripts/check_linux_repro_docs.py' + - 'tests/test_linux_repro_docs.py' - 'scripts/check_w1_repro_env.py' - 'scripts/run_w2_acceptance.py' - 'scripts/check_w2_ci_results.py' @@ -76,7 +78,7 @@ jobs: llm-deploy: name: "LLM Deploy v1.0 / W1 + W2" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -120,6 +122,9 @@ jobs: id: docs run: python3 scripts/check_llm_deploy_docs.py --week W1 + - name: "docs:linux-reproduction" + run: python3 scripts/check_linux_repro_docs.py --check-bash + # ── probe:qemu-matmul ───────────────────────────────────────────── - name: "probe:qemu-matmul" id: matmul @@ -409,7 +414,7 @@ jobs: # This downloads 1.24 GB and executes the 2.38 GB FP32 model. It is not # counted as a passing gate on PRs where it did not execute. if: github.event_name != 'pull_request' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -459,7 +464,7 @@ jobs: full-qwen3-frontend: name: "W2 / complete Qwen3 frontend" # Required on PRs as well as manual/nightly runs. No full-model ORT or IR inference. - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -507,7 +512,7 @@ jobs: name: "W2 / required acceptance" needs: [llm-deploy, full-qwen3-frontend] if: always() - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: diff --git a/.gitignore b/.gitignore index c12c3bc..d68b71f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ build/ models/ venv/ .venv/ +.venv-linux/ +.venv-*/ .claude/ .omo/ scratchv.egg-info/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 911b66b..faa8c1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,11 @@ pytest tests/ -v # run all tests - User-facing docs go in `docs/`. - Inline code comments are for *why* not *what*. - The README is the single source of truth for project-wide docs. +- Qwen3 deployment (W1 and later) is delivered and independently reproduced on + Linux. Use Ubuntu 24.04, Bash, Python 3.12 and Linux tool paths in public guides, + PR instructions and CI. See [the Linux reproduction contract](docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md). + Personal development environments are not acceptance evidence; preserve the + actual platform and commit identity of historical measurements. ## Code of Conduct diff --git a/benchmarks/test_regalloc/regalloc.md b/benchmarks/test_regalloc/regalloc.md index 21e1b12..0b35abe 100644 --- a/benchmarks/test_regalloc/regalloc.md +++ b/benchmarks/test_regalloc/regalloc.md @@ -284,20 +284,32 @@ convert_onnx_to_llvm(model) → LLVM IR (866K lines, 183MB) ## 7. 使用方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境和依赖： + +```bash +python3.12 -m venv .venv-regalloc +source .venv-regalloc/bin/activate +python -m pip install -e ".[all]" +``` + +CNN 用例还需要 `models/graph/cnn.onnx`；请先确认模型文件存在。LLVM 对比依赖 +`llvmlite`，报告中的 `llvm_available=false` 表示该项未执行，不能作为 LLVM 对比通过。 + ```bash -# 运行全部 4 项 benchmark，生成三格式报告（Windows PowerShell） -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json report.json ` - --output-html report.html ` - --output-md report.md +# 运行全部 4 项 benchmark，生成三格式报告 +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ + --output-md benchmark_reports/regalloc_bench.md # 单独运行某项 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_simple --repeats 100 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_dense --repeats 50 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_cnn ` +python -m benchmarks.test_regalloc.bench_simple --repeats 100 +python -m benchmarks.test_regalloc.bench_dense --repeats 50 +python -m benchmarks.test_regalloc.bench_cnn \ --cnn-path models/graph/cnn.onnx --repeats 30 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_pseudo --repeats 30 +python -m benchmarks.test_regalloc.bench_pseudo --repeats 30 ``` --- diff --git "a/docs/guide/04-\346\225\205\351\232\234\346\216\222\351\231\244FAQ.md" "b/docs/guide/04-\346\225\205\351\232\234\346\216\222\351\231\244FAQ.md" index 5006196..8644f11 100644 --- "a/docs/guide/04-\346\225\205\351\232\234\346\216\222\351\231\244FAQ.md" +++ "b/docs/guide/04-\346\225\205\351\232\234\346\216\222\351\231\244FAQ.md" @@ -282,7 +282,7 @@ git status # 手动编辑冲突文件，删除 <<<<<<<, =======, >>>>>>> 标记 # 然后 -git add <冲突文件> +git add path/to/resolved-file git commit ``` diff --git a/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md b/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md new file mode 100644 index 0000000..f41f9de --- /dev/null +++ b/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md @@ -0,0 +1,104 @@ +# Linux 交付与独立复现约定 + +从 W1/W2/W3 到后续阶段，正式交付、第二人复现和 CI 验收统一使用 **Linux**。 +推荐 Ubuntu 24.04 x86_64、Bash、Python 3.12、CPU PyTorch 和 Zig 0.14.1。 +作者可使用其他个人开发环境，但对外操作指南只提供 Linux 命令；个人路径、 +工具安装位置和本机通过记录不能作为 Linux 已通过的证明。旧记录保留实际环境、 +源码身份和历史链接，不能将旧平台名称改为 Linux。目标程序仍是 RISC-V 裸机程序： +这里的 Linux 是运行编译器、ORT、IR 解释器和 QEMU 的 **Host 环境**。 + +## 1. 获取要复现的确切版本 + +在独立目录克隆项目，获取待复现 PR 的准确版本。以下以 PR #94 为例； +复现 W3 时使用 96。将期望的完整提交 SHA 与 PR 页面核对，保留 Git 历史以运行旧版本 benchmark。 + +~~~bash +set -euo pipefail +sudo apt-get update +sudo apt-get install --no-install-recommends -y \ + git git-lfs python3.12 python3.12-venv clang lld qemu-user qemu-system-misc +git lfs install +git clone https://github.com/ScratchV-Compiler/ScratchV.git ScratchV-linux +cd ScratchV-linux +PR_NUMBER=94 +git fetch origin "pull/$PR_NUMBER/head" +git switch --detach FETCH_HEAD +git lfs pull +git rev-parse HEAD +git status --short +~~~ + +PR 分支会更新，报告必须记录实际完整 SHA。已解压的源码快照用随包校验入口验证； +它没有 Git 历史，不能宣称执行了依赖历史的 benchmark。 + +## 2. 安装固定环境 + +以下命令均在仓库根目录执行。虚拟环境和下载资产不提交 Git。 +首次安装需要网络；先安装 CPU Torch，避免解析到带 CUDA 的依赖。 + +~~~bash +set -euo pipefail +python3.12 -m venv .venv-linux +export SCRATCHV_PYTHON="$PWD/.venv-linux/bin/python" +"$SCRATCHV_PYTHON" -m pip install --upgrade pip +"$SCRATCHV_PYTHON" -m pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cpu +"$SCRATCHV_PYTHON" -m pip install -r requirements/qwen3-small-probe.txt +"$SCRATCHV_PYTHON" -m pip install -e . ziglang==0.14.1 PyYAML==6.0.3 +"$SCRATCHV_PYTHON" -m pip check +"$SCRATCHV_PYTHON" -c 'import torch; assert torch.__version__ == "2.7.1+cpu" and torch.version.cuda is None' +export SCRATCHV_CC="$("$SCRATCHV_PYTHON" -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" +export SCRATCHV_QEMU="$(command -v qemu-system-riscv64)" +export SCRATCHV_ZIG="$SCRATCHV_CC" +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export PYTHONIOENCODING=utf-8 +"$SCRATCHV_CC" version +"$SCRATCHV_QEMU" --version +if [[ -e .git ]]; then + "$SCRATCHV_PYTHON" -B -X utf8 scripts/check_w1_repro_env.py \ + --require-clean --output-dir output/linux-repro-env-01 +else + printf '%s\n' 'Source archive: verify its snapshot manifest; Git/LFS preflight does not apply.' +fi +~~~ + +Python 命令显式使用该解释器。重新打开终端后重新设置上述环境变量，不依赖全局 Python。 +环境预检输出目录必须是新目录；预检通过只证明前置条件，不代表数值验收。 + +## 3. 准备固定资产并选择阶段入口 + +W1 两层随机小模型和 W2 小算子不需要完整预训练权重。完整前端审计使用已发布 ONNX； +运行时使用固定官方 Tokenizer。已有资产默认离线校验；需要下载时显式选择 download： + +~~~bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 probes/w1_qwen3_export/run.py \ + --mode download --model-dir output/assets/onnx --output-dir output/linux-onnx-verify-01 +"$SCRATCHV_PYTHON" -B -X utf8 probes/w2_runtime/run.py \ + --mode download --tokenizer-dir output/assets/tokenizer --output-dir output/linux-tokenizer-01 +~~~ + +完整 ONNX 下载约 1.24 GB，并会解压权重、实际执行 ORT 校验。模型身份由仓库 manifest +固定，不能换成任意同名 checkpoint。W3 真实权重子图另需固定源 checkpoint； +按 [W3 PR #96 中的独立复现清单](https://github.com/ScratchV-Compiler/ScratchV/pull/96)准备，仅含 W2 的分支还没有 W3 入口。 + +| 阶段 | Linux 复现入口 | 判据 | +|---|---|---| +| W1 | [W1 README](W1/README.md) | 环境、MatMul、小模型、固定 ONNX 和文档分别验证 | +| W2 | [W2 README](W2/README.md) | 统一入口七项全部 PASS；部分执行不算通过 | +| W3（含 W3 的版本） | [W3 PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) | 准备五项与完整七组在同一版本全部通过 | + +每次执行使用新的输出目录，记录源码 SHA、资产哈希、依赖和系统资源。 +完整模型需要足够内存及原始数组磁盘空间；不要将小模型通过外推为完整模型容量可行。 +W3 已有 Linux 实测的资源、CPU 策略和耗时见其 Linux CI 文档。 + +## 4. 后续提交必须遵守 + +- 对外复现命令使用 Bash、POSIX 路径、Linux 工具及虚拟环境 bin/python。 +- 修改可执行代码后，以该提交的实际 Linux 测试为证据；本机验证与 Linux CI 分开记录。 +- 不改写历史数据的平台，不把旧提交的 PASS 冒充新提交复跑，也不把离线数组审计当成重新执行模型。 +- 文档检查命令如下；CI 必须执行 Bash 语法检查，缺少 Bash 不能算检查通过。 + +~~~bash +set -euo pipefail +"$SCRATCHV_PYTHON" scripts/check_linux_repro_docs.py --check-bash +~~~ diff --git "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\344\275\277\347\224\250\350\257\264\346\230\216.md" index dfb892f..8b053a2 100644 --- "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -1,5 +1,7 @@ # ScratchV IR 解释器使用文档 +正式交付、CI 和他人复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。命令中的 `python` 指已激活的 `.venv-linux/bin/python`。 + > 日期：2026-09-30 > 输入：直接构造的 Program、输入数组及初始化数组。 diff --git "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" index e9fa545..0a1777d 100644 --- "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -1,5 +1,7 @@ # ScratchV IR 解释器开发文档 +正式交付、CI 和他人复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。命令中的 `python` 指已激活的 `.venv-linux/bin/python`。 + > 版本：v0.1，开发说明；本轮实现已落地 > 日期：2026-09-30 > 依据：[IR 解释器设计文档（第二版·精简版）](IR解释器-设计文档.md) diff --git "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\350\256\276\350\256\241\346\226\207\346\241\243.md" index d2c8f1b..ae4f2e3 100644 --- "a/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/llm-deploy-v1.0/W1/IR\350\247\243\351\207\212\345\231\250-\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -1,5 +1,7 @@ # ScratchV IR 解释器设计文档（第二版·精简版） +正式交付、CI 和他人复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。命令中的 `python` 指已激活的 `.venv-linux/bin/python`。 + > 版本：v0.2，本轮实现已落地；实际支持规则见使用说明 > 日期：2026-09-30 diff --git a/docs/llm-deploy-v1.0/W1/README.md b/docs/llm-deploy-v1.0/W1/README.md index 35dbe0c..d38e667 100644 --- a/docs/llm-deploy-v1.0/W1/README.md +++ b/docs/llm-deploy-v1.0/W1/README.md @@ -1,30 +1,33 @@ -# W1 独立复现与验收记录 +# W1 Linux 独立复现与验收记录 本指南用于另一名成员从目标提交重新生成参考结果、编译产物并运行探测。当前已知本地结果见 [执行计划](W1-执行计划.md)，接口候选见 [interfaces.md](interfaces.md)，风险见 [risks.md](risks.md)。**Mastttttter 已于 2026-10-03 完成 `df18b02f` 的两层数值链路独立复现并本人确认；团队接口确认和 W1 出口评审仍待完成。** 原始证据见 [独立复现报告](https://github.com/ScratchV-Compiler/ScratchV/pull/91#issuecomment-5958157573)。该确认归属所列提交，不自动覆盖后续修复。 -PR #91 已合并，最终 head 为 `5903c1381d57f24062de7395f24385a59e279325`，合并提交为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。报告落盘失败和源码身份归属修复 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93) 已于 2026-10-04 合并为 `faab6a81562710e105d87265dafd2dc769a67c72`；发布前整合工作树记录见 [本轮修复与验收报告](../W2/W1修复与W2本地验收报告.md)。人工确认和完整模型第二人复现继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。下方原 PR 修复数据均为历史记录。 +PR #91 已合并，最终 head 为 `5903c1381d57f24062de7395f24385a59e279325`，合并提交为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。报告落盘失败和源码身份归属修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)；发布前整合工作树记录见 [本轮修复与验收报告](../W2/W1修复与W2本地验收报告.md)。人工确认和完整模型第二人复现继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。下方原 PR 修复数据均为历史记录。 + +正式交付、CI 和独立复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，环境总入口见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。Linux 是运行 ScratchV 和 QEMU 的宿主环境；当前 RISC-V guest 仍为裸机。 所有命令在仓库根目录执行。先记录 `git rev-parse HEAD` 和 `git status --short`；干净checkout便于关联提交，存在本地改动则保留diff并明确标记。不要使用第一人的预生成参考数组代替自己的运行。下述输出目录用完后保留，下一轮使用不同名称，不删除旧证据。 本次依据第二人报告修复两个复现问题：Linux 超时测试遗漏 `X` 死亡状态、错误解析含空格的进程名；三项优化集成测试依赖固定目录，导致自选产物目录和 CI 中被跳过。退出测试现在区分原进程身份，允许短暂退出过程，验证无关进程仍存活；CI 另连续运行 20 次真实进程用例，每次保留 JUnit，任一次失败即停止。没有修改编译器数值算法、模型配置或精度阈值，W2 扩展继续独立交付。 -修复工作树的本地验证：452 项专项回归全部通过、0 跳过（`output/pr91-final-regressions-zig.xml`）；显式产物集成单独实跑 3/3（`output/pr91-artifact-integration.xml`，与前者重叠）。本轮新导出及两层 QEMU 报告在 `output/pr91-repro-host/`、`output/pr91-repro-qemu/`，28/28 通过，最大目标误差 `1.6689300537109375e-6`，QEMU 进程累计 130.701 秒。以上为 Windows 上提交前的实际工作树证据，不冒充 Linux 退出竞态复验；新提交的 Linux 20 次执行结果以 PR CI 对应 SHA 为准。项目指定的 `.claude/harness/verify/run.py` 本地缺失，未宣称执行 L2。 +提交前个人环境的测量保留于 [固定提交中的历史报告](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/README.md)，不作为 Linux 验收数据。Linux 既有执行记录见第 5 节；新代码必须关联新的 run 和源码身份。 ## 0. 确定复现版本与领取任务 -`5ea22ecc7fd314025f6453c556dbb3e3f16a3175` 和 `3bb88e8` 是较早的 **历史基线**。下述已合并 W1 复现固定为包含 PR #93 的 `faab6a8`。若核对后续 PR 的追加修复，应另外取得该 PR 的提交并记录完整 SHA，不得将旧基线结果当作后续版本的复验。 +`3bb88e81498d6d9293aed70e25a4e5a2104465d0` 是 W1 的历史合并基线，`5ea22ec` 是更早的 Linux CI 记录。复现新交付时必须取得待验收 PR 的精确版本；下面以含 W1 基础能力的 PR #94 为例，W3 集成版本改用 96。历史提交只能按其自带的功能和证据验收。 -在新目录获取本次版本的示例（需已安装 Git 和 Git LFS，PowerShell/bash 均可逐条执行）： +在新目录获取本次版本的示例（需已安装 Git 和 Git LFS，以下为 Bash 命令）： ```bash git clone --no-checkout https://github.com/ScratchV-Compiler/ScratchV.git ScratchV-w1-repro cd ScratchV-w1-repro git lfs install --local -git fetch origin faab6a81562710e105d87265dafd2dc769a67c72 +REPRO_PR=94 +git fetch origin "pull/$REPRO_PR/head" git rev-parse FETCH_HEAD ``` -先确认输出与约定的合并 SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 +先确认输出与本次约定的完整 PR head SHA 完全一致，再执行以下命令；不一致时先确认目标版本，不要继续运行后误写为原约定提交的结果。 ```bash git checkout --detach FETCH_HEAD @@ -35,8 +38,6 @@ git status --short 将最终 `HEAD` 填入复现记录。若只复现历史 `5ea22ec`，请使用该提交自带的旧指南和脚本；不要检出旧 SHA 后调用本次新增入口。需要验证额外本地改动时，另保留完整 diff、增量文件哈希和交付方式，不能把 dirty 工作树写成精确提交复现。 -验收 [PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 中集成的最新 W1/W2/W3 代码时，在独立 checkout 中改用 `git fetch origin pull/96/head`，核对 `git rev-parse FETCH_HEAD` 后再 `git checkout --detach FETCH_HEAD`，并记录实际完整 SHA。PR 引用会随修复更新；其他人的报告和 CI 只覆盖各自记载的提交。 - E4 建议领取 MatMul/两层 QEMU，E5 建议领取合成图/真实结构 IR/完整 ONNX ORT；同一位其他成员也可执行全部探测。 开始前在群里记录领取人和目标 SHA，结束后填写第 6 节。维护者 review、CI、第一人再次运行都不自动替代第二人执行。 @@ -44,41 +45,30 @@ E4 建议领取 MatMul/两层 QEMU，E5 建议领取合成图/真实结构 IR/ 要求Python3.12，固定依赖在 `requirements/qwen3-small-probe.txt`。先安装CPU PyTorch，再装其余依赖，避免默认索引解析成CUDA包。 -Windows PowerShell： - -```powershell -python -m venv output/w1-repro-venv -$probePython = ".\output\w1-repro-venv\Scripts\python.exe" -& $probePython -m pip install --upgrade pip -& $probePython -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu -& $probePython -m pip install -r requirements/qwen3-small-probe.txt -& $probePython -m pip install --no-deps -e . -& $probePython -m pip install "ziglang==0.14.1" -& $probePython -m pip check -$env:SCRATCHV_CC = (& $probePython -c "from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / 'zig.exe')") -``` - -准备 `qemu-system-riscv64.exe` 便携工具，可放 `output/tools/` 由运行器自动查找；否则将 `SCRATCHV_QEMU` 设为已安装可执行文件的实际路径。QEMU Windows分发入口见 [QEMU下载页](https://www.qemu.org/download/#windows)。不需要WSL或Linux镜像。 - -Linux shell： +Ubuntu 24.04 / Bash： ```bash -python3.12 -m venv output/w1-repro-venv -. output/w1-repro-venv/bin/activate +set -euo pipefail +sudo apt-get update +sudo apt-get install --no-install-recommends python3.12-venv git-lfs qemu-system-misc +python3.12 -m venv .venv-linux +. .venv-linux/bin/activate python -m pip install --upgrade pip python -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu python -m pip install -r requirements/qwen3-small-probe.txt python -m pip install --no-deps -e . python -m pip install "ziglang==0.14.1" python -m pip check -sudo apt-get install --no-install-recommends qemu-system-misc export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" -qemu-system-riscv64 --version +export SCRATCHV_PYTHON="$(pwd)/.venv-linux/bin/python" +export SCRATCHV_QEMU="$(command -v qemu-system-riscv64)" +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +"$SCRATCHV_QEMU" --version ``` -后续代码块中的 `python` 均指上述环境。PowerShell请将行首 `python` 换成 `& $probePython`。CPU小模型无需官方权重下载；首次编译Zig会建立缓存，应允许足够编译时间，不要将首次构建耗时解释为推理性能。 +后续代码块中的 `python` 均指已激活的 `.venv-linux/bin/python`。在新终端先执行 `source .venv-linux/bin/activate` 并恢复工具环境变量。CPU小模型无需官方权重下载；首次编译Zig会建立缓存，应允许足够编译时间，不要将首次构建耗时解释为推理性能。 -本次更新提供环境预检入口（历史 `5ea22ec` 没有此脚本）： +当前交付提供环境预检入口（历史 `5ea22ec` 没有此脚本）： ```bash python -X utf8 -B scripts/check_w1_repro_env.py --require-clean --output-dir output/w1-repro-env @@ -88,7 +78,7 @@ python -X utf8 -B scripts/check_w1_repro_env.py --require-clean --output-dir out 完整 ONNX 的固定 ZIP 约 1.24 GB，解包模型约 2.41 GB；已有 ORT 验证进程峰值约 5.97 GiB。按实际机器预留下载、解包和运行余量；该内存记录不是导出峰值或最低内存承诺。完整模型可以单独安排，不能将未执行记为成功。 -运行脚本后立即检查退出码：PowerShell 用 `$LASTEXITCODE`，bash 用 `$?`。遇到非零退出先保存报告和日志，不要继续执行后用最后一条命令的成功覆盖前面的失败。 +Bash 使用 `set -euo pipefail`，任何命令失败即停止；记录失败码时使用 `if command; then ...; else status=$?; ...; fi`，不要用后续成功命令覆盖失败。数值失败仍须保存报告和日志。 ## 2. 文档、回归与原两层合成图 @@ -143,16 +133,6 @@ SCRATCHV_QWEN_ARTIFACT_DIR=output/w1-repro-qwen3-small python -X utf8 -B -m pyte -q -rs -o junit_family=xunit1 --junit-xml=output/w1-repro-qwen3-small/optimization-tests.xml ``` -PowerShell 使用同一固定解释器： - -```powershell -$env:SCRATCHV_QWEN_ARTIFACT_DIR = 'output/w1-repro-qwen3-small' -& $probePython -X utf8 -B -m pytest tests/test_optimizer_numeric_semantics.py::test_real_qwen_artifact_optimization_matches_ort -q -rs -o junit_family=xunit1 --junit-xml=output/w1-repro-qwen3-small/optimization-tests.xml -$artifactTestExit = $LASTEXITCODE -Remove-Item Env:SCRATCHV_QWEN_ARTIFACT_DIR -if ($artifactTestExit -ne 0) { throw "Qwen artifact integration failed: $artifactTestExit" } -``` - 必须 **3 passed、0 skipped**，分别对应 none/basic/all，每项执行全部 7 组输入。显式目录为空、缺文件、导出失败、模型/输入/源码指纹不符均失败；没有配置目录才允许普通单测跳过，不再查找历史 `output/qwen3-small-pr91`。 测试复用现有导出报告及模型校验，并验证实际 NPZ 文件。ORT 参考和未优化 IR 基线当次计算，不读取旧 `logits_*.npy` 作为真值；每种优化要求对 ORT 严格误差 `<1e-5`，同时与当次未优化 IR 数组逐元素相等。JUnit 的 testcase properties 记录模型、导出报告与源码哈希，以及导出/测试提交和实际 case 数。哈希一致只能证明内容配套；仍须在本轮新目录导出，保留执行命令及报告，不能把旧产物改名后宣称新导出。 @@ -215,12 +195,12 @@ peak_memory是验证器进程峰值RSS，**不包含导出子进程**，不能 工作流为 `.github/workflows/llm-deploy.yml`： -- PR 的小模型 job 执行原合成图、小模型 IR/QEMU 和 `tests/test_w1_qwen3_export.py` 的轻量回归；另一个 W2 完整前端 job 会下载完整模型并解析、审计，但不执行完整 IR/ORT 前向。 +- PR执行原合成图、小模型IR、小模型QEMU和 `tests/test_w1_qwen3_export.py` 的轻量回归，不下载完整模型。 - 手动或定时运行另启 `full-qwen3-onnx` job，使用 `--mode download --threads 2` 校验固定发布产物并真实运行ORT两case，上传 `output/qwen3-full-probe/` 报告。 - 完整模型job只需要固定NumPy2.2.6、ONNX1.18.0、ORT1.22.1、protobuf5.29.5，不需要PyTorch；这也意味着它不重新导出或比较PyTorch。 -- PR 中 `full-qwen3-onnx` 按条件未执行，应明确记录“完整 ORT 重型门禁未执行”，不能用完整解析或轻量测试通过代替。 +- PR中完整模型job按条件未执行，应明确记录“ORT重型门禁未执行”，不能用轻量测试通过代替。 -已确认本地28次两层QEMU执行，以及完整模型新入口verify的hash/checker/ORT两case通过（验证器峰值RSS约6.40 GB，报告 `output/qwen3-full-local/report.json`）。提交 `545e696` 已有两项Linux成功证据： +提交 `545e696` 已有两项 Linux 成功证据；历史个人测量不充当 Linux 报告： - [完整ONNX重型任务](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767)：真实固定release下载、校验、ORT两case及artifact上传。 - [小模型部署功能任务](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36983000988/job/110761571841)：所有部署步骤通过，含MatMul、合成图IR、官方Qwen3小模型IR和28次真实QEMU执行，artifacts上传成功。 diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" index 4814a0f..eb6dacf 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\211\247\350\241\214\350\256\241\345\210\222.md" @@ -5,6 +5,8 @@ > 本文记录实现事实和待办，不代表已召开冻结会议。命令与签字模板见 [独立复现指南](README.md)。 > PR #91 已合并为 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`。本地继续补齐报告发布与源码身份修复，结果见 [本轮报告](../W2/W1修复与W2本地验收报告.md)。W1 报告修复单独发布于 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付；人工确认和完整模型第二人复现由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 继续跟踪，旧 CI 不覆盖本地增量。 +正式交付与独立复现环境统一为 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。环境入口见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。本页 Linux 历史 run 保留原提交身份；个人环境测量归档后不计为 Linux 结果。 + ## 1. 目标和验收边界 W1 要确认 ScratchV 编译 Transformer 的可行性、明确风险和 Plan B、冻结前端/IR/后端/运行时四类接口。出口仍是实际执行的探测门禁、文档定稿及团队确认。文件存在、结构检查通过或跳过后的绿色 CI 都不等于数值验收。 @@ -28,16 +30,14 @@ ONNX → ONNXParser → shared Program IR → CompilerDriver / 优化 / verifier | IR 解释器 | 有直线图、循环/分支测试，并可执行两层图 | `scratchv/verification/ir_interpreter.py`、`tests/test_ir_interpreter_control_flow.py`；与旧 DSL 文本解释器不同 | | PR #89 两层合成图 | ONNX → ORT / IR 的数值验收已通过 | 保留 `probe:small-transformer`；该图没有 Q/K RMSNorm，使用 partial RoPE，不等于真实 Qwen3 | | 官方 Qwen3 类的两层小配置 | PyTorch → ONNX → ORT / IR，7 组输入、29 个检查点通过 | 固定随机权重、hidden=32、4 Q 头/2 KV 头、head_dim=16、L=256；不是预训练权重切片 | -| FP32 MatMul QEMU | 7 个矩阵/广播/向量用例通过；4×4 小数误差 `1.4901161193847656e-8` | 新 tensor-c 路线，`max_abs < 1e-5`、`rtol=0` | -| 两层 Qwen3 QEMU | 普通/诊断图 × none/all 优化 × 7 组输入，共 **28 次真实 QEMU 执行通过** | 29 个检查点，最大全张量绝对误差 `1.6689300537109375e-6`；另验 none/basic/all 优化后 IR | -| 新编译/运行时回归 | 此前本地 **237 项专项测试通过** | 历史专项集合计数，不代表本轮全仓测试或新增 Linux CI 已通过；本轮合并回归另列实际结果 | -| 完整 Qwen3-0.6B 导出产物 | 本轮 verify 模式重新通过四文件哈希、完整 ONNX checker、两组 ORT 执行，输出均为 `[1,256,151936]` | 新验证器峰值 RSS 6,401,679,360 bytes（约6.40 GB）；验证已有导出，未重新导出或比较PyTorch，不是完整 IR/QEMU 推理验收 | +| FP32 MatMul QEMU | `5ea22ec` Linux 部署任务的 7 个矩阵/广播/向量用例通过 | 新 tensor-c 路线，`max_abs < 1e-5`、`rtol=0` | +| 两层 Qwen3 QEMU | 普通/诊断图 × none/all 优化 × 7 组输入，共 **28 次真实 QEMU 执行通过** | `5ea22ec` Linux 任务：29 个检查点，最大全张量绝对误差 `1.7285346984863281e-6`；另验 none/basic/all 优化后 IR | | 完整ONNX Linux重型CI | 提交 `545e696` 的固定release下载、哈希/checker、ORT两case及artifact上传通过；验证器峰值RSS 6,409,670,656 bytes（约6.41 GB） | [实际job](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767)；只证明该任务/提交，不代表整个CI全绿 | -| 当前基线小模型 Linux CI | 提交 `5ea22ec` 的 MatMul、合成图 IR、官方小模型 IR 和 28 次 QEMU 均通过；QEMU 最大绝对误差 `1.7285346984863281e-6` | [部署工作流](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)；与 Windows 本地误差分开记录 | +| 当前基线小模型 Linux CI | 提交 `5ea22ec` 的 MatMul、合成图 IR、官方小模型 IR 和 28 次 QEMU 均通过；QEMU 最大绝对误差 `1.7285346984863281e-6` | [部署工作流](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)；仅归属该 Linux run | | 当前基线主 CI / benchmark | 提交 `5ea22ec` 的 Linux 全量测试 **2371 passed、6 skipped**，主 CI 后续步骤和 benchmark 通过 | [主 CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369611)；逐项 skip 不替代强制 QEMU 门禁 | | 当前基线其他检查 | Topic 06 和 AI review 通过 | [Topic 06](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369652)、[PR #91 Checks](https://github.com/ScratchV-Compiler/ScratchV/pull/91/checks)；自动 review 不代替维护者评审或第二人复现 | -原始本地 QEMU 证据位于 `output/qemu-matmul-final/report.json`、`output/qwen3-riscv-final/report.json`，本轮完整模型验证为 `output/qwen3-full-local/report.json`；这些忽略目录不会自动随 PR 发布。复现时生成新报告，比较其中的源码、模型、输入和 ELF 哈希。完整导出历史证据及 padding 数值边界见 [探测方案](W1-探测方案.md)。 +个人工作树测量见 [固定提交历史归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/W1-执行计划.md)，原始数值与平台身份保持不变。新的 Linux 复现须生成自己的报告，比较源码、模型、输入和 ELF 哈希。完整导出历史边界见 [探测方案](W1-探测方案.md)。 ## 3. 工具链与交付物 @@ -46,8 +46,7 @@ ONNX → ONNXParser → shared Program IR → CompilerDriver / 优化 / verifier | Python / 导出依赖 | Python 3.12；CPU PyTorch 2.7.1；其余固定版本见 `requirements/qwen3-small-probe.txt` | | C 交叉编译器 | Zig 0.14.1，使用 `zig cc`；`SCRATCHV_CC` 指向可执行文件 | | QEMU | `qemu-system-riscv64`、virt、TCG、RV64GC/LP64D；`SCRATCHV_QEMU` 可指定路径 | -| 本机已测工具 | Zig 0.14.1；QEMU 11.1.0；Windows 便携工具，不需要 WSL | -| Linux 复现 | `qemu-system-misc` 提供执行器；版本写入新报告，不能把 Windows 通过直接当成 Linux 通过 | +| Linux 复现 | Ubuntu 24.04 的 `qemu-system-misc` 提供执行器；实际版本写入本次 Linux 报告 | | 数学与内存 | Zig 随附 musl 数学库；禁用 fast-math/FMA；静态 arena 默认上限 256 MiB，当前 QEMU 512 MiB | | 交付物 | 仓库路径 | @@ -77,9 +76,9 @@ ONNX → ONNXParser → shared Program IR → CompilerDriver / 优化 / verifier ## 5. 出口验收清单 -- [x] 本地 IR 解释器和两层数值探测可用；没有把解析成功当成执行成功。 -- [x] 新 tensor-c 路线本地 MatMul 与两层 Qwen3 FP32 QEMU 数值探测通过。 -- [x] 完整 Qwen3 导出已有历史证据；本轮对固定产物完成哈希、结构和两组ORT的本地重验，保留两次验证的不同范围。 +- [x] 已有 Linux CI 中 IR 解释器和两层数值探测可用；没有把解析成功当成执行成功。 +- [x] `5ea22ec` Linux 部署任务中，新 tensor-c 路线的 MatMul 与两层 Qwen3 FP32 QEMU 数值探测通过。 +- [x] 完整 Qwen3 导出已有历史证据；固定产物在 `545e696` Linux 任务完成哈希、结构和两组 ORT 验证，与重新导出分别记录。 - [x] 提交 `5ea22ec` 的部署小模型 Linux 功能门禁实际通过：[run](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)，含 MatMul、IR 和 28 次真实 QEMU 执行；不是仅 IR 的 CI 证据。 - [x] 提交 `545e696` 的完整Qwen3重型门禁已在Linux实际通过：[run/job](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767)，含下载固定release、ORT两case、报告artifact。后续提交以对应Checks为准；PR仅跑轻量回归时不得算作新的重型通过。 - [x] 修复历史 Linux 测试竞争和旧 CNN smoke 命令；`5ea22ec` 的 [主 CI / benchmark](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369611) 已通过，Linux 全量测试 2371 passed、6 skipped。 @@ -113,43 +112,13 @@ ONNX → ONNXParser → shared Program IR → CompilerDriver / 优化 / verifier 工具链/网络阻塞时允许“本地先跑、CI 后补”，但 W1 状态保持未关闭。保留失败报告，按 [risks.md](risks.md) 触发方案；不得静默缩短序列、改精度、换模型或降级为 parse-only 后仍标原 gate 通过。 -## 7. 2026-10-02 回归收尾记录 - -本轮在 Windows / Python 3.12 上补齐真实 LLVM 18.1.8 工具后，运行 `pytest tests/`： -**2353 passed，0 failed，40 skipped**。跳过中 38 项需要本机没有的 Linux 用户态 -`qemu-riscv32`，另外两个是通用环境未安装 PyTorch 的模块；后者在固定 CPU 环境中 -与完整导出入口单测一起执行，**111 passed**。Linux CI 安装用户态与系统态 QEMU， -关键 MatMul 和真实 Qwen3 RISC-V 探测缺工具时必须失败。 - -独立代码审查随后发现并修复 Softmax 中间减法溢出没有立即报错的问题；追加 -Softmax、ReduceMean、MatMul、Mul 溢出和失败后恢复用例，后端/运行时专项 -**86 passed、0 skipped**。真实 RV64 ELF 对极端输入返回错误码 2，随后合法输入 -得到 `[0.5, 0.5]`。全量计数与之后的专项计数有重叠，不相加宣称总用例数。 - -对旧提交 `974ee37` 的同环境定点测试实际复现了 Windows 文件占用、路径、换行和 -`resource` 导入问题。完整源码归档基线也复现了这些问题；其两个依赖 Git 元数据的 -benchmark 在归档目录中不可比较，因此不把这两项失败归因于旧版本。当前工作树的 -这两项已通过。缺 `llvm-mca` 的问题通过准备真实工具解决，没有加入整体跳过策略。 - -CNN 调度 benchmark 的主要验收仍是 standalone 完整数值/安全路径；旧 CompilerDriver -的简化张量汇编不再计为成功，用 schema 5 的 `unsupported_cases` 明确列出。 -任何其他编译错误仍失败。IR verifier benchmark 改为对照优化后的 IR，计时仅代表 -验证开关的编译开销，不能与旧版占位汇编计时直接比较。 - -本机日志保存在忽略的 `output/closeout-final.*`、`output/pinned-closeout.*`、 -`output/tensor-overflow-review.*`、`output/windows-baseline.*`。这些是本地验证记录， -不是第二人复现记录。目标提交的 Linux 结果应查看 PR #91 Checks 和下载的 artifact； -仓库本地 `.claude/harness/verify/run.py` 不存在，未宣称执行该 L2 harness。 - -提交 `545e696` 的完整ONNX Linux重型任务已通过并上传 `qwen3-full-onnx-probe-output` -artifact，链接见§2。同一提交的小模型Linux功能job也已通过，包括28次QEMU执行与 -全部部署任务步骤。545e696首次通用主CI发生过测试竞争失败： -`tests/test_riscv_tensor_runtime.py` 对 `/proc` 的“存在检查后读取”遇到进程退出。 -随后仅修正测试的该处竞争处理并增加4个确定性用例，本地 **26 passed、0 skipped**； -运行时源码未因该修复改变。保留这次历史失败，**不把545e696写成整个CI全绿**； -后续 `bed2d32` 的 Linux 全量测试已通过，但旧 CNN verifier smoke 仍调用不支持张量的旧汇编后端；`5ea22ec` 将该 smoke 明确切换为 `--backend ir` 和 `.ir` 产物，仅验证 IR。最终 `5ea22ec` 主 CI、benchmark、部署任务和 Topic 06 均通过，链接见 §2。没有为了通过 CI 放松旧后端张量拒绝检查。 - -暂留阶段的修复曾通过2415项全仓测试、7次MatMul和28次两层QEMU执行，并重验完整ONNX ORT。本次将这些修复与新增仿真耗时统计一并集成到 PR #91；暂留阶段的历史结果、40项skip原因、数值与待决策事项见[本地审查报告](W1-本地收尾与审查报告.md)，新增改动的验证另行记录，不能沿用旧提交的 Linux CI 结论。 +## 7. Linux 回归与历史记录 + +提交 `545e696` 的完整 ONNX Linux 重型任务通过并保留 artifact；同提交小模型功能任务含 28 次 QEMU 通过，但通用主 CI 曾因 `/proc` 读取与进程退出竞争而失败。后续修复读取竞争，`5ea22ec` 将旧 CNN verifier smoke 明确切换为 IR 产物，通用 CI、benchmark、部署及 Topic 06 通过，链接见第 2 节。保留失败身份，不把 `545e696` 写成整套 CI 全绿。 + +已有语义修复包括 Softmax 等算子的非有限值检查、跨块 DCE、空输出 Gather 越界检查，以及上下游产物绑定和仿真时长。旧标量后端的张量缺口仍明确拒绝，不用占位汇编冒充成功。具体修复与未决数值错误契约见 [收尾与审查记录](W1-本地收尾与审查报告.md)。 + +个人环境的原始计数、计时及平台专属调查保留于 [固定提交历史归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/W1-执行计划.md)，不作为新的 Linux 测量。每次目标提交的正式验证以该 SHA 的 Linux Checks、artifact 及独立执行记录为准。 ## 8. 变更记录 diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" index 465338c..15f46c4 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\216\242\346\265\213\346\226\271\346\241\210.md" @@ -5,16 +5,18 @@ > 完整命令和验收模板见 [README](README.md)。保留各探测的范围，不用替代模型或解析成功冒充原数值门槛。 > 合并后的本地修复与 W2 增量见 [本轮验收报告](../W2/W1修复与W2本地验收报告.md)；其中 W1 报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)，W2 独立交付，远端 CI 以对应提交实际执行为准。人工待办继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。 +正式交付与独立复现环境为 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。当前页面保留既有 Linux run 的真实身份；个人环境数值只在固定提交历史归档中保存。 + ## 0. 总览 | 探测 | 核心判据 | 当前证据 | |---|---|---| -| probe:qemu-matmul | 真正执行 RISC-V 的 FP32 4×4 MatMul，绝对误差小于1e-5 | 新 tensor-c 路线本地通过，扩展7用例 | +| probe:qemu-matmul | 真正执行 RISC-V 的 FP32 4×4 MatMul，绝对误差小于1e-5 | `5ea22ec` Linux 部署任务通过，扩展 7 用例 | | probe:small-transformer | 两层 ONNX → IR解释器 vs ORT，绝对误差小于1e-5 | PR89合成图已通过；保留原入口 | -| numeric:ir-qwen3-small | 官方Qwen3类小配置，PyTorch/ORT/IR，7输入29检查点 | 本地已通过；不是预训练切片 | -| numeric:qemu-qwen3-small | 同一普通/诊断图经编译优化后生成RV64 ELF并执行 | 本地及 `5ea22ec` 的 Linux 部署任务 28 次 QEMU 通过，CI 链接见 §3.3 | -| probe:qwen3-onnx | 完整0.6B导出、固定输入/完整输出形状、ORT可执行 | 历史导出、本轮本地verify，以及545e696的Linux download+hash/checker/ORT两case均通过；CI链接见§4.4 | -| 环境与文档 | 工具可用、门禁实际执行、接口/风险确认 | 本地工具已验证；结构检查不能替代团队确认 | +| numeric:ir-qwen3-small | 官方Qwen3类小配置，PyTorch/ORT/IR，7输入29检查点 | `5ea22ec` Linux 部署任务通过；不是预训练切片 | +| numeric:qemu-qwen3-small | 同一普通/诊断图经编译优化后生成RV64 ELF并执行 | `5ea22ec` 的 Linux 部署任务 28 次 QEMU 通过，CI 链接见 §3.3 | +| probe:qwen3-onnx | 完整0.6B导出、固定输入/完整输出形状、ORT可执行 | 历史导出，以及545e696的Linux download+hash/checker/ORT两case均通过；CI链接见§4.4 | +| 环境与文档 | 工具可用、门禁实际执行、接口/风险确认 | Linux 工具与实际执行由对应 run 验证；结构检查不能替代团队确认 | ## 1. 当前执行配方与证据格式 @@ -42,13 +44,10 @@ | 项 | 已记录结果 | |---|---| -| 新路径结论 | 本地PASS，7/7 | -| 4×4小数最大绝对误差 | `1.4901161193847656e-8` | -| 工具 | Zig0.14.1、QEMU11.1.0；具体build字符串在报告 | -| 证据 | `output/qemu-matmul-final/report.json`；原始忽略产物不默认随PR上传 | +| 新路径结论 | `5ea22ec` Linux 部署任务 PASS，7/7 | | 旧路径结论 | H1/H2所述限制保留，不标成修复 | | Linux CI | `5ea22ec` 的[部署任务](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)已通过 MatMul 步骤；历史 `545e696` 同项也通过 | -| 第二人独立复现 | 待另一名成员执行和记录，不用CI自动运行替代 | +| 第二人独立复现 | Mastttttter 已确认历史 `df18b02f` 两层链路；新提交须按 README 留下新记录 | ## 3. 两层 Transformer 与真实 Qwen3 小配置 @@ -75,13 +74,10 @@ | 基础图/解释器 | 官方结构小配置已通过，不是parse-only | | 编译构建 / QEMU次数 | 普通/诊断 × none/all = 4构建；每个7输入，共28执行通过 | | 观察点 | 29检查点，含每层norm、RoPE、attention、MLP/residual以及最终logits | -| Windows本地QEMU最大绝对误差 | `1.6689300537109375e-6`，保持原1e-5阈值 | -| `5ea22ec` Linux QEMU最大绝对误差 | `1.7285346984863281e-6`（short_255），28次均通过；与历史 `545e696` 记录一致，仍单独对应本次 run，不覆盖本地值或改变1e-5阈值 | +| `5ea22ec` Linux QEMU最大绝对误差 | `1.7285346984863281e-6`（short_255），28次均通过；与历史 `545e696` 记录一致，仍单独对应本次 run，不改变1e-5阈值 | | 额外检查 | 普通/诊断logits一致；因果性与padding隔离通过 | -| 专项回归 | 此前本地237项通过，属于历史专项集合，不是本轮整个仓库测试计数 | -| 原始证据 | `output/qwen3-riscv-final/report.json`，含源码/模型/ELF/输入指纹、工具与优化信息 | | Linux CI | `5ea22ec` 的[部署任务](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)全部步骤通过，含官方小模型 IR、28 次真实 QEMU 执行；不是 IR-only 的 CI | -| 第二人独立复现 | 待另一名成员执行和记录；CI通过不等于人工签字 | +| 第二人独立复现 | 已有历史 `df18b02f` 确认；CI 通过不等于新提交人工签字 | 这不是两层预训练权重切片，也不具有语言生成能力；不是完整0.6B、完整28层误差/内存验证，更不是目标硬件性能证明。 @@ -93,66 +89,40 @@ 这是完整模型的ONNX/ORT门槛，不要求本项同时完成完整IR/QEMU推理。只检查protobuf或只运行两层小图不能完成此门槛。 -### 4.2 历史本机证据 - -历史 `models/qwen3-0.6b-onnx/verification.json` 记录如下（记录生成于2026-09-29，文件在原本机模型目录；新checkout须另准备，不能假定这些文件存在）： - -| 项 | 已记录结果 | -|---|---| -| 模型revision | `c1899de289a04d12100db370d81485cdf75e47ca` | -| 原始checkpoint SHA256 | `f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b` | -| 导出配置 | dynamo、opset18、FP32、eager、无Cache，IR version10 | -| ORT | CPUExecutionProvider，threads=4 | -| 形状 / 有限值 | 有效长度5及256两输入均为 `[1,256,151936]`，有限 | -| 数值比较范围 | 既有导出验证的有效token allclose通过，atol=rtol=1e-4；此容差不是小模型IR/QEMU的1e-5门槛 | -| 短输入5有效token | valid max_abs=`2.956390380859375e-5`；全张量max_abs=`0.0021619796752929688`、full_allclose=false | -| 全长256输入 | max_abs=`5.53131103515625e-5`；full_allclose=true | -| ORT单次前向 | 历史两case约1.365s / 1.785s，非QEMU时间或标准性能benchmark | -| 尚无记录 | 峰值内存、贪心文本输出；不能声称已经完成这些观察 | -| 第二人独立复现 | 待记录 | +### 4.2 导出来源与尚存边界 -保留padding位置全张量比较失败的事实，不能只展示valid通过而写“全部位置一致”。原计划要求记录峰值内存/生成文本作为观察；核心W1门槛仍是导出、形状与ORT可执行。新门禁若只核验现有导出产物，应报告“验证已有导出”，不冒称本轮重新导出或重新做过PyTorch对照。 +固定 revision 为 `c1899de289a04d12100db370d81485cdf75e47ca`，原始 checkpoint SHA256 为 `f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b`。发布产物来自 dynamo、opset 18、FP32、eager、无 KV cache 的导出，模型身份由 manifest 固定。 -### 4.3 本轮完整产物本地重验（2026-10-02） +原导出的有效 token PyTorch/ORT 对照通过，但短输入 padding 全张量对照曾失败。该历史差异仍需保留，不能因 Linux download/verify 模式通过就写成 PyTorch 与 ORT 全位置一致。这两种模式只验证既有产物，不重新导出或生成新的 PyTorch 参考。原始数值、依赖与当时环境见 [固定提交历史归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/W1-探测方案.md)；它们不属于新的 Linux 测量。 -`output/qwen3-full-local/report.json` 的 `mode=verify`、`passed=true`，记录时间为2026-10-02 16:08:45 +08:00。本轮重新读取真实完整文件并执行ORT，没有依赖历史JSON的PASS标志。 +### 4.3 Linux 独立执行要求 -| 项 | 本轮结果 | -|---|---| -| 来源 | 同一固定revision发布产物，`model.onnx`及三个external-data分片逐文件大小/SHA256通过 | -| 结构 | 完整ONNX checker通过，opset18/IR10；311个外部tensor、三个真实分片，静态输入输出符合约定 | -| 环境 | Windows、Python3.12.13、NumPy2.2.6、ONNX1.18.0、ORT1.22.1、protobuf5.29.5 | -| ORT配置 | CPUExecutionProvider、intra-op threads=2、inter-op threads=1、图优化关闭 | -| 输入 | 新生成有效长度5/256，seed分别20260929/20260930；两输出均FP32 `[1,256,151936]`、全部有限 | -| 时间 | 两次前向约2.6663s / 2.8343s；session初始化约0.7139s；整个验证约8.7384s，非稳定性能benchmark | -| 内存 | 验证器进程 `PeakWorkingSetSize=6,401,679,360 bytes`（约6.40 GB / 5.96 GiB）；不含导出子进程，不代表QEMU模型内存 | -| 边界 | 没有重新导出，没有重跑PyTorch对照，不验证语言质量或ScratchV完整模型执行；也没有刷新历史padding差异结论 | +按 [W1 README](README.md) 在新目录运行 `download` 或 `verify`；需要重验导出数值时执行 `export` 并保留两套结果的区别。源 snapshot、模型分片及输入身份必须核对，不能读取历史 JSON 的 PASS 代替真实执行。记录 Linux 系统、工具和依赖版本，峰值内存仅按实际测量范围报告。 ### 4.4 仓库门禁 `probes/w1_qwen3_export/run.py` 是统一门禁入口，具体命令见独立复现指南。PR运行该入口的轻量回归测试；手动/定时工作流的独立 `full-qwen3-onnx` job 用download模式验证固定发布产物并真实执行ORT两case。PR的轻量测试不替代完整ORT执行；PR中重型job未执行应如实显示。入口各模式均检查哈希、结构并运行ORT，缺权重/分片、哈希不符或执行失败时非零退出，不能以“尚未交付”成功退出。 -固定revision、模型/分片哈希、输入seed/有效长度、输出shape/dtype/有限值、provider/线程数和实际执行状态写入报告。新入口还以 `source_fingerprints` 记录wrapper、导出脚本和manifest的SHA256，成功/失败报告均保留，源码不可读取时为null并附错误。§4.3的既有本地报告生成较早，尚无该字段；必须按实际报告内容核对，不能事后补造源码指纹。报告不保存完整logits数组，checkout commit仍应与CI run关联。历史 `verification.json` 只作来源说明，不能作为新运行的成功标志。 +固定revision、模型/分片哈希、输入seed/有效长度、输出shape/dtype/有限值、provider/线程数和实际执行状态写入报告。新入口还以 `source_fingerprints` 记录wrapper、导出脚本和manifest的SHA256，成功/失败报告均保留，源码不可读取时为null并附错误。历史报告必须按实际字段核对，不能事后补造源码指纹。报告不保存完整logits数组，checkout commit仍应与CI run关联。历史 `verification.json` 只作来源说明，不能作为新运行的成功标志。 2026-10-02，提交 `545e696` 的 [full-qwen3-onnx Linux任务](https://github.com/yuki-328/ScratchV/actions/runs/36983119833/job/110761955767) **已通过**：真实下载固定release ZIP，校验完整产物、执行ORT两case，并成功上传 `qwen3-full-onnx-probe-output` artifact。这是该提交的完整 ONNX 交付物验证，不是重新导出或 ScratchV 完整模型执行。已核对 `probes/w1_qwen3_export/run.py`、`manifest.json`、`export_qwen3_onnx.py` 在 `545e696..5ea22ec` 无变化，因此该结果仍可作为当前已提交验证入口的既有证据；不写成 `5ea22ec` 又执行了一次重型任务。后续若修改这些源码、manifest 或依赖，应复跑受影响门禁。 -已取回的Linux artifact记录 `mode=download`、`download.reused=false`；下载阶段约32.216s，总验证40.193s，ORT两次前向约2.6874s/2.7554s。验证器进程峰值RSS为 **6,409,670,656 bytes（约6.41 GB）**，来自Linux `getrusage(RUSAGE_SELF).ru_maxrss`，与§4.3本地Windows约6.40 GB分开记录，均不包含导出子进程。`source_fingerprints` 三项已与提交545e696的对应源码blob核对一致。 +已取回的Linux artifact记录 `mode=download`、`download.reused=false`；下载阶段约32.216s，总验证40.193s，ORT两次前向约2.6874s/2.7554s。验证器进程峰值RSS为 **6,409,670,656 bytes（约6.41 GB）**，来自Linux `getrusage(RUSAGE_SELF).ru_maxrss`，不包含导出子进程。`source_fingerprints` 三项已与提交545e696的对应源码blob核对一致。 ## 5. 环境、CI与团队验收 | 检查 | 当前状态 / 下一步 | |---|---| -| Windows工具与真实执行 | 已通过；便携Zig/QEMU，不需要WSL | | 当前基线 Linux 部署工具链 | `5ea22ec` [部署工作流](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369623)通过 MatMul、官方小模型 IR 与 28 次 QEMU；固定 release 完整 ORT 有独立历史通过记录 | | 当前基线 Linux 主 CI / benchmark | `5ea22ec` [主工作流](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/36984369611)通过；全量测试 2371 passed、6 skipped。AI review 通过不替代人工验收 | | IR-only既有CI | 只证明相应提交和IR路径，不能外推新增QEMU路径 | -| 完整ONNX重型门禁 | 本地verify及545e696的Linux download重型任务通过，见§4.4；PR轻量回归不替代新的重型运行 | +| 完整ONNX重型门禁 | 545e696 的 Linux download 重型任务通过，见§4.4；PR轻量回归不替代新的重型运行 | | docs检查 | 只验证结构，不证明接口决定获批 | | 第二人复现 | 按README模板记录，不能用第一人重复跑/代理并行测试替代 | 原runner的sudo/HTTPS故障属于历史环境证据，不能假定仍然存在或已解决；以当前run为准。推进时先跑MatMul定位工具/数值链路，再运行真实结构小图，最后记录完整ONNX gate与人工签字。任一错误都保留报告，明确阶段和首次偏差。 -暂留阶段的修复曾通过2415项全仓测试、7次MatMul和28次两层QEMU执行，并重验完整ONNX ORT。本次将这些修复与新增仿真耗时统计一并集成到 PR #91；暂留阶段的历史结果、40项skip原因、数值与待决策事项见[本地审查报告](W1-本地收尾与审查报告.md)，新增改动的验证另行记录，不能沿用旧提交的 Linux CI 结论。 +个人工作树收尾测量见 [固定提交历史归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/W1-探测方案.md)；修改后的 Linux 验证需要对应 SHA 的新记录，不沿用旧 CI 结论。 ## 6. 当前结论与变更记录 diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\234\254\345\234\260\346\224\266\345\260\276\344\270\216\345\256\241\346\237\245\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\234\254\345\234\260\346\224\266\345\260\276\344\270\216\345\256\241\346\237\245\346\212\245\345\221\212.md" index c062a79..eb0935b 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\234\254\345\234\260\346\224\266\345\260\276\344\270\216\345\256\241\346\237\245\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\234\254\345\234\260\346\224\266\345\260\276\344\270\216\345\256\241\346\237\245\346\212\245\345\221\212.md" @@ -3,6 +3,8 @@ 日期：2026-10-02。基线：`5ea22ecc7fd314025f6453c556dbb3e3f16a3175`。 本地暂留阶段分支：`codex/w1-local-closeout`，不是最终 PR 分支。此前按要求完成本地收尾并保留未提交修改；此次经授权将这些修复、复现准备和新增时长记录整合更新 PR #91，其源分支为 `w2/qwen3-real-2layer-probe`。`5ea22ec` 仅为历史起点，本次发布的精确 SHA、测试和 CI 见 PR 说明及对应运行报告，不能沿用历史 PASS。 +正式交付和独立复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。文件名保留以兼容已有引用。 + ## 1. 结论与完成范围 此前本地阶段完成文档对齐、复现准备和明确缺陷修复，并重新执行本地数值探测。本次集成进一步补入成功/失败/超时可见的 QEMU 时长记录；最终结果须按本次提交的报告验收。 @@ -31,30 +33,11 @@ W1 仍不能标记为正式关闭：第二名成员独立复现、团队接口/ 全仓初次回归暴露一项旧测试依赖过时的检查顺序：缺工具测试使用空模型目录，现在会先在产物校验阶段失败。该测试已隔离前置产物检查，继续验证工具缺失的退出码、失败阶段和报告保留；产物错误由新的独立用例覆盖。 -## 3. 历史本地阶段验证 - -下表记录时长功能加入前的本地收尾验证，是本次集成的已有证据，不是新增计时代码或当前提交的最终验证。本次实测见 PR 说明和当前报告；受影响回归与真实 QEMU 探测需要重新执行。 - -| 验证 | 实际结果 | 本地证据（仓库根目录下） | -|---|---|---| -| 全量 `tests/` | **2415 passed、0 failed、40 skipped** | `output/w1-local-all-tests-final.log`、同名 `.xml` | -| 固定CPU环境的Qwen/导出相关回归 | **133 passed、0 skipped** | `output/w1-probe-closeout-20261002-cpu.xml` | -| 环境预检回归 | **26 passed**；真实环境ready=true | `output/w1-local-preflight-review-tests.xml`、`output/w1-local-preflight-reviewed/report.json` | -| 原PR89合成图 | ORT/NumPy max_abs约1.097e-6；IR/ORT约7.153e-7，均通过 | `output/w1-local-synthetic/report.json` | -| FP32 MatMul | **7次QEMU均通过**，最大绝对误差2.384185791015625e-7 | `output/w1-local-matmul/report.json` | -| 真实Qwen3两层小配置 | 新导出；7输入、29检查点、PyTorch/ORT/IR全部通过 | `output/w1-local-qwen3-small/report.json/md/html` | -| 新增后端与优化集成 | **4构建、28次QEMU均通过**；QEMU最大绝对误差1.6689300537109375e-6，优化后IR最大绝对误差1.9073486328125e-6 | `output/w1-local-qwen3-riscv/report.json/md/html` | -| 空输出Gather真实RV64专项 | 4次执行通过：正/负越界、正/负合法索引恢复 | `output/tensor-review-empty-gather-rv64/report.json` | -| 完整0.6B固定ONNX产物 | verify模式：4文件哈希、311个外部tensor、checker、两组ORT执行通过 | `output/w1-local-full-onnx/report.json/md` | -| 文档及workflow | 四接口/风险检查、链接检查、YAML与17段shell语法检查通过 | `scripts/check_llm_deploy_docs.py`、`scripts/check_docs_links.py`、`output/w1-local-workflow-check/` | - -全量40个skip中，38个需要本机没有的Linux用户态`qemu-riscv32`，另2个为通用环境缺Torch的模块，后者在固定CPU环境的133项回归中执行。回归集合有重叠，不相加作为总用例数。当前主机实际运行了系统态`qemu-system-riscv64`，两者不可混淆。 +## 3. Linux 验证与历史归档 -所有小模型数值门槛仍为`max_abs < 1e-5`、`rtol=0`。上述历史本地阶段的完整 ONNX verify 进程峰值 RSS 为 6,401,183,744 bytes，约 5.96 GiB；不含重新导出子进程。 -该阶段没有重新导出完整模型、没有重新生成全模型 PyTorch 参考，也没有执行完整 ScratchV IR/QEMU。 -导出验证器修复通过真实小型ONNX/ORT测试检查错误参考输入；它不能被描述为完整0.6B重新导出成功。 +个人工作树的原始回归计数、内存和耗时保留于 [固定提交历史归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/W1-本地收尾与审查报告.md)，不转换成 Linux 测量，也不视为新提交执行。已有 Linux 的 MatMul、两层 QEMU 和完整 ONNX ORT run 见 [W1 README](README.md) 第 5 节。后续复现按 Linux 指南在指定 SHA 重新运行，保存自己的报告与数组。 -本地 L2 harness 文件 `.claude/harness/verify/run.py` 不存在，未宣称执行。上表所处的历史阶段，新增 CI 步骤只完成本地语法与对应命令检查，尚未在远端执行；本次集成后的 Linux 结论以对应提交的 Checks 为准。`--require-clean` 在当时 dirty 工作树按设计应失败，本地数值测试保留修改状态。 +所有小模型数值门槛为 `max_abs < 1e-5`、`rtol=0`；完整 ONNX download/verify 不等于重新导出或完整模型 PyTorch 对比。Linux CI 和作者执行不能替代第二人确认。 ### 3.1 本次新增时长记录的读取与边界 @@ -62,23 +45,9 @@ W1 仍不能标记为正式关闭：第二名成员独立复现、团队接口/ 数值成功/失败、运行失败、超时、已尝试但未启动、尚未尝试等状态应与时长一起查看。QEMU 进程墙钟包含启动、guest 计算、输出打包/UART 和退出；超时还包含子进程清理，不含 host 输入准备/输出解码，不是纯 forward 计时；流水线时间不含最终报告渲染。失败保留已测耗时，无测量为 null；未启动不能记成零秒成功，汇总不能把未完成的尝试写成 28 次通过。CI Summary 展示相同 Markdown 数据，原始 JSON/HTML 随 artifact 保留。当前不据此给出加速比或性能通过结论。 -### 3.2 本次集成修改的本地复验 - -2026-10-02，在包含本次计时实现的工作树执行（提交前验证；源码和产物哈希由报告保留）： - -| 验证 | 结果 | 证据 | -|---|---|---| -| 全仓回归 | **2441 passed、0 failed、40 skipped**；skip 分类与上文相同 | `output/pr91-timing-all.log/xml` | -| 固定 CPU 环境回归 | **171 passed、0 skipped**，含真实模型导出与上游产物绑定 | `output/pr91-timing-cpu.log/xml` | -| 两层 QEMU | **4 构建、28 次通过**；最大绝对误差 **1.6689300537109375e-6** | `output/pr91-timing-qwen3-riscv/report.json/md/html` | -| MatMul | **7 次通过** | `output/pr91-timing-matmul/report.json` | -| 超时端到端负例 | 刻意设定 0.01 秒；进程非零退出，报告保留实际耗时、1 次 timeout、27 次未尝试、部分结果 | `output/pr91-timing-timeout/report.json/md/html` | -| 环境预检 | ready=true；提交前 dirty 状态如实记录 | `output/pr91-timing-env/report.json` | -| 完整已有 ONNX | verify 模式通过；仍不等于重新导出或完整模型 PyTorch 对比 | `output/pr91-timing-full-onnx/report.json/md` | - -本轮两层探测流程 **150.798 秒**，QEMU 进程合计 **139.350 秒**，成功交叉编译合计 **5.247 秒**。normal 的 none/all 平均分别为 **0.996/0.989 秒**，diagnostic 为 **9.130/8.792 秒**。这是一次运行的观察数据，期间本机也运行其他验证任务，不用于稳定性能或优化加速结论；诊断图的额外计算及输出开销包含在内。不同测试集合有重叠，不相加。 +### 3.2 Linux 复验记录 -受影响源码已经交叉审查，文档检查、workflow YAML/17 段 Bash 语法及 diff 空白检查通过。远端 CI 必须以更新后 PR head 的 Checks 为准；后续第二人验收仍须独立检出该 SHA 复现。 +运行前记录 SHA、dirty/diff 和 Linux 环境；运行后保留原始报告、退出码及 source/model/input/ELF 身份。完整七项 W2 的 Linux 入口与已记录 Linux CI 见 [W2 README](../W2/README.md)。不同批次、重叠集合或历史提交的计数不可相加，也不能迁移成当前提交结论。 ## 4. 留待决定或后续实施 diff --git a/docs/llm-deploy-v1.0/W1/interfaces.md b/docs/llm-deploy-v1.0/W1/interfaces.md index bfdf308..d69e7d5 100644 --- a/docs/llm-deploy-v1.0/W1/interfaces.md +++ b/docs/llm-deploy-v1.0/W1/interfaces.md @@ -1,5 +1,7 @@ # 接口基线 v1.1：待团队冻结确认 +正式交付、CI 和他人复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。命令中的 `python` 指已激活的 `.venv-linux/bin/python`。 + > 上游：[开发计划.md](../开发计划.md) §2.2、§4.3 W1。 > 状态（2026-10-03）：**按当前实现核对的候选基线，团队确认未完成**。PR #91 已合并；W2 运行时扩展与 W1 报告修复分开交付，D1–D8 不因此自动通过。 > `docs:interfaces` 只检查文档结构；“有版本号”不等于冻结会议已通过。正式决议见 §7。 diff --git a/docs/llm-deploy-v1.0/W1/risks.md b/docs/llm-deploy-v1.0/W1/risks.md index acd810a..c7ba7ff 100644 --- a/docs/llm-deploy-v1.0/W1/risks.md +++ b/docs/llm-deploy-v1.0/W1/risks.md @@ -25,7 +25,7 @@ | # | 风险 | 概率 | 影响 | 当前状态 / 证据 | 触发信号 | Plan B | 责任 | |---|---|---|---|---|---|---|---| -| R9 | 完整 Qwen3 ONNX 导出/载入失败 | 本地及指定CI已验证 | 高 | 历史导出、本地verify和545e696 Linux download重型任务通过；验证器峰值RSS分别为Windows约6.40 GB、Linux约6.41 GB，均非导出总峰值 | 缺/损坏分片、版本漂移、shape 不符或 ORT 失败 | 固定 revision/依赖与哈希，从已批准来源重新准备产物；必要时重导出。子图只能作定位工具，不代替完整导出 gate | E1/E5 | +| R9 | 完整 Qwen3 ONNX 导出/载入失败 | 本地及指定CI已验证 | 高 | 历史导出及 545e696 Linux download 重型任务通过；Linux 验证器峰值 RSS 约 6.41 GB，非导出总峰值 | 缺/损坏分片、版本漂移、shape 不符或 ORT 失败 | 固定 revision/依赖与哈希，从已批准来源重新准备产物；必要时重导出。子图只能作定位工具，不代替完整导出 gate | E1/E5 | | R10 | 完整权重/激活超出内存布局 | 当前配置容量不足已确认 | 高 | guest 512 MiB，constant/workspace 默认各上限 256 MiB；完整 logits 约 148.4 MiB，权重分片合计 2,384,201,728 bytes（约 2.22 GiB）。权重当前展开为 C 字面量；完整装载和 host IR 峰值均未验收 | C 源码/编译资源膨胀、arena/ELF 与输入区冲突、容量超限或 host/guest OOM | 先测各阶段峰值和完整预算，设计代码权重分离、地址/绑定校验及生命周期；评估 IR 权重复制和中间值驻留，检查点按需保存。mmap 须另有 Linux 路线；不能只放宽上限或增加 -m 就声称解决 | E2/E3/E4，W3–W4 | | R11 | L=256 无 KV Cache 前向耗时过长 | 待测完整规模 | 中 | 小模型 QEMU 时间只用于功能回归，不代表硬件性能 | 完整前向超时或生成循环不可用 | 记录真实耗时，按算子分块/优化；短 L 仅作诊断且另标配置，不降低固定 L=256 验收 | E3/E5 | | R12 | 151936×1024 LM Head 计算/输出开销 | 配置已确认 | 高 | 小词表128不覆盖完整词表开销 | 完整 LM Head 超时或输出传输过大 | 保留完整 logits oracle；生产生成路径可另设计最后位置/分块接口，并与完整输出对照，不能替换 W1 完整形状 gate | E3/E4 | diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.html" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.html" index 0077c1c..11fe9dd 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.html" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.html" @@ -1,674 +1,8 @@ - - -ONNX 算子支持与正确性对照 -
-

ONNX 算子支持与正确性：修改前后对照

-

归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。

-

总验收：通过。修改后 24/24 个用例通过；原先不支持、现通过的用例 20 个，原先错误、现通过的用例 2 个，出现回归的用例 0 个。用例数不等于新增算子种类数。

-
-
24 / 24修改后通过
-
20原先不支持、现通过的用例数
-
2原先错误、现通过的用例数
-
0出现回归的用例数
-

本次改动扩展算子支持并修正计算语义；计时用于本地观测，不代表性能优化结论。只有修改前后均 PASS 的用例才计算执行耗时比。不支持或缺少有效测量时显示“—”，不能解释为零耗时或无限加速。

-

基线兼容适配与输入绑定

-

输入绑定兼容适配不属于基线编译器实现；以下为 runner 的实际策略，需与数值结果一起审阅。

-

Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change.

-

按组通过情况

-
分组用例总数修改前 PASS修改后 PASS
独立算子160/1616/16
已有支持对照22/22/2
错误回归20/22/2
组合链路40/44/4
-

版本与测量条件

-
项目值
报告 schema1
基线提交20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6
基线源码 SHA-256a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c
当前 HEAD20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6
当前包含未提交改动是
当前源码 SHA-25657184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf
执行顺序ABBA：修改前 → 修改后 → 修改后 → 修改前
每版本轮数2
每轮预热次数1
每轮正式计时次数5
每版本正式样本总数10
计时统计合并各轮正式样本后计算中位数（ms），预热不计入；原始秒数见用例详情
runner 计时说明parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU.
-

执行计时覆盖完整 IRInterpreter.run，包括 IR 校验与输入/权重复制；不含 ONNX 解析和 ONNX Runtime 参考计算。ONNX → IR 解析单独计时。

-
运行环境 -
项目值
blas{"Build Dependencies": {"blas": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}, "lapack": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}}, "Compilers": {"c": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "c++": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "cython": {"commands": "cython", "linker": "cython", "name": "cython", "version": "3.3.0"}}, "Machine Information": {"build": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}, "host": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}}, "Python Information": {"version": "3.12"}, "SIMD Extensions": {"baseline": ["X86_V2"], "found": ["X86_V3"]}}
numpy2.5.3
onnx1.23.0
onnxruntime1.29.0
ortCPUExecutionProvider, sequential, one thread, optimizations disabled
platformWindows-11-10.0.26200-SP0
python3.12.13
threads{"MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1"}
-
-

全部用例

耗时比 = 修改前 / 修改后；大于 1 表示该次观测中修改后较快，小于 1 表示较慢。

- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
用例分组修改前修改后前最大绝对误差后最大绝对误差前解析 ms后解析 ms前执行 ms后执行 ms执行耗时比 前/后
add_float32前后均通过已有支持对照PASSPASS0.000e+000.000e+000.12710.37710.09700.10250.946×
matmul_32前后均通过已有支持对照PASSPASS0.000e+000.000e+000.12690.36940.09960.10740.927×
gather原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3609—0.1014—
sqrt原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3480—0.0959—
reduce_mean原先不支持 → 现通过独立算子UNSUPPORTEDPASS—2.980e-08—0.3785—0.1077—
transpose原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3687—0.1007—
concat原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3693—0.0978—
slice原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3966—0.1037—
unsqueeze原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3842—0.0848—
expand原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3765—0.1045—
abs原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3495—0.0859—
cast原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3603—0.0802—
constant原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3612—0.0613—
cos原先不支持 → 现通过独立算子UNSUPPORTEDPASS—5.960e-08—0.3408—0.0865—
identity原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3380—0.0520—
pow原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.4307—0.1330—
reciprocal原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3487—0.0909—
sin原先不支持 → 现通过独立算子UNSUPPORTEDPASS—0.000e+00—0.3411—0.0864—
scalar_fp32_add_chain原先错误 → 现正确错误回归NUMERIC_ERRORPASS2.000e+000.000e+00—0.4024—0.1478—
singleton_initializer原先错误 → 现正确错误回归EXECUTION_ERRORPASS—0.000e+00—0.3394—0.0623—
constant_cast_abs_expand原先不支持 → 现通过组合链路UNSUPPORTEDPASS—0.000e+00—0.4442—0.1341—
qwen_rmsnorm原先不支持 → 现通过组合链路UNSUPPORTEDPASS—0.000e+00—0.5022—0.2821—
rope_numeric原先不支持 → 现通过组合链路UNSUPPORTEDPASS—2.980e-08—0.5084—0.2474—
pr89_transformer_2l原先不支持 → 现通过组合链路UNSUPPORTEDPASS—7.153e-07—4.5905—10.1120—

用例与错误详情

-
add_float32 · 前后均通过 -

FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs.

-
项目值
算子["Add"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-2561a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba
输入 SHA-2568d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036
参考输出 SHA-256 (expected_sha256)9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00012459990102797747, 0.00012260000221431255, 0.00012610002886503935, 0.00012240000069141388, 0.00011799996718764305, 0.00017060001846402884, 0.00017589994240552187, 0.0001556000206619501, 0.00013669999316334724, 0.0001281999284401536]
执行原始样本 s[9.820004925131798e-05, 9.109999518841505e-05, 9.620003402233124e-05, 9.129999671131372e-05, 8.999998681247234e-05, 0.0001316999550908804, 0.0001227000029757619, 0.00012240000069141388, 9.77999297901988e-05, 9.400001727044582e-05]
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0004192000487819314, 0.00039460009429603815, 0.00038620003033429384, 0.0003771999618038535, 0.00038330000825226307, 0.00037699996028095484, 0.00035340001340955496, 0.00035879993811249733, 0.00034790008794516325, 0.0003451999509707093]
执行原始样本 s[0.00011930009350180626, 0.00010579999070614576, 0.00010310008656233549, 0.00010159995872527361, 0.0001004999503493309, 0.00010600010864436626, 0.00010289996862411499, 0.0001020999625325203, 0.00010160007514059544, 0.00010059995111078024]
-
-
-
matmul_32 · 前后均通过 -

FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs.

-
项目值
算子["MatMul"]
绝对容差 atol1e-05
相对容差 rtol1e-05
模型 SHA-256c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203
输入 SHA-2563c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0
参考输出 SHA-256 (expected_sha256)d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[32, 32] / float32
参考 shape / dtype[32, 32] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00012470001820474863, 0.00012370001059025526, 0.0001233000075444579, 0.0001382000045850873, 0.00012390001211315393, 0.00015219999477267265, 0.00013090006541460752, 0.0001327999634668231, 0.00012919993605464697, 0.00012370001059025526]
执行原始样本 s[0.00010169995948672295, 9.799993131309748e-05, 9.8800053820014e-05, 9.50000248849392e-05, 9.69000393524766e-05, 0.00011190003715455532, 0.00010029994882643223, 0.00010699999984353781, 0.00010060006752610207, 9.679992217570543e-05]
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[32, 32] / float32
参考 shape / dtype[32, 32] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.000394199974834919, 0.00036610011011362076, 0.00036529998760670424, 0.0003619999624788761, 0.00037210003938525915, 0.00037340004928410053, 0.00036669999826699495, 0.0003709999145939946, 0.00040339992847293615, 0.000367900007404387]
执行原始样本 s[0.00011040002573281527, 0.00010629999451339245, 0.00010939990170300007, 0.00010389997623860836, 0.00011880008969455957, 0.00010979990474879742, 0.00010659999679774046, 0.00010810000821948051, 0.00010469998233020306, 0.0001050999853760004]
-
-
-
gather · 原先不支持 → 现通过 -

FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4].

-
项目值
算子["Gather"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e
输入 SHA-256dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf
参考输出 SHA-256 (expected_sha256)4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Gather
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3, 4] / float32
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.000382099999114871, 0.0003637999761849642, 0.00036070006899535656, 0.00037329993210732937, 0.0003640999784693122, 0.0003535000141710043, 0.0003445000620558858, 0.00034570007119327784, 0.00036100007127970457, 0.000354600022546947]
执行原始样本 s[0.00010419997852295637, 0.00010269996710121632, 0.00010080006904900074, 0.0001020999625325203, 0.00010219996329396963, 0.00010059995111078024, 0.00010079995263367891, 0.00010109995491802692, 0.00010179996024817228, 0.00010019994806498289]
-
-
-
sqrt · 原先不支持 → 现通过 -

Positive FP32 x[2,4] -> Sqrt [2,4].

-
项目值
算子["Sqrt"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97
输入 SHA-256dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6
参考输出 SHA-256 (expected_sha256)2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Sqrt
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0003466999623924494, 0.0003605000674724579, 0.0003635999746620655, 0.00034799997229129076, 0.00034380005672574043, 0.00040040002204477787, 0.0003725000424310565, 0.0003456000704318285, 0.0003480000887066126, 0.0003408000338822603]
执行原始样本 s[0.00010339997243136168, 9.490002412348986e-05, 9.290000889450312e-05, 9.370001498609781e-05, 9.099999442696571e-05, 9.990006219595671e-05, 9.940005838871002e-05, 9.69000393524766e-05, 9.69000393524766e-05, 9.26000066101551e-05]
-
-
-
reduce_mean · 原先不支持 → 现通过 -

FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1].

-
项目值
算子["ReduceMean"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-25620a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132
输入 SHA-256037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17
参考输出 SHA-256 (expected_sha256)43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7
初始化器 SHA-256 (initializers_sha256)31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3, 1] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: ReduceMean
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3, 1] / float32
参考 shape / dtype[2, 3, 1] / float32
最大绝对误差2.980e-08
最大相对误差9.933e-08
执行 IR 步数2
解析原始样本 s[0.0003934999695047736, 0.00037949997931718826, 0.00037489994429051876, 0.000372299924492836, 0.00037619995418936014, 0.0003952999832108617, 0.00040259992238134146, 0.0003915999550372362, 0.0003774999640882015, 0.0003698000218719244]
执行原始样本 s[0.00011030002497136593, 0.00010720000136643648, 0.0001078000059351325, 0.00010830000974237919, 0.00010569998994469643, 0.00010750000365078449, 0.00010809989180415869, 0.00010909989941865206, 0.00010730000212788582, 0.00010609999299049377]
-
-
-
transpose · 原先不支持 → 现通过 -

FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4].

-
项目值
算子["Transpose"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6
输入 SHA-256c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a
参考输出 SHA-256 (expected_sha256)b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 3, 2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Transpose
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 3, 2, 4] / float32
参考 shape / dtype[1, 3, 2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0003711999161168933, 0.00037619995418936014, 0.0003662999952211976, 0.00041259999852627516, 0.0003730000462383032, 0.00035260000731796026, 0.0003722000401467085, 0.0003524000057950616, 0.0003497001016512513, 0.0003529998939484358]
执行原始样本 s[0.00010429997928440571, 0.00010159995872527361, 9.989994578063488e-05, 0.00010140007361769676, 0.00010170007590204477, 0.00010349997319281101, 9.93999419733882e-05, 9.840005077421665e-05, 9.790004696696997e-05, 9.710004087537527e-05]
-
-
-
concat · 原先不支持 → 现通过 -

FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4].

-
项目值
算子["Concat"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977
输入 SHA-25609f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9
参考输出 SHA-256 (expected_sha256)c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Concat
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3, 4] / float32
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0003693000180646777, 0.00038620003033429384, 0.0003693000180646777, 0.0003645999822765589, 0.00036700000055134296, 0.00041009997949004173, 0.0003698000218719244, 0.00037619995418936014, 0.00035749992821365595, 0.00035550002939999104]
执行原始样本 s[9.770004544407129e-05, 0.00010369997471570969, 9.69000393524766e-05, 9.790004696696997e-05, 9.599991608411074e-05, 0.00010299996938556433, 9.889993816614151e-05, 9.699992369860411e-05, 9.910005610436201e-05, 9.550002869218588e-05]
-
-
-
slice · 原先不支持 → 现通过 -

FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4].

-
项目值
算子["Slice"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30
输入 SHA-25604457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa
参考输出 SHA-256 (expected_sha256)15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df
初始化器 SHA-256 (initializers_sha256)6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Slice
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00040770007763057947, 0.0004149000160396099, 0.00039679999463260174, 0.000395399983972311, 0.000394199974834919, 0.0004005999071523547, 0.0003872000379487872, 0.0003947000950574875, 0.00039820000529289246, 0.0003963999915868044]
执行原始样本 s[0.00010409997776150703, 0.00010469998233020306, 0.00010289996862411499, 0.00010169995948672295, 0.00010259996633976698, 0.00010469998233020306, 0.00010290008503943682, 0.0001039999770000577, 0.00010349997319281101, 0.00010409997776150703]
-
-
-
unsqueeze · 原先不支持 → 现通过 -

INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1].

-
项目值
算子["Unsqueeze"]
绝对容差 atol0.0
相对容差 rtol0.0
模型 SHA-2563af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd
输入 SHA-2564a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50
参考输出 SHA-256 (expected_sha256)d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193
初始化器 SHA-256 (initializers_sha256)a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 2, 3, 1] / int64
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Unsqueeze
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 2, 3, 1] / int64
参考 shape / dtype[1, 2, 3, 1] / int64
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0004278999986127019, 0.00037889997474849224, 0.0003894000547006726, 0.0004752000095322728, 0.00046479993034154177, 0.00037020002491772175, 0.00035470002330839634, 0.0003568999236449599, 0.0003622000804170966, 0.0003958999877795577]
执行原始样本 s[8.939998224377632e-05, 8.649996016174555e-05, 8.329993579536676e-05, 8.98001017048955e-05, 8.560006972402334e-05, 8.510006591677666e-05, 8.279993198812008e-05, 8.359993807971478e-05, 8.44000605866313e-05, 8.329993579536676e-05]
-
-
-
expand · 原先不支持 → 现通过 -

FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4].

-
项目值
算子["Expand"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-25665a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab
输入 SHA-2564fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6
参考输出 SHA-256 (expected_sha256)eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6
初始化器 SHA-256 (initializers_sha256)7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Expand
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3, 4] / float32
参考 shape / dtype[2, 3, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0004062999505549669, 0.0003779999678954482, 0.0004357000580057502, 0.00038139999378472567, 0.0004078999627381563, 0.0003739000530913472, 0.0003749999450519681, 0.0003727000439539552, 0.00036589999217540026, 0.000359599944204092]
执行原始样本 s[0.00011890009045600891, 0.00010760000441223383, 0.00010830000974237919, 0.00010639999527484179, 0.0001023000804707408, 0.00010429997928440571, 0.00010469998233020306, 0.00010419997852295637, 0.00010289996862411499, 0.00010410009417682886]
-
-
-
abs · 原先不支持 → 现通过 -

Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype.

-
项目值
算子["Abs"]
绝对容差 atol0.0
相对容差 rtol0.0
模型 SHA-2562446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf
输入 SHA-256898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a
参考输出 SHA-256 (expected_sha256)7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3] / int64
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Abs
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3] / int64
参考 shape / dtype[2, 3] / int64
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00036099995486438274, 0.0003568999236449599, 0.0003541000187397003, 0.0003519000019878149, 0.0003469999646767974, 0.00033569999504834414, 0.00035450002178549767, 0.00034719996619969606, 0.0003350999904796481, 0.000327100045979023]
执行原始样本 s[8.889997843652964e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.420005906373262e-05, 8.320005144923925e-05, 8.759996853768826e-05, 8.549995254725218e-05, 8.499994874000549e-05, 8.68999632075429e-05, 8.470006287097931e-05]
-
-
-
cast · 原先不支持 → 现通过 -

INT64 position IDs[1,8] -> FP32 [1,8].

-
项目值
算子["Cast"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5
输入 SHA-25693f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f
参考输出 SHA-256 (expected_sha256)5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 8] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Cast
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 8] / float32
参考 shape / dtype[1, 8] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0003632999723777175, 0.00036179996095597744, 0.0004943999228999019, 0.0005131000652909279, 0.00037509994581341743, 0.00034440006129443645, 0.00033980002626776695, 0.00034899997990578413, 0.00033760000951588154, 0.00035889993887394667]
执行原始样本 s[7.98999099060893e-05, 0.00017800007481127977, 0.00011080002877861261, 8.82000895217061e-05, 8.230004459619522e-05, 7.829989772289991e-05, 7.860001642256975e-05, 7.840001489967108e-05, 8.060003165155649e-05, 7.970002479851246e-05]
-
-
-
constant · 原先不支持 → 现通过 -

Constant tensor attribute FP32 [2,3]; no runtime inputs.

-
项目值
算子["Constant"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227
输入 SHA-2568739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
参考输出 SHA-256 (expected_sha256)5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Constant
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3] / float32
参考 shape / dtype[2, 3] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数1
解析原始样本 s[0.0003632999723777175, 0.000381799996830523, 0.00039549998473376036, 0.000381799996830523, 0.0003598000621423125, 0.00036259996704757214, 0.0003591999411582947, 0.0003554000286385417, 0.0003502999898046255, 0.0003415999235585332]
执行原始样本 s[6.250001024454832e-05, 6.170000415295362e-05, 6.10999995842576e-05, 6.200000643730164e-05, 6.0399994254112244e-05, 6.150000263005495e-05, 6.219989154487848e-05, 6.0399994254112244e-05, 6.049999501556158e-05, 5.909998435527086e-05]
-
-
-
cos · 原先不支持 → 现通过 -

FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4].

-
项目值
算子["Cos"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-2563af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298
输入 SHA-2568c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998
参考输出 SHA-256 (expected_sha256)7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Cos
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差5.960e-08
最大相对误差8.024e-08
执行 IR 步数2
解析原始样本 s[0.0003530000103637576, 0.00034549995325505733, 0.00033980002626776695, 0.000340400030836463, 0.0003451999509707093, 0.00034739996772259474, 0.000332699972204864, 0.0003412999212741852, 0.00033599999733269215, 0.00033199996687471867]
执行原始样本 s[8.690007962286472e-05, 8.670007809996605e-05, 8.889997843652964e-05, 8.739996701478958e-05, 8.559995330870152e-05, 8.60999571159482e-05, 8.629995863884687e-05, 8.690007962286472e-05, 8.579995483160019e-05, 8.400005754083395e-05]
-
-
-
identity · 原先不支持 → 现通过 -

INT64 indices[2,3] -> Identity [2,3]; frontend alias.

-
项目值
算子["Identity"]
绝对容差 atol0.0
相对容差 rtol0.0
模型 SHA-256e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b
输入 SHA-25619f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413
参考输出 SHA-256 (expected_sha256)a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3] / int64
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Identity
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3] / int64
参考 shape / dtype[2, 3] / int64
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数1
解析原始样本 s[0.000341000035405159, 0.00034380005672574043, 0.0003355999942868948, 0.0003361001145094633, 0.0003378000110387802, 0.00033820001408457756, 0.0003301999531686306, 0.00032760004978626966, 0.0003744999412447214, 0.0003630999708548188]
执行原始样本 s[5.170004442334175e-05, 5.280005279928446e-05, 5.219993181526661e-05, 5.2200048230588436e-05, 5.080003757029772e-05, 5.179992876946926e-05, 5.079992115497589e-05, 5.139992572367191e-05, 6.809993647038937e-05, 5.4600066505372524e-05]
-
-
-
pow · 原先不支持 → 现通过 -

Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3].

-
项目值
算子["Pow"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-2569184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771
输入 SHA-256b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1
参考输出 SHA-256 (expected_sha256)dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Pow
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3] / float32
参考 shape / dtype[2, 3] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.0004491000436246395, 0.00043999997433274984, 0.00040019990410655737, 0.00037569995038211346, 0.00038789992686361074, 0.0003453000681474805, 0.00048369995784014463, 0.00044920004438608885, 0.00047619990073144436, 0.0004214000655338168]
执行原始样本 s[0.00014999997802078724, 0.00014180003199726343, 0.00011959997937083244, 0.00011729996185749769, 0.00011809996794909239, 0.00011589995119720697, 0.00015820004045963287, 0.00015029998030513525, 0.00015840004198253155, 0.0001241000136360526]
-
-
-
reciprocal · 原先不支持 → 现通过 -

Nonzero signed FP32 x[2,4] -> Reciprocal [2,4].

-
项目值
算子["Reciprocal"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-2563d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3
输入 SHA-256fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f
参考输出 SHA-256 (expected_sha256)c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Reciprocal
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00035170000046491623, 0.0003496999852359295, 0.00034989998675882816, 0.00034769997000694275, 0.00033870001789182425, 0.00037759996484965086, 0.00035119999665766954, 0.0003469000803306699, 0.00033660000190138817, 0.00033329997677356005]
执行原始样本 s[9.190000128000975e-05, 9.019998833537102e-05, 9.029998909682035e-05, 9.14999982342124e-05, 8.899997919797897e-05, 9.609991684556007e-05, 9.510002564638853e-05, 9.06999921426177e-05, 9.109999518841505e-05, 9.059999138116837e-05]
-
-
-
sin · 原先不支持 → 现通过 -

FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4].

-
项目值
算子["Sin"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a
输入 SHA-256efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e
参考输出 SHA-256 (expected_sha256)e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 4] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Sin
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 4] / float32
参考 shape / dtype[2, 4] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数2
解析原始样本 s[0.00034879997838288546, 0.0003532000118866563, 0.00035470002330839634, 0.0003448999486863613, 0.00033730000723153353, 0.00033710000570863485, 0.0003510999958962202, 0.0003322999691590667, 0.00032819993793964386, 0.000327700050547719]
执行原始样本 s[8.570007048547268e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.480006363242865e-05, 8.730008266866207e-05, 8.71000811457634e-05, 8.659996092319489e-05, 8.569995407015085e-05, 8.390005677938461e-05, 8.750008419156075e-05]
-
-
-
scalar_fp32_add_chain · 原先错误 → 现正确 -

FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output.

-
项目值
算子["Add"]
绝对容差 atol0.0
相对容差 rtol0.0
模型 SHA-2564ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7
输入 SHA-2568739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
参考输出 SHA-256 (expected_sha256)87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a
初始化器 SHA-256 (initializers_sha256)32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a
-
-

修改前 NUMERIC_ERROR

-
项目值
状态 / 阶段NUMERIC_ERROR / compare
实际 shape / dtype[] / float32
参考 shape / dtype[] / float32
最大绝对误差2.000e+00
最大相对误差1.192e-07
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

NumericMismatch: output differs from ORT: max_abs_error=2
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[] / float32
参考 shape / dtype[] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数5
解析原始样本 s[0.0005631999811157584, 0.0004637000383809209, 0.0004089000867679715, 0.00039579998701810837, 0.0003945999778807163, 0.000381799996830523, 0.00041089998558163643, 0.00040959997568279505, 0.0003945999778807163, 0.0003902000607922673]
执行原始样本 s[0.00019599997904151678, 0.00015169999096542597, 0.00014780007768422365, 0.0001476999605074525, 0.0001452999422326684, 0.00015129998791962862, 0.00014889996964484453, 0.0001441999338567257, 0.00014589994680136442, 0.0001444999361410737]
-
-
-
singleton_initializer · 原先错误 → 现正确 -

FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar.

-
项目值
算子[]
绝对容差 atol0.0
相对容差 rtol0.0
模型 SHA-25652473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722
输入 SHA-2568739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
参考输出 SHA-256 (expected_sha256)3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8
初始化器 SHA-256 (initializers_sha256)7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df
-
-

修改前 EXECUTION_ERROR

-
项目值
状态 / 阶段EXECUTION_ERROR / execute
实际 shape / dtype— / —
参考 shape / dtype[1] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1] / float32
参考 shape / dtype[1] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数1
解析原始样本 s[0.0003521000035107136, 0.00034039991442114115, 0.00033650000113993883, 0.0003445999464020133, 0.0003336999798193574, 0.0003494999837130308, 0.0003339999821037054, 0.00035889993887394667, 0.00032849994022399187, 0.00033849989995360374]
执行原始样本 s[6.290001329034567e-05, 6.280001252889633e-05, 6.140000186860561e-05, 6.009999196976423e-05, 6.079999729990959e-05, 6.370001938194036e-05, 6.200000643730164e-05, 6.330001633614302e-05, 6.260001100599766e-05, 6.0399994254112244e-05]
-
-
-
constant_cast_abs_expand · 原先不支持 → 现通过 -

Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3].

-
项目值
算子["Constant", "Cast", "Abs", "Expand"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3
输入 SHA-2569b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822
参考输出 SHA-256 (expected_sha256)eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[2, 3] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Constant
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[2, 3] / float32
参考 shape / dtype[2, 3] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数4
解析原始样本 s[0.00043970008846372366, 0.00044329999946057796, 0.0004450000124052167, 0.0004384000785648823, 0.0006199999479576945, 0.00047199998516589403, 0.00045169994700700045, 0.00044700002763420343, 0.00044139998499304056, 0.00043539993930608034]
执行原始样本 s[0.00013560010120272636, 0.0001338999718427658, 0.00013189995661377907, 0.0002367999404668808, 0.00017560005653649569, 0.00013229995965957642, 0.0001343999756500125, 0.000130399945192039, 0.00013109995052218437, 0.00013479997869580984]
-
-
-
qwen_rmsnorm · 原先不支持 → 现通过 -

FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8].

-
项目值
算子["Pow", "ReduceMean", "Add", "Sqrt", "Reciprocal", "Mul"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-256cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237
输入 SHA-256f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48
参考输出 SHA-256 (expected_sha256)5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec
初始化器 SHA-256 (initializers_sha256)6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 4, 8] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Pow
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 4, 8] / float32
参考 shape / dtype[1, 4, 8] / float32
最大绝对误差0.000e+00
最大相对误差0.000e+00
执行 IR 步数10
解析原始样本 s[0.000506900018081069, 0.0005015999777242541, 0.0005096999229863286, 0.0005014999769628048, 0.0004960000514984131, 0.0005388999124988914, 0.0005028999876230955, 0.000512199942022562, 0.000493900035507977, 0.0004851999692618847]
执行原始样本 s[0.0002813000464811921, 0.0002788000274449587, 0.0002808000426739454, 0.00028170004952698946, 0.00027820002287626266, 0.0002930000191554427, 0.00028490007389336824, 0.00028249993920326233, 0.0002837000647559762, 0.0002848999574780464]
-
-
-
rope_numeric · 原先不支持 → 现通过 -

INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8].

-
项目值
算子["Cast", "Mul", "Cos", "Sin", "Sub", "Add", "Concat"]
绝对容差 atol1e-06
相对容差 rtol1e-06
模型 SHA-2567aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa
输入 SHA-256823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a
参考输出 SHA-256 (expected_sha256)eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd
初始化器 SHA-256 (initializers_sha256)8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 4, 8] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Cast
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 4, 8] / float32
参考 shape / dtype[1, 4, 8] / float32
最大绝对误差2.980e-08
最大相对误差1.699e-07
执行 IR 步数12
解析原始样本 s[0.0005266999360173941, 0.000520200002938509, 0.0005088000325486064, 0.0006041000597178936, 0.0006553001003339887, 0.0005080000264570117, 0.0005031999899074435, 0.0005010999739170074, 0.0005029999883845448, 0.00049919995944947]
执行原始样本 s[0.0002498000394552946, 0.00024560000747442245, 0.0002465000143274665, 0.0003208999987691641, 0.00031370006036013365, 0.00025370006915181875, 0.0002449000021442771, 0.00024610001128166914, 0.0002482000272721052, 0.0002460000105202198]
-
-
-
pr89_transformer_2l · 原先不支持 → 现通过 -

PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3

-
项目值
算子["Add", "Concat", "Div", "Expand", "Gather", "MatMul", "Mul", "Neg", "ReduceMean", "Reshape", "Sigmoid", "Slice", "Softmax", "Sqrt", "Transpose", "Unsqueeze"]
绝对容差 atol1e-05
相对容差 rtol0
模型 SHA-25617a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1
输入 SHA-256bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457
参考输出 SHA-256 (expected_sha256)413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c
初始化器 SHA-256 (initializers_sha256)955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de
-
-

修改前 UNSUPPORTED

-
项目值
状态 / 阶段UNSUPPORTED / parse
实际 shape / dtype— / —
参考 shape / dtype[1, 256, 128] / float32
最大绝对误差—
最大相对误差—
执行 IR 步数—
解析原始样本 s[]
执行原始样本 s[]
-

错误详情

ONNXParseError: Unsupported ONNX op type: Gather
-
-

修改后 PASS

-
项目值
状态 / 阶段PASS / complete
实际 shape / dtype[1, 256, 128] / float32
参考 shape / dtype[1, 256, 128] / float32
最大绝对误差7.153e-07
最大相对误差5.722e-04
执行 IR 步数159
解析原始样本 s[0.004760700045153499, 0.004594899946823716, 0.004717700066976249, 0.004990800051018596, 0.004586199996992946, 0.004221900017000735, 0.004203699994832277, 0.004572000005282462, 0.004674999974668026, 0.004224899923428893]
执行原始样本 s[0.010629900032654405, 0.011460799956694245, 0.009812200092710555, 0.009985200013034046, 0.010765800019726157, 0.009604100021533668, 0.010238799965009093, 0.009660700103268027, 0.010715999989770353, 0.009805700043216348]
-
-
-
独立 HTML 报告，无外部资源依赖。误差、状态及版本指纹由同一份 runner JSON 提供。
- + +ONNX 算子 benchmark Linux 复跑入口 +

ONNX 算子 benchmark：Linux 复跑入口

+

正式环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。本页是入口，不是新测量报告。

+

请按环境与单测指南准备依赖、固定 Git 基线和 LFS 模型，再按Linux benchmark 命令生成自己的 JSON、Markdown 和 HTML。

+

当前同名 JSON 为 historical_benchmark_index，不能作为 benchmark 结果。历史个人测量不改名为 Linux 数据，也不能与不同环境的耗时直接计算加速比。

+

查看固定提交中的原始 HTML 测量

+ diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" index 9b68647..1ce68a3 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.json" @@ -1,2388 +1,14 @@ { + "document_kind": "historical_benchmark_index", "schema_version": 1, - "baseline": { - "commit": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "source_sha256": "a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c" + "measurement_available": false, + "reproduction_target": "Ubuntu 24.04 x86_64 / Bash / Python 3.12", + "guide": "算子补齐与单测-benchmark.md", + "original_commit": "2d07ccc42936dcfde19d552bca02a324cd33821c", + "historical_reports": { + "md": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md", + "json": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json", + "html": "https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html" }, - "current": { - "head": "20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6", - "dirty": true, - "source_sha256": "57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf" - }, - "environment": { - "python": "3.12.13", - "platform": "Windows-11-10.0.26200-SP0", - "numpy": "2.5.3", - "onnx": "1.23.0", - "onnxruntime": "1.29.0", - "threads": { - "OMP_NUM_THREADS": "1", - "OPENBLAS_NUM_THREADS": "1", - "MKL_NUM_THREADS": "1", - "NUMEXPR_NUM_THREADS": "1", - "VECLIB_MAXIMUM_THREADS": "1" - }, - "blas": { - "Compilers": { - "c": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - }, - "cython": { - "name": "cython", - "linker": "cython", - "version": "3.3.0", - "commands": "cython" - }, - "c++": { - "name": "msvc", - "linker": "link", - "version": "19.44.35228", - "commands": "cl" - } - }, - "Machine Information": { - "host": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - }, - "build": { - "cpu": "x86_64", - "family": "x86_64", - "endian": "little", - "system": "windows" - } - }, - "Build Dependencies": { - "blas": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - }, - "lapack": { - "name": "scipy-openblas", - "found": true, - "version": "0.3.34.106.0", - "detection method": "pkgconfig", - "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24" - } - }, - "Python Information": { - "version": "3.12" - }, - "SIMD Extensions": { - "baseline": [ - "X86_V2" - ], - "found": [ - "X86_V3" - ] - } - }, - "ort": "CPUExecutionProvider, sequential, one thread, optimizations disabled" - }, - "warmup": 1, - "repeats": 10, - "repeats_per_round": 5, - "rounds_per_variant": 2, - "execution_order": [ - "before", - "after", - "after", - "before" - ], - "timing_scope": "parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU.", - "binding_policy": "Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change.", - "results": [ - { - "name": "add_float32", - "group": "control", - "description": "FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs.", - "operators": [ - "Add" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba", - "inputs_sha256": "8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036", - "expected_sha256": "9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012459990102797747, - 0.00012260000221431255, - 0.00012610002886503935, - 0.00012240000069141388, - 0.00011799996718764305, - 0.00017060001846402884, - 0.00017589994240552187, - 0.0001556000206619501, - 0.00013669999316334724, - 0.0001281999284401536 - ], - "run_samples_s": [ - 9.820004925131798e-05, - 9.109999518841505e-05, - 9.620003402233124e-05, - 9.129999671131372e-05, - 8.999998681247234e-05, - 0.0001316999550908804, - 0.0001227000029757619, - 0.00012240000069141388, - 9.77999297901988e-05, - 9.400001727044582e-05 - ], - "parse_median_s": 0.00012714997865259647, - "run_median_s": 9.699998190626502e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00011799996718764305, - "parse_max_s": 0.00017589994240552187, - "run_min_s": 8.999998681247234e-05, - "run_max_s": 0.0001316999550908804 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004192000487819314, - 0.00039460009429603815, - 0.00038620003033429384, - 0.0003771999618038535, - 0.00038330000825226307, - 0.00037699996028095484, - 0.00035340001340955496, - 0.00035879993811249733, - 0.00034790008794516325, - 0.0003451999509707093 - ], - "run_samples_s": [ - 0.00011930009350180626, - 0.00010579999070614576, - 0.00010310008656233549, - 0.00010159995872527361, - 0.0001004999503493309, - 0.00010600010864436626, - 0.00010289996862411499, - 0.0001020999625325203, - 0.00010160007514059544, - 0.00010059995111078024 - ], - "parse_median_s": 0.0003770999610424042, - "run_median_s": 0.00010249996557831764, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003451999509707093, - "parse_max_s": 0.0004192000487819314, - "run_min_s": 0.0001004999503493309, - "run_max_s": 0.00011930009350180626 - }, - "fixture_files": { - "model": "add_float32.onnx", - "inputs": "add_float32.inputs.npz", - "expected": "add_float32.expected.npy", - "initializers": "add_float32.initializers.npz" - } - }, - { - "name": "matmul_32", - "group": "control", - "description": "FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs.", - "operators": [ - "MatMul" - ], - "atol": 1e-05, - "rtol": 1e-05, - "model_sha256": "c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203", - "inputs_sha256": "3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0", - "expected_sha256": "d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00012470001820474863, - 0.00012370001059025526, - 0.0001233000075444579, - 0.0001382000045850873, - 0.00012390001211315393, - 0.00015219999477267265, - 0.00013090006541460752, - 0.0001327999634668231, - 0.00012919993605464697, - 0.00012370001059025526 - ], - "run_samples_s": [ - 0.00010169995948672295, - 9.799993131309748e-05, - 9.8800053820014e-05, - 9.50000248849392e-05, - 9.69000393524766e-05, - 0.00011190003715455532, - 0.00010029994882643223, - 0.00010699999984353781, - 0.00010060006752610207, - 9.679992217570543e-05 - ], - "parse_median_s": 0.0001269499771296978, - "run_median_s": 9.955000132322311e-05, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0001233000075444579, - "parse_max_s": 0.00015219999477267265, - "run_min_s": 9.50000248849392e-05, - "run_max_s": 0.00011190003715455532 - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000394199974834919, - 0.00036610011011362076, - 0.00036529998760670424, - 0.0003619999624788761, - 0.00037210003938525915, - 0.00037340004928410053, - 0.00036669999826699495, - 0.0003709999145939946, - 0.00040339992847293615, - 0.000367900007404387 - ], - "run_samples_s": [ - 0.00011040002573281527, - 0.00010629999451339245, - 0.00010939990170300007, - 0.00010389997623860836, - 0.00011880008969455957, - 0.00010979990474879742, - 0.00010659999679774046, - 0.00010810000821948051, - 0.00010469998233020306, - 0.0001050999853760004 - ], - "parse_median_s": 0.0003694499609991908, - "run_median_s": 0.00010735000250861049, - "executed_steps": 2, - "actual_shape": [ - 32, - 32 - ], - "actual_dtype": "float32", - "expected_shape": [ - 32, - 32 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003619999624788761, - "parse_max_s": 0.00040339992847293615, - "run_min_s": 0.00010389997623860836, - "run_max_s": 0.00011880008969455957 - }, - "fixture_files": { - "model": "matmul_32.onnx", - "inputs": "matmul_32.inputs.npz", - "expected": "matmul_32.expected.npy", - "initializers": "matmul_32.initializers.npz" - } - }, - { - "name": "gather", - "group": "operators", - "description": "FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4].", - "operators": [ - "Gather" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e", - "inputs_sha256": "dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf", - "expected_sha256": "4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000382099999114871, - 0.0003637999761849642, - 0.00036070006899535656, - 0.00037329993210732937, - 0.0003640999784693122, - 0.0003535000141710043, - 0.0003445000620558858, - 0.00034570007119327784, - 0.00036100007127970457, - 0.000354600022546947 - ], - "run_samples_s": [ - 0.00010419997852295637, - 0.00010269996710121632, - 0.00010080006904900074, - 0.0001020999625325203, - 0.00010219996329396963, - 0.00010059995111078024, - 0.00010079995263367891, - 0.00010109995491802692, - 0.00010179996024817228, - 0.00010019994806498289 - ], - "parse_median_s": 0.00036085007013753057, - "run_median_s": 0.0001014499575830996, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003445000620558858, - "parse_max_s": 0.000382099999114871, - "run_min_s": 0.00010019994806498289, - "run_max_s": 0.00010419997852295637 - }, - "fixture_files": { - "model": "gather.onnx", - "inputs": "gather.inputs.npz", - "expected": "gather.expected.npy", - "initializers": "gather.initializers.npz" - } - }, - { - "name": "sqrt", - "group": "operators", - "description": "Positive FP32 x[2,4] -> Sqrt [2,4].", - "operators": [ - "Sqrt" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97", - "inputs_sha256": "dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6", - "expected_sha256": "2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sqrt", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003466999623924494, - 0.0003605000674724579, - 0.0003635999746620655, - 0.00034799997229129076, - 0.00034380005672574043, - 0.00040040002204477787, - 0.0003725000424310565, - 0.0003456000704318285, - 0.0003480000887066126, - 0.0003408000338822603 - ], - "run_samples_s": [ - 0.00010339997243136168, - 9.490002412348986e-05, - 9.290000889450312e-05, - 9.370001498609781e-05, - 9.099999442696571e-05, - 9.990006219595671e-05, - 9.940005838871002e-05, - 9.69000393524766e-05, - 9.69000393524766e-05, - 9.26000066101551e-05 - ], - "parse_median_s": 0.0003480000304989517, - "run_median_s": 9.590003173798323e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003408000338822603, - "parse_max_s": 0.00040040002204477787, - "run_min_s": 9.099999442696571e-05, - "run_max_s": 0.00010339997243136168 - }, - "fixture_files": { - "model": "sqrt.onnx", - "inputs": "sqrt.inputs.npz", - "expected": "sqrt.expected.npy", - "initializers": "sqrt.initializers.npz" - } - }, - { - "name": "reduce_mean", - "group": "operators", - "description": "FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1].", - "operators": [ - "ReduceMean" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132", - "inputs_sha256": "037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17", - "expected_sha256": "43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7", - "initializers_sha256": "31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: ReduceMean", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 9.933077874436559e-08, - "parse_samples_s": [ - 0.0003934999695047736, - 0.00037949997931718826, - 0.00037489994429051876, - 0.000372299924492836, - 0.00037619995418936014, - 0.0003952999832108617, - 0.00040259992238134146, - 0.0003915999550372362, - 0.0003774999640882015, - 0.0003698000218719244 - ], - "run_samples_s": [ - 0.00011030002497136593, - 0.00010720000136643648, - 0.0001078000059351325, - 0.00010830000974237919, - 0.00010569998994469643, - 0.00010750000365078449, - 0.00010809989180415869, - 0.00010909989941865206, - 0.00010730000212788582, - 0.00010609999299049377 - ], - "parse_median_s": 0.0003784999717026949, - "run_median_s": 0.0001076500047929585, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003698000218719244, - "parse_max_s": 0.00040259992238134146, - "run_min_s": 0.00010569998994469643, - "run_max_s": 0.00011030002497136593 - }, - "fixture_files": { - "model": "reduce_mean.onnx", - "inputs": "reduce_mean.inputs.npz", - "expected": "reduce_mean.expected.npy", - "initializers": "reduce_mean.initializers.npz" - } - }, - { - "name": "transpose", - "group": "operators", - "description": "FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4].", - "operators": [ - "Transpose" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6", - "inputs_sha256": "c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a", - "expected_sha256": "b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Transpose", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003711999161168933, - 0.00037619995418936014, - 0.0003662999952211976, - 0.00041259999852627516, - 0.0003730000462383032, - 0.00035260000731796026, - 0.0003722000401467085, - 0.0003524000057950616, - 0.0003497001016512513, - 0.0003529998939484358 - ], - "run_samples_s": [ - 0.00010429997928440571, - 0.00010159995872527361, - 9.989994578063488e-05, - 0.00010140007361769676, - 0.00010170007590204477, - 0.00010349997319281101, - 9.93999419733882e-05, - 9.840005077421665e-05, - 9.790004696696997e-05, - 9.710004087537527e-05 - ], - "parse_median_s": 0.00036874995566904545, - "run_median_s": 0.00010065000969916582, - "executed_steps": 2, - "actual_shape": [ - 1, - 3, - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 3, - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003497001016512513, - "parse_max_s": 0.00041259999852627516, - "run_min_s": 9.710004087537527e-05, - "run_max_s": 0.00010429997928440571 - }, - "fixture_files": { - "model": "transpose.onnx", - "inputs": "transpose.inputs.npz", - "expected": "transpose.expected.npy", - "initializers": "transpose.initializers.npz" - } - }, - { - "name": "concat", - "group": "operators", - "description": "FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4].", - "operators": [ - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977", - "inputs_sha256": "09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9", - "expected_sha256": "c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Concat", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003693000180646777, - 0.00038620003033429384, - 0.0003693000180646777, - 0.0003645999822765589, - 0.00036700000055134296, - 0.00041009997949004173, - 0.0003698000218719244, - 0.00037619995418936014, - 0.00035749992821365595, - 0.00035550002939999104 - ], - "run_samples_s": [ - 9.770004544407129e-05, - 0.00010369997471570969, - 9.69000393524766e-05, - 9.790004696696997e-05, - 9.599991608411074e-05, - 0.00010299996938556433, - 9.889993816614151e-05, - 9.699992369860411e-05, - 9.910005610436201e-05, - 9.550002869218588e-05 - ], - "parse_median_s": 0.0003693000180646777, - "run_median_s": 9.780004620552063e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00035550002939999104, - "parse_max_s": 0.00041009997949004173, - "run_min_s": 9.550002869218588e-05, - "run_max_s": 0.00010369997471570969 - }, - "fixture_files": { - "model": "concat.onnx", - "inputs": "concat.inputs.npz", - "expected": "concat.expected.npy", - "initializers": "concat.initializers.npz" - } - }, - { - "name": "slice", - "group": "operators", - "description": "FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4].", - "operators": [ - "Slice" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30", - "inputs_sha256": "04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa", - "expected_sha256": "15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df", - "initializers_sha256": "6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Slice", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00040770007763057947, - 0.0004149000160396099, - 0.00039679999463260174, - 0.000395399983972311, - 0.000394199974834919, - 0.0004005999071523547, - 0.0003872000379487872, - 0.0003947000950574875, - 0.00039820000529289246, - 0.0003963999915868044 - ], - "run_samples_s": [ - 0.00010409997776150703, - 0.00010469998233020306, - 0.00010289996862411499, - 0.00010169995948672295, - 0.00010259996633976698, - 0.00010469998233020306, - 0.00010290008503943682, - 0.0001039999770000577, - 0.00010349997319281101, - 0.00010409997776150703 - ], - "parse_median_s": 0.00039659999310970306, - "run_median_s": 0.00010374997509643435, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003872000379487872, - "parse_max_s": 0.0004149000160396099, - "run_min_s": 0.00010169995948672295, - "run_max_s": 0.00010469998233020306 - }, - "fixture_files": { - "model": "slice.onnx", - "inputs": "slice.inputs.npz", - "expected": "slice.expected.npy", - "initializers": "slice.initializers.npz" - } - }, - { - "name": "unsqueeze", - "group": "operators", - "description": "INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1].", - "operators": [ - "Unsqueeze" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd", - "inputs_sha256": "4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50", - "expected_sha256": "d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193", - "initializers_sha256": "a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Unsqueeze", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004278999986127019, - 0.00037889997474849224, - 0.0003894000547006726, - 0.0004752000095322728, - 0.00046479993034154177, - 0.00037020002491772175, - 0.00035470002330839634, - 0.0003568999236449599, - 0.0003622000804170966, - 0.0003958999877795577 - ], - "run_samples_s": [ - 8.939998224377632e-05, - 8.649996016174555e-05, - 8.329993579536676e-05, - 8.98001017048955e-05, - 8.560006972402334e-05, - 8.510006591677666e-05, - 8.279993198812008e-05, - 8.359993807971478e-05, - 8.44000605866313e-05, - 8.329993579536676e-05 - ], - "parse_median_s": 0.00038415001472458243, - "run_median_s": 8.475006325170398e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 2, - 3, - 1 - ], - "actual_dtype": "int64", - "expected_shape": [ - 1, - 2, - 3, - 1 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00035470002330839634, - "parse_max_s": 0.0004752000095322728, - "run_min_s": 8.279993198812008e-05, - "run_max_s": 8.98001017048955e-05 - }, - "fixture_files": { - "model": "unsqueeze.onnx", - "inputs": "unsqueeze.inputs.npz", - "expected": "unsqueeze.expected.npy", - "initializers": "unsqueeze.initializers.npz" - } - }, - { - "name": "expand", - "group": "operators", - "description": "FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4].", - "operators": [ - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab", - "inputs_sha256": "4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6", - "expected_sha256": "eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6", - "initializers_sha256": "7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Expand", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004062999505549669, - 0.0003779999678954482, - 0.0004357000580057502, - 0.00038139999378472567, - 0.0004078999627381563, - 0.0003739000530913472, - 0.0003749999450519681, - 0.0003727000439539552, - 0.00036589999217540026, - 0.000359599944204092 - ], - "run_samples_s": [ - 0.00011890009045600891, - 0.00010760000441223383, - 0.00010830000974237919, - 0.00010639999527484179, - 0.0001023000804707408, - 0.00010429997928440571, - 0.00010469998233020306, - 0.00010419997852295637, - 0.00010289996862411499, - 0.00010410009417682886 - ], - "parse_median_s": 0.00037649995647370815, - "run_median_s": 0.00010449998080730438, - "executed_steps": 2, - "actual_shape": [ - 2, - 3, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000359599944204092, - "parse_max_s": 0.0004357000580057502, - "run_min_s": 0.0001023000804707408, - "run_max_s": 0.00011890009045600891 - }, - "fixture_files": { - "model": "expand.onnx", - "inputs": "expand.inputs.npz", - "expected": "expand.expected.npy", - "initializers": "expand.initializers.npz" - } - }, - { - "name": "abs", - "group": "operators", - "description": "Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype.", - "operators": [ - "Abs" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf", - "inputs_sha256": "898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a", - "expected_sha256": "7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Abs", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00036099995486438274, - 0.0003568999236449599, - 0.0003541000187397003, - 0.0003519000019878149, - 0.0003469999646767974, - 0.00033569999504834414, - 0.00035450002178549767, - 0.00034719996619969606, - 0.0003350999904796481, - 0.000327100045979023 - ], - "run_samples_s": [ - 8.889997843652964e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.420005906373262e-05, - 8.320005144923925e-05, - 8.759996853768826e-05, - 8.549995254725218e-05, - 8.499994874000549e-05, - 8.68999632075429e-05, - 8.470006287097931e-05 - ], - "parse_median_s": 0.0003495499840937555, - "run_median_s": 8.585001341998577e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.000327100045979023, - "parse_max_s": 0.00036099995486438274, - "run_min_s": 8.320005144923925e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "abs.onnx", - "inputs": "abs.inputs.npz", - "expected": "abs.expected.npy", - "initializers": "abs.initializers.npz" - } - }, - { - "name": "cast", - "group": "operators", - "description": "INT64 position IDs[1,8] -> FP32 [1,8].", - "operators": [ - "Cast" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5", - "inputs_sha256": "93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f", - "expected_sha256": "5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.00036179996095597744, - 0.0004943999228999019, - 0.0005131000652909279, - 0.00037509994581341743, - 0.00034440006129443645, - 0.00033980002626776695, - 0.00034899997990578413, - 0.00033760000951588154, - 0.00035889993887394667 - ], - "run_samples_s": [ - 7.98999099060893e-05, - 0.00017800007481127977, - 0.00011080002877861261, - 8.82000895217061e-05, - 8.230004459619522e-05, - 7.829989772289991e-05, - 7.860001642256975e-05, - 7.840001489967108e-05, - 8.060003165155649e-05, - 7.970002479851246e-05 - ], - "parse_median_s": 0.00036034994991496205, - "run_median_s": 8.02499707788229e-05, - "executed_steps": 2, - "actual_shape": [ - 1, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033760000951588154, - "parse_max_s": 0.0005131000652909279, - "run_min_s": 7.829989772289991e-05, - "run_max_s": 0.00017800007481127977 - }, - "fixture_files": { - "model": "cast.onnx", - "inputs": "cast.inputs.npz", - "expected": "cast.expected.npy", - "initializers": "cast.initializers.npz" - } - }, - { - "name": "constant", - "group": "operators", - "description": "Constant tensor attribute FP32 [2,3]; no runtime inputs.", - "operators": [ - "Constant" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003632999723777175, - 0.000381799996830523, - 0.00039549998473376036, - 0.000381799996830523, - 0.0003598000621423125, - 0.00036259996704757214, - 0.0003591999411582947, - 0.0003554000286385417, - 0.0003502999898046255, - 0.0003415999235585332 - ], - "run_samples_s": [ - 6.250001024454832e-05, - 6.170000415295362e-05, - 6.10999995842576e-05, - 6.200000643730164e-05, - 6.0399994254112244e-05, - 6.150000263005495e-05, - 6.219989154487848e-05, - 6.0399994254112244e-05, - 6.049999501556158e-05, - 5.909998435527086e-05 - ], - "parse_median_s": 0.00036120001459494233, - "run_median_s": 6.130000110715628e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003415999235585332, - "parse_max_s": 0.00039549998473376036, - "run_min_s": 5.909998435527086e-05, - "run_max_s": 6.250001024454832e-05 - }, - "fixture_files": { - "model": "constant.onnx", - "inputs": "constant.inputs.npz", - "expected": "constant.expected.npy", - "initializers": "constant.initializers.npz" - } - }, - { - "name": "cos", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4].", - "operators": [ - "Cos" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298", - "inputs_sha256": "8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998", - "expected_sha256": "7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cos", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 5.960464477539063e-08, - "max_rel_error": 8.023689783612319e-08, - "parse_samples_s": [ - 0.0003530000103637576, - 0.00034549995325505733, - 0.00033980002626776695, - 0.000340400030836463, - 0.0003451999509707093, - 0.00034739996772259474, - 0.000332699972204864, - 0.0003412999212741852, - 0.00033599999733269215, - 0.00033199996687471867 - ], - "run_samples_s": [ - 8.690007962286472e-05, - 8.670007809996605e-05, - 8.889997843652964e-05, - 8.739996701478958e-05, - 8.559995330870152e-05, - 8.60999571159482e-05, - 8.629995863884687e-05, - 8.690007962286472e-05, - 8.579995483160019e-05, - 8.400005754083395e-05 - ], - "parse_median_s": 0.0003408499760553241, - "run_median_s": 8.650001836940646e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033199996687471867, - "parse_max_s": 0.0003530000103637576, - "run_min_s": 8.400005754083395e-05, - "run_max_s": 8.889997843652964e-05 - }, - "fixture_files": { - "model": "cos.onnx", - "inputs": "cos.inputs.npz", - "expected": "cos.expected.npy", - "initializers": "cos.initializers.npz" - } - }, - { - "name": "identity", - "group": "operators", - "description": "INT64 indices[2,3] -> Identity [2,3]; frontend alias.", - "operators": [ - "Identity" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b", - "inputs_sha256": "19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413", - "expected_sha256": "a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Identity", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000341000035405159, - 0.00034380005672574043, - 0.0003355999942868948, - 0.0003361001145094633, - 0.0003378000110387802, - 0.00033820001408457756, - 0.0003301999531686306, - 0.00032760004978626966, - 0.0003744999412447214, - 0.0003630999708548188 - ], - "run_samples_s": [ - 5.170004442334175e-05, - 5.280005279928446e-05, - 5.219993181526661e-05, - 5.2200048230588436e-05, - 5.080003757029772e-05, - 5.179992876946926e-05, - 5.079992115497589e-05, - 5.139992572367191e-05, - 6.809993647038937e-05, - 5.4600066505372524e-05 - ], - "parse_median_s": 0.0003380000125616789, - "run_median_s": 5.1999930292367935e-05, - "executed_steps": 1, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "int64", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "int64", - "parse_min_s": 0.00032760004978626966, - "parse_max_s": 0.0003744999412447214, - "run_min_s": 5.079992115497589e-05, - "run_max_s": 6.809993647038937e-05 - }, - "fixture_files": { - "model": "identity.onnx", - "inputs": "identity.inputs.npz", - "expected": "identity.expected.npy", - "initializers": "identity.initializers.npz" - } - }, - { - "name": "pow", - "group": "operators", - "description": "Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3].", - "operators": [ - "Pow" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771", - "inputs_sha256": "b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1", - "expected_sha256": "dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0004491000436246395, - 0.00043999997433274984, - 0.00040019990410655737, - 0.00037569995038211346, - 0.00038789992686361074, - 0.0003453000681474805, - 0.00048369995784014463, - 0.00044920004438608885, - 0.00047619990073144436, - 0.0004214000655338168 - ], - "run_samples_s": [ - 0.00014999997802078724, - 0.00014180003199726343, - 0.00011959997937083244, - 0.00011729996185749769, - 0.00011809996794909239, - 0.00011589995119720697, - 0.00015820004045963287, - 0.00015029998030513525, - 0.00015840004198253155, - 0.0001241000136360526 - ], - "parse_median_s": 0.00043070001993328333, - "run_median_s": 0.00013295002281665802, - "executed_steps": 2, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0003453000681474805, - "parse_max_s": 0.00048369995784014463, - "run_min_s": 0.00011589995119720697, - "run_max_s": 0.00015840004198253155 - }, - "fixture_files": { - "model": "pow.onnx", - "inputs": "pow.inputs.npz", - "expected": "pow.expected.npy", - "initializers": "pow.initializers.npz" - } - }, - { - "name": "reciprocal", - "group": "operators", - "description": "Nonzero signed FP32 x[2,4] -> Reciprocal [2,4].", - "operators": [ - "Reciprocal" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3", - "inputs_sha256": "fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f", - "expected_sha256": "c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Reciprocal", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00035170000046491623, - 0.0003496999852359295, - 0.00034989998675882816, - 0.00034769997000694275, - 0.00033870001789182425, - 0.00037759996484965086, - 0.00035119999665766954, - 0.0003469000803306699, - 0.00033660000190138817, - 0.00033329997677356005 - ], - "run_samples_s": [ - 9.190000128000975e-05, - 9.019998833537102e-05, - 9.029998909682035e-05, - 9.14999982342124e-05, - 8.899997919797897e-05, - 9.609991684556007e-05, - 9.510002564638853e-05, - 9.06999921426177e-05, - 9.109999518841505e-05, - 9.059999138116837e-05 - ], - "parse_median_s": 0.0003486999776214361, - "run_median_s": 9.089999366551638e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00033329997677356005, - "parse_max_s": 0.00037759996484965086, - "run_min_s": 8.899997919797897e-05, - "run_max_s": 9.609991684556007e-05 - }, - "fixture_files": { - "model": "reciprocal.onnx", - "inputs": "reciprocal.inputs.npz", - "expected": "reciprocal.expected.npy", - "initializers": "reciprocal.initializers.npz" - } - }, - { - "name": "sin", - "group": "operators", - "description": "FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4].", - "operators": [ - "Sin" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a", - "inputs_sha256": "efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e", - "expected_sha256": "e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Sin", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00034879997838288546, - 0.0003532000118866563, - 0.00035470002330839634, - 0.0003448999486863613, - 0.00033730000723153353, - 0.00033710000570863485, - 0.0003510999958962202, - 0.0003322999691590667, - 0.00032819993793964386, - 0.000327700050547719 - ], - "run_samples_s": [ - 8.570007048547268e-05, - 8.719996549189091e-05, - 8.620007429271936e-05, - 8.480006363242865e-05, - 8.730008266866207e-05, - 8.71000811457634e-05, - 8.659996092319489e-05, - 8.569995407015085e-05, - 8.390005677938461e-05, - 8.750008419156075e-05 - ], - "parse_median_s": 0.0003410999779589474, - "run_median_s": 8.640001760795712e-05, - "executed_steps": 2, - "actual_shape": [ - 2, - 4 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 4 - ], - "expected_dtype": "float32", - "parse_min_s": 0.000327700050547719, - "parse_max_s": 0.00035470002330839634, - "run_min_s": 8.390005677938461e-05, - "run_max_s": 8.750008419156075e-05 - }, - "fixture_files": { - "model": "sin.onnx", - "inputs": "sin.inputs.npz", - "expected": "sin.expected.npy", - "initializers": "sin.initializers.npz" - } - }, - { - "name": "scalar_fp32_add_chain", - "group": "regressions", - "description": "FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output.", - "operators": [ - "Add" - ], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a", - "initializers_sha256": "32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a", - "before": { - "status": "NUMERIC_ERROR", - "phase": "compare", - "error": "NumericMismatch: output differs from ORT: max_abs_error=2", - "max_abs_error": 2.0, - "max_rel_error": 1.1920928955078125e-07, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0005631999811157584, - 0.0004637000383809209, - 0.0004089000867679715, - 0.00039579998701810837, - 0.0003945999778807163, - 0.000381799996830523, - 0.00041089998558163643, - 0.00040959997568279505, - 0.0003945999778807163, - 0.0003902000607922673 - ], - "run_samples_s": [ - 0.00019599997904151678, - 0.00015169999096542597, - 0.00014780007768422365, - 0.0001476999605074525, - 0.0001452999422326684, - 0.00015129998791962862, - 0.00014889996964484453, - 0.0001441999338567257, - 0.00014589994680136442, - 0.0001444999361410737 - ], - "parse_median_s": 0.00040235003689303994, - "run_median_s": 0.00014775001909583807, - "executed_steps": 5, - "actual_shape": [], - "actual_dtype": "float32", - "expected_shape": [], - "expected_dtype": "float32", - "parse_min_s": 0.000381799996830523, - "parse_max_s": 0.0005631999811157584, - "run_min_s": 0.0001441999338567257, - "run_max_s": 0.00019599997904151678 - }, - "fixture_files": { - "model": "scalar_fp32_add_chain.onnx", - "inputs": "scalar_fp32_add_chain.inputs.npz", - "expected": "scalar_fp32_add_chain.expected.npy", - "initializers": "scalar_fp32_add_chain.initializers.npz" - } - }, - { - "name": "singleton_initializer", - "group": "regressions", - "description": "FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar.", - "operators": [], - "atol": 0.0, - "rtol": 0.0, - "model_sha256": "52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722", - "inputs_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "expected_sha256": "3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8", - "initializers_sha256": "7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df", - "before": { - "status": "EXECUTION_ERROR", - "phase": "execute", - "error": "IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.0003521000035107136, - 0.00034039991442114115, - 0.00033650000113993883, - 0.0003445999464020133, - 0.0003336999798193574, - 0.0003494999837130308, - 0.0003339999821037054, - 0.00035889993887394667, - 0.00032849994022399187, - 0.00033849989995360374 - ], - "run_samples_s": [ - 6.290001329034567e-05, - 6.280001252889633e-05, - 6.140000186860561e-05, - 6.009999196976423e-05, - 6.079999729990959e-05, - 6.370001938194036e-05, - 6.200000643730164e-05, - 6.330001633614302e-05, - 6.260001100599766e-05, - 6.0399994254112244e-05 - ], - "parse_median_s": 0.00033944990718737245, - "run_median_s": 6.230000872164965e-05, - "executed_steps": 1, - "actual_shape": [ - 1 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00032849994022399187, - "parse_max_s": 0.00035889993887394667, - "run_min_s": 6.009999196976423e-05, - "run_max_s": 6.370001938194036e-05 - }, - "fixture_files": { - "model": "singleton_initializer.onnx", - "inputs": "singleton_initializer.inputs.npz", - "expected": "singleton_initializer.expected.npy", - "initializers": "singleton_initializer.initializers.npz" - } - }, - { - "name": "constant_cast_abs_expand", - "group": "composite", - "description": "Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3].", - "operators": [ - "Constant", - "Cast", - "Abs", - "Expand" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3", - "inputs_sha256": "9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822", - "expected_sha256": "eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Constant", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.00043970008846372366, - 0.00044329999946057796, - 0.0004450000124052167, - 0.0004384000785648823, - 0.0006199999479576945, - 0.00047199998516589403, - 0.00045169994700700045, - 0.00044700002763420343, - 0.00044139998499304056, - 0.00043539993930608034 - ], - "run_samples_s": [ - 0.00013560010120272636, - 0.0001338999718427658, - 0.00013189995661377907, - 0.0002367999404668808, - 0.00017560005653649569, - 0.00013229995965957642, - 0.0001343999756500125, - 0.000130399945192039, - 0.00013109995052218437, - 0.00013479997869580984 - ], - "parse_median_s": 0.00044415000593289733, - "run_median_s": 0.00013414997374638915, - "executed_steps": 4, - "actual_shape": [ - 2, - 3 - ], - "actual_dtype": "float32", - "expected_shape": [ - 2, - 3 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00043539993930608034, - "parse_max_s": 0.0006199999479576945, - "run_min_s": 0.000130399945192039, - "run_max_s": 0.0002367999404668808 - }, - "fixture_files": { - "model": "constant_cast_abs_expand.onnx", - "inputs": "constant_cast_abs_expand.inputs.npz", - "expected": "constant_cast_abs_expand.expected.npy", - "initializers": "constant_cast_abs_expand.initializers.npz" - } - }, - { - "name": "qwen_rmsnorm", - "group": "composite", - "description": "FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8].", - "operators": [ - "Pow", - "ReduceMean", - "Add", - "Sqrt", - "Reciprocal", - "Mul" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237", - "inputs_sha256": "f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48", - "expected_sha256": "5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec", - "initializers_sha256": "6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Pow", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 0.0, - "max_rel_error": 0.0, - "parse_samples_s": [ - 0.000506900018081069, - 0.0005015999777242541, - 0.0005096999229863286, - 0.0005014999769628048, - 0.0004960000514984131, - 0.0005388999124988914, - 0.0005028999876230955, - 0.000512199942022562, - 0.000493900035507977, - 0.0004851999692618847 - ], - "run_samples_s": [ - 0.0002813000464811921, - 0.0002788000274449587, - 0.0002808000426739454, - 0.00028170004952698946, - 0.00027820002287626266, - 0.0002930000191554427, - 0.00028490007389336824, - 0.00028249993920326233, - 0.0002837000647559762, - 0.0002848999574780464 - ], - "parse_median_s": 0.0005022499826736748, - "run_median_s": 0.0002820999943651259, - "executed_steps": 10, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.0004851999692618847, - "parse_max_s": 0.0005388999124988914, - "run_min_s": 0.00027820002287626266, - "run_max_s": 0.0002930000191554427 - }, - "fixture_files": { - "model": "qwen_rmsnorm.onnx", - "inputs": "qwen_rmsnorm.inputs.npz", - "expected": "qwen_rmsnorm.expected.npy", - "initializers": "qwen_rmsnorm.initializers.npz" - } - }, - { - "name": "rope_numeric", - "group": "composite", - "description": "INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8].", - "operators": [ - "Cast", - "Mul", - "Cos", - "Sin", - "Sub", - "Add", - "Concat" - ], - "atol": 1e-06, - "rtol": 1e-06, - "model_sha256": "7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa", - "inputs_sha256": "823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a", - "expected_sha256": "eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd", - "initializers_sha256": "8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Cast", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 2.9802322387695312e-08, - "max_rel_error": 1.698927025647682e-07, - "parse_samples_s": [ - 0.0005266999360173941, - 0.000520200002938509, - 0.0005088000325486064, - 0.0006041000597178936, - 0.0006553001003339887, - 0.0005080000264570117, - 0.0005031999899074435, - 0.0005010999739170074, - 0.0005029999883845448, - 0.00049919995944947 - ], - "run_samples_s": [ - 0.0002498000394552946, - 0.00024560000747442245, - 0.0002465000143274665, - 0.0003208999987691641, - 0.00031370006036013365, - 0.00025370006915181875, - 0.0002449000021442771, - 0.00024610001128166914, - 0.0002482000272721052, - 0.0002460000105202198 - ], - "parse_median_s": 0.000508400029502809, - "run_median_s": 0.00024735002079978585, - "executed_steps": 12, - "actual_shape": [ - 1, - 4, - 8 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 4, - 8 - ], - "expected_dtype": "float32", - "parse_min_s": 0.00049919995944947, - "parse_max_s": 0.0006553001003339887, - "run_min_s": 0.0002449000021442771, - "run_max_s": 0.0003208999987691641 - }, - "fixture_files": { - "model": "rope_numeric.onnx", - "inputs": "rope_numeric.inputs.npz", - "expected": "rope_numeric.expected.npy", - "initializers": "rope_numeric.initializers.npz" - } - }, - { - "name": "pr89_transformer_2l", - "group": "composite", - "operators": [ - "Add", - "Concat", - "Div", - "Expand", - "Gather", - "MatMul", - "Mul", - "Neg", - "ReduceMean", - "Reshape", - "Sigmoid", - "Slice", - "Softmax", - "Sqrt", - "Transpose", - "Unsqueeze" - ], - "description": "PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3", - "atol": 1e-05, - "rtol": 0, - "model_sha256": "17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1", - "inputs_sha256": "bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457", - "expected_sha256": "413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c", - "initializers_sha256": "955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de", - "before": { - "status": "UNSUPPORTED", - "phase": "parse", - "error": "ONNXParseError: Unsupported ONNX op type: Gather", - "max_abs_error": null, - "max_rel_error": null, - "parse_samples_s": [], - "run_samples_s": [], - "parse_median_s": null, - "run_median_s": null, - "executed_steps": null, - "actual_shape": null, - "actual_dtype": null, - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": null, - "parse_max_s": null, - "run_min_s": null, - "run_max_s": null - }, - "after": { - "status": "PASS", - "phase": "complete", - "error": null, - "max_abs_error": 7.152557373046875e-07, - "max_rel_error": 0.0005722416551695097, - "parse_samples_s": [ - 0.004760700045153499, - 0.004594899946823716, - 0.004717700066976249, - 0.004990800051018596, - 0.004586199996992946, - 0.004221900017000735, - 0.004203699994832277, - 0.004572000005282462, - 0.004674999974668026, - 0.004224899923428893 - ], - "run_samples_s": [ - 0.010629900032654405, - 0.011460799956694245, - 0.009812200092710555, - 0.009985200013034046, - 0.010765800019726157, - 0.009604100021533668, - 0.010238799965009093, - 0.009660700103268027, - 0.010715999989770353, - 0.009805700043216348 - ], - "parse_median_s": 0.004590549971908331, - "run_median_s": 0.01011199998902157, - "executed_steps": 159, - "actual_shape": [ - 1, - 256, - 128 - ], - "actual_dtype": "float32", - "expected_shape": [ - 1, - 256, - 128 - ], - "expected_dtype": "float32", - "parse_min_s": 0.004203699994832277, - "parse_max_s": 0.004990800051018596, - "run_min_s": 0.009604100021533668, - "run_max_s": 0.011460799956694245 - }, - "fixture_files": { - "model": "pr89_transformer_2l.onnx", - "inputs": "pr89_transformer_2l.inputs.npz", - "expected": "pr89_transformer_2l.expected.npy", - "initializers": "pr89_transformer_2l.initializers.npz" - } - } - ], - "passed": true, - "archive_note": "归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。" + "note": "Index only; not a benchmark measurement or a Linux execution result. Generate a new report with benchmarks.bench_onnx_operators." } diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" index 87d5a06..2428b17 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213-benchmark.md" @@ -1,1228 +1,25 @@ -# ONNX 算子支持与正确性：修改前后对照 +# ONNX 算子补齐 benchmark：Linux 复跑入口 -> 归档说明（2026-10-01）：本文件保留原始测量的全部结果、计时样本、误差、源码及数据指纹。current.head 与 current.dirty 记录测量时的工作区状态；dirty=true 不描述归档或 PR 提交后的状态。已移除本机源码快照、manifest、fixture、解释器及依赖构建目录的绝对路径；fixture_files 仅记录文件名，相关临时文件未随报告提交。复跑请在仓库根目录执行 python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5，由 CLI 重新生成 fixtures；依赖、历史基线和 LFS 准备见同目录《算子补齐与单测报告.txt》。源码 SHA-256 是测量快照的原始字节指纹，跨平台换行符可能影响该值。 +正式交付与复现使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。本页是入口，**不是新测量报告**。环境、Git 历史和 LFS 准备见 [算子补齐与单测报告](算子补齐与单测报告.txt)，通用交付约定见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。 -总验收：通过。修改后 24/24 个用例通过；原先不支持、现通过的用例 20 个，原先错误、现通过的用例 2 个，出现回归的用例 0 个。用例数不等于新增算子种类数。 +在仓库根目录激活 `.venv-operators` 后执行： -本次改动扩展算子支持并修正计算语义；计时用于本地观测，不代表性能优化结论。只有修改前后均 PASS 的用例才计算执行耗时比。不支持或缺少有效测量时显示“—”，不能解释为零耗时或无限加速。 - -## 基线兼容适配与输入绑定 - -输入绑定兼容适配不属于基线编译器实现；以下为 runner 的实际策略，需与数值结果一起审阅。 - -> Both versions receive original ONNX initializer arrays without reshaping. The current parser's arrays must match dtype/shape/bytes, and its extra Constant globals are added. The baseline parser has no initializers property; supplying this existing interpreter argument is a harness adapter, not a baseline implementation change. - -## 按组通过情况 - -| 分组 | 用例总数 | 修改前 PASS | 修改后 PASS | -| --- | --- | --- | --- | -| 独立算子 | 16 | 0/16 | 16/16 | -| 已有支持对照 | 2 | 2/2 | 2/2 | -| 错误回归 | 2 | 0/2 | 2/2 | -| 组合链路 | 4 | 0/4 | 4/4 | - -## 版本与测量条件 - -| 项目 | 值 | -| --- | --- | -| 报告 schema | 1 | -| 基线提交 | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 基线源码 SHA-256 | a189ff5bd14d9f0c534009fbfa90416a554facb4ee047fe6288c2567723a067c | -| 当前 HEAD | 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 | -| 当前包含未提交改动 | 是 | -| 当前源码 SHA-256 | 57184d30623c30ddcc0c59db74e0a37a74171ca4a85e45adf1dae1dd578ffeaf | -| 执行顺序 | ABBA：修改前 → 修改后 → 修改后 → 修改前 | -| 每版本轮数 | 2 | -| 每轮预热次数 | 1 | -| 每轮正式计时次数 | 5 | -| 每版本正式样本总数 | 10 | -| 计时统计 | 合并各轮正式样本后计算中位数（ms），预热不计入；原始秒数见用例详情 | -| runner 计时说明 | parse: fresh ONNXParser.parse per sample; run: IRInterpreter.run including verification, planning, binding copies, NumPy computation and return copy. ORT reference generation, common initializer loading, external verification and comparisons are excluded. Each round starts with one untimed correctness qualification, then the configured warmups and timed repetitions. Sequential ABBA order, two rounds per version; warmup is per round. Host CPU, not QEMU. | - -执行计时覆盖完整 IRInterpreter.run，包括 IR 校验与输入/权重复制；不含 ONNX 解析和 ONNX Runtime 参考计算。ONNX → IR 解析单独计时。 - -环境： - -| 项目 | 值 | -| --- | --- | -| blas | {"Build Dependencies": {"blas": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}, "lapack": {"detection method": "pkgconfig", "found": true, "name": "scipy-openblas", "openblas configuration": "OpenBLAS 0.3.34.106.0 USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell MAX_THREADS=24", "version": "0.3.34.106.0"}}, "Compilers": {"c": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "c++": {"commands": "cl", "linker": "link", "name": "msvc", "version": "19.44.35228"}, "cython": {"commands": "cython", "linker": "cython", "name": "cython", "version": "3.3.0"}}, "Machine Information": {"build": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}, "host": {"cpu": "x86_64", "endian": "little", "family": "x86_64", "system": "windows"}}, "Python Information": {"version": "3.12"}, "SIMD Extensions": {"baseline": ["X86_V2"], "found": ["X86_V3"]}} | -| numpy | 2.5.3 | -| onnx | 1.23.0 | -| onnxruntime | 1.29.0 | -| ort | CPUExecutionProvider, sequential, one thread, optimizations disabled | -| platform | Windows-11-10.0.26200-SP0 | -| python | 3.12.13 | -| threads | {"MKL_NUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1"} | - -## 全部用例 - -耗时比 = 修改前 / 修改后；大于 1 表示该次观测中修改后较快，小于 1 表示较慢。 - -| 用例 | 分组 | 修改前 | 修改后 | 前最大绝对误差 | 后最大绝对误差 | 前解析 ms | 后解析 ms | 前执行 ms | 后执行 ms | 执行耗时比 前/后 | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| add_float32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1271 | 0.3771 | 0.0970 | 0.1025 | 0.946× | -| matmul_32 | 已有支持对照 | PASS | PASS | 0.000e+00 | 0.000e+00 | 0.1269 | 0.3694 | 0.0996 | 0.1074 | 0.927× | -| gather | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3609 | — | 0.1014 | — | -| sqrt | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3480 | — | 0.0959 | — | -| reduce_mean | 独立算子 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.3785 | — | 0.1077 | — | -| transpose | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3687 | — | 0.1007 | — | -| concat | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3693 | — | 0.0978 | — | -| slice | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3966 | — | 0.1037 | — | -| unsqueeze | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3842 | — | 0.0848 | — | -| expand | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3765 | — | 0.1045 | — | -| abs | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3495 | — | 0.0859 | — | -| cast | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3603 | — | 0.0802 | — | -| constant | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3612 | — | 0.0613 | — | -| cos | 独立算子 | UNSUPPORTED | PASS | — | 5.960e-08 | — | 0.3408 | — | 0.0865 | — | -| identity | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3380 | — | 0.0520 | — | -| pow | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4307 | — | 0.1330 | — | -| reciprocal | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3487 | — | 0.0909 | — | -| sin | 独立算子 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.3411 | — | 0.0864 | — | -| scalar_fp32_add_chain | 错误回归 | NUMERIC_ERROR | PASS | 2.000e+00 | 0.000e+00 | — | 0.4024 | — | 0.1478 | — | -| singleton_initializer | 错误回归 | EXECUTION_ERROR | PASS | — | 0.000e+00 | — | 0.3394 | — | 0.0623 | — | -| constant_cast_abs_expand | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.4442 | — | 0.1341 | — | -| qwen_rmsnorm | 组合链路 | UNSUPPORTED | PASS | — | 0.000e+00 | — | 0.5022 | — | 0.2821 | — | -| rope_numeric | 组合链路 | UNSUPPORTED | PASS | — | 2.980e-08 | — | 0.5084 | — | 0.2474 | — | -| pr89_transformer_2l | 组合链路 | UNSUPPORTED | PASS | — | 7.153e-07 | — | 4.5905 | — | 10.1120 | — | - -## 用例与错误详情 - -
add_float32 · 前后均通过 - -FP32 x[2,4] + bias[4] -> [2,4]; both tensors are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 1a1905eb33b4789ee579dbd7d5115bb6b0a8a873debffe328373747fdabf31ba | -| 输入 SHA-256 | 8d89e6f89fd7071d90cac47c89d8ded538f4b4aae762624573a6a39e6955a036 | -| 参考输出 SHA-256 (expected_sha256) | 9e9c9737447763b3066c69b737e0d08b98cfb8790a5b17b01d46887bc9dc84da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012459990102797747, 0.00012260000221431255, 0.00012610002886503935, 0.00012240000069141388, 0.00011799996718764305, 0.00017060001846402884, 0.00017589994240552187, 0.0001556000206619501, 0.00013669999316334724, 0.0001281999284401536] | -| 执行原始样本 s | [9.820004925131798e-05, 9.109999518841505e-05, 9.620003402233124e-05, 9.129999671131372e-05, 8.999998681247234e-05, 0.0001316999550908804, 0.0001227000029757619, 0.00012240000069141388, 9.77999297901988e-05, 9.400001727044582e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004192000487819314, 0.00039460009429603815, 0.00038620003033429384, 0.0003771999618038535, 0.00038330000825226307, 0.00037699996028095484, 0.00035340001340955496, 0.00035879993811249733, 0.00034790008794516325, 0.0003451999509707093] | -| 执行原始样本 s | [0.00011930009350180626, 0.00010579999070614576, 0.00010310008656233549, 0.00010159995872527361, 0.0001004999503493309, 0.00010600010864436626, 0.00010289996862411499, 0.0001020999625325203, 0.00010160007514059544, 0.00010059995111078024] | - -
- -
matmul_32 · 前后均通过 - -FP32 x[32,32] @ w[32,32] -> [32,32]; both matrices are runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["MatMul"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 1e-05 | -| 模型 SHA-256 | c973f94731864d1635c9bddb470d5c4cd94f0419f1f94128cb4898969e28b203 | -| 输入 SHA-256 | 3c92a5b12a96c82867e7404232374651a9dd92c1d9d7797c1139f311f082f0a0 | -| 参考输出 SHA-256 (expected_sha256) | d5c0096f9040d6b2358be41e18aff6195c7fda4791f54e38a0f7267c4461285a | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00012470001820474863, 0.00012370001059025526, 0.0001233000075444579, 0.0001382000045850873, 0.00012390001211315393, 0.00015219999477267265, 0.00013090006541460752, 0.0001327999634668231, 0.00012919993605464697, 0.00012370001059025526] | -| 执行原始样本 s | [0.00010169995948672295, 9.799993131309748e-05, 9.8800053820014e-05, 9.50000248849392e-05, 9.69000393524766e-05, 0.00011190003715455532, 0.00010029994882643223, 0.00010699999984353781, 0.00010060006752610207, 9.679992217570543e-05] | - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [32, 32] / float32 | -| 参考 shape / dtype | [32, 32] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000394199974834919, 0.00036610011011362076, 0.00036529998760670424, 0.0003619999624788761, 0.00037210003938525915, 0.00037340004928410053, 0.00036669999826699495, 0.0003709999145939946, 0.00040339992847293615, 0.000367900007404387] | -| 执行原始样本 s | [0.00011040002573281527, 0.00010629999451339245, 0.00010939990170300007, 0.00010389997623860836, 0.00011880008969455957, 0.00010979990474879742, 0.00010659999679774046, 0.00010810000821948051, 0.00010469998233020306, 0.0001050999853760004] | - -
- -
gather · 原先不支持 → 现通过 - -FP32 table[8,4], INT64 indices[2,3] including -1; axis=0 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Gather"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 698871efdf7efb0830f4743bcad6bdf661ea15cb920effbb6625d9b314442f6e | -| 输入 SHA-256 | dc4e4b0e613fdd64091fc2d9960083e55125cd961bab6c40151bca09801a71cf | -| 参考输出 SHA-256 (expected_sha256) | 4bd588e72cbd3de896e1fe60ba48ef5969f272be01c201c4ecfe0e8c7306c974 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.000382099999114871, 0.0003637999761849642, 0.00036070006899535656, 0.00037329993210732937, 0.0003640999784693122, 0.0003535000141710043, 0.0003445000620558858, 0.00034570007119327784, 0.00036100007127970457, 0.000354600022546947] | -| 执行原始样本 s | [0.00010419997852295637, 0.00010269996710121632, 0.00010080006904900074, 0.0001020999625325203, 0.00010219996329396963, 0.00010059995111078024, 0.00010079995263367891, 0.00010109995491802692, 0.00010179996024817228, 0.00010019994806498289] | - -
- -
sqrt · 原先不支持 → 现通过 - -Positive FP32 x[2,4] -> Sqrt [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sqrt"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cb10a2140fd47284e844b3d6a335b751c80b4647c12543a7e84133363855ce97 | -| 输入 SHA-256 | dd7b2461264e89c3ed6592836c2c66c4131208d2ad1f2af1a1d04fc368d2e8a6 | -| 参考输出 SHA-256 (expected_sha256) | 2940435af5f548faedd548a4c8752a60c9dcb25a05f00abbb1387ab26024f0da | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sqrt -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003466999623924494, 0.0003605000674724579, 0.0003635999746620655, 0.00034799997229129076, 0.00034380005672574043, 0.00040040002204477787, 0.0003725000424310565, 0.0003456000704318285, 0.0003480000887066126, 0.0003408000338822603] | -| 执行原始样本 s | [0.00010339997243136168, 9.490002412348986e-05, 9.290000889450312e-05, 9.370001498609781e-05, 9.099999442696571e-05, 9.990006219595671e-05, 9.940005838871002e-05, 9.69000393524766e-05, 9.69000393524766e-05, 9.26000066101551e-05] | - -
- -
reduce_mean · 原先不支持 → 现通过 - -FP32 x[2,3,4], constant INT64 axes[1]=[-1], keepdims=1 -> [2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["ReduceMean"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 20a01c6a789029d8138071b2ac1e692a4402d74d6ae928319351aca39c708132 | -| 输入 SHA-256 | 037f24d9d4527b017605afa283f05dddb03773049338c106ffa47fa642d45b17 | -| 参考输出 SHA-256 (expected_sha256) | 43dcac99f0a069df3610b20c7d4c8e59f9d5c1248ac3578e2ec0e5c5124216d7 | -| 初始化器 SHA-256 (initializers_sha256) | 31bd8476b5e08b62c88590c3a19c07e7c89306b3027fee871f28a188701f0793 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: ReduceMean -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 1] / float32 | -| 参考 shape / dtype | [2, 3, 1] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 9.933e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003934999695047736, 0.00037949997931718826, 0.00037489994429051876, 0.000372299924492836, 0.00037619995418936014, 0.0003952999832108617, 0.00040259992238134146, 0.0003915999550372362, 0.0003774999640882015, 0.0003698000218719244] | -| 执行原始样本 s | [0.00011030002497136593, 0.00010720000136643648, 0.0001078000059351325, 0.00010830000974237919, 0.00010569998994469643, 0.00010750000365078449, 0.00010809989180415869, 0.00010909989941865206, 0.00010730000212788582, 0.00010609999299049377] | - -
- -
transpose · 原先不支持 → 现通过 - -FP32 x[1,2,3,4], perm=[0,2,1,3] -> [1,3,2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Transpose"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e6c4bcca556bf025f9035ef090e35bba44c4dbcbc9175d3bf087590a3f7318e6 | -| 输入 SHA-256 | c2c2c890613dd624400f084697ecf07d2251cd6956cf57234da0551d2b56508a | -| 参考输出 SHA-256 (expected_sha256) | b610b4d2f5c3976762d1a7c04a0d9690f4ec3d39ab51fa9cc7fc9d5f628f6b75 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Transpose -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 3, 2, 4] / float32 | -| 参考 shape / dtype | [1, 3, 2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003711999161168933, 0.00037619995418936014, 0.0003662999952211976, 0.00041259999852627516, 0.0003730000462383032, 0.00035260000731796026, 0.0003722000401467085, 0.0003524000057950616, 0.0003497001016512513, 0.0003529998939484358] | -| 执行原始样本 s | [0.00010429997928440571, 0.00010159995872527361, 9.989994578063488e-05, 0.00010140007361769676, 0.00010170007590204477, 0.00010349997319281101, 9.93999419733882e-05, 9.840005077421665e-05, 9.790004696696997e-05, 9.710004087537527e-05] | - -
- -
concat · 原先不支持 → 现通过 - -FP32 left[2,1,4] and right[2,2,4], axis=-2 -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c96a7b79f82cd23c6c943f61a9d675161ee47102c1251259b7d5d1bfbbc8c977 | -| 输入 SHA-256 | 09f2f52e4a0c7c9c0cbb4a3962eff5a53214d57e4445b1ce4fce07bc6acae6d9 | -| 参考输出 SHA-256 (expected_sha256) | c94ae1852d9ca136be5084962a6333495a5a4df0f7f7db13d5ea0d3d41ce80cf | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Concat -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003693000180646777, 0.00038620003033429384, 0.0003693000180646777, 0.0003645999822765589, 0.00036700000055134296, 0.00041009997949004173, 0.0003698000218719244, 0.00037619995418936014, 0.00035749992821365595, 0.00035550002939999104] | -| 执行原始样本 s | [9.770004544407129e-05, 0.00010369997471570969, 9.69000393524766e-05, 9.790004696696997e-05, 9.599991608411074e-05, 0.00010299996938556433, 9.889993816614151e-05, 9.699992369860411e-05, 9.910005610436201e-05, 9.550002869218588e-05] | - -
- -
slice · 原先不支持 → 现通过 - -FP32 x[2,8], INT64 vectors: start=-1,end=INT64_MIN,axis=-1,step=-2 -> [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Slice"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | b9d3881f01090a4f8ec8ab828d2ef4d9f5460d37c9dce469e03577525ebb3b30 | -| 输入 SHA-256 | 04457bea47baae4c174cbb9964c15e2173ed21476eea164fdfa83f0c66b3f3aa | -| 参考输出 SHA-256 (expected_sha256) | 15916501385783e26c51e19e641bcd04e59f27d8f241438d4c392f02a2da72df | -| 初始化器 SHA-256 (initializers_sha256) | 6fb5ce1e94fa98118310be1dbbec731166552b05cf34b063477f2d54ea3bda7a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Slice +```bash +set -euo pipefail +source .venv-operators/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 ``` -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00040770007763057947, 0.0004149000160396099, 0.00039679999463260174, 0.000395399983972311, 0.000394199974834919, 0.0004005999071523547, 0.0003872000379487872, 0.0003947000950574875, 0.00039820000529289246, 0.0003963999915868044] | -| 执行原始样本 s | [0.00010409997776150703, 0.00010469998233020306, 0.00010289996862411499, 0.00010169995948672295, 0.00010259996633976698, 0.00010469998233020306, 0.00010290008503943682, 0.0001039999770000577, 0.00010349997319281101, 0.00010409997776150703] | - -
- -
unsqueeze · 原先不支持 → 现通过 - -INT64 x[2,3], constant INT64 axes[2]=[0,-1] -> [1,2,3,1]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Unsqueeze"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 3af8a32c4178606972d98c12b3e1bf28d1fd8301587a5efa3b1dd3e10d4545cd | -| 输入 SHA-256 | 4a7719bc73234f93e26685e50c1a40a771fe9465af4fa82ee75df54ceba5ea50 | -| 参考输出 SHA-256 (expected_sha256) | d96e4f88f5e537f71a56023d5684c075b9d81e960a60fdb1ceef46d3434e7193 | -| 初始化器 SHA-256 (initializers_sha256) | a8ff6dd261ccbd0a49d97047d600420769bfd677d4891898103e1d823513dcdf | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Unsqueeze -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 2, 3, 1] / int64 | -| 参考 shape / dtype | [1, 2, 3, 1] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004278999986127019, 0.00037889997474849224, 0.0003894000547006726, 0.0004752000095322728, 0.00046479993034154177, 0.00037020002491772175, 0.00035470002330839634, 0.0003568999236449599, 0.0003622000804170966, 0.0003958999877795577] | -| 执行原始样本 s | [8.939998224377632e-05, 8.649996016174555e-05, 8.329993579536676e-05, 8.98001017048955e-05, 8.560006972402334e-05, 8.510006591677666e-05, 8.279993198812008e-05, 8.359993807971478e-05, 8.44000605866313e-05, 8.329993579536676e-05] | - -
- -
expand · 原先不支持 → 现通过 - -FP32 x[2,1,4], constant INT64 shape[3]=[1,3,4], bidirectional broadcast -> [2,3,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 65a0c9a5fe1045ad4511d3099009989574b5b294a542a2716eb014aa0526a6ab | -| 输入 SHA-256 | 4fdf47ab1aae83654c3b631bf596434cdad68ae0ea325af90a5feb832173d3d6 | -| 参考输出 SHA-256 (expected_sha256) | eec80e44c7acb11e603e87f01a7570d0e15a45d9b2db4c7d148a5c5e379ddfd6 | -| 初始化器 SHA-256 (initializers_sha256) | 7eccb03cb24eb87dec4c8282d83c3bf88de5ed2ad14006e861cb239668ff0f3a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Expand -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3, 4] / float32 | -| 参考 shape / dtype | [2, 3, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004062999505549669, 0.0003779999678954482, 0.0004357000580057502, 0.00038139999378472567, 0.0004078999627381563, 0.0003739000530913472, 0.0003749999450519681, 0.0003727000439539552, 0.00036589999217540026, 0.000359599944204092] | -| 执行原始样本 s | [0.00011890009045600891, 0.00010760000441223383, 0.00010830000974237919, 0.00010639999527484179, 0.0001023000804707408, 0.00010429997928440571, 0.00010469998233020306, 0.00010419997852295637, 0.00010289996862411499, 0.00010410009417682886] | - -
- -
abs · 原先不支持 → 现通过 - -Signed INT64 x[2,3] -> Abs [2,3], preserving integer dtype. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Abs"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 2446ec6d9d36c8ab37f7a443113228b6773c8a6e05822c33e638e10338a58cdf | -| 输入 SHA-256 | 898fdb420a5c2a481f896b1f932dbbd6a45ef50a482d7c2d3ef7263b830e043a | -| 参考输出 SHA-256 (expected_sha256) | 7443fff444e2f78e27d69236ee9c0023e49acfc2d0158234069c0ca7d02bb640 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Abs -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00036099995486438274, 0.0003568999236449599, 0.0003541000187397003, 0.0003519000019878149, 0.0003469999646767974, 0.00033569999504834414, 0.00035450002178549767, 0.00034719996619969606, 0.0003350999904796481, 0.000327100045979023] | -| 执行原始样本 s | [8.889997843652964e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.420005906373262e-05, 8.320005144923925e-05, 8.759996853768826e-05, 8.549995254725218e-05, 8.499994874000549e-05, 8.68999632075429e-05, 8.470006287097931e-05] | - -
- -
cast · 原先不支持 → 现通过 - -INT64 position IDs[1,8] -> FP32 [1,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | e5bf2bf24fd5667082da808f92a835bf3b6412a8a494bee301ee042c31167dd5 | -| 输入 SHA-256 | 93f57a04397de81ea2aabce45687351a7238a5b72a8b4a625de522bfd69df92f | -| 参考输出 SHA-256 (expected_sha256) | 5296d1fd5eb039a4b0ccb36f0498e194eb76696167b82a590cb30defc016141f | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 8] / float32 | -| 参考 shape / dtype | [1, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003632999723777175, 0.00036179996095597744, 0.0004943999228999019, 0.0005131000652909279, 0.00037509994581341743, 0.00034440006129443645, 0.00033980002626776695, 0.00034899997990578413, 0.00033760000951588154, 0.00035889993887394667] | -| 执行原始样本 s | [7.98999099060893e-05, 0.00017800007481127977, 0.00011080002877861261, 8.82000895217061e-05, 8.230004459619522e-05, 7.829989772289991e-05, 7.860001642256975e-05, 7.840001489967108e-05, 8.060003165155649e-05, 7.970002479851246e-05] | - -
- -
constant · 原先不支持 → 现通过 - -Constant tensor attribute FP32 [2,3]; no runtime inputs. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | c9d931c03dd507060befe5387ac7a6e07c4de2f8970c6fe4009ad2799a7e7227 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 5a1a9e658f38c6a788419b7f7321353352ffeb929be456073dca9d36511f2a67 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003632999723777175, 0.000381799996830523, 0.00039549998473376036, 0.000381799996830523, 0.0003598000621423125, 0.00036259996704757214, 0.0003591999411582947, 0.0003554000286385417, 0.0003502999898046255, 0.0003415999235585332] | -| 执行原始样本 s | [6.250001024454832e-05, 6.170000415295362e-05, 6.10999995842576e-05, 6.200000643730164e-05, 6.0399994254112244e-05, 6.150000263005495e-05, 6.219989154487848e-05, 6.0399994254112244e-05, 6.049999501556158e-05, 5.909998435527086e-05] | - -
- -
cos · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Cos [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cos"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3af95b0a4f4cd19cd254fb139b753cfa99171c187c492d6a322a8e31b8866298 | -| 输入 SHA-256 | 8c196fdf42e2e25a11894ee88e8d73e31e59a061f7e76bead86793c428ddc998 | -| 参考输出 SHA-256 (expected_sha256) | 7a6831ace7c14d1ee2ec5d6d7ad497adc95914e56eecf8f384c018012e36e0ed | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cos -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 5.960e-08 | -| 最大相对误差 | 8.024e-08 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0003530000103637576, 0.00034549995325505733, 0.00033980002626776695, 0.000340400030836463, 0.0003451999509707093, 0.00034739996772259474, 0.000332699972204864, 0.0003412999212741852, 0.00033599999733269215, 0.00033199996687471867] | -| 执行原始样本 s | [8.690007962286472e-05, 8.670007809996605e-05, 8.889997843652964e-05, 8.739996701478958e-05, 8.559995330870152e-05, 8.60999571159482e-05, 8.629995863884687e-05, 8.690007962286472e-05, 8.579995483160019e-05, 8.400005754083395e-05] | - -
- -
identity · 原先不支持 → 现通过 - -INT64 indices[2,3] -> Identity [2,3]; frontend alias. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Identity"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | e16a4cbfc11442ded30de5b7503512fe1d179b3c64dfcc65140d17f81cab3a4b | -| 输入 SHA-256 | 19f74d0f712a161d1f093b37f6d7d1464fb4a730dd83d1069f5cbf5e38e13413 | -| 参考输出 SHA-256 (expected_sha256) | a700997366bef11ca0353f21bc678822148eb06156a05ed32d1ce76b73ad1cc8 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Identity -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / int64 | -| 参考 shape / dtype | [2, 3] / int64 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.000341000035405159, 0.00034380005672574043, 0.0003355999942868948, 0.0003361001145094633, 0.0003378000110387802, 0.00033820001408457756, 0.0003301999531686306, 0.00032760004978626966, 0.0003744999412447214, 0.0003630999708548188] | -| 执行原始样本 s | [5.170004442334175e-05, 5.280005279928446e-05, 5.219993181526661e-05, 5.2200048230588436e-05, 5.080003757029772e-05, 5.179992876946926e-05, 5.079992115497589e-05, 5.139992572367191e-05, 6.809993647038937e-05, 5.4600066505372524e-05] | - -
- -
pow · 原先不支持 → 现通过 - -Signed FP32 x[2,3] ** INT64 scalar exponent 2 -> FP32 [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 9184fbaccd896e171a8ca025b6745d0674bf1493913f34b77f8bc0f218751771 | -| 输入 SHA-256 | b33fcf5b3bd13b2fa3bacb0388119b95ed138163db9b168dd3993654533b4cd1 | -| 参考输出 SHA-256 (expected_sha256) | dcada1e53c419cefb8b09810f4494beb138bcecdce47c5c0f48466ee49d5d51e | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.0004491000436246395, 0.00043999997433274984, 0.00040019990410655737, 0.00037569995038211346, 0.00038789992686361074, 0.0003453000681474805, 0.00048369995784014463, 0.00044920004438608885, 0.00047619990073144436, 0.0004214000655338168] | -| 执行原始样本 s | [0.00014999997802078724, 0.00014180003199726343, 0.00011959997937083244, 0.00011729996185749769, 0.00011809996794909239, 0.00011589995119720697, 0.00015820004045963287, 0.00015029998030513525, 0.00015840004198253155, 0.0001241000136360526] | - -
- -
reciprocal · 原先不支持 → 现通过 - -Nonzero signed FP32 x[2,4] -> Reciprocal [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Reciprocal"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 3d0e2326e2db7ff3ad97b6a1e7f2e59dc713fd2e70ca1a4d067c04ed699a04e3 | -| 输入 SHA-256 | fb2145f899f1258c7169e1f7a540b9b94625c689d770793b142cda8ae4bf722f | -| 参考输出 SHA-256 (expected_sha256) | c31992656b1ac1d73d6fc90936ecaf8d816946286b2c6170097a6cf9823d818d | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Reciprocal -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00035170000046491623, 0.0003496999852359295, 0.00034989998675882816, 0.00034769997000694275, 0.00033870001789182425, 0.00037759996484965086, 0.00035119999665766954, 0.0003469000803306699, 0.00033660000190138817, 0.00033329997677356005] | -| 执行原始样本 s | [9.190000128000975e-05, 9.019998833537102e-05, 9.029998909682035e-05, 9.14999982342124e-05, 8.899997919797897e-05, 9.609991684556007e-05, 9.510002564638853e-05, 9.06999921426177e-05, 9.109999518841505e-05, 9.059999138116837e-05] | - -
- -
sin · 原先不支持 → 现通过 - -FP32 angles[2,4] spanning positive/negative radians -> Sin [2,4]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Sin"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cd93a789d0f603c48706203da065f2451a65cf345581f9772d34fff8bca88a2a | -| 输入 SHA-256 | efb400ff25087f013055f64abc4fcf498072fb9d417b825f6e9e292742b3ef9e | -| 参考输出 SHA-256 (expected_sha256) | e2a29d3f9d4bc5d9003ed673b7e4cb70997b5048a77aff3b181721c9461d13ee | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Sin -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 4] / float32 | -| 参考 shape / dtype | [2, 4] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 2 | -| 解析原始样本 s | [0.00034879997838288546, 0.0003532000118866563, 0.00035470002330839634, 0.0003448999486863613, 0.00033730000723153353, 0.00033710000570863485, 0.0003510999958962202, 0.0003322999691590667, 0.00032819993793964386, 0.000327700050547719] | -| 执行原始样本 s | [8.570007048547268e-05, 8.719996549189091e-05, 8.620007429271936e-05, 8.480006363242865e-05, 8.730008266866207e-05, 8.71000811457634e-05, 8.659996092319489e-05, 8.569995407015085e-05, 8.390005677938461e-05, 8.750008419156075e-05] | - -
- -
scalar_fp32_add_chain · 原先错误 → 现正确 - -FP32 scalar initializers: (16777216 + 1) + 1 must round after each Add; exact scalar output. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add"] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 4ff5c4c55a16938c3a8c30f16a2b8e6662e6f7cc38413473bc00bd50a36abce7 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 87da7d0335886bc568cddf1844505676699d5476a6bd72d0d6292c3ee42c253a | -| 初始化器 SHA-256 (initializers_sha256) | 32823c7b66e5511946296130ba5b03f3e897d3939bd738d14f84d41f7e80923a | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | NUMERIC_ERROR / compare | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 2.000e+00 | -| 最大相对误差 | 1.192e-07 | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -NumericMismatch: output differs from ORT: max_abs_error=2 -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [] / float32 | -| 参考 shape / dtype | [] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 5 | -| 解析原始样本 s | [0.0005631999811157584, 0.0004637000383809209, 0.0004089000867679715, 0.00039579998701810837, 0.0003945999778807163, 0.000381799996830523, 0.00041089998558163643, 0.00040959997568279505, 0.0003945999778807163, 0.0003902000607922673] | -| 执行原始样本 s | [0.00019599997904151678, 0.00015169999096542597, 0.00014780007768422365, 0.0001476999605074525, 0.0001452999422326684, 0.00015129998791962862, 0.00014889996964484453, 0.0001441999338567257, 0.00014589994680136442, 0.0001444999361410737] | - -
- -
singleton_initializer · 原先错误 → 现正确 - -FP32 initializer weight[1]=[7] returned directly: retain tensor data and shape [1], not scalar. - -| 项目 | 值 | -| --- | --- | -| 算子 | [] | -| 绝对容差 atol | 0.0 | -| 相对容差 rtol | 0.0 | -| 模型 SHA-256 | 52473f6b8b73062f1bc8eb98f6a61a2dffb467a6c9966cb985c8b53037c2f722 | -| 输入 SHA-256 | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | -| 参考输出 SHA-256 (expected_sha256) | 3948d58d392cf3b813d8cc0773e2ed0af42344e5e96ff9eff386c223ce2ac1f8 | -| 初始化器 SHA-256 (initializers_sha256) | 7da88694441abcf15dddeff15e70aaa41eb8e0ee336a5b08e02b61215b3bb6df | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | EXECUTION_ERROR / execute | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -IRExecutionError: BindingError function=singleton_initializer value=weight: initializer disagrees with scalar constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1] / float32 | -| 参考 shape / dtype | [1] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 1 | -| 解析原始样本 s | [0.0003521000035107136, 0.00034039991442114115, 0.00033650000113993883, 0.0003445999464020133, 0.0003336999798193574, 0.0003494999837130308, 0.0003339999821037054, 0.00035889993887394667, 0.00032849994022399187, 0.00033849989995360374] | -| 执行原始样本 s | [6.290001329034567e-05, 6.280001252889633e-05, 6.140000186860561e-05, 6.009999196976423e-05, 6.079999729990959e-05, 6.370001938194036e-05, 6.200000643730164e-05, 6.330001633614302e-05, 6.260001100599766e-05, 6.0399994254112244e-05] | - -
- -
constant_cast_abs_expand · 原先不支持 → 现通过 - -Constant INT32 [-2,3] -> Cast INT64 -> Abs target[2]=[2,3]; Expand FP32 x[1,3] -> [2,3]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Constant", "Cast", "Abs", "Expand"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | bd8568ed91a153a8c061bf51d0ff3512314db206860531456cb11aa0735938b3 | -| 输入 SHA-256 | 9b2c7512831a4cace198062de741146fbeedb181b006cf32431c4fce833a9822 | -| 参考输出 SHA-256 (expected_sha256) | eed5f694fcfe47f46f2a01790cff931b624565234b061b5c64a278c2ed1eac53 | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Constant -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [2, 3] / float32 | -| 参考 shape / dtype | [2, 3] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 4 | -| 解析原始样本 s | [0.00043970008846372366, 0.00044329999946057796, 0.0004450000124052167, 0.0004384000785648823, 0.0006199999479576945, 0.00047199998516589403, 0.00045169994700700045, 0.00044700002763420343, 0.00044139998499304056, 0.00043539993930608034] | -| 执行原始样本 s | [0.00013560010120272636, 0.0001338999718427658, 0.00013189995661377907, 0.0002367999404668808, 0.00017560005653649569, 0.00013229995965957642, 0.0001343999756500125, 0.000130399945192039, 0.00013109995052218437, 0.00013479997869580984] | - -
- -
qwen_rmsnorm · 原先不支持 → 现通过 - -FP32 x[1,4,8], gamma[8]; Pow(INT64 scalar 2)/ReduceMean(-1)/Add(eps=1e-6)/Sqrt/Reciprocal/Mul -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Pow", "ReduceMean", "Add", "Sqrt", "Reciprocal", "Mul"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | cf11b7c966672ab3f03eb3d92a8e45fc87968d70e33a4a500964ce52d02a7237 | -| 输入 SHA-256 | f9c2bb681cd4b72783d29572f427b6e237b6c7bf40252ab5b88a2c6174751a48 | -| 参考输出 SHA-256 (expected_sha256) | 5fe5157ae0bc87a41a4a9db76a037f52f8b7e5aa4f2c64e8674e0bdbcb8dfaec | -| 初始化器 SHA-256 (initializers_sha256) | 6bf5dd34dd2bb7907f3d53ccd995420cfd6a610b463eb41cf53d0698308ab92c | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Pow -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 0.000e+00 | -| 最大相对误差 | 0.000e+00 | -| 执行 IR 步数 | 10 | -| 解析原始样本 s | [0.000506900018081069, 0.0005015999777242541, 0.0005096999229863286, 0.0005014999769628048, 0.0004960000514984131, 0.0005388999124988914, 0.0005028999876230955, 0.000512199942022562, 0.000493900035507977, 0.0004851999692618847] | -| 执行原始样本 s | [0.0002813000464811921, 0.0002788000274449587, 0.0002808000426739454, 0.00028170004952698946, 0.00027820002287626266, 0.0002930000191554427, 0.00028490007389336824, 0.00028249993920326233, 0.0002837000647559762, 0.0002848999574780464] | - -
- -
rope_numeric · 原先不支持 → 现通过 - -INT64 positions[1,4,1] -> Cast FP32; frequencies[1,1,4], Sin/Cos rotate FP32 left/right[1,4,4] -> [1,4,8]. - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Cast", "Mul", "Cos", "Sin", "Sub", "Add", "Concat"] | -| 绝对容差 atol | 1e-06 | -| 相对容差 rtol | 1e-06 | -| 模型 SHA-256 | 7aed8255b36ac1866b0f27cd399e045010f0b82b1308036bbdbb849e02476cfa | -| 输入 SHA-256 | 823f1be624262313583ea77e2dc14efcb3ea0696d059c2cc89643266d78f944a | -| 参考输出 SHA-256 (expected_sha256) | eb0681ce4f70f4032e2806727ea5eb56e3d9f4036729cba48950643b393a39cd | -| 初始化器 SHA-256 (initializers_sha256) | 8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Cast -``` - -**修改后** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 4, 8] / float32 | -| 参考 shape / dtype | [1, 4, 8] / float32 | -| 最大绝对误差 | 2.980e-08 | -| 最大相对误差 | 1.699e-07 | -| 执行 IR 步数 | 12 | -| 解析原始样本 s | [0.0005266999360173941, 0.000520200002938509, 0.0005088000325486064, 0.0006041000597178936, 0.0006553001003339887, 0.0005080000264570117, 0.0005031999899074435, 0.0005010999739170074, 0.0005029999883845448, 0.00049919995944947] | -| 执行原始样本 s | [0.0002498000394552946, 0.00024560000747442245, 0.0002465000143274665, 0.0003208999987691641, 0.00031370006036013365, 0.00025370006915181875, 0.0002449000021442771, 0.00024610001128166914, 0.0002482000272721052, 0.0002460000105202198] | - -
- -
pr89_transformer_2l · 原先不支持 → 现通过 - -PR #89 原始两层合成 Transformer，FP32，输出 [1,256,128]；非完整 Qwen3 - -| 项目 | 值 | -| --- | --- | -| 算子 | ["Add", "Concat", "Div", "Expand", "Gather", "MatMul", "Mul", "Neg", "ReduceMean", "Reshape", "Sigmoid", "Slice", "Softmax", "Sqrt", "Transpose", "Unsqueeze"] | -| 绝对容差 atol | 1e-05 | -| 相对容差 rtol | 0 | -| 模型 SHA-256 | 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 | -| 输入 SHA-256 | bd3f848d2679e66691a7701a84e39f35a02db58c0874e65b023b561f4f507457 | -| 参考输出 SHA-256 (expected_sha256) | 413dd44ea90d93a25cbe0f89bc3703ffeb8ee7b0774c8fa955bcd7521b24ef5c | -| 初始化器 SHA-256 (initializers_sha256) | 955cb24a985e3aa13cb93e62af4e51e92ff27d14bdfdeac1d5a6107f65a8e3de | - -**修改前** - -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | UNSUPPORTED / parse | -| 实际 shape / dtype | — / — | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | — | -| 最大相对误差 | — | -| 执行 IR 步数 | — | -| 解析原始样本 s | [] | -| 执行原始样本 s | [] | - -错误详情： - -```text -ONNXParseError: Unsupported ONNX op type: Gather -``` +输出目录每次换新名字，生成本次 JSON、Markdown、HTML 及原始样本。仅前后均 PASS 的用例比较耗时；缺失、不支持或失败不等于零耗时。结果用于支持/正确性对比，IR 解释器计时不等于 QEMU 或目标硬件性能。 -**修改后** +原始个人测量保持原平台身份和数据，分别保存于固定提交： -| 项目 | 值 | -| --- | --- | -| 状态 / 阶段 | PASS / complete | -| 实际 shape / dtype | [1, 256, 128] / float32 | -| 参考 shape / dtype | [1, 256, 128] / float32 | -| 最大绝对误差 | 7.153e-07 | -| 最大相对误差 | 5.722e-04 | -| 执行 IR 步数 | 159 | -| 解析原始样本 s | [0.004760700045153499, 0.004594899946823716, 0.004717700066976249, 0.004990800051018596, 0.004586199996992946, 0.004221900017000735, 0.004203699994832277, 0.004572000005282462, 0.004674999974668026, 0.004224899923428893] | -| 执行原始样本 s | [0.010629900032654405, 0.011460799956694245, 0.009812200092710555, 0.009985200013034046, 0.010765800019726157, 0.009604100021533668, 0.010238799965009093, 0.009660700103268027, 0.010715999989770353, 0.009805700043216348] | +- [原始 MD](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.md) +- [原始 JSON](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.json) +- [原始 HTML](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测-benchmark.html) -
+当前同名 JSON 的 `document_kind=historical_benchmark_index`，只保存归档链接，不含 PASS、环境测量或计时字段，不能作为 benchmark 结果输入。新 Linux 测量必须从 CLI 输出目录读取。 diff --git "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" index 13b7df2..4d516da 100644 --- "a/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" +++ "b/docs/llm-deploy-v1.0/W1/\347\256\227\345\255\220\350\241\245\351\275\220\344\270\216\345\215\225\346\265\213\346\212\245\345\221\212.txt" @@ -1,148 +1,92 @@ -Qwen3 ONNX 算子补齐与单测报告 -日期：2026-10-01 - -工作目录：仓库根目录；本文使用仓库相对路径。 -主干基线：20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 -历史测量状态：原始算子探测和 benchmark 在实现尚未提交时执行；归档报告保留测量时的 HEAD、dirty 和源码指纹。 -本文用于随算子实现及测试 PR 归档，不以历史 dirty 标记描述当前提交或远程仓库状态。 -本报告更新上一阶段“真实 Qwen3 尚缺八类前端算子”的临时结论。 - -一、完成范围 - -完成 ONNX 前端 → 共享 IR → NumPy 解释器路径所需的两组算子： -1. PR #89 探测图八类：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 -2. 真实 Qwen3 导出图额外八类：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 -第一组复用已有 IR 算子与内核，补齐前端和边界测试；第二组新增六个 IR opcode，Constant 绑定为全局常量，Identity 复用输入值。 - -二、本轮代码改动 - -scratchv/frontend/onnx_parser.py -- 补齐第二组前端映射；支持 Constant 的 tensor、value_int/value_ints、value_float/value_floats。 -- 保持零维标量、单元素向量与 INT64 值的区别，支持外部权重及 Constant 外部数据。 -- 求值 Constant → Cast/Abs/Reshape 等静态 shape/axis 链，适配真实 Qwen3 导出形式。 -- 不将中间结果冒充 initializer；限制静态参数求值规模，避免超大 Expand 与尺寸乘积溢出。 -- Add/Mul 统一发出 IR，修复 Python 标量折叠造成的 FP32 舍入与整数宽度错误。 -- 预留全图 v_N 名字，修复 ONNX 原始名字和临时结果名碰撞。 -- 明确拒绝当前 IR 不支持的数据类型。 - -scratchv/ir/types.py、scratchv/ir/builder.py -- 新增 ABS、CAST、COS、SIN、POW、RECIPROCAL 及 builder 方法。 -- Cast 的结果 dtype 表示目标类型；Pow 允许底数与指数类型不同，结果保持底数类型。 - -scratchv/analysis/ir_verifier.py -- 增加新算子签名、浮点算子类型约束和 Pow 输出类型校验。 - -scratchv/verification/ir_numpy_ops.py -- 增加新算子的实际 NumPy 求值和数值错误诊断。 -- Cast 整数缩窄保留低位；浮点转整数按截断后范围检查，避免 INT64 边界错误。 -- Pow 支持广播、混合类型以及有界的精确整数计算。 -- Slice 使用 ONNX 的正反向钳制规则，修复过小负 start 的反向切片，并处理空维和 INT64 哨兵。 - -三、测试 - -本轮新增四个测试文件，共 282 个用例： -- tests/test_onnx_shape_ops.py：116 个，第一组八类算子与 ORT 对照、边界和非法参数。 -- tests/test_onnx_qwen_numeric_ops.py：78 个，第二组算子、外部常量、实际静态参数链及 RMSNorm 组合。 -- tests/test_ir_qwen_numeric_ops.py：74 个，IR builder、类型签名、广播、整数精度、数值边界与错误定位。 -- tests/test_onnx_constant_regressions.py：14 个，FP32 舍入、整数宽度、名字碰撞和静态求值规模限制。 -更新 tests/test_ir_verifier.py 的穷举签名表，覆盖六个新增 opcode。 -延续上一阶段的 tests/test_onnx_transformer_execution.py 和 initializer 测试修复。 - -算子及原有相关回归共 715 项，加上 22 项 benchmark 自检；提交前合并重跑结果为 737 passed，无失败。 -该次合并运行有 7 条 NumPy 配置提示 warnings，未影响测试通过。 -其中 715 项算子及原有相关回归的分组记录如下： -- IR、解释器、校验器、优化器、PassManager 相关：439 passed。 -- 前端、ONNX 算子、Transformer 组合、诊断和相关 benchmark：262 passed。 -- Constant 专项回归：14 passed（另与已计入的 85 项用例联合运行，结果 99 passed）。 -这是相关回归集合，未声称运行整个项目测试全集。 -单测测量环境：Python 3.12.13，NumPy 2.5.3、ONNX 1.23.0、ORT 1.29.0，Python -X utf8 -B。 - -四、模型验证 - -1. PR #89 原始 LFS 模型未修改： - probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx - SHA-256：17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1 - 本轮重新运行 seed=1：ORT vs 独立 NumPy 参考最大绝对误差 1.09718e-6；IR vs ORT 为 7.1525574e-7。 - 均通过最大绝对误差 <1e-5；IR 执行 159 步。 - 另有 seed=0/1/42 的组合回归通过。 - 原始探测 JSON 保留在本机，未随本 PR 归档；此处保留模型指纹和数值结论。 - -2. 本地真实 Qwen3-0.6B 导出模型： - models/qwen3-0.6b-onnx/model.onnx（额外交付模型，不包含在本次算子 PR 中） - 7847 个 ONNX 节点、23 类算子；未识别算子为 0。 - 完整载入外部权重，得到 2148 个全局定义、5910 条 IR 指令。 - verify_ir 通过，问题数为 0。 - 原始结构审计 JSON 保留在本机，未随本 PR 归档。 - 此项是完整图解析及结构/类型校验，不是全模型 IR 推理的数值对照。 -模型探测使用独立导出依赖环境（NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1）。 - -五、支持边界与下一步 - -本次范围是部署方案所需的静态图及 FP32 路径，同时覆盖 IR 已有的 FLOAT64、INT32、INT64 类型。 -没有实现 ONNX 的全部 dtype/opset 扩展；动态 shape/axis 参数及 Reshape allowzero=1 仍明确拒绝。 -解释器沿用严格数值策略：非法定义域、非有限结果以及新增 Abs/整数 Pow 的溢出会报错。 -整数 Pow 对大整数采用精确计算，ORT CPU 在超 2**53 的整数幂上可能存在浮点精度差异；实际 Qwen 使用 FP32 底数和 INT64 指数 2。 - -下一步应运行真实 Qwen3 结构的小模型，包含 Q/K RMSNorm、完整 RoPE、GQA、SwiGLU，逐层对照 PyTorch/ORT/IR。 -之后再验证优化流水线、完整 0.6B 推理的数值/内存，以及 RISC-V 后端和 QEMU。 -本轮没有补 RISC-V 降低和执行支持，不能据此前端/IR 结果判断 QEMU 已可运行。 - -六、修改前后 benchmark 对比 - -归档文件（同目录）： -- 算子补齐与单测-benchmark.json：移除本机路径后的测量数据，保留全部样本、误差及指纹。 -- 算子补齐与单测-benchmark.md：GitHub 可直接阅读的结果报告。 -- 算子补齐与单测-benchmark.html：独立 HTML 报告，下载后用浏览器打开，可筛选并展开错误详情。 - -共 24 个固定用例，修改前 2/24 PASS，修改后 24/24 PASS： -- 16 个独立算子用例：0/16 → 16/16。 -- 2 个已有支持的对照用例：2/2 → 2/2。 -- 2 个错误回归用例：0/2 → 2/2。 -- 4 个组合链路用例：0/4 → 4/4，包含 PR #89 的原始两层 Transformer。 -因此，“原先不支持、现通过”是 20 个用例，不是新增 20 种算子。 - -使用同一批模型、输入、ORT 参考及初始化器，并校验指纹；基线通过解释器已有 initializers 参数接受原始 ONNX 数组，属于基准适配，不改基线实现，也不重塑权重以掩盖旧错误。 -ABBA 顺序为修改前、修改后、修改后、修改前；每版本 2 轮，每轮预热 1 次、正式计时 5 次，合计每版本 10 个正式样本。 -IR.run 完整计时包括校验、执行规划、绑定复制、NumPy 计算及返回复制；ONNX 解析单独计时，ORT 参考计算不计入。 -只在前后均 PASS 时显示执行耗时比；本报告证明支持和正确性改善，不宣称性能优化或 QEMU 性能。 -新增 tests/test_onnx_operator_benchmark.py 的 22 项自检与原 715 项回归已合并运行，共 737 passed。 -CI 的测试 checkout 获取完整历史，以便历史基线集成测试实际运行。 - -七、复跑准备与命令（在仓库根目录 PowerShell 执行） - -建议 Python 3.12，并使用独立虚拟环境。依赖安装： -python -m pip install -e ".[all]" pytest -其中项目 all 依赖包含 onnxruntime；基准和测试直接依赖 NumPy、ONNX、onnxruntime、pytest。 -本次单测/benchmark 版本为 NumPy 2.5.3、ONNX 1.23.0、onnxruntime 1.29.0；解释器路径不是复跑前提。 - -确保 Git 历史中存在固定基线提交，浅克隆需补充该历史；origin 应指向包含该提交的项目仓库或 fork： +Qwen3 ONNX 算子补齐与单测：Linux 复现说明 +更新：2026-10-06 + +正式交付与他人复现环境：Ubuntu 24.04 x86_64 / Bash / Python 3.12。 +所有命令在仓库根目录执行。通用环境约定见 docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md。 +个人本机测量的原始版本保留在固定提交： +https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W1/算子补齐与单测报告.txt +原始平台、计数、计时和数值不改名为 Linux 数据；新结果必须由下面的 Linux 命令重新生成。 + +一、实现范围 + +ONNX 前端 → 共享 IR → NumPy 解释器覆盖两组算子： +1. PR #89 探测图：Gather、Sqrt、ReduceMean、Transpose、Concat、Slice、Unsqueeze、Expand。 +2. 真实 Qwen3 导出额外需要：Abs、Cast、Constant、Cos、Identity、Pow、Reciprocal、Sin。 +第一组复用已有 IR 与内核，补前端/边界；第二组新增 ABS、CAST、COS、SIN、POW、RECIPROCAL，Constant 使用绑定，Identity 复用输入值。 + +语义修复包括零维标量与向量区别、INT64 精度、静态 shape/axis 参数链、外部权重和 Constant、FP32 常量舍入、临时值命名冲突、静态求值规模限制、Cast 截断/溢出、Pow 广播与整数精度，以及 Slice 的反向边界和 INT64 哨兵。前端不将中间值伪装为 initializer。 + +二、验收用例 + +tests/test_onnx_shape_ops.py：八类形状/索引算子，ORT 对照、边界和非法参数。 +tests/test_onnx_qwen_numeric_ops.py：数值算子、常量、静态参数链及 RMSNorm 组合。 +tests/test_ir_qwen_numeric_ops.py：IR 类型、广播、整数精度与错误定位。 +tests/test_onnx_constant_regressions.py：FP32 舍入、整数宽度、名字碰撞、求值规模。 +tests/test_onnx_operator_benchmark.py：基准自身的身份、正确性及失败传播。 +实际 passed/skipped/failed 以本次 Linux 输出为准，不沿用历史计数。 + +三、Linux 环境与资产准备 + +以下为 Bash，已存在环境时可复用，但必须核对依赖版本。benchmark 使用单独环境，以保持历史前后对照依赖；它不替代完整模型门禁的固定导出环境。 + +set -euo pipefail +sudo apt-get update +sudo apt-get install --no-install-recommends python3.12-venv git-lfs +python3.12 -m venv .venv-operators +source .venv-operators/bin/activate +python -m pip install "numpy==2.5.3" "onnx==1.23.0" "onnxruntime==1.29.0" "pytest==9.1.1" +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export NUMEXPR_NUM_THREADS=1 + +git rev-parse HEAD +git status --short git fetch origin 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 - -安装 Git LFS 后，下载已由 PR #89 提交的模型实体，不能把 LFS 指针作为 ONNX 运行： -git lfs install +git lfs install --local git lfs pull --include="probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx" -以下 24 用例基准不需要完整 Qwen3-0.6B 权重；完整模型结构审计需要另外准备本文第四节所述模型。 -生成新的基准报告及临时 fixtures，不覆盖 docs 中的历史归档： -python -X utf8 -B -m benchmarks.bench_onnx_operators --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 --output-dir benchmark_reports/onnx_operators --warmup 1 --repeats 5 - -复跑相关测试： - -$operatorTests = @( - 'tests/test_ir.py', 'tests/test_ir_qwen_numeric_ops.py', - 'tests/test_ir_interpreter.py', 'tests/test_ir_interpreter_ops.py', - 'tests/test_ir_interpreter_control_flow.py', 'tests/test_ir_verifier.py', - 'tests/test_ir_verifier_integration.py', 'tests/test_optimizer.py', - 'tests/test_optimizer_advanced.py', 'tests/test_pass_manager.py', - 'tests/test_pass_registry.py', 'tests/test_parser.py', - 'tests/test_onnx_initializers.py', 'tests/test_onnx_shape_ops.py', - 'tests/test_onnx_qwen_numeric_ops.py', 'tests/test_onnx_transformer_execution.py', - 'tests/test_ir_interpreter_benchmark.py', 'tests/test_ir_verifier_benchmark.py', - 'tests/test_ir_diagnostics.py', 'tests/test_onnx_constant_regressions.py', - 'tests/test_onnx_operator_benchmark.py' +基线为固定提交 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6，不能用不同分支 HEAD 代替。浅克隆必须补齐历史。原模型 SHA256 应为 17a8f191ee65a9d97143131016058e8a3c10a7d5c06e8152bf16dd8941cd19e1，不可把 LFS pointer 当 ONNX。 + +四、生成修改前后 benchmark + +python -X utf8 -B -m benchmarks.bench_onnx_operators \ + --baseline-ref 20b105e80ba3dbe13cb01a3d4ca18c32b6c31ed6 \ + --output-dir benchmark_reports/onnx_operators-linux-01 \ + --warmup 1 --repeats 5 + +每轮选择新的输出目录。CLI 生成自己的 fixtures、参考和 JSON/Markdown/HTML 报告，不读取 docs 目录中的历史索引作为测量。源码、模型、输入、环境身份与错误详情由新报告保存。 + +24 个固定用例包含 16 独立算子、2 已支持的对照、2 错误回归和 4 组合链路，含原 PR #89 两层 Transformer。ABBA 顺序比较修改前/后，每版本两轮；前后都通过的用例才计算耗时比。不支持、失败或缺测不能解释为零耗时或无限加速。 + +IR.run 计时包含验证、执行规划、绑定复制、NumPy 计算与返回复制；ONNX 解析单独计时，ORT 参考不计入。此基准验证支持与正确性变化，不证明 QEMU 性能改善。重新生成的 Linux 样本不可与不同平台的历史耗时直接算加速比。 + +五、运行相关回归与合成图 + +operator_tests=( + tests/test_ir.py tests/test_ir_qwen_numeric_ops.py + tests/test_ir_interpreter.py tests/test_ir_interpreter_ops.py + tests/test_ir_interpreter_control_flow.py tests/test_ir_verifier.py + tests/test_ir_verifier_integration.py tests/test_optimizer.py + tests/test_optimizer_advanced.py tests/test_pass_manager.py + tests/test_pass_registry.py tests/test_parser.py + tests/test_onnx_initializers.py tests/test_onnx_shape_ops.py + tests/test_onnx_qwen_numeric_ops.py tests/test_onnx_transformer_execution.py + tests/test_ir_interpreter_benchmark.py tests/test_ir_verifier_benchmark.py + tests/test_ir_diagnostics.py tests/test_onnx_constant_regressions.py + tests/test_onnx_operator_benchmark.py ) -python -X utf8 -B -m pytest @operatorTests -q --basetemp=output/pytest-operator-completion +mkdir -p output +python -X utf8 -B -m pytest "${operator_tests[@]}" -q \ + --junit-xml=output/operator-tests-linux-01.xml +python -X utf8 -B probes/w1_tiny_transformer/run.py \ + --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx \ + --output-dir output/operator-completion-linux-01 --seed 1 + +六、范围与后续门禁 + +这是静态图和 FP32 所需前端/IR 能力，同时覆盖现有 FLOAT64、INT32、INT64；不宣称全部 ONNX dtype/opset 或动态 shape/axis。Reshape allowzero=1 等不支持语义明确拒绝；非法定义域、非有限值及部分整数溢出明确报错。当前固定 Qwen 图使用 FP32 底数与整数指数 2；通用 Pow 范围需要分别验证。 -python -X utf8 -B probes/w1_tiny_transformer/run.py --model probes/w1_tiny_transformer/out/tiny_transformer_2l.onnx --output-dir output/operator-completion-probe --seed 1 +完整 28 层解析、真实两层 Qwen3 数值与 RV64/QEMU 分别从 W1/README.md、W2/README.md 和 probes/w2_qwen3_small/README.md 进入；基础算子 benchmark 通过不能代替这些门禁。 -上述探测命令默认使用已下载的 ONNX，不要求重新导出；切换导出依赖环境时请遵循模型探测的已记录版本。 +同目录 benchmark.md 为 Linux 复跑入口，benchmark.html 为可打开的指引，benchmark.json 为明确标识的历史索引，不包含新测量。原始完整测量保留在固定提交历史中。 diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index 13f45a9..0db79de 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -1,264 +1,85 @@ -# W2：完整前端解析、基础后端与 Host 运行时 +# W2 Linux 独立复现与验收 -本目录记录开发计划中的 W2 验收：完整 28 层 Qwen3-0.6B 的前端解析与审计、四类模型模式回归、基础算子 QEMU，以及 Tokenizer/输入/mask/greedy。已有真实两层模型的 IR/ORT/QEMU 门禁继续保留。 +正式交付、CI 和第二人复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**。完整环境准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)。Linux 为宿主环境；QEMU 内的 RV64GC/LP64D 程序仍使用当前裸机 ABI。 -W2 完整图的范围止于前端验收。完整 IR 与 ORT 的数值对照及真实维度预训练子图已在独立 [W3 交付](../W3/README.md) 中取得本地证据；权重独立装载、完整 QEMU 前向和生成循环仍按后续阶段推进。W2 不自动关闭 W1 的完整模型第二人复现、接口确认和出口评审。 +W2 的范围是完整前端解析、小模型 IR 数值、基础后端和 Host 单步接口。完整 28 层模型的 IR 数值前向属于 W3；完整模型 QEMU 与生成循环属于后续阶段。W1 的人工接口决议继续见 [W1 记录](../W1/README.md) 和 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92)。 -**交付状态核对（2026-10-05）**：W1 修复 PR #93 已合并；W2 [PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 尚待评审合并和第二人七项统一复现。初次 W2 交付 `65616a8` 已有三项远端工作流成功记录；集成 W3 的 [PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 在 `b5a4f58` 上也已通过通用 CI、完整前端及 W1/W2 专项。上述 CI 不能代替第二人执行。初次独立分支的范围与实测见 [分支验收与发布说明](W2-分支验收与发布说明.md)，下方本地整合记录保留各自日期及源码归属，不改写成当前提交的结果。 +## 独立复现版本 -## W2 独立技术验收入口 +[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 的既有 Linux 记录对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。后续文档或源码更新不自动继承该提交的执行结果。开始前记录约定 SHA、`git rev-parse HEAD` 和 `git status --short`；有未提交增量则保留完整 diff 和新增文件，不能只报告 HEAD。 -使用已有的 Python 3.12 固定 CPU 环境（[依赖](../../../requirements/qwen3-small-probe.txt)）、Zig 0.14.1、QEMU RV64，以及固定完整 ONNX 和官方 Tokenizer 资产。在仓库或完整源码副本根目录执行： +发布前个人工作树的详细测量按原始身份保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/README.md)。当前页面只提供 Linux 交付入口与可关联到具体提交的 Linux CI 记录，不把历史个人测量改名为 Linux 结果。 -```powershell -$w2Python = 'D:/path/to/qwen3-venv/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-new ` - --tokenizer-dir D:/path/to/qwen3-tokenizer ` - --model-dir D:/path/to/qwen3-0.6b-onnx ` - --cc D:/path/to/zig.exe ` - --qemu D:/path/to/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -Linux 使用相同参数，将解释器和工具替换为本机路径即可。程序不安装依赖；默认两个资产目录均使用 `verify` 离线校验。无资产时可显式传 `--tokenizer-mode download` 或 `--model-mode download`，后者需要下载约 1.24 GB 的完整模型发布包。所有结果使用本次全新的输出目录，不能复用已有目录，即使为空。 - -| 执行顺序 | 独立门禁 | 要求 | -|---|---|---| -| 1 | `frontend` | 22 项 RMSNorm/RoPE/SwiGLU/GQA 模式单测，不接受 skip | -| 2 | `backend` | 16 个小图 × none/all，32 次实际 RV64 QEMU | -| 3 | `runtime` | 官方 Tokenizer 20 组/480 对照，以及 5 组输入、mask、采样与停止条件 | -| 4 | `runtime-model` | 官方文本 ID 直入完整词表的真实两层小配置，Torch/ORT/IR none/all 单步接口集成 | -| 5 | `small-ir` | 从固定种子重新导出原两层普通/诊断图，7 组输入、29 检查点与不变性验证 | -| 6 | `small-qemu` | 使用本轮新导出的模型与报告，28 次两层实际 QEMU | -| 7 | `full-parse` | 固定 28 层完整图解析、逐节点/权重绑定审计与 IR verifier | - -只有七项全部实际执行并通过才返回 **PASS / 退出 0**。`--gates frontend backend` 等子集支持开发期间定位，但最多返回 **PARTIAL / 退出 2**；失败或阻断返回 **FAIL / 退出 1**。small-qemu 自动包含 small-ir，runtime-model 自动包含 runtime。默认每门禁截止 1800 秒、每次 QEMU 截止 180 秒、完整解析 worker 截止 300 秒，可用 `--gate-timeout`、`--qemu-timeout`、`--parse-timeout` 明确调整。 - -总报告在 `output/w2-acceptance-new/report.json/md/html`，每项有独立目录和 `logs/.log`。总入口核对退出码、报告身份、必要计数、实际输出和源码指纹；运行期间源文件变化也会失败。报告记录 HEAD/dirty diff（可用时）及本次源码 SHA256；没有 `.git` 的源码包允许运行，但必须明确标为未知 Git 身份，不能借用父目录仓库的提交。 - -新 [单步模型集成探测](../../../probes/w2_runtime_model/README.md) 保留完整词表 151936，hidden32/FFN96/两层随机权重。原文本 ID 不裁剪、不取模、不重新映射。逐元素检查完整 logits（含 padding），校验最后有效位置、原始 greedy ID 和官方解码；不要求随机权重生成有意义的文本。本地会保存较大的参考数组（约 614 MiB），CI 仅上传摘要报告，不上传这些参考数组或权重。该探测不执行完整 0.6B、完整词表 QEMU 或多步生成。 - -当前本地 workflow 已加入单步集成、完整 28 层解析必跑任务和汇总检查。`W2 / required acceptance` 要求轻量/数值任务与完整解析任务均为 success；失败、取消、跳过、缺失结果均不通过。完整 ONNX 的 ORT 前向仍是独立的手动/夜间 W1 任务。配置随本 W2 分支发布，远端 CI 以对应提交的实际执行为准；分支保护规则未变更。源码隔离复现共用本机 Python/工具链时，只能证明不依赖旧源码目录与旧产物，不能代替第二人或另一台机器复现。 - -统一验收的 Linux 超时路径会先发 SIGTERM，让运行器清理独立会话内的编译器、QEMU 或解析 worker，再执行进程组兜底清理。Windows 继续使用进程树/Job 清理。Linux 专属进程集成回归在 Windows 上明确跳过，发布前须在 Linux 实跑；不能用 Windows 结果替代它。 - -## 本轮继续交付与最终实测（2026-10-02，本地未提交） - -本轮新增完整词表的单步模型接口集成、七项统一验收入口及回归，并补齐原两层 IR 门禁的报告原子写入：MD/HTML 或 JSON 发布失败时必须返回失败，保留原数值错误，不遗留成功 JSON。此前四条报告路径的修复继续保留。 - -从当前工作树复制 **553 个源码文件**到全新的 `output/w2-independent-source-v2`，包含未提交改动，不复制 `.git`、历史 output、Python 环境或模型产物；从该目录真实运行统一入口，最终 **7/7 PASS，退出 0**。固定 ONNX/Tokenizer 资产重新验证哈希，Python 环境与 Zig/QEMU 工具仍共用本机现有安装。导入路径确认来自新副本，模型重新导出、ELF 重新编译、所有结果重新执行；总报告的 **134 项源码指纹与最终工作树一致**。 - -总证据：`output/w2-independent-source-v2/output/w2-acceptance/report.json/md/html`；每项明细、误差位置、日志和实际命令位于同一目录下对应子目录。快照来源及逐文件哈希在 `output/w2-independent-source-v2-manifest.json`，外层日志在 `output/w2-independent-acceptance-v2.log`。这些本机产物不进入版本库。 - -| 门禁 | 最终实测 | -|---|---| -| 前端四模式 | **22/22** 通过，未跳过 | -| 基础后端 | **32/32 QEMU**；最大绝对误差 **2.384185791015625e-7** | -| 运行时 | **20/20** 官方语料、**480/480** 编码/解码对照、**5/5** 输入与采样场景 | -| 单步模型集成 | **2 个原始文本、8 次模型调用**（2 Torch + 2 ORT + 4 IR）；完整输出 `[1,256,151936]`；最大绝对误差 **2.8312206268310547e-7**，采样 ID/解码一致 | -| 两层 IR | **7 组输入、29 检查点、6 项不变性**全部通过；所有数值比较最大绝对误差 **1.9073486328125e-6** | -| 两层 QEMU | **28/28**，普通/诊断图及 none/all 全通过；普通 logits 最大误差 **3.129243850708008e-7**，诊断检查点最大误差 **1.6689300537109375e-6** | -| 完整前端 | **28/28 层、7,847 节点、8,161 value、2,148 绑定**全部通过；5,910 条 IR 指令、verifier 无错误；绑定数据 **2,384,216,772 bytes** | - -统一流程 **264.340 秒**；两层 QEMU 进程累计 **157.642 秒**，其门禁全流程 **172.264 秒**；完整解析门禁 **9.538 秒**，其中 parser **2.031 秒**，worker 峰值 RSS **5,064,372,224 bytes（约 4.72 GiB）**。QEMU 计时包含进程启动、guest 运算和输出传输，诊断图包含检查点工作；这些是单次主机观察，不是纯推理性能，也不能据此声称提速。 - -首次源码副本实跑保存在 `output/w2-independent-source`，总结果正确为 FAIL：前端 pytest 临时目录的父目录未创建，导致 22 个 fixture setup 错误，其余六项通过。修复为使用统一输出根下的临时目录，新增真实 pytest 子进程回归后从新副本完整重跑，没有复用第一次的 PASS。另修复源码包向上查找 Git 的身份错误：运行时、两层 IR 和完整解析入口检查自身 `.git`；缺失时保留源码指纹并明确未知提交。最终实跑不依赖 `GIT_CEILING_DIRECTORIES` 等外部补救设置。 - -专项回归先执行 **631 passed / 0 skipped**（`output/w2-closeout-combined-tests.xml`）。最终修正上述入口后，对受影响两组分别复验 **72 passed / 0 skipped**（`output/w2-source-identity-all-tests.xml`）与 **47 passed / 0 skipped**（`output/w2-acceptance-tests-d.xml`）；批次重叠，不相加。工作流 YAML、25 段 Bash 语法、文档链接和 diff 格式检查通过。本轮未执行全仓测试或远端 CI，未提交、未推送、未更新 PR #91。 - -本轮补齐的是 W2 的独立技术验收和单步接口集成。第二人独立复现、团队接口确认以及发布后对应源码的远端 CI 仍需完成；完整预训练模型的 IR 数值、真实维度后端、独立权重装载和多步生成仍属于后续阶段。 - -## 前一轮：拆分后的本地 Review（2026-10-02） - -本轮审查完整解析、四模式前端、基础后端、Tokenizer、输入/mask/greedy，以及本地分支携带的两层 QEMU 修复。发现一类 P2 验收漏洞并修复四条报告路径：必需的报告写入失败时，可能留下可解析的 `passed=true` JSON。它会使仅消费 artifact 的脚本误判，不能以数值计算正确为理由忽略。 - -| 入口 | 复现与修复 | -|---|---| -| `probes/w2_backend_ops/run.py` | 原来先写 PASS JSON，随后 MD/HTML 失败；改为保存视图结果后发布 JSON，保存失败使门禁非零退出，保留原错误 | -| `probes/w2_qwen3_parse/run.py` | 同类先发布成功状态问题；保留验证阶段、failure_context 和主错误，报告保存异常单列，JSON 最后原子发布 | -| `probes/w2_runtime/run.py` | 虽然原来已最后写 JSON，完整内容写入后关闭/flush 抛异常仍可留下 PASS；改为同目录临时文件成功关闭后原子替换，失败不发布成功 JSON | -| `probes/w2_qwen3_small/riscv.py` | 既有两层 QEMU 门禁存在同类问题；视图与 JSON 原子写入，JSON 最后发布，保留原执行错误与时间证据。此修复只在本地，未同步 PR #91 | - -新增 **30 项**失败注入回归，覆盖 MD/HTML/JSON 无法保存、完整写入后报错、JSON 发布失败、原探测已失败及 CLI 状态。没有修改数值算法、精度阈值或模型配置。在本轮实际核对的路径和样本中，未发现新的前端/MatMul/Tokenizer 计算错误；这不构成完整模型数值正确性的证明。 - -| 本轮验证 | 结果与证据 | -|---|---| -| W2 与两层报告相关专项 | **396 passed / 0 skipped**；`output/w2-review-combined-tests.xml` | -| 前端模式、C 后端与 RISC-V 运行时相关回归 | **138 passed / 0 skipped**；`output/w2-review-compiler-regression.xml`。与上一行有重叠，不相加 | -| 基础后端实际 RV64 QEMU | **32/32**；对 ORT/独立 NumPy 最大绝对误差均 **2.384185791015625e-7**；流程 **23.588 秒**，QEMU 累计 **5.800 秒**；`output/w2-review-backend-real/report.json` | -| 官方 Tokenizer 离线探测 | **20/20** 语料、**480/480** 对照、**5/5** 输入与采样场景；流程 **3.739 秒**；`output/w2-review-runtime-offline-fixed/report.json` | - -已核对上述后端 30 项、运行时 5 项源码 SHA256 与修复后代码一致。额外 500 组随机 Unicode/特殊 token 文本编码和 4,000 次解码对照也通过；它们是本轮探索性检查，不算新增 CI 用例。本轮未重跑完整 28 层模型、整两层 QEMU 或全仓测试，也没有远端 CI；历史结果仍按原 SHA 归属。计时为单次观察，不用于声称性能改善。 - -### 当时可以继续的工作（历史规划） - -以下清单保留本地审查时的规划，不作为 2026-10-05 的当前待办：独立 W2 PR #94 已提交，第二至四项已由 W3 PR #96 实现并取得本地证据。当前仍待 W2 第二人复现和评审合并，以及 W3 专项 Linux、Nightly 与团队验收；见本页顶部和 [W3 进度](../W3/W3-工作总结与验收进度.md)。 - -1. **W2 交付收尾**：第二人用自己的环境执行统一验收入口；未来按用户指示整理独立 W2 PR，明确哪些两层门禁修复应单独回补 W1。当前 PR #91 的绿色状态不是本地 W2 的验收；同机源码副本复现也不能代替第二人验收。 -2. **真实维度与预训练子图**：先验证投影 MatMul、RMSNorm、RoPE、Attention、SwiGLU，保留 Torch/ORT/IR 和适当小子图的 QEMU 误差。固定源配置为 hidden=1024、head_dim=128、16 个 Q head/8 个 KV head、FFN=3072。现有两层缩小配置和 3-token GQA case 不能证明这些长累加维度满足误差要求。优先建立基线，再决定 MatMul 分块/布局优化。 -3. **参数化中等模型与逐层定位（W3）**：将两层配置和固定 29 检查点推广成配置驱动，跑计划中的六层中等模型，记录首个偏差、最大绝对误差、耗时与峰值内存。Attention 可先沿用现有分解 IR，融合不是正确性前置。 -4. **完整 IR 内存准备（W3）**：当前解释器会复制 initializer，并在 `values` 中保留中间结果，没有最后使用后回收。先测权重、活跃张量与 RSS，设计只读权重复用和按生命周期回收；再执行完整 IR/ORT，沿用计划中的 `<1e-4` 门槛。 -5. **完整后端与生成集成（W4/W5）**：当前常量/工作区默认各 256 MiB，QEMU 为 512 MiB，权重展开为 C 字面量；仅真实 embedding 就约 593.5 MiB。需要代码与权重分离、装载 ABI、地址布局和容量验证，再接完整前向与生成循环。Host helper 与完整词表两层随机模型的单步调用现已集成；真实预训练模型调用、逐 token 循环、chat template 与流式解码仍未完成。 - -建议下一轮从第 2 项的真实维度数值基线开始，同时准备第 3 项的可配置探测，避免直接以完整模型试跑替代可定位的小范围验证。 - -## W2 基础能力补齐（2026-10-02，历史提交 ff193c7） - -该次在已通过完整解析和两层数值探测的 `dfc9437` 上补齐三个独立入口，曾更新 PR #91，现已按 W1/W2 拆分要求撤出该 PR 并保留在本地。表中“通过”必须读取对应提交的报告或 CI；本页不把旧提交或跳过项计作当前本地修改通过。 - -| W2 门禁 | 当前入口与范围 | -|---|---| -| `unit:frontend-ops` | `tests/test_qwen3_frontend_patterns.py`：RMSNorm、RoPE、SwiGLU、GQA 四类模式，含错误 epsilon、旋转符号、门控分支与 KV 重复顺序反例 | -| `unit:backend-ops` | `probes/w2_backend_ops/run.py`：16 小图，MatMul/逐元素/Softmax/RMSNorm/RoPE/SwiGLU/GQA 七类；none/all 共 32 次实际 RV64 QEMU；独立 NumPy 公式和 ORT 双参考 | -| `unit:runtime` | `probes/w2_runtime/run.py`：固定官方七份 tokenizer/配置文件，20 组语料、fast/slow 双参考与 golden IDs/解码文本；mask/最后有效位置/greedy/EOS/容量/预算 | -| `numeric:ir-small` | 既有 `probes/w2_qwen3_small/run.py`：真实两层缩小配置随机权重，继续验收 IR 对照；另保留两层 QEMU 门禁 | -| `frontend:parse-qwen3` | 既有 `probes/w2_qwen3_parse/run.py`：完整 28 层固定产物解析与绑定审计；手动/定时完整模型 CI 实际执行 | +## 七项统一验收 -具体命令见 [基础后端门禁](../../../probes/w2_backend_ops/README.md) 和 [运行时门禁](../../../probes/w2_runtime/README.md)。生产新增 `scratchv/runtime/llm_inputs.py` 与 `scratchv/runtime/qwen3_tokenizer.py`；Tokenizer 只依赖可选 `tokenizers`，不在生产模块引入 Torch/Transformers。纯文本编码、完整模型输入准备与单步采样为可复用函数，尚未组合成端到端生成循环。 - -交叉 review 修复了非默认 prefix/special-token 配置被静默接受而与官方语义不同、缺省 unk token 偏离参考的问题；门禁也增加词表/特殊 ID 元数据与默认参数对照，并修正报告写入失败遗留 PASS 的路径。官方 NFC 规范化是保留行为，不把组合 Unicode 归一化误判成编译器错误。未知或空余 logits 行解码时失败，不静默丢弃或改变 greedy 候选。 - -本次 CI 增加明确的三个门禁、固定环境单测、独立 Summary 与 `w2-unit-gates-output` artifact。运行时下载约 15.2 MiB 的 tokenizer/config/许可证，按固定 revision 和既有 manifest SHA256 校验，不下载模型权重。源码和测试文件进入 Git，模型、下载缓存、ELF 与本地 output 不进入 Git。 - -### 本轮本地验收 - -| 验证 | 结果 | 本地证据 | -|---|---|---| -| 新增专项测试（固定 CPU 环境） | **228 passed / 0 skipped**，含四类前端模式 22 项、后端门禁 18 项、运行时 188 项 | `output/w2-final-unit-tests.xml` | -| 全仓回归（通用开发环境） | **2,732 passed / 41 skipped / 0 failed** | `output/w2-final-full-ready-tests.xml` | -| 基础后端真实 RV64 QEMU | **32 / 32** 通过；对 ORT 与独立 NumPy 参考的最大绝对误差均为 **2.384185791015625e-7** | `output/w2-backend-ops-final/report.json/md/html` | -| 官方 Tokenizer 下载与运行时门禁 | **20 / 20** 语料、**480 / 480** 比较、**5 / 5** 输入与采样场景通过 | `output/w2-runtime-final-download/report.json/md/html` | -| 同一执行身份离线复验 | 禁用 Hugging Face 在线访问，复用经过哈希校验的本地资源；上述 **20 / 480 / 5** 全部通过 | `output/w2-runtime-final-offline-same-user/report.json/md/html` | - -41 个跳过项包括本机缺少 `qemu-riscv32` 的 38 项，以及通用环境没有 Torch/Tokenizers 的 3 个模块；新 Tokenizer 模块已在上述固定 CPU 环境执行。两个既有 Torch 模块在本轮未本地重跑，PR CI 的固定环境继续执行原两层数值探测。各批次测试有重叠，不能相加成通过总数。 - -首次全仓运行因子进程未获得 LLVM 工具路径失败，修正测试进程的 PATH 后得到上表结果；首次跨执行身份的离线复验因 Windows 目录读取权限失败，同一身份复验通过。失败报告仍保留，未通过修改数值门槛或跳过检查消除失败。 - -后端探测全流程 **22.194 秒**，32 次 QEMU 进程墙钟累计 **5.660 秒**；Tokenizer 下载门禁 **14.824 秒**，同一身份离线复验 **4.198 秒**。均为本机单次观察，不是性能改善结论。已核对后端 30 项、运行时 5 项源码指纹与本次源码一致。仓库指定的本地 L2 harness 文件缺失，未宣称执行；本轮以实际专项、全仓和真实探测结果交付。 - -## 完整前端解析的本地复现 - -详细环境、Windows/Linux 命令、下载模式和报告说明见 [探测入口 README](../../../probes/w2_qwen3_parse/README.md)。所有命令在仓库根目录执行；使用 Python 3.12 和固定 NumPy/ONNX/ORT/protobuf 环境，无需 torch、Zig 或 QEMU。 - -已有完整模型时： +先完成环境准备，激活 `.venv-linux`，并设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU`。模型及官方 Tokenizer 目录必须包含固定 manifest 对应的真实资产。在仓库根目录执行： ```bash -python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -``` - -PowerShell 可使用实际虚拟环境解释器： - -```powershell -& .\output\qwen3-parse-venv\Scripts\python.exe -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE +set -euo pipefail +source .venv-linux/bin/activate +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export PYTHONIOENCODING=utf-8 +export SCRATCHV_PYTHON="$(pwd)/.venv-linux/bin/python" +export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" +export SCRATCHV_QEMU="$(command -v qemu-system-riscv64)" +W2_TOKENIZER=/absolute/path/to/fixed-qwen3-tokenizer +W2_MODEL=/absolute/path/to/fixed-qwen3-onnx + +git rev-parse HEAD +git status --short +"$SCRATCHV_PYTHON" -B -X utf8 scripts/run_w2_acceptance.py \ + --python "$SCRATCHV_PYTHON" \ + --output-dir output/w2-acceptance-linux-01 \ + --tokenizer-dir "$W2_TOKENIZER" \ + --model-dir "$W2_MODEL" \ + --cc "$SCRATCHV_CC" \ + --qemu "$SCRATCHV_QEMU" ``` -没有完整模型时，将 `verify` 改成 `download`，仍使用同一固定 W1 manifest。模型必须包含真实 external-data 分片。每次使用新的报告目录，失败时保留已有文件与日志。 - -## 如何判断本阶段完成 - -| 项目 | 应保留的证据 | -|---|---| -| 源码身份 | checkout SHA、是否有本地改动、实际 diff 与新增文件 | -| 固定模型身份 | 发布产物和 external-data 文件校验结果 | -| 完整结构与层覆盖 | 本次结构审计摘要及 28 层检查结果 | -| 实际节点转换 | `nodes.json` 与审计摘要 | -| initializer 绑定 | `bindings.json` 与审计摘要 | -| 共享 IR | `ir.txt` 与 IR 验证器结果 | -| 本次执行 | 命令、退出码、`report.json/md/html`、实测时间及内存（如平台可测） | -| CI | 对应提交和实际 run URL；未执行时明确写未执行 | - -退出 0 和 `passed=true` 必须来自本次执行。轻量 fixture 测试、历史 ORT 通过或一个可打开的报告文件都不足以代替完整图审计。失败发生较早时，某些明细可能尚未产生；按实际阶段和错误定位，不将缺失记录补成成功。 +把两项资产路径改成 Linux 实际目录。默认使用 `verify` 离线校验；无资产时可显式增加 `--tokenizer-mode download` 与 `--model-mode download`。后者下载约 1.24 GB 的固定完整发布包，解包及解析需要额外磁盘与数 GiB 空闲内存。入口不安装依赖。`--output-dir` 必须是本次全新的路径，连已存在的空目录也会被拒绝。 -本地 W2 workflow 现在让 PR、手动和定时任务均执行独立的 `full-qwen3-frontend` 完整解析任务，无须等待完整 ONNX 的 ORT 前向。该任务下载约 1.24 GB 的固定发布包，解析时需约 5 GB RSS；手动/夜间同时运行 ORT 任务时，两台 runner 会各自下载。汇总任务拒绝失败或跳过的依赖任务。PR #91 已合并；未来 W2 远端结果须关联其独立提交和实际 run，不能继承旧提交的 PASS。 - -## 完整解析 Review 与实测（2026-10-02，历史提交 dfc9437） - -该次同时复核本地完整解析门禁和 [PR #91](https://github.com/ScratchV-Compiler/ScratchV/pull/91)。核对时 PR head 为 `df18b02f152878b1ea838bbf6073def01d367456`，base 为 `d1c2b5f04cad817acf0b2f14d3bfa4ff56dbfa11`，当时差异涉及 52 个文件。审查覆盖编译入口、IR 优化、tensor-c 后端、RISC-V 运行时、数值探测、CI 和复现说明。下述完整解析与 Review 修复曾在 `dfc9437` 纳入 PR #91，现已撤出该 PR 并保留于本地；当时实际执行源码由报告中的 SHA256 确定。 - -### 完整解析门禁修复 - -| 问题 | 修复与验证 | +| 门禁 | 实际执行与判断 | |---|---| -| 属性审计复用生产 parser 的属性解码，可能让相同错误在转换和审计两端同时通过 | 直接从原 ONNX protobuf 独立读取属性，拒绝重复属性；增加错误 Transpose 轴和错误 epsilon 的反例 | -| 未检查函数参数名称与常量标记 | 对照原 ONNX 输入校验名称、顺序、shape/dtype，并拒绝被标记为常量的输入 | -| 标量辅助 `LOAD_CONST` 校验不完整 | 对 initializer 和 Constant 两条路径核对 opcode、目标 Value、空操作数、shape/dtype、常量标记、常量值及指令属性 | -| 审计时隐式 `np.asarray` 可能掩盖错误的绑定类型 | 要求实际绑定是 `numpy.ndarray`；list 等错误类型直接失败，合法只读 ndarray 仍接受 | -| 日志写入失败可能覆盖 worker 原始错误 | stdout/stderr 独立尝试保存；保留原错误、阶段及退出码，附加 `log_errors`；成功路径缺少日志也不能报 PASS | -| 文档链接指向被忽略的本地输出，干净 checkout 中无法访问 | 改为明确的本地输出路径说明，并增加模拟缺少 output 目录的文档链接回归测试 | - -进一步修正失败明细写入的同类问题：保存 `nodes.partial.json` 等失败证据时，写入异常记录到 `evidence_errors`，不覆盖原始解析或绑定错误；成功路径缺少必需证据仍失败。supervisor 在启动 worker 前保存 `attempt`，记录命令、模型路径、解释器、checkout 和源码指纹，便于定位过早超时、worker 未能启动等情况；该字段不代表验证已经执行。 - -### PR #91 修复 - -| 问题 | 修复与验证 | -|---|---| -| tensor-c 接受与 IR 标量常量值不同的外部绑定，可能与解释器行为不一致 | 校验绑定与常量声明一致；增加 FP32/INT32/INT64 错值、错误向量形状和合法标量执行的回归测试 | -| 编译或 QEMU 失败时，保存日志的异常可能掩盖编译错误、超时或 guest 错误 | 独立保存两路日志；保留主错误及原耗时，并附加日志错误。覆盖编译失败/成功与 QEMU 超时/失败/成功等组合 | -| RISC-V 报告不足以确定本次 ORT/IR 对照的实际执行环境和完整源码身份 | 顶层 `environment` 记录当前进程，`export_evidence.environment` 保留导出时环境；源码指纹覆盖 parser、IR、优化、后端和运行时，共 30 项 | - -未发现本次审查范围内需要额外修改的 MatMul 实现问题；下列结果只证明已测试输入和路径。未调整精度门槛、模型权重或 D8 等团队待确认决策。 +| `frontend` | RMSNorm、RoPE、SwiGLU、GQA 四种基础算子分解图；独立公式与反例 | +| `backend` | 七类基础算子的 16 个小图 × none/all，共 32 次 QEMU；误差 `<1e-5` | +| `runtime` | 官方 Tokenizer 的 20 组语料、480 次对照，以及输入、掩码、greedy、停止边界 | +| `runtime-model` | 完整词表两层随机小配置；2 文本、8 次 Torch/ORT/IR 调用及单步 greedy/解码 | +| `small-ir` | 官方 Qwen3 类两层小配置，7 输入、29 检查点、none/basic/all，误差 `<1e-5` | +| `small-qemu` | 同一本轮普通/诊断图 × none/all × 7 输入，共 28 次真实 QEMU，误差 `<1e-5` | +| `full-parse` | 固定完整 28 层 ONNX 的结构、逐节点转换、真实权重绑定与 IR verifier 审计 | -### 修复后的实测 +所有数值门禁同时检查 shape、dtype、有限值；上述阈值使用最大绝对误差、`rtol=0`，包括 padding 输出。Attention 使用基础算子组合，不要求新增融合 opcode。Tokenizer 的“往返 100%”指与官方 NFC/特殊 Token 规则一致，不是任意字节可逆。 -| 验证 | 当前结果 | 本地证据 | -|---|---|---| -| 全仓回归 | 2,563 passed / 40 skipped / 0 failed | `output/review-fix-full-tests.xml`、同名 `.log` | -| 固定 CPU 环境最终相关测试 | 286 passed / 0 skipped，包含全部 114 项完整解析测试 | `output/review-fix-pinned-tests.xml`、同名 `.log` | -| 真实 Qwen3 结构两层小模型重新导出与 PyTorch/ORT/IR 对照 | 7 / 7 组输入通过；全部数值比较最大绝对误差 `1.9073486328125e-6` | `output/review-fix-qwen3-small/report.json` | -| RISC-V QEMU 两层探测 | 7 组输入 × 普通/诊断图 × none/all 优化，共 28 / 28 执行通过；QEMU 对照最大绝对误差 `1.6689300537109375e-6`，门槛 `atol=1e-5, rtol=0` | `output/review-fix-qwen3-riscv/report.json`、`report.md` | -| QEMU MatMul | 7 / 7 通过，最大绝对误差 `2.384185791015625e-7` | `output/review-fix-matmul/report.json` | -| 完整 Qwen3 前端解析 | 28 / 28 层；7,847 节点、23 类算子；5,910 条 IR；2,148 项绑定；IR verifier 0 问题 | `output/review-fix-full-qwen3-final/report.json`、`report.md`、`report.html` | +仅七项全部执行并通过才会退出 **0 / PASS**；`--gates` 选择部分门禁即使成功，也只会退出 **2 / PARTIAL**。失败退出 1，中断按实际退出状态记录。`runtime-model` 自动带上 `runtime`，`small-qemu` 自动带上 `small-ir`。缺失、跳过、非有限误差、复用旧输出或执行中源码变化均不能冒充完整通过。 -全仓回归中的 38 个 RV32 执行测试因缺少 `qemu-riscv32` 跳过，另有 2 个依赖 torch 的模块因通用开发环境不含 torch 而跳过；最终固定 CPU 环境相关测试已覆盖这两个模块。全仓测试收集后补充了 3 项失败证据回归测试，均包含在最后的 286 项测试内；没有把不同批次或重叠测试相加为一个通过总数。RV64 tensor 真实运行时集成测试和上述实际 QEMU 探测已执行。 +## 报告与独立复现记录 -本次 QEMU 进程墙钟时间累计 **160.450 秒**，探测全流程 **174.634 秒**，4 个成功构建累计 **6.586 秒**。普通图每次平均约 **1.143 / 1.147 秒**（none/all），诊断图约 **10.138 / 10.494 秒**。QEMU 时间包含进程启动、guest 计算、输出打包/UART 和退出；诊断图还包含中间张量输出，不能当作纯前向耗时或真实硬件性能。 +总报告位于本次 `--output-dir` 下的 `report.json`、`report.md`、`report.html`；子目录保存各门禁原始报告、命令、日志、数组和模型/ELF 身份。源码指纹核对的是实际执行文件，不能把别处安装的模块误当当前 checkout。 -完整模型本次纯解析 **3.636 秒**，入口至验证及日志保存结束 **12.772 秒**，worker 峰值 RSS **5,064,331,264 bytes，约 4.72 GiB**。完整模型输出静态契约为 FP32 `[1,256,151936]`，绑定共 **2,384,216,772 bytes**；这些是解析与绑定审计结果，本次未执行完整模型数值前向。以上耗时均为本机单次运行，期间存在并行测试，不用于声称性能改善。 +- 保存 checkout SHA、dirty/diff、Linux 发行版、CPU、Python/NumPy/ORT/Torch、Zig/QEMU 版本。 +- 保存每项实际退出码、PASS/FAIL/PARTIAL/未执行、原始误差、首次失败阶段及报告位置。 +- QEMU 时长是进程墙钟，包含启动、guest 执行、UART 和退出；不是目标硬件性能或纯 forward 时间。 +- 重新生成自己的模型、参考与输出，不能下载第一人的数组后称为独立执行。 +- 独立复现和团队接口/出口确认分开记录，作者再次执行或 CI 自动运行不替代第二人。 -已核对完整解析报告的 20 项、RISC-V 报告的 30 项源码 SHA256 与当前源码一致；完整解析的 `nodes.json`、`bindings.json`、`ir.txt` 的文件大小和 SHA256 均与报告一致。测试后仅补充说明文档，执行源码未再修改。输出目录被 Git 忽略，仅在本机保存；他人复现需运行入口生成自己的报告。 +## 已有 Linux CI 证据 -最终还实跑了 0.1 秒截止和模型缺失两个失败路径：均正确退出 1，分别记录 `timeout`、`hashes` 阶段，且保留启动前的 20 项源码指纹；报告在 `output/review-fix-final-expected-timeout/` 和 `output/review-fix-final-expected-missing/`。文档检查包含尚未跟踪的新文档，共检查 97 个文件、196 个相对链接，0 个失效；LLM workflow YAML 与 19 段 shell 命令语法检查通过，`git diff --check` 通过。 +以下全部归属历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c`，不作为这次文档更新的新执行结果： -当前没有已确认且尚未修复的本轮阻断性问题。完整 IR/ORT 数值与容量验收、完整模型 QEMU 前向、权重独立装载、W1 第二人复现和接口签字仍未由本次工作完成。本页实测均为本地证据，远端 CI 状态及 run 链接以 PR 描述和 Checks 为准。 - -## 首次交付结果（2026-10-02，历史记录） - -首次交付工作在独立本地分支 `codex/w2-full-qwen3-parse`，基于 PR #91 的 `df18b02f152878b1ea838bbf6073def01d367456`。实际工作目录为 `D:/cyq/code/ScratchV/output/w1-pr89-review`；当时修改未暂存、未提交、未推送，尚未更新 PR #91。上层原工作目录中的已有改动未纳入本轮。 - -首次交付的完整实物执行退出 0，结果 **PASS**。本地证据保存在 `output/w2-full-qwen3-parse-complete/`：`report.md`、`report.json`、`nodes.json`、`bindings.json` 和 `ir.txt`。这些是该次 checkout 的本地输出，`output/` 不进入版本库；其他机器需执行入口重新生成，不能依赖这些历史路径存在。 - -| 验收项 | 首次交付实测 | +| 工作流 | 已记录结果 | |---|---| -| 固定模型 | Qwen3-0.6B，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 / L=256 / 无 KV Cache | -| 完整层链 | 28 / 28 层，含最终归一化与共享 embedding / LM Head | -| ONNX 节点 | 7,847 / 7,847；23 类基础算子；全部可达 logits | -| ONNX value | 8,161 项静态 shape / dtype 审计通过 | -| 实际 IR | 5,910 条指令；IR verifier 0 个问题 | -| 真实绑定 | 312 个 initializer + 1,836 个 Constant，共 2,148 项；逐项内容哈希通过 | -| 绑定字节数 | 2,384,216,772 bytes | -| 合法别名 | 112 个 Identity，均核对实际对象映射 | -| 返回值 | `logits`，FP32 `[1,256,151936]` | -| 纯解析耗时 | 2.493 秒 | -| 全入口耗时 | 10.533 秒，包含检查、解析、审计与子进程开销 | -| 验证进程峰值 RSS | 5,063,430,144 bytes，约 4.72 GiB，Windows PeakWorkingSetSize | - -耗时来自单次本地运行，不是平均值或性能承诺；本轮没有做解析性能优化。报告记录运行环境、实际源码 SHA256 和产物哈希。工作树有本地改动，不能将此结果仅归属于基准 HEAD。 - -首次交付的轻量测试 **91 / 91 通过**，在固定 CPU 环境和现有开发环境各执行一次，分别记录于 `output/w2-gates-pinned-delivery.xml` 和 `output/w2-gates-general-delivery.xml`。测试含真实两层 fixture 经文件哈希、结构检查、生产 parser、IR verifier 和绑定审计的贯通，以及漏节点、错权重、错 shape/dtype、外部数据损坏、解析异常、缺报告与超时等反例。当时全仓回归 **2,521 passed / 40 skipped**，记录于 `output/w2-regression-final.xml`；首次交付最后修正后重跑上述全部 91 项相关测试。当前修复后的验证以本页上方结果为准；跳过项不计为通过。 - -实际进程反例也已执行：模型缺失在 `hashes` 阶段退出 1；0.1 秒截止测试在 `timeout` 阶段退出 1，实测总耗时约 0.184 秒，均保存 FAIL 报告。记录分别位于 `output/w2-expected-missing-model/` 和 `output/w2-expected-timeout/`。 - -首次交付新增三个探测模块、三组测试与复现文档，并更新本地 LLM CI 配置。没有发现这份完整固定图需要新增前端算子，当时未修改通用 ONNX parser 或编译后端。首次互审修正了验收实现中的两个重点问题：控制参数必须从原 ONNX 独立求值，避免解析器错误自证通过；超时日志写入失败、进程异常退出及 verifier 失败时应尽量保留原始错误和节点证据。后续 Review 的更多修复见本页上方。 +| [通用 CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915411) | 成功；2927 passed / 7 skipped。Linux `test_w2_acceptance.py` 55/55、无跳过 | +| [LLM Deploy](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915420) | 成功；32/32 基础 QEMU、28/28 两层 QEMU、3/3 显式产物、20/20 超时清理；两层诊断最大误差 `1.7285346984863281e-6` | +| [Topic 06](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37277915428) | 成功；范围以该 run 的实际步骤与 artifact 为准 | -本阶段已具备完整前端解析的本地验收证据；远端 CI、第二人复现、完整模型 IR 数值及完整模型 QEMU 前向尚未由本轮验证。 +PR 中按条件未运行的完整 ONNX ORT 重型任务仍是未执行，不因上述绿色记录而算作完整 ORT 重验。新的提交需要新的 Checks；已有 Linux CI 不替代本次七项统一入口的第二人独立复现。 -## 后续衔接 +## 分项入口与后续边界 -1. 前端审计通过后，依据实际图和权重规模建立完整 IR 数值与峰值内存基线，逐步扩大预训练模型验证范围。 -2. 设计代码与权重分离、权重绑定 ABI 和内存布局；当前 C 常量展开及小模型 guest 配置不能直接承担完整 0.6B 模型。 -3. 完整 IR 数值验收通过后，再验证完整 QEMU 单次前向；随后将已有 Tokenizer、输入准备、最后有效 token logits、采样和停止条件组合成生成循环，并验证真实文本输出。 +- [完整前端解析](../../../probes/w2_qwen3_parse/README.md)：无须 Torch 或 QEMU，可单独准备固定解析环境。报告含 `nodes.json`、`bindings.json`、`ir.txt`。 +- [前端模式与基础后端](../../../probes/w2_backend_ops/README.md)：NumPy 独立公式、ORT、IR 和 QEMU 四路检查。 +- [Tokenizer 与输入](../../../probes/w2_runtime/README.md) 和 [单步模型接口](../../../probes/w2_runtime_model/README.md)：模型词表 151936 与实际可解码 ID 集合分开处理，不取模重映射。 +- [两层数值探测](../../../probes/w2_qwen3_small/README.md)：普通图、诊断图、产物绑定、仿真计时与精度边界。 +- [发布范围与修复](W2-分支验收与发布说明.md)：共享后端、日志和 Linux 取消清理的交付归属。 -前端解析通过不能跳过这些阶段，也不能据此宣布完整 Qwen3 已在 ScratchV 上运行。开发范围见 [总开发计划](../开发计划.md)，W1 人工验收状态见 [W1 独立复现记录](../W1/README.md)。 +完整解析通过不能替代完整 IR 数值、完整权重独立装载、完整模型 QEMU 前向或真实文本生成。阶段安排见 [开发计划](../开发计划.md)。 diff --git "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" index e08da37..1cde2ae 100644 --- "a/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W2/W1\344\277\256\345\244\215\344\270\216W2\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -1,125 +1,9 @@ -# W1 修复与 W2 本地验收报告 +# W1 修复与 W2 历史验收记录索引 -日期：2026-10-03。**历史整合工作树报告**：记录的是发布前 W1+W2 合并在本地的验证；其中“未提交”等为当时状态。后来按 W1/W2 拆分独立发布，当前 W2 分支结果见 [分支验收与发布说明](W2-分支验收与发布说明.md)，不能将下述组合源码哈希当作当前分支哈希。 +本文件名保留以兼容已有链接。当前正式交付、CI 和独立复现统一使用 **Ubuntu 24.04 x86_64 / Bash / Python 3.12**，从 [Linux 复现约定](../LINUX_REPRODUCTION.md) 和 [W2 七项统一入口](README.md) 开始。 -## 1. 结论与源码位置 +原报告记录 2026-10-03 发布前的个人混合工作树，包含 W1 修复与 W2 增量，既不是独立 W2 分支的精确源码，也不是 Linux 实测。原始测量和当时状态完整保留于 [固定提交归档](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W1修复与W2本地验收报告.md)。没有修改原始平台标签、计数、误差或耗时来生成新的通过结论。 -W1 的报告可靠性与源码归属修复已补入；W2 已在 PR #91 合并基线上整合，并重新执行七项技术验收，结果 **7/7 PASS**。完整图的通过范围是前端解析、结构和权重绑定审计；完整 0.6B 的 IR 数值、QEMU 前向与生成循环仍未完成。本报告不替代团队冻结接口、第二人复现或远端 CI。 +已交付的修复包括报告写入失败与源码归属、标量常量绑定一致性、Linux 嵌套会话退出清理、七项统一验收及 CI 缺失/跳过检测。具体代码归属与 Linux run 见 [W2 分支验收与发布说明](W2-分支验收与发布说明.md)；W1 报告修复单独见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。 -本轮源码在 `D:/cyq/code/ScratchV/output/w1-w2-local-closeout`，本地分支 `codex/w1-w2-local-closeout`。基线是 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，所有增量均未提交。原 `output/w1-pr89-review` 的 W2 修改、`output/w1-pr91-repro-fixes` 和最外层工作目录的代码保持原样。 - -下述 `output/...` 均相对于本轮源码目录。它们是本机证据，不是仓库发布资产；复现者应使用新的目录生成自己的报告。 - -## 2. W1 补完了什么 - -| 修复 | 原问题 | 当前行为与主要文件 | -|---|---|---| -| 两层报告完整发布 | MD/HTML 写入失败或 JSON 发布失败可能留下成功证据 | 视图先写、JSON 最后原子发布；失败返回非零，尽力保存 FAIL，保留原始数值错误。`probes/w2_qwen3_small/run.py`、`riscv.py` | -| 源码与环境归属 | 无 `.git` 的源码副本可能向父目录查找 Git，误认其他 checkout；QEMU 对照环境可能与导出环境混淆 | 无本目录元数据时明确未知 Git 身份，保留源码 SHA256；分别记录当前执行与导出环境，并补齐 parser/IR/优化/运行时指纹 | -| 回归覆盖 | 成功路径测试不足以验证失败时的证据 | 故障注入覆盖写入、关闭、替换失败与原始错误保留；补源码副本及当前环境归属回归 | -| 原有算子用例与固定 ORT 环境兼容 | ORT 1.22.1 无 DOUBLE Cos 内核，ReduceMean 的 omitted axes + noop=1 路径报内核错误 | 三项用例显式使用 ONNX 官方 ReferenceEvaluator，继续实际运行生产 parser/IR，ReduceMean 再检查与输入完全相等；其余用例继续 ORT 对照，不捕获或忽略数值错误 | -| 共享后端与运行时防错 | 外部绑定可能覆盖标量 IR 常量；日志 I/O 失败可能遮蔽原编译/QEMU 错误 | `tensor_c_codegen.py` 校验常量绑定一致；`riscv_tensor.py` 独立保存两路日志，保留主错误和耗时；一并整合之前 W2 分支暂存的修复 | -| 文档状态 | 仍使用 PR 未合并、旧提交或“所有第二人复现待做”的描述 | 固定已合并基线，保留 `df18b02f` 第二人两层复现事实，区分完整模型复现与接口签字待办 | - -在整合 W2 前，W1 四个源码/测试文件的专项测试已执行 **120 passed / 0 skipped**，证据为 `output/w1-fix-tests-v2.xml`。W1 独立检查点在 `output/w1-fix-checkpoint/`：包含补丁、四文件快照与哈希，便于后续按 W1/W2 分拆提交。这次没有重新打开或更新已合并的 PR #91。 - -最终共享修复的建议拆分补丁为 `output/w1-final-review/w1-fixes.patch`，含上述报告/归属、标量绑定、运行时清理/日志及 ORT 用例兼容的 10 个源码/测试文件，配有 `manifest.json`；它是供审阅的本地补丁，不是已暂存或已提交的改动。 - -最后还单独实跑原 W1 入口：MatMul **7/7**（4×4 小数最大误差 `1.4901161193847656e-8`，七例最大 `2.384185791015625e-7`）；PR #89 合成两层图的 IR vs ORT 最大误差约 `7.153e-7`，ORT vs 独立 NumPy 约 `1.097e-6`。证据在 `output/w1-matmul-final/`、`output/w1-transformer-final/`。完整 0.6B ONNX 的 ORT 前向本轮没有重跑，其历史验收仍按原记录归属。 - -W1 人工出口继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪：团队 D1–D8 接口决议、风险处置责任与 Plan B、完整模型第二人复现和最终出口评审。本轮没有代签或关闭这些事项。 - -## 3. W2 工作与开发计划逐项对照 - -| 开发计划要求 | 当前实现与本轮证据 | 状态边界 | -|---|---|---| -| `unit:frontend-ops`：RMSNorm/RoPE/SwiGLU/GQA | 22 项真实 ONNX 模式测试全部通过 | 通过基础算子组合解析，未新增同名融合 IR 指令 | -| `unit:backend-ops`：基础算子 QEMU | MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA，7 类/16 小图 × none/all，共 32 次通过 | 超过表中六类最低覆盖；仅证明这些 shape 与输入 | -| `numeric:ir-small`：两层 IR vs ORT | 真实 Qwen3 类的两层缩小配置，7 输入、29 检查点、6 项不变性通过；none/basic/all 优化对照 | 随机固定权重，不是预训练权重切片 | -| Attention IR 定义 | 候选接口文档明确基础算子组合、Q/K 独立 RMSNorm、全维 RoPE、GQA 分组与加性 mask 语义 | 当前不强制新增融合 ATTENTION opcode；这项接口决议仍需团队确认 | -| `unit:runtime`：Tokenizer、greedy、mask | 固定官方 Tokenizer 20 组语料/480 对照，5 组输入、mask、采样与停止条件通过 | 往返要求按官方 tokenizer 的特殊 token/规范化语义校验，不声称任意文本字节级恒等 | -| `frontend:parse-qwen3`：完整 28 层 | 固定 FP32/L=256 产物完整解析；7,847 节点、8,161 value、2,148 绑定、5,910 IR 指令通过审计，verifier 无问题 | 不代表完整 IR 数值或 QEMU 已运行 | -| 单步运行时与模型接口集成 | 2 个文本、完整词表 151936；2 Torch + 2 ORT + 4 IR，共 8 次调用；原始 token ID、最后有效位置、greedy ID 与解码一致 | hidden32/FFN96 两层随机小配置；未执行完整词表 QEMU 或多步生成 | -| W1 数值链路不退化 | 两层普通/诊断图 × none/all × 7 输入，28 次真实 QEMU 全通过 | 原 W1 的 20 次 Linux 超时回归与 3 项显式产物集成 CI 配置继续保留 | -| CI 全绿 | 本地配置增加完整 28 层解析 PR 必跑任务与汇总检查，拒绝跳过/失败/缺失 | **尚未提交，所以没有本轮远端 CI 通过结论** | - -本轮整合原先仅本地保存的完整解析、后端和运行时工作，新增修复 CI 验收缺口与嵌套进程超时清理。生产后端还保留审查所得的标量常量绑定一致性检查，拒绝调用方用不同绑定覆盖 IR 常量；日志写入失败保留原编译/QEMU 错误与耗时。这些此前 W2 分支的修复均已在本轮基线上重新验证。 - -## 4. 本轮重新执行的验收数据 - -总入口：`scripts/run_w2_acceptance.py`。总报告：`output/w2-acceptance-20261003/report.json`，另有同名 Markdown/HTML。每项包含自己的原始报告、日志、命令、输出数组、模型或 ELF 身份。总入口核对 **136 项源码指纹**，拒绝旧目录、缺失门禁、skip、非有限误差及运行中代码变化。 - -| 门禁 | 结果 | 最大绝对误差 | 顶层观测耗时 | -|---|---|---:|---:| -| frontend | 22/22 | 不适用 | 1.157 秒 | -| backend | 32/32 QEMU | `2.384185791015625e-7` | 23.769 秒 | -| runtime | 20/20、480/480、5/5 | 离散结果精确对照 | 4.905 秒 | -| runtime-model | 2 文本、8 次调用 | `2.8312206268310547e-7` | 14.889 秒 | -| small-ir | 7 输入/29 检查点/6 不变性 | `1.9073486328125e-6` | 6.971 秒 | -| small-qemu | 28/28 QEMU | 普通图 `3.129243850708008e-7`；诊断图 `1.6689300537109375e-6` | 153.658 秒 | -| full-parse | 28/28 层、逐节点/绑定审计通过 | 本项不执行数值前向 | 11.111 秒 | - -数值门槛保持 `max_abs < 1e-5`、`rtol=0`，没有放宽精度、裁剪 token ID 或用历史产物替代本次执行。small-qemu 报告也包含主机 IR 参考，其全报告最大误差 `1.9073486328125e-6` 不应误写成 QEMU 自身误差。 - -整轮 **216.597 秒（约 3 分 37 秒）**。基础后端 32 次 QEMU 进程墙钟累计 **5.545 秒**；两层 28 次累计 **140.312 秒**，4 次 C→RV64 编译累计 **6.189 秒**。两层普通图平均约 1.06/1.07 秒（none/all），诊断图约 8.94/8.97 秒；诊断输出多，不能当作纯前向速度。 - -完整前端 worker 的纯解析 **2.579 秒**，验证流程 **10.818 秒**，峰值 RSS **5,064,474,624 bytes，约 4.72 GiB**；绑定数据 **2,384,216,772 bytes**。这些是本机单次观察，不是优化前后基准对比，也不等同真实硬件性能。 - -## 5. Review、回归及 CI 检查 - -本轮 Review 先修复两个新发现的问题： - -1. 完整解析原来只在手动/夜间任务运行，PR 绿色无法证明 W2 全部门禁。现在 `full-qwen3-frontend` 在匹配路径的 PR 上必跑；`w2-acceptance` 对所有依赖要求 success。对应 21 项结果校验测试覆盖失败、跳过、取消、缺失和畸形 JSON。分支保护设置未修改。 -2. 顶层探测的 Linux 超时直接杀进程组，可能遗留 `_run_process` 创建的独立编译器/QEMU 会话。现在先 SIGTERM 触发运行器清理，再强制清理原组；增加实际进程中断回归和 Linux 嵌套会话回归。Windows 路径沿用 Job/进程树处理。 - -扩大回归时还修复了上文三项 ORT 参考内核兼容问题；相关四个测试文件最终 **251 passed / 0 skipped**，证据 `output/review-regression-fixes.xml`。没有修改编译器计算算法或数值门槛来适应参考执行器。 - -超时与 CI 专项结果 **128 passed / 1 skipped**，证据 `output/timeout-ci-tests.xml`；跳过的是 Linux 专属嵌套会话集成用例，本机 Windows 无法执行，不能计作通过。W2 集成/报告/CI 结果校验前一轮为 **111 passed / 0 skipped**，证据 `output/w2-integration-tests.xml`。各批次有重叠，不能相加。 - -最终 `tests/` 全量回归 **3020 passed / 39 skipped / 0 failed，177.66 秒**，证据为 `output/final-all-tests-v4.xml` 和同名 `.log`。39 个 skip 明确分为：本机无 `qemu-riscv32` 的 38 个旧 RV32 用户态执行测试，以及 1 个 Linux 专属嵌套会话清理测试。真实 RV64 tensor 运行时测试、三项显式产物优化集成及本轮所有可在 Windows 执行的 W1/W2 测试均已执行；不能把 39 个 skip 写成通过。 - -最终工作流 YAML、29 段 Bash 语法、文档相对链接、W1 四类接口/21 条风险及 Plan B 的结构检查、`git diff --check` 通过；详见 `output/final-static-review.json`。七项验收记录的 136 项源码指纹仍与最终执行代码一致。报告及状态文档在数值执行结束后补写，不改变该数值结果对应的执行代码。项目约定的 `.claude/harness/verify/run.py` 不存在，未宣称运行 L2 harness。 - -失败尝试也保留：W1 首次测试因自定义 pytest 临时目录的父目录尚未创建而产生 82 个 setup error，创建目录后 120 项通过；全仓第一次在收集阶段发现固定模型环境缺少可选 `llvmlite`。后者使用已有开发环境的 llvmlite 0.49.0 与 tinyfive 1.0.0 的本地副本补入仅本轮测试的路径，未改固定模型环境或全局安装。 - -中间全仓尝试 `final-all-tests-v2` 为 228 failed / 2791 passed / 40 skipped：224 项是 LLVM 工具未发现，3 项是上述 ORT 参考限制，1 项文档链接失败发生在报告文件尚未写完时。`v3` 显式指定 llvm-mca 后为 5 failed / 3014 passed / 40 skipped，剩余 5 项均因 clang/ld.lld 未从 PATH 找到。最终通过 `output/run_local_regressions.py` 在 Python 进程内配置路径，并确认三项 LLVM 工具实际可运行；不把这些中间失败隐藏或计算成通过。 - -## 6. 尚待完成与后续工作 - -- 本轮 Linux 嵌套会话清理集成回归、修改后的 GitHub workflow 和完整解析 runner 的实际容量，需要发布后取得对应源码的 Linux CI 证据。源码/配置本地通过不等于远端已通过。 -- W2 需要第二人用自己的环境和全新输出目录运行七项统一验收；W1 已确认的是旧 SHA 的两层链路，不能自动覆盖这次新增修复或 W2。 -- W1 接口冻结与风险责任决议保持候选，继续按 Issue #92 讨论；不要因技术门禁通过代替团队确认。 -- 后续按 W3 先做真实维度、预训练权重子图和更深层 IR 数值/内存基线，再做权重与代码分离、权重加载 ABI、内存复用和完整模型 QEMU 前向；最后接多步生成。原 `backend=riscv` 标量选择器并未因此支持大模型，本路线使用 `tensor-c`。 - -## 7. 本机复现命令 - -在本轮源码目录执行，输出目录必须是全新的。下面使用现有工具与固定模型资产，不自动安装或下载。 - -```powershell -$w2Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -& $w2Python -B -X utf8 scripts/run_w2_acceptance.py ` - --python $w2Python ` - --output-dir output/w2-acceptance-review-new ` - --tokenizer-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` - --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` - --cc D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/zig-python/ziglang/zig.exe ` - --qemu D:/cyq/code/ScratchV/output/w1-pr89-review/output/tools/qemu/qemu-system-riscv64.exe -$LASTEXITCODE -``` - -人工首先检查本报告、七项总报告和文件清单，再决定后续拆分提交方式。本轮没有创建、修改或关闭 GitHub PR/Issue。 - -## 8. 修改文件审阅顺序 - -| 范围 | 主要文件 | -|---|---| -| W1 两层证据可靠性 | `probes/w2_qwen3_small/run.py`、`riscv.py` 及对应 gate 单测 | -| 编译/运行时基础修复 | `scratchv/backend/tensor_c_codegen.py`、`scratchv/runtime/riscv_tensor.py` 及对应单测 | -| 固定 ORT 测试兼容 | `tests/test_onnx_qwen_numeric_ops.py`、`tests/test_onnx_shape_ops.py` | -| 完整图审计 | `probes/w2_qwen3_parse/{run,validation,audit}.py`、三组 `test_qwen3_full_*.py` | -| 基础后端与前端模式 | `probes/w2_backend_ops/`、`tests/test_qwen3_frontend_patterns.py`、后端 gate 测试 | -| Host 运行时与单步模型集成 | `scratchv/runtime/{qwen3_tokenizer,llm_inputs}.py`、`probes/w2_runtime/`、`probes/w2_runtime_model/` 及对应测试 | -| 验收入口与 CI | `scripts/run_w2_acceptance.py`、`scripts/check_w2_ci_results.py`、`.github/workflows/llm-deploy.yml` 及对应测试 | -| 文档与依赖声明 | W1/W2 指南、执行/探测计划、接口/风险候选、总开发计划、各 probe README、`pyproject.toml` 的可选 tokenizer 依赖 | - -最终全文件清单与哈希保存在 `output/final-review-manifest.json`；完整已跟踪文件差异在 `output/final-tracked.patch`，新增文件保留在上表目录。模型、参考数组、ELF、可选测试依赖副本均在被忽略目录，未纳入源码差异。 +新复现必须在指定 SHA 上自行生成报告，记录 Linux 环境、实际退出码、源码与模型身份以及误差。旧原始数据可用于调查历史问题，不能代替新提交执行或第二人确认。 diff --git "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" index 4e95d1e..fc5b57d 100644 --- "a/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" +++ "b/docs/llm-deploy-v1.0/W2/W2-\345\210\206\346\224\257\351\252\214\346\224\266\344\270\216\345\217\221\345\270\203\350\257\264\346\230\216.md" @@ -1,27 +1,27 @@ -# W2 独立分支验收与发布说明 +# W2 Linux 分支验收与发布说明 -W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。本分支直接基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`，不包含独立 W1 PR 中的六个两层报告/参考测试文件。两部分可分别审阅和合并。共享后端的标量绑定、运行时日志和取消清理加固随 W2 交付。 +正式复现环境为 Ubuntu 24.04 x86_64 / Bash / Python 3.12。环境与工具准备见 [Linux 复现约定](../LINUX_REPRODUCTION.md)，七项统一入口和实际 CI 链接见 [W2 README](README.md)。 -## 范围 +W1 两层报告与参考测试修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。W2 初次独立交付基于 PR #91 合并提交 `3bb88e81498d6d9293aed70e25a4e5a2104465d0`；[PR #94](https://github.com/ScratchV-Compiler/ScratchV/pull/94) 既有 Linux 执行对应 `2d07ccc42936dcfde19d552bca02a324cd33821c`。新修改按新 SHA 验收，不能继承旧 CI PASS。 -- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和权重绑定审计。 -- RMSNorm/RoPE/SwiGLU/GQA 前端模式，七类基础后端的 16 小图/32 次 QEMU。 -- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层小配置的单步模型集成。 -- 七项统一验收入口，完整前端 PR 必跑任务，以及拒绝失败、取消、跳过、缺失的 CI 汇总。 -- 标量 IR 常量外部绑定一致性；日志写入失败保留主错误；取消时清理独立编译器/QEMU 会话。 +## 交付范围 -本分支 Windows 固定 CPU 环境重新执行：**629 passed / 1 skipped**（仅 Linux 嵌套会话清理用例跳过）；**七项统一验收 7/7 PASS**，实际 32 次基础后端 + 28 次两层 QEMU 全通过。完整图为 28 层、7,847 节点、2,148 绑定、5,910 条 IR，verifier 与逐项审计通过。运行时为 20 组语料/480 对照/5 场景；完整词表单步集成为 2 文本、8 次调用。136 项源码指纹与发布前代码一致。 +- 固定完整 28 层 ONNX 的结构、前端解析、IR verifier、逐节点和真实权重绑定审计。 +- RMSNorm/RoPE/SwiGLU/GQA 前端模式；七类基础后端的 16 小图、32 次 QEMU。 +- 官方 Tokenizer、输入/mask、最后有效位置 greedy/停止条件，以及完整词表两层随机小配置的单步模型集成。 +- 七项统一验收、完整前端 PR 必跑任务，以及拒绝失败、取消、跳过和缺失的 CI 汇总。 +- 标量 IR 常量外部绑定一致性；保存日志失败时保留主错误；Linux 取消时清理独立编译器/QEMU 会话。 -总入口耗时 251.494 秒，两层 QEMU 进程累计 154.125 秒，基础后端 QEMU 累计 6.194 秒；并行验收时的本机单次观察，不用作性能对比。精度最大误差：基础后端 `2.384185791015625e-7`，单步集成 `2.8312206268310547e-7`，两层 IR `1.9073486328125e-6`，两层目标诊断 `1.6689300537109375e-6`，均小于 `1e-5`。 +## Linux 验证与源码归属 -另外，以本分支新生成的模型显式执行 none/basic/all 三项产物优化集成，**3/3 通过、0 跳过**，证据为 `output/w2-publish-artifact.xml`。 +历史 SHA `2d07ccc42936dcfde19d552bca02a324cd33821c` 的通用 CI、LLM Deploy、Topic 06 均成功。通用 CI 为 2927 passed / 7 skipped；Linux 统一验收单测 55/55、无跳过；LLM 记录基础 QEMU 32/32、两层 QEMU 28/28、显式优化产物 3/3、退出清理 20/20。两层 QEMU 诊断最大绝对误差 `1.7285346984863281e-6`，小于 `1e-5`。原始 run 链接在 [W2 README](README.md)。 -证据在 `output/w2-publish-tests.xml`、`output/w2-publish-acceptance/report.json/md/html` 及各子目录。工作流 YAML、29 段 Bash、文档链接及 diff 格式检查通过。本分支没有重跑全仓 tests；原整合工作树 3020 项通过属于下述历史记录，不能用其代替本分支验证。固定 ORT 环境下三个原有算子参考用例的兼容修复归属 PR #93。项目的本地 L2 harness 文件缺失，未声称运行。此前 [W1 修复与 W2 本地验收报告](W1修复与W2本地验收报告.md) 记录的是两部分合在一起的历史工作树，不应冒充本分支的精确源码证据。 +跳过项不计为通过；按 PR 条件未运行的完整 ONNX ORT 重型任务不计为完整 ORT 重验。上述工作流集合与作者运行统一七项 CLI、第二人完整独立执行，是不同证据，分别记录。 -## 复现与限制 +发布前个人工作树的计数、数值和耗时保留于 [固定提交中的原报告](https://github.com/ScratchV-Compiler/ScratchV/blob/2d07ccc42936dcfde19d552bca02a324cd33821c/docs/llm-deploy-v1.0/W2/W2-分支验收与发布说明.md)，不迁移成 Linux 测量值。早期 W1/W2 混合工作树记录见 [历史归档说明](W1修复与W2本地验收报告.md)，不代表独立 W2 分支的精确源码。 -运行 [统一入口](README.md) 的七项门禁；每次使用新的输出目录。源文件哈希、模型/ELF 身份、原始命令、误差和仿真墙钟时间均保存在生成的 JSON/Markdown/HTML 中。 +## 复现与限制 -完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合，无新增融合 opcode。第二人复现、接口冻结、完整模型权重独立装载和生成循环仍需后续完成。 +运行统一入口时每次使用不存在的新输出目录。保留 Linux 版本、CPU、依赖和工具版本、源文件哈希、模型/ELF 身份、原始命令、误差及仿真墙钟；七项完整通过才能记为 PASS。新的提交需要新的实际运行证据。 -Linux 嵌套会话清理用例在 Windows 上明确跳过；CI 配置已经提供入口，但只有对应提交的实际远端执行才能证明 Linux 通过。完整 ONNX ORT 前向继续由独立手动/夜间任务验证。 +完整图解析不等于完整 0.6B 的 IR/QEMU 数值前向。当前 Attention 使用已定义语义的基础算子组合；完整词表单步模型使用随机小配置，不具备预训练模型语言能力。第二人独立复现和团队接口确认仍由参与者完成；完整权重装载、生成循环等后续阶段不能由本分支的小模型结果代替。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 917e598..57536a7 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -11,6 +11,8 @@ --- +**交付环境约定（2026-10-06）**：W1 及后续阶段的公开指南、第二人复现和 CI 统一使用 Linux（Ubuntu 24.04 / Bash / Python 3.12）。作者个人环境不构成 Linux 验收；保留历史记录的真实平台和提交身份。环境安装、资产和后续提交规则见 [Linux 复现约定](LINUX_REPRODUCTION.md)。这只规定 Host 环境，目标仍为 RISC-V 裸机程序。 + ## 1. 项目概述 ### 1.1 背景 diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" index b39dab2..b1395f8 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -17,14 +17,22 @@ DSL -> DSLInterpreter -> 解释器结果 -> 期望值对比 ## 运行依赖 -测试命令需要在 ScratchV 仓库根目录运行。`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 +复现环境统一为 Linux / Bash，使用 Python 3.12。测试命令需要在 ScratchV 仓库根目录运行。 +首次使用时创建并激活虚拟环境： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +`scripts/run_topic06_benchmarks.py` 会导入 `scratchv` 包，并通过 ScratchV 编译器把每个 DSL 用例编译为 RISC-V 汇编。 调用真实 TinyFive 所需的配套接口位于 `scratchv/simulator/tinyfive.py`、 `scratchv/backend/register_alloc.py`、`scratchv/compiler.py` 和 `scratchv/main.py`。 安装基础测试依赖： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 基础模式需要以下依赖： @@ -34,8 +42,8 @@ pip install -e ".[topic06]" 如需生成 HTML 和 PNG 完整报告，再安装可选依赖： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` 其中 `jinja2` 用于生成 HTML，`matplotlib` 用于生成性能图表。 @@ -67,13 +75,13 @@ build/topic06/ # 运行时生成，不提交 在仓库根目录执行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` 默认同时执行解释器与 TinyFive，并分别记录状态。也可以显式选择： -```powershell +```bash python scripts/run_topic06_benchmarks.py --verification-backend both python scripts/run_topic06_benchmarks.py --verification-backend interpreter python scripts/run_topic06_benchmarks.py --verification-backend tinyfive @@ -102,13 +110,13 @@ python scripts/run_topic06_benchmarks.py --verification-backend tinyfive 更新性能基线： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` 性能退化阈值默认是 5%，可以按场景调整： -```powershell +```bash # 超过 2% 即判定为退化 python scripts/run_topic06_benchmarks.py --regression-threshold 2 @@ -122,19 +130,19 @@ python scripts/run_topic06_benchmarks.py --regression-threshold 10 按类别运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按用例名称进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 组合筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -160,7 +168,7 @@ benchmarks/topic06/baseline.json 测试和报告生成采用两阶段流程。先运行测试并生成与展示格式无关的元数据： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -172,7 +180,7 @@ benchmark_reports/topic06/metadata.json 随后由独立报告工具读取元数据并生成轻量报告： -```powershell +```bash python scripts/generate_topic06_report.py ``` @@ -187,7 +195,7 @@ benchmark_reports/topic06/cases/*.json 需要 HTML 和 PNG 时运行： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` diff --git "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" index b5a687c..a95e6b0 100644 --- "a/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/06-\346\200\247\350\203\275\345\237\272\345\207\206\345\245\227\344\273\266/06-\346\200\247\350\203\275\346\265\213\350\257\225\345\245\227\344\273\266\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -366,7 +366,7 @@ Markdown 和 HTML 报告固定展示相同核心指标。CI 推荐读取 `benchm 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -374,7 +374,7 @@ python scripts/run_topic06_benchmarks.py 运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` @@ -386,7 +386,7 @@ benchmarks/topic06/baseline.json 后续再次运行： -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -398,7 +398,7 @@ python scripts/run_topic06_benchmarks.py 如果变化率超过配置阈值，则认为发生性能退化。默认阈值仍为 5%，可以使用命令行参数覆盖： -```powershell +```bash python scripts/run_topic06_benchmarks.py --regression-threshold 2 ``` @@ -478,48 +478,57 @@ benchmark_reports/topic06/course_report_instructions.png 测试运行时仍使用绝对路径调用编译器、读取用例和写入文件，避免工作目录变化影响执行。写入报告前，`_report_path()` 会将仓库内路径转换为相对于 ScratchV 仓库根目录的 POSIX 格式。 -例如，原报告中的： +例如，Linux 工作区中的绝对路径： ```text -D:\PycharmProjects\ScratchV\ScratchV\tests\topic06\cases\activation\relu_only.dsl +/home/reviewer/ScratchV/tests/topic06/cases/activation/relu_only.dsl ``` -现在写为： +在报告中写为： ```text tests/topic06/cases/activation/relu_only.dsl ``` -该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖本机盘符与工作区位置，可以在 Windows、Linux、macOS 和 CI artifact 中查看。 +该规则应用于测试用例路径、汇编文件、编译失败日志，以及 Markdown 概览中的测试目录、构建目录和基线文件。生成的 Markdown、JSON 和 HTML 报告不再依赖作者机器的工作区位置，可在其他 Linux 工作区和 CI artifact 中查看。 --- ## 六、运行方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行，使用 Python 3.12： + +```bash +python3.12 -m venv .venv-topic06 +source .venv-topic06/bin/activate +``` + +各节保留的历史测试结果不因命令格式调整而变成新的 Linux 执行证据；新结果以本次生成的报告为准。 + ### 6.1 安装依赖 基础测试： -```powershell -pip install -e ".[topic06]" +```bash +python -m pip install -e ".[topic06]" ``` 完整报告： -```powershell -pip install -e ".[topic06-report]" +```bash +python -m pip install -e ".[topic06-report]" ``` ### 6.2 普通测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py python scripts/generate_topic06_report.py ``` 第一条命令生成 `metadata.json`，第二条命令读取元数据并生成 `report.md`、`report.json` 和单用例报告。生成 HTML 和 PNG 完整报告： -```powershell +```bash python scripts/generate_topic06_report.py --full-report ``` @@ -527,19 +536,19 @@ python scripts/generate_topic06_report.py --full-report 按测试目录类别筛选： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category activation ``` 按 DSL 文件名进行大小写不敏感的包含匹配： -```powershell +```bash python scripts/run_topic06_benchmarks.py --filter matmul ``` 两个条件可以组合，组合时必须同时满足： -```powershell +```bash python scripts/run_topic06_benchmarks.py --category tensor --filter relu ``` @@ -549,7 +558,7 @@ python scripts/run_topic06_benchmarks.py --category tensor --filter relu ### 6.4 性能回归测试 -```powershell +```bash python scripts/run_topic06_benchmarks.py ``` @@ -557,7 +566,7 @@ python scripts/run_topic06_benchmarks.py ### 6.5 更新性能基线 -```powershell +```bash python scripts/run_topic06_benchmarks.py --update-baseline ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" index 9cd0e8c..bf93541 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-CI\344\270\216Benchmark.md" @@ -8,7 +8,11 @@ push 使用 self-hosted）、Python 3.12 ## 专项测试 -```powershell +以下复现命令统一使用 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境： + +```bash +python3.12 -m venv .venv-dsl-diagnostics +source .venv-dsl-diagnostics/bin/activate python -m pip install -e . "pytest>=7,<10" python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_diagnostics_benchmark.py -v --tb=short ``` @@ -23,7 +27,7 @@ python -m pytest tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test 先为基线建立独立 checkout。以下 `997d2aa` 是本 PR 加入诊断实现之前的历史版本； rebase 后与最新上游比较时，应改用对应的 PR base SHA。 -```powershell +```bash git worktree add --detach ../ScratchV-dsl-baseline 997d2aa python -m benchmarks.bench_dsl_diagnostics --baseline-root ../ScratchV-dsl-baseline --json-output benchmark_reports/dsl_diagnostics.json --markdown benchmark_reports/dsl_diagnostics.md --html benchmark_reports/dsl_diagnostics.html ``` diff --git "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" index dd5e243..2346026 100644 --- "a/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" +++ "b/docs/topics/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250/09-DSL\351\224\231\350\257\257\346\217\220\347\244\272\347\276\216\345\214\226\345\231\250-\345\274\200\345\217\221\346\226\207\346\241\243.md" @@ -507,6 +507,9 @@ scratchv bad.dsl -o output.s 2> error.txt ## 16. 测试计划 +本节复现命令统一使用 Linux / Bash，在仓库根目录和 Python 3.12 虚拟环境中执行； +依赖安装见 [CI 与 Benchmark](09-DSL错误提示美化器-CI与Benchmark.md#专项测试)。 + ### 16.1 快速测试 开发过程中每个小步骤运行： @@ -526,12 +529,6 @@ python -m pytest \ tests/test_dsl_diagnostics_cli.py -q ``` -Windows PowerShell 可把路径放在同一行执行。 - -```powershell -python -m pytest tests/test_parser.py tests/test_dsl_extended.py tests/test_dsl_errors.py tests/test_dsl_validator.py tests/test_dsl_diagnostics_cli.py -q -``` - ### 16.3 项目验证 ```bash diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" index 9fed5ea..bd89c20 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-Benchmark\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -286,20 +286,32 @@ convert_onnx_to_llvm(model) → LLVM IR (866K lines, 183MB) ## 7. 使用方式 +以下复现命令面向 Linux / Bash，在仓库根目录执行。先准备 Python 3.12 虚拟环境和依赖： + +```bash +python3.12 -m venv .venv-regalloc +source .venv-regalloc/bin/activate +python -m pip install -e ".[all]" +``` + +CNN 用例还需要 `models/graph/cnn.onnx`；请先确认模型文件存在。LLVM 对比依赖 +`llvmlite`，报告中的 `llvm_available=false` 表示该项未执行，不能作为 LLVM 对比通过。 + ```bash -# 运行全部 4 项 benchmark，生成三格式报告（Windows PowerShell） -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json report.json ` - --output-html report.html ` - --output-md report.md +# 运行全部 4 项 benchmark，生成三格式报告 +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ + --output-md benchmark_reports/regalloc_bench.md # 单独运行某项 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_simple --repeats 100 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_dense --repeats 50 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_cnn ` +python -m benchmarks.test_regalloc.bench_simple --repeats 100 +python -m benchmarks.test_regalloc.bench_dense --repeats 50 +python -m benchmarks.test_regalloc.bench_cnn \ --cnn-path models/graph/cnn.onnx --repeats 30 -& .\.venv\Scripts\python.exe -m benchmarks.test_regalloc.bench_pseudo --repeats 30 +python -m benchmarks.test_regalloc.bench_pseudo --repeats 30 ``` --- diff --git "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" index 8f5e0f3..a4c2dda 100644 --- "a/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" +++ "b/docs/topics/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215/17-\345\257\204\345\255\230\345\231\250\345\210\206\351\205\215-\344\277\256\346\224\271\346\265\213\350\257\225\350\207\252\346\237\245\346\212\245\345\221\212.md" @@ -137,13 +137,16 @@ ### 5.2 30 次运行结果 -命令： - -```powershell -python -m benchmarks.test_regalloc.bench_regalloc_linear ` - --repeats 30 ` - --output-json benchmark_reports/regalloc_bench.json ` - --output-html benchmark_reports/regalloc_bench.html ` +以下是原验证命令的 Linux / Bash 等价写法，在仓库根目录和已安装依赖的 Python 3.12 +虚拟环境中执行。环境准备见 [Benchmark 设计文档](17-寄存器分配-Benchmark设计文档.md#7-使用方式)。 +下表保留原历史测量，不代表使用这份 Linux 命令重新运行过。 + +```bash +mkdir -p benchmark_reports +python -m benchmarks.test_regalloc.bench_regalloc_linear \ + --repeats 30 \ + --output-json benchmark_reports/regalloc_bench.json \ + --output-html benchmark_reports/regalloc_bench.html \ --output-md benchmark_reports/regalloc_bench.md ``` diff --git a/probes/w1_qwen3_export/README.md b/probes/w1_qwen3_export/README.md index 02d5da2..f72c019 100644 --- a/probes/w1_qwen3_export/README.md +++ b/probes/w1_qwen3_export/README.md @@ -15,24 +15,19 @@ for blocked positions. Short-input predictions use the last valid token. ## Environment -Use Python 3.12. The existing `requirements/qwen3-small-probe.txt` CPU +The published reproduction target is Ubuntu 24.04 x86_64 with Bash and Python 3.12. See the [Linux reproduction contract](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md). The existing `requirements/qwen3-small-probe.txt` CPU environment also satisfies this gate. For a separate reproducible export environment: -```sh -python -m venv output/qwen3-export-venv -source output/qwen3-export-venv/bin/activate +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate python -m pip install torch==2.7.1 --index-url https://download.pytorch.org/whl/cpu python -m pip install -r requirements/qwen3-export.txt ``` -Keep this environment activated for all `python` commands below. On Windows -PowerShell, create it with `py -3.12 -m venv output/qwen3-export-venv`, then -activate it with `& .\output\qwen3-export-venv\Scripts\Activate.ps1` and run -the two installation commands above. If activation is unavailable, replace -every later `python` with `& .\output\qwen3-export-venv\Scripts\python.exe` -(or `output/qwen3-export-venv/bin/python` on Linux); put multiline commands on -one line in PowerShell rather than using shell backslash continuations. +Keep this environment activated for all `python` commands below; the interpreter is `.venv-linux/bin/python`. Record the checkout SHA and actual dependency versions with each new run. Verify/download mode enforces NumPy 2.2.6, ONNX 1.18.0, ONNX Runtime 1.22.1 and protobuf 5.29.5; @@ -41,7 +36,7 @@ export mode additionally enforces the complete fixed export stack, including ## Verify the fixed release -```sh +```bash python probes/w1_qwen3_export/run.py --mode download \ --model-dir output/qwen3-full-model \ --output-dir output/qwen3-full-probe --threads 2 @@ -59,7 +54,7 @@ uses a previously downloaded ZIP with the same size/hash checks. For an already available complete model directory, skip network access: -```sh +```bash python probes/w1_qwen3_export/run.py --mode verify \ --model-dir models/qwen3-0.6b-onnx \ --output-dir output/qwen3-full-probe --threads 2 @@ -78,7 +73,7 @@ ScratchV's full-model execution or language quality. Obtain the official `Qwen/Qwen3-0.6B` Hugging Face snapshot at revision `c1899de289a04d12100db370d81485cdf75e47ca`. For example, in the pinned environment: -```sh +```bash python -c "from huggingface_hub import snapshot_download; snapshot_download('Qwen/Qwen3-0.6B', revision='c1899de289a04d12100db370d81485cdf75e47ca', local_dir='output/qwen3-source')" python probes/w1_qwen3_export/run.py --mode export \ --source-dir output/qwen3-source \ @@ -110,7 +105,7 @@ measurement. Export mode also saves `export.log` and its reference artifacts. Source fingerprints are included in failed reports as well as successful ones. Upload the report directory as CI evidence, not the multi-gigabyte model. -```sh +```bash python -m pytest tests/test_w1_qwen3_export.py -q ``` diff --git a/probes/w2_backend_ops/README.md b/probes/w2_backend_ops/README.md index a535d14..dd26184 100644 --- a/probes/w2_backend_ops/README.md +++ b/probes/w2_backend_ops/README.md @@ -1,9 +1,11 @@ # W2 前端模式与基础后端算子验收 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 准备并激活环境。下文 `python` 为 `.venv-linux/bin/python`。 + `unit:frontend-ops` 明确检查 RMSNorm、RoPE、SwiGLU、GQA 四种分解图模式。 这些模式由现有 ONNX 基础算子表达，不声明已有独立融合 Attention/RMSNorm IR。 -```powershell +```bash python -m pytest tests/test_qwen3_frontend_patterns.py -q ``` @@ -11,12 +13,12 @@ python -m pytest tests/test_qwen3_frontend_patterns.py -q 交叉编译为 RV64GC ELF，然后在 QEMU `virt` 裸机执行。16 个用例分别运行 `none`、`all` 两个优化级别，共 32 次目标执行；不下载模型权重。 -```powershell -python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc path/to/zig --qemu path/to/qemu-system-riscv64 +```bash +python probes/w2_backend_ops/run.py --output-dir output/w2-backend-ops --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" python -m pytest tests/test_w2_backend_ops_gate.py -q ``` -Linux 可省略工具路径，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 +也可省略工具参数，使用 PATH 或 `SCRATCHV_CC`、`SCRATCHV_QEMU`。 使用现有 `requirements/qwen3-small-probe.txt` 中的 NumPy/ONNX/ORT 环境即可， 本探测不需要 PyTorch。输出目录须不存在或为空；依赖、工具、编译、执行或 比较失败均非零退出，不将跳过当通过。 diff --git a/probes/w2_qwen3_parse/README.md b/probes/w2_qwen3_parse/README.md index 23a69e7..d303efe 100644 --- a/probes/w2_qwen3_parse/README.md +++ b/probes/w2_qwen3_parse/README.md @@ -6,29 +6,18 @@ ## 准备环境 -使用 Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12，固定 NumPy 2.2.6、ONNX 1.18.0、ONNX Runtime 1.22.1 和 protobuf 5.29.5。可直接复用 W1 的固定环境；单独准备时无需安装 torch、Zig 或 QEMU。 -Windows PowerShell： - -```powershell -py -3.12 -m venv output/qwen3-parse-venv -$parsePython = ".\output\qwen3-parse-venv\Scripts\python.exe" -& $parsePython -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" -& $parsePython -m pip check -& $parsePython -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -$LASTEXITCODE -``` - -Linux shell： +Linux Bash： ```bash +set -euo pipefail python3.12 -m venv output/qwen3-parse-venv . output/qwen3-parse-venv/bin/activate python -m pip install "numpy==2.2.6" "onnx==1.18.0" "onnxruntime==1.22.1" "protobuf==5.29.5" python -m pip check python -X utf8 -B probes/w2_qwen3_parse/run.py --mode verify \ --model-dir output/qwen3-full-model --output-dir output/qwen3-parse --timeout 300 -echo $? ``` 所有命令在仓库根目录执行。`--model-dir` 必填，应指向同目录保存 `model.onnx` 和三个真实 external-data 分片的完整产物，不能只有 ONNX 文件或 LFS pointer。`verify` 是默认模式，不访问网络；不存在完整模型时，将命令中的 `--mode verify` 改成 `--mode download`，由入口获取并校验 W1 固定发布产物。不要用新导出或其他 revision 的文件替换固定产物。 @@ -78,10 +67,10 @@ python -m pip install "pytest==9.1.1" python -m pytest tests/test_qwen3_full_structure.py tests/test_qwen3_full_audit.py tests/test_qwen3_full_parse.py -q ``` -PowerShell 将上面命令中的 `python` 替换为 `& $parsePython`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 +命令中的 `python` 指已激活的 `output/qwen3-parse-venv/bin/python`。这些测试使用小型 fixture 和故障注入，不下载或加载完整权重，不能代替完整产物实跑。 [LLM 工作流](../../.github/workflows/llm-deploy.yml) 的 PR 路径接入上述三组测试，并在独立 `full-qwen3-frontend` job 以 `--mode download` 实跑完整固定图。手动/定时同样运行该解析任务。解析有独立 Summary，并以 `always()` 上传 `output/qwen3-parse/`，artifact 名为 `qwen3-full-frontend-parse-output`。下载约 1.24 GB，解析 worker 峰值 RSS 约 5 GB；runner 的实际容量与耗时仍须远端验证。 `w2-acceptance` 汇总任务要求数值任务和完整解析任务均实际成功，失败、取消、跳过或缺失都不能通过。手动/定时的 `full-qwen3-onnx` job 独立保留 W1 完整 ONNX ORT 验证；解析不依赖它成功。并行的两个完整模型任务会各自下载资产。 -PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存本地实跑与 Review 修复结果，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 +PR #91 已合并为 W1 基线；这些 W2 代码、测试、文档与 CI 配置独立发布，W1 两层报告修复见 [PR #93](https://github.com/ScratchV-Compiler/ScratchV/pull/93)。[W2 交付记录](../../docs/llm-deploy-v1.0/W2/README.md) 保存 Linux 复现入口与已记录的 Linux CI 证据，CI 配置存在不等于远端已通过。W1 已有旧提交的两层第二人复现；完整模型第二人复现和团队接口确认仍待完成，见 [W1 记录](../../docs/llm-deploy-v1.0/W1/README.md)。 diff --git a/probes/w2_qwen3_small/README.md b/probes/w2_qwen3_small/README.md index 66ccc2b..bda658d 100644 --- a/probes/w2_qwen3_small/README.md +++ b/probes/w2_qwen3_small/README.md @@ -27,20 +27,22 @@ RV64GC ELF，在 QEMU `virt` 裸机执行。下文“不是 RISC-V 验证”的 ONNX 输入为 `input_ids: INT64 [1,256]` 与 `attention_mask: FP32 [1,1,256,256]`。mask 是加性因果和 key-padding 掩码（允许位置为 0，屏蔽位置为 FP32 最小有限值），不是 tokenizer 常用的二维 0/1 mask；位置编号固定为 `0..255`。普通图输出为 `logits: FP32 [1,256,128]`。 -## 本地运行 - -先确认用于建环境的 `python` 是 Python 3.12。下面是 PowerShell 命令；Linux/macOS 可将 `& $probePython` 换成 `output/qwen3-probe-venv/bin/python`。 - -```powershell -python -m venv output/qwen3-probe-venv -$probePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $probePython -m pip install --upgrade pip -& $probePython -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu -& $probePython -m pip install -r requirements/qwen3-small-probe.txt -& $probePython -m pip install --no-deps -e . -& $probePython -m pip check -& $probePython probes/w2_qwen3_small/run.py --output-dir output/qwen3-small -& $probePython -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q +## Linux 复现 + +正式交付使用 Ubuntu 24.04 x86_64 / Bash / Python 3.12。通用工具准备见 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)；在仓库根目录执行： + +```bash +set -euo pipefail +python3.12 -m venv .venv-linux +source .venv-linux/bin/activate +python -m pip install --upgrade pip +python -m pip install "torch==2.7.1" --index-url https://download.pytorch.org/whl/cpu +python -m pip install -r requirements/qwen3-small-probe.txt +python -m pip install --no-deps -e . +python -m pip check +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +python probes/w2_qwen3_small/run.py --output-dir output/qwen3-small +python -m pytest tests/test_qwen3_small_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_gate.py -q ``` CPU PyTorch 必须在安装依赖文件前单独安装；随后 `torch==2.7.1` 会保留已安装的 `2.7.1+cpu`。不要只执行依赖文件安装而让 pip 从默认索引另选 CUDA 依赖。导出相关包采用已验证的固定组合，避免环境升级改变导出图或数值基线。 @@ -121,15 +123,13 @@ ScratchV 承担 ONNX/IR、IR 优化、张量循环生成、内存规划与输入 Linux 工具准备（Python 导出依赖仍按前面的固定版本安装）： ```bash +sudo apt-get update sudo apt-get install --no-install-recommends qemu-system-misc python -m pip install ziglang==0.14.1 export SCRATCHV_CC="$(python -c 'from pathlib import Path; import ziglang; print(Path(ziglang.__file__).parent / "zig")')" ``` -Windows 可使用便携 Zig 和 QEMU，设置 `SCRATCHV_CC` 为 `zig.exe` 的完整路径、 -`SCRATCHV_QEMU` 为 `qemu-system-riscv64.exe` 的完整路径。也会自动搜索仓库的 -`output/tools/`。不需要 WSL、Linux 镜像或系统安装；QEMU Windows 分发来源见 -[QEMU 下载页](https://www.qemu.org/download/#windows)。工具仅在本机准备，脚本不会隐式联网安装。 +设置 `SCRATCHV_QEMU="$(command -v qemu-system-riscv64)"`；所有工具均为 Linux 可执行文件。脚本不会隐式联网安装工具。 先验证矩阵乘，再导出模型并运行完整验收；输出目录必须为空，避免覆盖旧证据： diff --git a/probes/w2_runtime/README.md b/probes/w2_runtime/README.md index 462324a..3ed2242 100644 --- a/probes/w2_runtime/README.md +++ b/probes/w2_runtime/README.md @@ -8,18 +8,10 @@ 使用 [两层模型指南](../w2_qwen3_small/README.md) 中的 Python 3.12 固定 CPU 环境；其中已包含 `numpy==2.2.6`、`tokenizers==0.21.4`、`transformers==4.51.3`。仅使用生产 tokenizer 时可安装 `pip install -e ".[llm]"`，无需 Torch；复现 gate 仍使用固定版本和官方参考依赖。 -在仓库根目录运行，PowerShell： - -```powershell -$runtimePython = ".\output\qwen3-probe-venv\Scripts\python.exe" -& $runtimePython -B -X utf8 probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime -$LASTEXITCODE -``` - -Linux： +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活环境，在仓库根目录运行： ```bash -output/qwen3-probe-venv/bin/python -B -X utf8 probes/w2_runtime/run.py \ +.venv-linux/bin/python -B -X utf8 probes/w2_runtime/run.py \ --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime ``` diff --git a/probes/w2_runtime_model/README.md b/probes/w2_runtime_model/README.md index 794fa0e..d58cd4f 100644 --- a/probes/w2_runtime_model/README.md +++ b/probes/w2_runtime_model/README.md @@ -4,9 +4,9 @@ 这是 W2 的接口集成证据；权重随机，不代表预训练 Qwen3-0.6B 的语言能力。执行单步前向与一个 greedy token，不执行生成循环、KV cache 或 QEMU。 -## 本地复现 +## Linux 复现 -使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 +正式复现使用 Ubuntu 24.04 x86_64 / Bash；先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 激活 `.venv-linux`。使用仓库 [固定 CPU 环境](../../requirements/qwen3-small-probe.txt)，Python 3.12。首先安装对应 CPU Torch，再安装固定依赖。Tokenizer 资产复用 [W2 Runtime 探测](../w2_runtime/README.md)下载的目录，或固定模型源目录；每次运行都会核验 [W1 manifest](../w1_qwen3_export/manifest.json) 的七个资产 SHA256，只读取小文件，不加载预训练权重。 ```bash python probes/w2_runtime/run.py --mode download --tokenizer-dir output/qwen3-tokenizer --output-dir output/w2-runtime diff --git a/scripts/check_linux_repro_docs.py b/scripts/check_linux_repro_docs.py new file mode 100644 index 0000000..dfe8b6f --- /dev/null +++ b/scripts/check_linux_repro_docs.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Reject host-specific reproduction instructions; optionally parse Bash blocks. + +This checks documentation, not execution or numerical acceptance. Historical +platform descriptions and portable implementation details remain valid. +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import re +import shutil +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] +FENCE = re.compile(r"^\s*(`{3,}|~{3,})\s*([\w+-]*)\s*$") +HOST_PATTERNS = ( + (re.compile(r"(?= len(active) and not match.group(2): + if language in ("bash", "sh", "shell"): + blocks.append((start, "\n".join(body) + "\n")) + active = None + elif active: + body.append(line) + if language in ("bash", "sh", "shell") and re.search(r"\b[\w.-]+\.exe\b", line, re.I): + problems.append(f"{path}:{number}: non-Linux executable in shell example") + if active: + problems.append(f"{path}:{start}: unterminated fenced block") + if bash: + for start, content in blocks: + try: + result = subprocess.run([bash, "--noprofile", "--norc", "-n"], input=content, + capture_output=True, text=True, encoding="utf-8", timeout=10) + except (OSError, subprocess.TimeoutExpired) as exc: + problems.append(f"{path}:{start}: Bash validation could not run: {exc}") + continue + if result.returncode: + problems.append(f"{path}:{start}: invalid Bash: {result.stderr.strip()}") + return problems, len(blocks) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument("--check-bash", action="store_true", help="Require Bash and syntax-check every shell block") + parser.add_argument("--bash", help="Explicit Bash executable; defaults to PATH") + args = parser.parse_args(argv) + bash = (args.bash or shutil.which("bash")) if args.check_bash else None + if args.check_bash and not bash: + parser.exit(1, "FAIL: Bash is required for --check-bash\n") + paths = document_paths(args.root.resolve()) + if not paths: + parser.exit(1, "FAIL: no reproduction documents found\n") + problems, blocks = [], 0 + for path in paths: + findings, count = inspect_document(path, bash=bash) + problems.extend(findings) + blocks += count + for problem in problems: + print(problem) + print(f"{'FAIL' if problems else 'PASS'}: {len(paths)} documents, {blocks} shell blocks; " + f"Bash syntax {'checked' if bash else 'not requested'}; no model execution") + return int(bool(problems)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_linux_repro_docs.py b/tests/test_linux_repro_docs.py new file mode 100644 index 0000000..b528a79 --- /dev/null +++ b/tests/test_linux_repro_docs.py @@ -0,0 +1,67 @@ +"""Check rejection of unusable public reproduction commands, not doc wording.""" +import shutil + +import pytest + +from scripts import check_linux_repro_docs as docs + + +@pytest.mark.parametrize("content", [ + "```powershell\npython run.py\n```\n", + "~~~pwsh\npython run.py\n~~~\n", + "Use D:/private/models for the checkpoint\n", + "python C:\\private\\run.py\n", + "$env:OMP_NUM_THREADS = '1'\n", + "$LASTEXITCODE\n", + "venv/Scripts/python.exe\n", + "```bash\nzig.exe version\n```\n", + "```bash\necho unfinished\n", +]) +def test_rejects_non_linux_or_broken_examples(tmp_path, content): + path = tmp_path / "README.md" + path.write_text(content, encoding="utf-8") + assert docs.inspect_document(path)[0] + + +def test_linux_examples_and_truthful_history_are_allowed(tmp_path): + path = tmp_path / "README.md" + path.write_text('Historical Windows results are not Linux evidence.\n' + 'https://example.com/source\n' + '```bash\nset -euo pipefail\n' + '"$SCRATCHV_PYTHON" run.py --model-dir /srv/models/qwen3\n```\n', encoding="utf-8") + assert docs.inspect_document(path) == ([], 1) + + +def test_required_bash_cannot_silently_skip(monkeypatch): + monkeypatch.setattr(docs.shutil, "which", lambda _: None) + with pytest.raises(SystemExit) as exc: + docs.main(["--check-bash"]) + assert exc.value.code == 1 + + +def test_empty_input_is_not_success(tmp_path): + with pytest.raises(SystemExit) as exc: + docs.main(["--root", str(tmp_path)]) + assert exc.value.code == 1 + + +def test_inventory_covers_shared_guides_and_benchmarks(tmp_path): + names = ("README.md", "CONTRIBUTING.md", "docs/guide/setup.md", + "docs/topics/topic/reproduce.md", "probes/w4/example.md", "benchmarks/sample/README.md") + for name in names: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("Linux reproduction\n", encoding="utf-8") + assert {p.relative_to(tmp_path).as_posix() for p in docs.document_paths(tmp_path)} == set(names) + + +def test_actual_bash_rejects_bad_syntax_without_executing(tmp_path): + bash = shutil.which("bash") + if not bash: + pytest.skip("Bash is required; Linux CI exercises this case") + path = tmp_path / "README.md" + marker = tmp_path / "must-not-exist" + path.write_text(f'```bash\ntouch "{marker.as_posix()}"\nif then\n```\n', encoding="utf-8") + problems, count = docs.inspect_document(path, bash=bash) + assert count == 1 and any("invalid Bash" in item for item in problems) + assert not marker.exists() diff --git a/tests/test_tensor_c_codegen.py b/tests/test_tensor_c_codegen.py index 23dd265..06b037f 100644 --- a/tests/test_tensor_c_codegen.py +++ b/tests/test_tensor_c_codegen.py @@ -1,7 +1,7 @@ """Execute generated tensor C against ORT, plus static/numeric rejection cases. -Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Repository -local output/tools/zig-python/ziglang/zig.exe is also recognized on Windows. +Host compilation uses clang/gcc, or SCRATCHV_ZIG (a zig executable). Optional +repository-local tools are resolved for the actual host platform. Validation tests do not require a C toolchain; execution tests skip explicitly when no compiler is available rather than reporting unexecuted cases as passes. """ @@ -35,7 +35,7 @@ def builder(*params): @pytest.fixture(scope="module") def compiler(): zig = os.environ.get("SCRATCHV_ZIG") - local = ROOT / "output/tools/zig-python/ziglang/zig.exe" + local = ROOT / "output/tools/zig-python/ziglang" / ("zig.exe" if os.name == "nt" else "zig") if zig or local.is_file(): return [zig or str(local), "cc"] for name in ("clang", "gcc", "cc"): From 6326237dc372096f7f803dbdb8dac3e2e2eb1099 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Tue, 6 Oct 2026 17:22:15 +0800 Subject: [PATCH 14/30] docs(w3): deliver Linux-only reproduction guides and source handoff --- ...40\346\200\247\344\277\256\345\244\215.md" | 6 +- .../W3/Linux-CI\344\270\216Nightly.md" | 22 +-- docs/llm-deploy-v1.0/W3/README.md | 25 ++-- ...12\346\226\255\346\212\245\345\221\212.md" | 115 +++++---------- ...14\346\224\266\350\277\233\345\272\246.md" | 46 +++--- ...14\346\224\266\346\212\245\345\221\212.md" | 100 +++---------- ...14\346\224\266\346\270\205\345\215\225.md" | 131 ++++++++++++------ ...00\345\217\221\350\256\241\345\210\222.md" | 4 +- probes/w3_attention/README.md | 7 +- probes/w3_full_preflight/README.md | 8 +- probes/w3_layer_diff/README.md | 7 +- probes/w3_qwen3_full/EVIDENCE.md | 14 +- probes/w3_qwen3_full/LOCALIZE.md | 24 ++-- probes/w3_qwen3_full/README.md | 22 +-- probes/w3_qwen3_medium/README.md | 18 ++- probes/w3_qwen3_subgraphs/README.md | 19 ++- scripts/package_w3_repro.py | 15 +- tests/test_w3_source_package.py | 23 ++- tests/test_w3_workflows.py | 2 +- 19 files changed, 298 insertions(+), 310 deletions(-) diff --git "a/docs/llm-deploy-v1.0/W1/W1-\346\212\245\345\221\212\345\217\257\351\235\240\346\200\247\344\277\256\345\244\215.md" "b/docs/llm-deploy-v1.0/W1/W1-\346\212\245\345\221\212\345\217\257\351\235\240\346\200\247\344\277\256\345\244\215.md" index 7a5b29e..f393af2 100644 --- "a/docs/llm-deploy-v1.0/W1/W1-\346\212\245\345\221\212\345\217\257\351\235\240\346\200\247\344\277\256\345\244\215.md" +++ "b/docs/llm-deploy-v1.0/W1/W1-\346\212\245\345\221\212\345\217\257\351\235\240\346\200\247\344\277\256\345\244\215.md" @@ -11,11 +11,15 @@ ## 验收与边界 -发布前在本分支 Windows 固定 CPU 环境重新运行：**262 passed / 0 skipped**；两层 IR 的 7 组输入、29 个检查点及不变性通过；普通/诊断图 × none/all × 7 输入共 **28/28 次实际 QEMU 通过**，目标最大绝对误差 `1.6689300537109375e-6`。QEMU 进程墙钟累计 154.702 秒、探测流程 189.065 秒；并行验收时的单次观察，不作性能结论。证据在 `output/w1-publish-tests.xml`、`output/w1-publish-ir/`、`output/w1-publish-qemu/`；执行源码指纹与发布内容核对一致。远端 CI 以 PR 对应提交的实际 Checks 为准。项目指定的本地 L2 harness 文件缺失，未宣称运行。源码没有修改计算算法、模型配置或 `max_abs < 1e-5` 门槛；不自动代表第二人复现或团队接口冻结通过。人工待办继续由 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92) 跟踪。 +正式复现使用 [Linux 统一环境](../LINUX_REPRODUCTION.md)。本修复已随 PR #93 合并为 `faab6a81562710e105d87265dafd2dc769a67c72`。当前 W3 集成分支已有 Linux 验证：提交 `77c3aa86e9794e5bfe3c81fd873a1506d14bc753` 的 [LLM 工作流](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37303495582)完成两层 QEMU 28/28、显式产物 3/3；两层最大误差 `1.7881393432617188e-6 < 1e-5`。这些属于该次实际运行，不作为本次文档更新的新执行结果。 + +发布前个人环境的原始记录保留在 [固定历史原文](https://github.com/ScratchV-Compiler/ScratchV/blob/77c3aa86e9794e5bfe3c81fd873a1506d14bc753/docs/llm-deploy-v1.0/W1/W1-报告可靠性修复.md)，不改写成 Linux 数据。后续提交应使用对应 SHA 的实际 Linux Checks。修复不修改计算算法、模型配置或 `max_abs < 1e-5` 门槛；人工接口冻结与独立复现继续按 [Issue #92](https://github.com/ScratchV-Compiler/ScratchV/issues/92)验收。 输出保存在本机新目录，复现者应自行重新生成： ```bash +set -euo pipefail +source .venv-linux/bin/activate python -B -X utf8 -m pytest tests/test_qwen3_small_gate.py tests/test_qwen3_small_riscv_gate.py tests/test_onnx_qwen_numeric_ops.py tests/test_onnx_shape_ops.py -q python -B -X utf8 probes/w2_qwen3_small/run.py --output-dir output/w1-ir-new python -B -X utf8 probes/w2_qwen3_small/riscv.py --model-dir output/w1-ir-new --output-dir output/w1-qemu-new diff --git "a/docs/llm-deploy-v1.0/W3/Linux-CI\344\270\216Nightly.md" "b/docs/llm-deploy-v1.0/W3/Linux-CI\344\270\216Nightly.md" index c4d2e3f..92f3698 100644 --- "a/docs/llm-deploy-v1.0/W3/Linux-CI\344\270\216Nightly.md" +++ "b/docs/llm-deploy-v1.0/W3/Linux-CI\344\270\216Nightly.md" @@ -1,5 +1,7 @@ # W3 Linux CI 与 Nightly +对外复现统一使用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 统一环境与资产准备](../LINUX_REPRODUCTION.md)，并在同一个 Bash 会话、仓库根目录执行以下命令；`SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU` 由该指南设置。 + 配置已随 [PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 提交。2026-10-05，作者在提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)中完成同提交 preparation、完整七组 full-numeric 和汇总，均为 success。preparation 下载已复核；完整 reports/raw 均已下载验真，七组原始数组离线 audit 通过。前三轮失败与历史数值 profile 原样保留。下面区分实际运行记录与通用命令模板；没有设置 Nightly 远端开关，手动调用不等于 schedule，也不替代他人独立复现。团队确认内容见 [独立复现与验收清单](独立复现与验收清单.md)。 ## 2026-10-05 Linux 实测记录 @@ -10,7 +12,7 @@ | 检查 | 实际结果 | 用时 | |---|---|---:| -| preparation 回归 | 1203 passed、1 skipped；跳过项仅适用于 Windows Job ownership | 46.80 s | +| preparation 回归 | 1203 passed、1 skipped；跳过项仅适用于非 Linux 平台进程所有权兼容测试 | 46.80 s | | 六层 medium | 7/7 输入、81 检查点、18 不变量；各项对比最大绝对误差 `2.771615982055664e-6 < 1e-5` | 97.052790 s | | 真实权重子图 | 14/14 输入；各项对比最大绝对误差 `3.0517578125e-5 < 1e-4` | 5.894554 s | | 小 Attention 后端 | 12/12 执行；QEMU vs ORT 最大绝对误差 `3.5762786865234375e-7 < 1e-4` | 22.863963 s | @@ -51,7 +53,7 @@ preparation 产物上传成功且已下载核对： | 检查 | 第三轮实际结果 | 用时 | |---|---|---:| -| preparation 回归 | 1270 passed、1 skipped；skip 仅为 Windows Job ownership 测试 | 49.64 s | +| preparation 回归 | 1270 passed、1 skipped；skip 仅为非 Linux 平台进程所有权兼容测试 | 49.64 s | | 六层 medium | 7/7 输入、81 检查点、18 不变量；全部比较最大绝对误差 `2.771615982055664e-6 < 1e-5` | 99.530785 s | | 真实权重子图 | 14/14 输入、10 不变量；全部比较最大绝对误差 `3.0517578125e-5 < 1e-4` | 5.807397 s | | 小 Attention 后端 | 12/12 执行；QEMU vs ORT 最大绝对误差 `3.5762786865234375e-7 < 1e-4` | 23.216017 s | @@ -75,7 +77,7 @@ preparation 主报告 SHA256 为 `2f12bdf97f18356829c6222bc86e2da7175983619e17c3 | 检查 | 第四轮实际结果 | 用时 | |---|---|---:| -| preparation 回归 | 1297 passed、1 skipped；skip 为 Windows Job ownership 测试；真实 Haswell BLAS 回归通过 | JUnit 50.074 s | +| preparation 回归 | 1297 passed、1 skipped；skip 为非 Linux 平台进程所有权兼容测试；真实 Haswell BLAS 回归通过 | JUnit 50.074 s | | 六层 medium | 7/7 输入、81 检查点、18 不变量；全部比较最大绝对误差 `2.771615982055664e-6 < 1e-5` | 97.368098 s | | 真实权重子图 | 14/14 输入、10 不变量；全部比较最大绝对误差 `3.0517578125e-5 < 1e-4` | 5.775279 s | | 小 Attention 后端 | 12/12 QEMU 执行；QEMU vs ORT 最大绝对误差 `3.5762786865234375e-7 < 1e-4` | 22.837814 s | @@ -127,6 +129,7 @@ Nightly 使用仓库内相对路径调用，因此被调用工作流与调用者 若 W3 新入口尚未进入默认分支，但仓库已注册并运行过 `LLM Deploy v1.0`，可从该既有手动入口选择包含 W3 代码的分支，显式设置 `run_w3_validation=true`。它从同一提交复用 W3 完整编排，执行 preparation、七组 full-numeric 和汇总；原 W1/W2 任务也照常执行。默认值为 false，普通 PR 和原 schedule 不会因此自动增加 W3 重型任务。首次远端通过前仍须按实际日志核对，手动调用不算定时 Nightly 记录。 ```bash +set -euo pipefail gh workflow run llm-deploy.yml --repo YOUR_ACCOUNT/ScratchV --ref YOUR_W3_BRANCH -f run_w3_validation=true ``` @@ -141,6 +144,7 @@ gh workflow run llm-deploy.yml --repo YOUR_ACCOUNT/ScratchV --ref YOUR_W3_BRANCH 需要命令行时，可在确定运行仓库和分支后使用以下模板；替换占位符。实际已执行记录见本文首节，命令模板本身不是运行证据： ```bash +set -euo pipefail gh workflow run w3-nightly.yml --repo YOUR_ACCOUNT/ScratchV --ref YOUR_W3_BRANCH gh run list --repo YOUR_ACCOUNT/ScratchV --workflow w3-nightly.yml gh run view RUN_ID --repo YOUR_ACCOUNT/ScratchV @@ -181,6 +185,7 @@ CPU 的 SIMD 支持、IR 选择的 `cpu_strategy` 和 NumPy 实际 BLAS 内核 完整 raw artifact 内的相对根路径为 `w3-full/`。用 `gh run download` 指定单个 `-n` 时，它直接解压到 `--dir`；所以传给离线工具的是 `--dir` 下的 `w3-full`，不能把下载根目录直接当作模型证据根。下面使用新目录，先分别下载 reports 与 raw，再要求两份主报告字节哈希一致： ```bash +set -euo pipefail W3_REPO=YOUR_ACCOUNT/ScratchV W3_RUN_ID=RUN_ID W3_DOWNLOAD=output/downloaded-w3-RUN_ID @@ -188,15 +193,15 @@ gh run download "$W3_RUN_ID" --repo "$W3_REPO" \ -n w3-full-numeric-reports --dir "$W3_DOWNLOAD/reports" gh run download "$W3_RUN_ID" --repo "$W3_REPO" \ -n w3-full-numeric-raw --dir "$W3_DOWNLOAD/raw" -W3_REPORT_SHA=$(python -c 'import hashlib,pathlib,sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "$W3_DOWNLOAD/reports/w3-full/report.json") +W3_REPORT_SHA=$("$SCRATCHV_PYTHON" -c 'import hashlib,pathlib,sys; print(hashlib.sha256(pathlib.Path(sys.argv[1]).read_bytes()).hexdigest())' "$W3_DOWNLOAD/reports/w3-full/report.json") printf '%s\n' "$W3_REPORT_SHA" -python -B -X utf8 scripts/verify_w3_evidence.py \ +"$SCRATCHV_PYTHON" -B -X utf8 scripts/verify_w3_evidence.py \ --evidence-dir "$W3_DOWNLOAD/raw/w3-full" \ --output-dir output/w3-ci-evidence-audit-RUN_ID \ --expected-report-sha256 "$W3_REPORT_SHA" ``` -先核对 Actions run 的完整 SHA、attempt、结果和 artifact 身份，再单独保存 `W3_REPORT_SHA`。这里从 reports 产物取哈希，用它校验 raw 中的报告，能发现两包混用；两包本身来自同一次运行，不构成额外的发布者身份认证。若发件人已通过另一可信渠道给出报告 SHA，应直接用该值作为 `--expected-report-sha256`。Windows 可用 `(Get-FileHash -Algorithm SHA256 -LiteralPath '下载目录/reports/w3-full/report.json').Hash.ToLowerInvariant()` 获取同一值。每次 audit 使用新输出目录，检查返回码以及生成的 `report.json`。 +先核对 Actions run 的完整 SHA、attempt、结果和 artifact 身份，再单独保存 `W3_REPORT_SHA`。这里从 reports 产物取哈希，用它校验 raw 中的报告，能发现两包混用；两包本身来自同一次运行，不构成额外的发布者身份认证。若发件人已通过另一可信渠道给出报告 SHA，应直接用该值作为 `--expected-report-sha256`。Linux 也可用 `sha256sum` 读取报告哈希。每次 audit 使用新输出目录，检查返回码以及生成的 `report.json`。 完整原始数组可以用 [复现清单](独立复现与验收清单.md) 中的离线复核入口重新核对。仅有报告和哈希不能重新计算数组误差；选定 preparation 轨迹也不能替代完整七组数组。v4 auditor 依据报告保存的 CPU 策略核对完整规范 profile（包括 MatMul 布局字段），不根据下载者 CPU 推断生产策略，不需要为数组核对更改本机 CPU 策略；旧 v2/v3 证据必须使用对应历史源码的验证器，不能手改报告升级。`audit:w3-full-saved-evidence` PASS 表示保存证据满足其契约，`source_comparison` 明列生成时与当前审计源码的差异；它不执行 ORT/IR，也不能替代第二人在其环境重新执行模型。 @@ -219,10 +224,11 @@ W3_NIGHTLY_ENABLED=true ## 本地配置验证 ```bash -python -B -X utf8 -m pytest tests/test_w3_workflows.py -q -p no:cacheprovider +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 -m pytest tests/test_w3_workflows.py -q -p no:cacheprovider actionlint -shellcheck= -pyflakes= .github/workflows/w3-preparation.yml .github/workflows/w3-full-numeric.yml .github/workflows/w3-nightly.yml ``` -测试读取实际 YAML，校验默认全覆盖、手动/复用入口、opt-in 条件、独立并发组、失败汇总和产物范围；通过 Bash 执行实际参数组装及汇总片段，确保 `FAIL/1`、`PARTIAL/2` 不被吞掉，并对所有 shell 块做语法检查。Windows 可使用已有 Git Bash；没有 Bash 时 shell 执行测试会明确跳过，不能据此声称该部分已经验证。 +测试读取实际 YAML，校验默认全覆盖、手动/复用入口、opt-in 条件、独立并发组、失败汇总和产物范围；通过 Bash 执行实际参数组装及汇总片段，确保 `FAIL/1`、`PARTIAL/2` 不被吞掉，并对所有 shell 块做语法检查。Linux 复现环境必须安装 Bash 并实际执行这些检查；跳过 shell 检查不能作为该部分通过。 本轮使用 actionlint 1.7.12 对上述三个文件静态检查通过；工具放在本地忽略的 output 下，未安装系统服务。actionlint 不执行下载、Python 环境安装、QEMU 或数值模型；实际 Linux 执行与产物核对另见本文首节。 diff --git a/docs/llm-deploy-v1.0/W3/README.md b/docs/llm-deploy-v1.0/W3/README.md index c326525..4893b52 100644 --- a/docs/llm-deploy-v1.0/W3/README.md +++ b/docs/llm-deploy-v1.0/W3/README.md @@ -1,12 +1,14 @@ # W3 完整 Qwen3 IR 验证与交付 +对外复现统一使用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 统一环境与资产准备](../LINUX_REPRODUCTION.md)，并在同一个 Bash 会话、仓库根目录执行以下命令；`SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU` 由该指南设置。 + 本目录汇总完整 Qwen3 IR 数值验证、准备探测、诊断与复现交付。W3 已提交至 [PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96)，本地开发工作树为 `codex/w3-preparation`，原 W2 基线为 `65616a8cde6a581661ac3734378da78a9fcbe5ed`。当前同提交 Linux 全部硬门槛和下载数组 audit 已通过，下一步由团队确认口径、独立复现并验收 Nightly。PR 可能继续更新，复现时按清单获取 head 并记录实际完整 SHA，不能只记录分支名或旧基线。 提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已完成 preparation 五项、完整七组 full 及汇总，全部通过；v4/avx2-fma3 下普通/诊断完整 logits 最大误差为 0，完整 runner 约 24.6 分钟。通用 CI 为 3821 passed、5 skipped，W1/W2 和 Topic06 通过。preparation 下载诊断已复核；完整 raw 已下载验真、七组数组离线 audit 通过。前三轮失败记录与历史 v2/v3 证据均保留，不能跨提交拼接。定时 Nightly 未启用，E1/E2/E5 团队独立复现尚未登记。各轮结果和下载方法见 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。 -初次预备阶段的数据见 [历史预备工作本地验收报告](W3-预备工作本地验收报告.md)。这些历史记录不会随代码修改自动更新；本次验收应使用当前代码实际生成的报告、源码指纹和数值模式。 +Linux 预备与完整执行的数据分别见 [预备验收记录](W3-预备工作本地验收报告.md) 和 [完整模型执行记录](W3-完整模型执行与数值诊断报告.md)。早期个人开发机记录只保留为固定提交历史索引，不用于 Linux 验收；本次说明调整没有重新执行模型。 -本地交付入口： +Linux 交付入口： - [W3 工作总结与验收进度](W3-工作总结与验收进度.md)：开发计划对照、改动分组、实测证据及待团队推进的事项。 - [独立复现与验收清单](独立复现与验收清单.md)：候选验收口径、源码快照、实际执行命令，以及 E1/E2/E5 各自填写的记录。 @@ -36,19 +38,18 @@ medium 候选配置为 6 层、hidden64、FFN192、Q4/KV2、head_dim16、vocab12 ## 环境与统一复现 -使用 Python 3.12、[固定 CPU 依赖](../../../requirements/qwen3-small-probe.txt)、Zig 0.14.1 和 qemu-system-riscv64。环境安装方法见 [Qwen3 小模型说明](../../../probes/w2_qwen3_small/README.md)。固定 HF snapshot 及 ONNX 发布目录均需真实完整资产，不能是 LFS 指针。统一入口离线校验已有资产，不会安装依赖或下载模型。 +使用 Python 3.12、[固定 CPU 依赖](../../../requirements/qwen3-small-probe.txt)、Zig 0.14.1 和 qemu-system-riscv64。环境安装方法见 [Linux 统一环境与资产准备](../LINUX_REPRODUCTION.md)。固定 HF snapshot 及 ONNX 发布目录均需真实完整资产，不能是 LFS 指针。统一入口离线校验已有资产，不会安装依赖或下载模型。 在这个 W3 checkout 的根目录执行；输出目录必须不存在： -~~~powershell -$w3Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -& $w3Python -B -X utf8 scripts/run_w3_preparation.py ` - --source-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` - --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` - --output-dir output/w3-team-preparation-new ` - --cc D:/path/to/zig.exe ` - --qemu D:/path/to/qemu-system-riscv64.exe -$LASTEXITCODE +~~~bash +set -euo pipefail +W3_SOURCE=/absolute/path/to/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca +W3_MODEL=/absolute/path/to/qwen3-0.6b-onnx +"$SCRATCHV_PYTHON" -B -X utf8 scripts/run_w3_preparation.py \ + --source-dir "$W3_SOURCE" --model-dir "$W3_MODEL" \ + --output-dir output/w3-team-preparation-new \ + --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" ~~~ 五项都实际执行并通过、源码指纹一致、必需报告/产物齐全时，才返回 preparation:w3 的 PASS/0。它不是 numeric:ir-full-qwen3 的通过。失败保留子日志和 FAIL 报告，后续复跑必须换新目录。 diff --git "a/docs/llm-deploy-v1.0/W3/W3-\345\256\214\346\225\264\346\250\241\345\236\213\346\211\247\350\241\214\344\270\216\346\225\260\345\200\274\350\257\212\346\226\255\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W3/W3-\345\256\214\346\225\264\346\250\241\345\236\213\346\211\247\350\241\214\344\270\216\346\225\260\345\200\274\350\257\212\346\226\255\346\212\245\345\221\212.md" index 4597f43..f20ddbc 100644 --- "a/docs/llm-deploy-v1.0/W3/W3-\345\256\214\346\225\264\346\250\241\345\236\213\346\211\247\350\241\214\344\270\216\346\225\260\345\200\274\350\257\212\346\226\255\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W3/W3-\345\256\214\346\225\264\346\250\241\345\236\213\346\211\247\350\241\214\344\270\216\346\225\260\345\200\274\350\257\212\346\226\255\346\212\245\345\221\212.md" @@ -1,101 +1,48 @@ -# W3 完整模型执行与数值诊断报告 +# W3 完整模型 Linux 执行与验收记录 -> 历史记录：下文保留当时两组数值失败、未提交状态和原始诊断结果，不代表后续 `reference-v2` 的最新结果。W3 后续已提交至 PR #96，当前进度见 [工作总结与验收进度](W3-工作总结与验收进度.md)。原数据与源码指纹保持原样，不能将历史报告改写成当前 PR 提交的新执行记录。 +复现采用 Ubuntu 24.04 x86_64、Bash、Python 3.12；环境、模型资产与 Linux 工具链按 [统一指南](../LINUX_REPRODUCTION.md) 准备，实际命令见 [完整模型入口](../../../probes/w3_qwen3_full/README.md)。 -本轮继续完成了完整 28 层 Qwen3 的实际 ORT/IR 执行、内存受控模式、逐层证据和局部数值诊断。完整模型已经能够执行到 logits；数值门槛仍有两组失败，不能宣称 W3 验收通过。 +本文整理 **2026-10-05 已执行的 Linux 证据**。固定代码 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 在作者 [run 37299235103 / attempt 1](https://github.com/yuki-328/ScratchV/actions/runs/37299235103) 中同时通过 preparation、完整七组和汇总。本次文档迁移没有重新执行模型，也不将旧证据认定为新代码或第二人执行。 -改动保存在 `D:/cyq/code/ScratchV/output/w3-preparation` 的 `codex/w3-preparation` 分支，基线仍为 `65616a8cde6a581661ac3734378da78a9fcbe5ed`。本轮只在本地工作，没有提交、推送或更新 PR。此前五项预备检查的数据见 [预备验收报告](W3-预备工作本地验收报告.md)。 +## 执行与数值范围 -## 新增能力 +固定真实 Qwen3-0.6B ONNX 为 28 层、FP32、L256、无 KV cache。每组输入分别启动 ORT 和 IR worker，两个后端各运行普通图及诊断图。ORT 使用 CPUExecutionProvider、关闭图优化与 CPU memory arena、intra/inter threads 均为 1。IR 使用优化等级 `none`、`numpy-fp32-reference-v4/avx2-fma3`。 -| 交付 | 实际行为 | -|---|---| -| 完整执行入口 | 七组固定 L256 输入，ORT 和 IR 分进程；每个后端分别运行普通、诊断前向 | -| 完整数值核对 | 比较全部 `[1,256,151936]` logits，含 padding，严格 `<1e-4`，不使用相对容差 | -| 逐层证据 | embedding、28 层 residual、final norm 共 30 点；保存实际张量、输入、模型和源码哈希 | -| 内存模式 | parser 显式只读 mmap，解释器显式借用 initializer 并在最后一次使用后回收 binding | -| 局部诊断 | 同一层输入送入 ORT 和 IR，区分本层产生的差异与上游误差传播；FP64 仅作诊断参考 | -| 资源与失败管理 | 子进程超时、内存采样、中断清理、失败阶段和原始日志；完整数值失败保持 FAIL | -| 手动 CI 配置 | 新增完整模型手动验证配置；尚未提交或在远端执行，未启用 Nightly | +| 检查 | Linux 实测结果 | 验收含义 | +|---|---|---| +| 七组输入 | 7/7 PASS：两组满长、one_token、short_17、short_255、changed_future、changed_padding | 完整覆盖，没有缩减失败 case | +| 普通/诊断完整 logits | 最大绝对误差 0，全部有限，形状 `[1,256,151936]` | 正式比较包含全部 padding query，严格 `<1e-4`、rtol=0 | +| 每组 30 检查点 | 本次全部最大绝对误差 0 | 保存中间定位证据；不改变最终 logits 门槛 | +| 因果和 padding 不变量 | 两后端共四项，最大绝对误差均为 0 | 修改未来或填充 token 后，受保护的有效前缀保持一致 | +| full worker 回归 | 581 passed、0 skipped | 该工作流专项测试，不等于全部仓库测试 | +| 同提交 preparation | 五项 PASS | 见 [Linux 预备记录](W3-预备工作本地验收报告.md) | -核心既有代码新增修改为 `scratchv/frontend/onnx_parser.py` 和 `scratchv/verification/ir_interpreter.py`。默认调用仍使用原加载、复制、保留策略；新策略仅显式启用，限制在已验证的纯单 block 图，不删除死指令执行或 D8 数值错误检查。所有返回值仍是独立拷贝。 +零误差是该模型、代码与环境的实测值，不是跨机器承诺。`reference-v4` 约定 FP32 计算顺序及数学近似；IR 独立执行，不在算子中调用 ORT。解释器默认 API 的 `native` 模式仍需单独验证，reference 通过不等于 native 或目标后端通过。 -本轮保持固定 ONNX、权重和生产 FP32 数学实现不变，没有清零 padding logits、切换参考后端、放宽阈值或以局部结果代替完整结果。 +## Linux 环境、时长与内存 -## 完整七组输入的结果 +实际 runner 为 Ubuntu 24.04.5、Python 3.12.14、4 vCPU AMD EPYC 7763、约 16.77 GB RAM。NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1、Torch 2.7.1+cpu；NumPy 诊断为 AVX2/FMA3、OpenBLAS 0.3.29 / Haswell、`OPENBLAS_CORETYPE=null`。这些字段不能推断 ORT 内部 dispatch。 -最终完整证据位于 `output/w3-full-suite-02/`，整套耗时 **222.339 秒**。七组输入均实际执行完整模型：7 个 ORT worker、7 个 IR worker，各执行普通和诊断前向，共 **28 次完整前向**。两个后端的普通/诊断 logits 在各自内部逐元素一致。正式 IR/ORT 比较结果如下，与首轮 `output/w3-full-suite-01/` 相同；普通和诊断 logits 的跨后端误差相同。 +- 完整 runner 墙钟时长 `1477.314971525 s`，约 24.6 分钟，包含加载、普通/诊断执行、比较和证据写盘。 +- 每组 ORT worker 总时长约 `15.20–15.53 s`；每组 IR worker 总时长约 `193.45–195.88 s`。每个 worker 都有两次前向，不是单次推理 benchmark。 +- worker 自报全生命周期 RSS 峰值：ORT `4,443,017,216–4,443,987,968` bytes，IR `3,102,253,056–3,102,715,904` bytes。 +- 父监控的进程树 RSS 采样峰值：ORT `4,443,328,512–4,444,389,376` bytes，IR `2,943,205,376–3,057,094,656` bytes。采样可能错过瞬时峰值，不与 worker 自报值混用。 -| 输入 | 全位置最大绝对误差 | 有效位置最大绝对误差 | padding 最大绝对误差 | `<1e-4` | -|---|---:|---:|---:|---| -| full_seed_0 | 4.67300415e-5 | 4.67300415e-5 | 无 padding | PASS | -| full_seed_42 | 4.86373901e-5 | 4.86373901e-5 | 无 padding | PASS | -| one_token | 4.10079956e-5 | 4.10079956e-5 | 2.81333923e-5 | PASS | -| short_17 | 2.34365463e-4 | 4.10079956e-5 | 2.34365463e-4 | **FAIL** | -| short_255 | 6.34193420e-5 | 6.34193420e-5 | 2.14576721e-5 | PASS | -| changed_future | 5.14984131e-5 | 5.14984131e-5 | 无 padding | PASS | -| changed_padding | 2.53677368e-4 | 4.10079956e-5 | 2.53677368e-4 | **FAIL** | +完整入口采用只读权重映射、initializer 借用和中间 binding 最后使用后回收。Linux 通过独立进程组管理本次拥有的进程树，超时/采样内存超限/中断时清理。采样终止阈值不是操作系统硬配额，也不是整台机器总内存预算。默认解释器调用行为保持兼容；本报告不将容量准备当作 W6 性能优化收益。 -因果性和 padding 隔离共四项检查均通过，比较的前缀在两个后端内分别逐元素相同，误差为 0。改变 padding token 后只要求有效 query 的输出不变；正式 IR/ORT 精度比较仍覆盖全部 256 个位置。 +## 原始产物与核对 -七次普通前向的执行时间合计：ORT **27.811 秒**，IR **42.794 秒**。整套 222.339 秒还包含诊断前向、解析、模型/源码哈希、进程启动、写盘和产物复核，不能当作单次推理耗时。这些是本机 CPU 上的 ORT/NumPy IR 测量，完整 28 层尚未在 RISC-V/QEMU 执行。 +主报告 SHA256：`0ddb97d13b6bab44442fb0b453d1e0132ef7a02c96be9c8caf70ae2862e31746`。 -中间检查点的 `1e-4` 标记用于定位，正式门槛适用于完整 logits。第 2 层一些 residual 值约为 6872，单个 FP32 ULP 约 `4.88e-4`；中间值绝对误差超过 `1e-4` 本身不能证明矩阵乘或解析语义有错，也不能用于取消 logits 的正式门槛。 +| Artifact | ID | 已核对 ZIP SHA256 | +|---|---|---| +| `w3-full-numeric-reports` | 11341033619 | `46047d41334f3336eaf2e39385bffca37551a726891ad6c671c23df1e60f88c5` | +| `w3-full-numeric-raw` | 11341924764 | `b99c06f3e39c7c465d31763eba98350682961de1541cab719ce3da1df2ee4171` | -## 数值差异定位 +完整 raw ZIP 为 `4,350,864,947` bytes，包含 140 个文件。作者下载后完成七组、四不变量的数组离线复核，固定报告 hash 与模型文件也通过核对，用时 `21.234900 s`；`model_executed=false`、`independent_reproduction=false`。历史源码换行差异的只读核对说明以及原始 audit 字段完整保留在 [Linux 实测记录](Linux-CI与Nightly.md)，不改写报告中的源码哈希。产物保留期限有限，需另行留存或重新运行。 -short_17 的 embedding 完全一致，第 0 层 residual 已出现约 `6.97e-6` 的差异，随后逐层传播。其最终最大 logits 差异位于 query 80、token 126661，属于 padding query。final norm 最大差异约 `5.36e-4`。 +## 尚待验收的边界 -使用完全相同的局部输入，实际运行原 ONNX 算子经 parser/IR 的结果与 ORT 比较： +团队仍需确认 medium 配置、reference 模式与比较契约；E1/E2/E5 各自在 Linux 上执行；定时 Nightly 尚未启用。`w3_exit_accepted=false`。既有 PyTorch/ORT 导出差异单独记录，IR/ORT 通过不表示它自动消失。W3 的小 Attention QEMU 与完整 host IR 分别验收，完整模型的 RISC-V 权重加载和 QEMU 前向属于 W4。 -| 第 0 层局部操作 | IR/ORT 最大绝对误差 | 解释 | -|---|---:|---| -| input RMSNorm 的 ReduceMean | 2.32830644e-10 | Pow 输出相同后首次出现舍入差异，此时尚未进入 Attention/mask | -| Q 投影 MatMul | 3.81469727e-6 | 存在 FP32 数值差异，未发现形状或转置语义错误 | -| K 投影 MatMul | 1.66893005e-6 | 同上 | -| Attention QK MatMul | 0 | 当前相同输入下逐元素一致 | -| Attention Softmax | 1.78813934e-7 | 双方所有屏蔽概率严格为 0 | - -独立 FP64 局部诊断中，ReduceMean 的 IR 误差约 `8.00e-11`，ORT 约 `2.59e-10`；IR 在这个位置反而更接近高精度参考。Q/K 投影的 ORT 局部误差较小。证据支持多种 FP32 归约和矩阵乘舍入差异叠加的解释，尚未证明所有最终误差只来自某一个算子。不能仅为贴近 ORT 而硬编码某一输入的累加顺序。 - -可复用的 [局部诊断入口](../../../probes/w3_qwen3_full/LOCALIZE.md) 已实际运行，证据为 `output/w3-full-localize-01/report.json`：读取首轮 short_17 的固定证据，检查第 0/21 层及五个原语，耗时 **5.258 秒**，进程峰值 RSS **888,676,352 字节**。同输入下这两层的 residual 最大差异分别约 `6.97e-6`、`6.10e-5`。报告的 `COMPLETE` 只表示诊断完成，`numeric_gate_passed=false`；它不替代完整七组验收。 - -历史 short5 的 Torch/ORT padding 差异也已独立复现：逐层官方 Torch 加同一分块 head 与旧完整 Torch logits 全部逐元素相同；ORT 层状态经过该共享 head 与 Torch 相差约 `0.00216293`，接近历史 `0.00216198`。这条实验与本轮正式 IR/ORT gate 分开保存；它不代替正式参考或验收结果。实验脚本、逐层数组和说明位于 `output/w3-padding-diagnosis/`。 - -## 内存对照与测量边界 - -在相同 short_17 输入上，默认 parser 加复制/保留全部中间值的解释器实际峰值 RSS 为 **10,341,670,912 字节，约 9.63 GiB**。本轮新模式的完整 worker 实测约 **2.80 GiB**，其普通和诊断执行都保留完整 logits。默认策略输出与新策略 logits **逐元素完全一致**，支持内存策略没有改变本输入数值结果。对照证据为 `output/w3-full-default-memory-01/report.json`。 - -解释器默认策略的去重 buffer 峰值为 `5,451,086,568` 字节；新模式约 `2,397,770,952` 字节。该计数包括当时已绑定的借用数组，不包括已解除绑定却仍由调用方保留的对象、内核临时缓冲或观察器拷贝。Windows mmap 的对齐前缀及独立映射也会计入 buffer 估算。它不等同实际 RSS。 - -内存对照同时发现并修复了首版监控缺陷：Windows venv Python 的 `Popen.pid` 是启动器，实际计算发生在另一个进程，旧父监控只看到约 6 MiB，未执行到原定 8 GiB 的终止条件。因此 `w3-full-suite-01` 和 `w3-full-default-memory-monitor-01.json` 的父进程采样不能作资源限制有效性的证据；实际 worker 的自测 RSS、张量和数值比较不受该采样问题影响。 - -修复后先挂起创建 Windows 启动器，将其加入本次拥有的 Job Object，再恢复运行；内存采样和终止均覆盖整个 Job。实际 venv 启动器加计算子进程的回归测试，在 96 MiB 上限下采样到 RSS `145,768,448` 字节、私有提交 `134,787,072` 字节，正确触发超限终止，并通过持有的实际子进程句柄确认子进程退出。超时、中断、启动器先退出但后代仍运行也有回归覆盖，证据为 `output/w3-job-tree-01.xml`。 - -最终 suite02 的 14 个 worker 均记录 `ownership=windows_job`，每次实际采样到至少两个进程，没有清理错误；运行上限为 10 GiB。ORT/IR 的 worker 自测峰值 RSS 分别为 **4,247,453,696 / 3,003,027,456 字节**（约 **3.96 / 2.80 GiB**）；父监控采样的进程树 RSS 峰值分别为 `4,243,689,472 / 2,962,190,336` 字节。采样可能错过短峰值，进程树共享页也可能重复计数，两个口径不能混用；该措施不是操作系统硬内存配额。 - -同一进程树管理也接入五项准备检查的总入口，修复 Windows venv 后代在超时或中断后可能残留的问题。准备入口只记录内存并限制耗时，没有声称为这五项配置实际内存额度。 - -## 最终本地回归 - -| 验证 | 结果与证据 | -|---|---| -| 完整七组 IR/ORT | **5 PASS、2 FAIL**；`output/w3-full-suite-02/report.json` | -| 五项准备检查 | **5 PASS**，总耗时 **57.173 秒**；`output/w3-preparation-after-full-01/report.json` | -| 其中 Attention RISC-V/QEMU | **12/12 PASS**，QEMU 进程墙钟累计 **2.253 秒**；与完整模型 CPU 执行分开统计 | -| 扩大单测回归 | **933 PASS、3 FAIL、0 SKIP**，耗时 **43.24 秒**；`output/w3-full-regression-final-02.xml` 和同名 `.log` | -| 三项失败的未修改基线复核 | 相同三项失败；`output/w3-old-ort-baseline-01.xml` | - -扩大回归包含所有 `test_w3*.py`、`test_qwen3_medium*.py`、`test_ir_interpreter*.py`、`test_onnx*.py`、`test_qwen3_full*.py`，以及原两层模型、探测、IR gate、RISC-V gate 和 runtime-model 测试。不能把上述结果写成“整个项目测试全绿”。 - -各执行报告保存实际源码 SHA256。完整 suite02 运行期间源码保持一致；之后仅 `scripts/run_w3_preparation.py` 补上进程树生命周期修复，完整模型入口、数学计算、parser、解释器及资源监控代码均与 suite02 相同。最终五项准备检查和扩大回归是在这项修复后运行，准备报告的源码指纹与当前代码一致。 - -## 验收与后续 - -复现命令、完整输出及耗时字段见 [完整模型入口](../../../probes/w3_qwen3_full/README.md)。完整执行成功与精度通过是两件分别记录的事实；统一报告的 `full_ir_executed=true` 不会自动令 `passed=true` 或 `w3_exit_accepted=true`。 - -当前最重要的剩余项是 short_17 和 changed_padding 的完整 logits 数值超标。继续定位时应固定相同层输入，比较原语及高精度诊断，并用完整七组回归验证任何候选修改；不得只看有效位置。若最终需要调整计算策略或验收契约，应保留当前失败基线并交由团队评审，不能直接修改已有门槛。 - -扩展旧 ONNX 回归还发现三项基线已有的 ORT 参考失败：float64 Cos 缺少 CPU kernel，以及 opset18 ReduceMean omitted axes 与 noop 两项 ORT 内部断言。在未修改的 `65616a8` 工作树上以相同环境独立复现了相同三项失败，证据为 `output/w3-old-ort-baseline-01.xml`。它们与本轮内存策略无关，没有通过跳过或降低阈值隐藏。 - -W1 人工前置验收、团队契约确认、E1/E2/E5 独立复现和 Nightly 仍由团队推进。完整 IR 数值硬门槛未通过前，不宣称可以完成 W3 收尾或进入 W4 完整后端集成。 +早期个人开发机的失败定位、内存实验和进程兼容记录保留在 [固定提交历史原文](https://github.com/ScratchV-Compiler/ScratchV/blob/77c3aa86e9794e5bfe3c81fd873a1506d14bc753/docs/llm-deploy-v1.0/W3/W3-完整模型执行与数值诊断报告.md)，不计入 Linux 验收；没有将其历史结果换成 Linux 标签。Linux 前三轮失败与第四轮同提交通过记录见 [Linux CI 与 Nightly](Linux-CI与Nightly.md)，不能跨提交拼接通过项。 diff --git "a/docs/llm-deploy-v1.0/W3/W3-\345\267\245\344\275\234\346\200\273\347\273\223\344\270\216\351\252\214\346\224\266\350\277\233\345\272\246.md" "b/docs/llm-deploy-v1.0/W3/W3-\345\267\245\344\275\234\346\200\273\347\273\223\344\270\216\351\252\214\346\224\266\350\277\233\345\272\246.md" index 43eb040..5bc8007 100644 --- "a/docs/llm-deploy-v1.0/W3/W3-\345\267\245\344\275\234\346\200\273\347\273\223\344\270\216\351\252\214\346\224\266\350\277\233\345\272\246.md" +++ "b/docs/llm-deploy-v1.0/W3/W3-\345\267\245\344\275\234\346\200\273\347\273\223\344\270\216\351\252\214\346\224\266\350\277\233\345\272\246.md" @@ -1,6 +1,8 @@ # W3 工作总结与验收进度 -更新日期：2026-10-05。W3 已在第四轮同提交 Linux 实跑中通过 preparation、完整七组及汇总，下载原始数组 audit 也通过；[PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 可交给团队复现。剩余为团队契约确认、E1/E2/E5 独立执行及实际定时 Nightly 验收，正式阶段出口仍为 false。 +对外复现统一使用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 统一环境与资产准备](../LINUX_REPRODUCTION.md)，并在同一个 Bash 会话、仓库根目录执行以下命令；`SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU` 由该指南设置。 + +更新日期：2026-10-06；仅调整 Linux 交付说明，未重新执行模型。W3 已在第四轮同提交 Linux 实跑中通过 preparation、完整七组及汇总，下载原始数组 audit 也通过；[PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 可交给团队复现。剩余为团队契约确认、E1/E2/E5 独立执行及实际定时 Nightly 验收，正式阶段出口仍为 false。 本地代码位于 `codex/w3-preparation` 工作树，原基线为 `65616a8cde6a581661ac3734378da78a9fcbe5ed`。当前实测代码为 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a`，[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)的 preparation、完整七组 full 和汇总均通过；通用 CI 为 3821 passed、5 skipped，W1/W2 与 Topic06 通过。preparation 产物迁移已复核，完整 raw 已下载验真、离线 audit 通过；实际 schedule 未启用。复现应获取 PR head 并记录实际完整 SHA，不能只记录分支或旧基线。前三轮失败记录保留，各轮环境、耗时和 artifact 身份见 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。 @@ -17,7 +19,7 @@ | CI 与 Nightly | 两套手动/可复用工作流，以及显式启用的定时编排；既有入口可显式调用 W3 | 第四轮 preparation、完整 full 和汇总均通过；完整 raw audit 已通过 | 同提交两套 gate 和汇总通过；完整原始数组 audit 已通过，定时 schedule 未启用 | | 阶段出口 | 验收口径提案、复现步骤、每人记录模板 | [独立复现与验收清单](独立复现与验收清单.md) | 团队确认及 E1/E2/E5 记录待填写 | -完整模型的正式数值门槛是全部普通/诊断 logits 的严格 `max_abs < 1e-4`、`rtol=0`，包括全部 256 个位置及 padding query。每组另外保存 30 个检查点用于定位；历史 Windows 记录的检查点实际误差也均为 0，但不把它们改成额外的全中间激活硬门槛，也不据此预判后续 Linux 复跑结果。 +完整模型的正式数值门槛是全部普通/诊断 logits 的严格 `max_abs < 1e-4`、`rtol=0`，包括全部 256 个位置及 padding query。每组另外保存 30 个检查点用于定位；第四轮 Linux 记录的检查点实际误差也均为 0，但不把它们改成额外的全中间激活硬门槛，也不据此预判新代码或其他 Linux 环境的复跑结果。 ## 2. 改动如何组成完整工作 @@ -52,40 +54,26 @@ Nightly 默认不执行定时数值任务，需要 `W3_NIGHTLY_ENABLED=true`； ## 3. 验证记录及证据边界 -以下本地路径相对 W3 工作树；`output/` 是生成产物，不作为 Git 源码提交。历史记录保留原版本和环境，本轮 Linux 的 run URL、artifact 与下载后的核对结果另见 [Linux 实测记录](Linux-CI与Nightly.md#2026-10-05-linux-实测记录)。 +对外验收使用实际 Linux 记录。下表同属于提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的 [第四轮 run 37299235103](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)，不把早期个人开发机测试改名为 Linux 证据。本次说明调整未重新执行模型。 -| 记录 | 结果 | 原始位置 | +| 记录 | 已保存的结果 | 证据边界 | |---|---|---| -| 完整 28 层七组真实执行 | PASS；约 664.26 秒；普通/诊断完整 logits 最大误差 0 | `output/w3-full-review-v2-01/report.json` | -| 上轮交付快照的五项 preparation | 全部 PASS；约 99.49 秒 | `output/w3-local-handoff/preparation-final/report.json` | -| 本次预览代码的 Git 工作树扩展回归 | 1280 passed，1 skipped | `output/w3-pr-preview/regression.xml` | -| 上轮解压源码包扩展回归 | 1277 passed，3 skipped | `output/w3-local-handoff/archive-regression-final.xml` | -| 七组完整模型数组离线复核 | PASS；七组、四项不变量；没有执行模型 | `output/w3-local-handoff/full-evidence-audit/report.json` | -| 解压源码包单例模型执行 | `short_17` 数值正确，状态为 PARTIAL | `output/w3-local-handoff/full-short17/report.json` | -| 工作流静态检查 | actionlint 1.7.12，三份配置零诊断 | `output/w3-actionlint-final.json` | -| 本轮 PR 通用 CI | `2e5f9f4`：3727 passed、5 skipped；W1/W2、Topic06 通过 | [CI run 37290861920](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37290861920)；不替代 W3 专项 | -| 本轮作者 Linux preparation | 五项 PASS，132.844285 秒；相关回归 1203 passed、1 skipped | [run 37290898463](https://github.com/yuki-328/ScratchV/actions/runs/37290898463)，同提交 `2e5f9f4` | -| 本轮 preparation 产物迁移核对 | 63 个诊断证据文件哈希匹配、无遗漏；选定 medium 数组 81/81 点离线复算通过 | 该 run 的 preparation reports/diagnostics artifact；没有重新执行模型 | -| 第二轮 Linux 完整模型 | v3/avx512 下 7/7 输入、4 不变量 PASS；普通/诊断 logits 最大误差 0；456.723772664 秒 | [run 37294081510](https://github.com/yuki-328/ScratchV/actions/runs/37294081510)，提交 `ca110be`；报告已下载，该轮 preparation 失败，整体不能验收 | -| 第三轮 Linux preparation | 五项 PASS，135.615884 秒；相关回归 1270 passed、1 skipped | [run 37295229214](https://github.com/yuki-328/ScratchV/actions/runs/37295229214)，提交 `8132f3c` | -| 第三轮 preparation 产物复核 | 63 文件大小/哈希匹配、无遗漏；下载数组 81/81 点离线复算通过 | `output/w3-linux-37295229214/layer-diff-download-audit-01/report.json`；没有执行模型 | -| 第四轮 Linux preparation | 五项 PASS，132.923089 秒；相关回归 1297 passed、1 skipped；同提交完整模型亦通过 | [run 37299235103](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)，提交 `1c49e8a` | -| 第四轮 preparation 产物复核 | 63 文件大小/哈希匹配、无遗漏；下载数组 81/81 点离线复算通过 | `output/w3-linux-37299235103/preparation-independent-audit.json` 与 `layer-diff-download-audit-01/report.json`；没有重新执行模型 | -| 第四轮代码通用 CI | 3821 passed、5 skipped；真实 Haswell 回归及 BLAS 环境证据测试在 Linux 通过 | [CI run 37299215953](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37299215953)，同提交 `1c49e8a` | -| 第四轮完整模型与手动编排 | v4/avx2-fma3 下 7/7、4 不变量及两套 gate 汇总 PASS；1477.314972 秒，普通/诊断 logits 最大误差 0 | `output/w3-linux-37299235103/full-reports/extracted/w3-full/report.json`；完整 raw audit 已通过 | -| 第四轮 raw 下载与完整离线复核 | 140 文件 ZIP digest 匹配；7 组、4 不变量、固定报告 hash 与模型文件核对通过，21.234900 秒 | `output/w3-linux-37299235103/full-evidence-download-audit-01/report.json`；没有重新执行模型 | - -工作树的跳过项是本机无法创建符号链接；源码包额外跳过缺少 Git 历史的集成测试和未打包的 CNN 资产测试。真实历史比较已在 Git 工作树执行通过。上述耗时包含各自的加载、编译、诊断或写盘开销，且可能与其他测试并行，不是优化前后性能 benchmark。 - -本轮总结所引用的是各自生成时的实测记录，保留原始源码哈希。之后的文档和兼容性维护不覆盖原报告，也不把旧报告改写成新版本重新运行。完整执行、单例执行和离线重验是三种不同证据；本地原始记录不能替代第二人独立执行。 - -Python 3.8 路径 API 兼容性的小修复另有 265 项相关回归通过，并重新完成了固定完整模型的映射解析与 IR 校验；本次 PR 预览版本的记录另存于 `output/w3-pr-preview/`。W3 实验环境仍固定为 Python 3.12，不据此宣称完整 Python 3.8 环境已测试。 +| Linux preparation | 五项 PASS，132.923089 秒；回归 1297 passed、1 项非 Linux 平台兼容性测试 skipped | 准备范围；不包含完整 IR gate | +| Linux 完整模型 | 七组、四不变量 PASS，1477.314971525 秒；v4/avx2-fma3、IR none；普通/诊断完整 logits 最大误差 0 | 固定完整模型、代码及环境；不是 native 或任意 CPU 的通过保证 | +| 完整 worker 回归 | 581 passed、0 skipped | 专项测试范围 | +| 同提交通用 CI | 3821 passed、5 skipped | [CI run 37299215953](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/37299215953)，与专项范围分别记录 | +| preparation 下载复核 | 63 文件大小/哈希匹配、无遗漏；选定数组 81/81 点重算通过 | 作者离线复核，没有执行模型 | +| 完整 raw 下载复核 | 140 文件 ZIP digest 匹配；7 组、4 不变量及固定报告 hash、模型文件核对通过；21.234900 秒 | `model_executed=false`、`independent_reproduction=false` | + +运行环境、报告 SHA256、artifact 身份、历史失败轮次和保存产物的方法集中在 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。[预备记录](W3-预备工作本地验收报告.md) 和 [完整记录](W3-完整模型执行与数值诊断报告.md) 给出各自的数据与范围；早期个人开发机资料通过其中的固定提交索引留存，不计入 Linux 验收。 + +每次重新执行保存新的目录、实际 SHA、环境及模型指纹。旧报告不因文档或代码更新而自动成为新版本通过证据；完整执行、单例执行和离线数组重验是三种不同证据。耗时包含加载、编译、诊断或写盘，不能当作目标硬件性能或优化前后 benchmark。 ## 4. 评审顺序与兼容边界 建议先确认完整 logits 的范围和误差门槛，再评审前端/解释器可选接口、FP32 模式、完整 runner 的验收拒绝逻辑、其他探测与证据工具，最后核对 CI、文档及第三方许可。 -- 历史 Windows v2、第二轮 Linux v3/avx512 与第四轮 Linux v4/avx2-fma3 的零误差分别属于其原提交和环境；不是默认 `native`、其他 CPU/BLAS 策略或未来提交的通过保证。v4 应记录解析后的 `cpu_strategy`、MatMul 布局字段及 `environment.numeric_runtime` 中的 `OPENBLAS_CORETYPE` 原值和 `numpy_openblas` 实际库诊断；CPU SIMD 不等于实际 BLAS 内核，可选诊断缺失不豁免数值门槛；离线 audit 按保存的策略核对，旧 v2/v3 数组则用对应历史源码核对。 +- 第二轮 Linux v3/avx512 与第四轮 Linux v4/avx2-fma3 的零误差分别属于其原提交和环境；不是默认 `native`、其他 CPU/BLAS 策略或未来提交的通过保证。v4 应记录解析后的 `cpu_strategy`、MatMul 布局字段及 `environment.numeric_runtime` 中的 `OPENBLAS_CORETYPE` 原值和 `numpy_openblas` 实际库诊断；CPU SIMD 不等于实际 BLAS 内核，可选诊断缺失不豁免数值门槛；离线 audit 按保存的策略核对，旧 v2/v3 数组则用对应历史源码核对。 - medium 为六层随机小配置候选，需要团队认可通道尺寸；真实子图仅使用第一层预训练权重和受控激活。 - 映射的权重文件在调用期间必须保持不变；映射接口本身不替代固定资产哈希验证。 - 解释器统计不包含全部临时缓冲、分配器及调用方持有的数组；进程 RSS 和采样内存另行报告。 diff --git "a/docs/llm-deploy-v1.0/W3/W3-\351\242\204\345\244\207\345\267\245\344\275\234\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" "b/docs/llm-deploy-v1.0/W3/W3-\351\242\204\345\244\207\345\267\245\344\275\234\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" index d96b33c..b6a7adf 100644 --- "a/docs/llm-deploy-v1.0/W3/W3-\351\242\204\345\244\207\345\267\245\344\275\234\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" +++ "b/docs/llm-deploy-v1.0/W3/W3-\351\242\204\345\244\207\345\267\245\344\275\234\346\234\254\345\234\260\351\252\214\346\224\266\346\212\245\345\221\212.md" @@ -1,92 +1,38 @@ -# W3 预备工作本地验收报告 +# W3 Linux 预备工作验收记录 -> 历史记录：下文保留当时的代码状态、覆盖范围和原始数据，其中“尚未提交”和完整模型待开展均指该次验证时点。W3 后续已提交至 PR #96，当前进度见 [工作总结与验收进度](W3-工作总结与验收进度.md)。旧报告不证明 PR 后续提交已经重新执行通过。 +对外复现统一采用 Ubuntu 24.04 x86_64、Bash、Python 3.12，环境安装见 [Linux 统一指南](../LINUX_REPRODUCTION.md)，执行步骤见 [W3 入口](README.md) 与 [独立复现清单](独立复现与验收清单.md)。保留当前文件名以兼容已有链接。 -2026 年 10 月 4 日完成本轮本地验证：五项预备检查全部通过，403 项相关回归测试通过、无失败或跳过。现在已有六层模型、真实权重子图、组合 Attention 后端、逐层误差定位和内存观测的可复现入口。完整 28 层 Qwen3 的 IR 数值执行仍待开展，因此本报告不作 W3 阶段出口通过的结论。 +本文整理 **2026-10-05 已执行的 Linux 记录**，不是本次文档调整后的重新运行，也不是团队独立复现。固定代码为 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a`；来源为作者 [run 37299235103 / attempt 1](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)。该提交的 preparation、完整七组及汇总全部通过。 -改动位于 `D:/cyq/code/ScratchV/output/w3-preparation`，分支为 `codex/w3-preparation`，基线为 W2 提交 `65616a8cde6a581661ac3734378da78a9fcbe5ed`。全部改动尚未提交、推送或更新 PR。原工作目录及 W1/W2 发布工作树保留原状。 +## 环境与身份 -## 本轮交付 +实际 runner 为 Ubuntu 24.04.5、Python 3.12.14、4 vCPU AMD EPYC 7763；NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1、Torch 2.7.1+cpu。NumPy 诊断记录 AVX2/FMA3、OpenBLAS 0.3.29 / Haswell，`OPENBLAS_CORETYPE=null`。CPU 能力、IR 参考策略与实际 BLAS 内核分别记录。 -| 修改 | 作用 | -|---|---| -| `probes/w3_qwen3_medium/` | 导出并执行官方 Qwen3 类的六层随机模型，保存普通图、诊断图和逐层张量 | -| `probes/w3_qwen3_subgraphs/` | 用固定 checkpoint 第一层的真实权重检查真实维度的投影、RMSNorm、RoPE、GQA 和 SwiGLU | -| `probes/w3_attention/` | 将组合 Attention 编译为 RISC-V，在 QEMU 中与 ORT 和独立 NumPy 参考比较 | -| `probes/w3_layer_diff/` | 独立读取张量文件和检查点定义，定位首次超阈值及最大误差元素 | -| `probes/w3_full_preflight/` | 校验完整 ONNX 资产、接口契约和静态尺寸，为完整执行准备输入 | -| `probes/w3_common.py` | 统一源码指纹、环境、进程峰值内存、输出目录和报告写入规则 | -| `scripts/run_w3_preparation.py` | 串行运行五项检查，核对覆盖范围、原始产物、数值结果和源码一致性 | -| `.github/workflows/w3-preparation.yml` 和证据收集脚本 | 提供手动 CI 配置，保存报告、原始比较张量、QEMU 日志及失败证据 | +preparation 主报告 SHA256 为 `ceeeb239ac7b88c22372d0e76f5506841e1b634a81b5b1567af35ca84d443046`。完整产物身份、历史失败轮次和下载方法见 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。 -已有公共代码只修改两处：`probes/w2_qwen3_small/model.py` 增加按配置构造模型和检查点的能力，保留 small 默认行为；`scratchv/verification/ir_interpreter.py` 增加默认关闭的内存统计，不调整数值计算、输入复制或张量保留行为。 +## 五项验收 -本轮没有修改 MatMul 后端、放宽 W1/W2 阈值或实现内存回收优化。新增测试覆盖数值契约、检查点覆盖、内存计数、失败退出、报告一致性和证据留存。 - -## 本地验收数据 - -最终统一运行目录为工作树中的 `output/w3-preparation-final-02/`。下表的数据来自该目录的子报告，最大误差均为绝对误差。 - -| 项目 | 实际覆盖 | 验收门槛 | 实测结果 | +| 项目 | 实际覆盖 | 严格门槛 | Linux 实测结果 | |---|---|---|---| -| 六层 medium | 7 组输入，每组 81 个检查点；普通/诊断双图；IR none/basic/all；18 项不变量 | 全部位置含 padding，严格 `<1e-5` | PASS；IR/ORT 最大 `2.5033950805664062e-6` | -| 真实维度子图 | 14 个 L256 子图，10 项因果和 padding 不变量；Torch、ORT、IR 三方比较 | 严格 `<1e-4` | PASS；IR/ORT 最大 `1.52587890625e-5`，Torch/ORT 最大 `3.0517578125e-5` | -| 组合 Attention 后端 | 6 组图 × none/all，共 12 次实际 QEMU；含 L256 边界及因果/padding 检查 | 严格 `<1e-4` | PASS；QEMU/ORT 最大 `3.5762786865234375e-7` | -| 独立逐层比较 | medium 的 short_17，优化 all 的诊断 IR 与诊断 ORT，81/81 检查点 | 严格 `<1e-5` | PASS；最大 `2.5033950805664062e-6`，无超阈值检查点 | -| 完整资产预检 | 固定资产哈希、ONNX 输入输出及静态大小 | 资产与契约符合 manifest | PASS；未执行完整 ORT 或 IR | - -medium 配置为六层、hidden64、FFN192、Q4/KV2、head_dim16、vocab128、L256，共 304128 个随机参数。它使用官方 Qwen3 实现，但不是预训练六层模型。开发计划尚未定义 medium 的具体宽度，此配置是本轮候选，仍需团队确认。 - -真实子图使用 hidden1024、FFN3072、16Q/8KV、head_dim128，Q 投影宽度为 2048。权重来自固定 Qwen3-0.6B checkpoint 的第一层，原 BF16 权重提升到 FP32；输入激活使用固定随机种子。它验证了这些真实尺寸子图，尚不等于完整 decoder 层或完整模型前向。K RMSNorm 的误差高于 `1e-5`，符合本轮 W3 `1e-4` 的约束；不能据此声称满足 W2 更严格的门槛。 - -逐层比较的最大误差位于零起始编号的 `layer_3.q_norm`，索引 `[0,191,2,7]`。IR 值为 `-1.3136119842529297`，ORT 值为 `-1.3136144876480103`。工具可输出这种位置和数值；仅比较选定层时会返回 PARTIAL，不能误报全图通过。 - -## 耗时与内存 - -本次统一预备检查耗时 **63.484 秒**。以下为一次本机观测，包含校验、导出、文件读写等开销，不作为性能优化收益或真实硬件推理速度。 - -| 子项 | 子报告耗时 | 该进程全生命周期峰值 RSS | -|---|---:|---:| -| medium | 40.228 秒 | 620.82 MiB | -| 真实子图 | 5.968 秒 | 686.16 MiB | -| Attention | 9.697 秒 | 94.01 MiB | -| 完整资产预检 | 2.089 秒 | 139.71 MiB | - -Attention 的 **12 次 QEMU 累计进程时长为 2.265 秒**，字段为 `attention/report.json` 的 `qemu_process_wall_seconds.total`，计数为 `measured_count=12`。该时长包含 QEMU 启动、guest 计算、UART 传输与退出。统一报告的 `elapsed_seconds` 是整轮墙钟时长，`gates[].elapsed_seconds` 是各子进程调用时长；它们与子报告内部计时范围不同。 - -medium 在优化 all 下，普通图的 NumPy 去重存储峰值为 **59,548,664 字节**，诊断图为 **81,372,152 字节**。数据在各 case 的 `ir.all.normal.memory_stats.peak_numpy_storage_bytes` 和 `ir.all.diagnostic.memory_stats.peak_numpy_storage_bytes`。统计包括解释器持有的底层数组和返回值拷贝，不包括 NumPy 内核临时缓冲、Python 对象及分配器缓存，不能直接与进程 RSS 等同。 - -完整模型三份外部数据文件合计 **2,384,201,728 字节**；FP32 logits `[1,256,151936]` 单份为 **155,582,464 字节**。预检能计算 3008 个有静态尺寸记录的值，另有 5153 个图中值缺少可用的持久化静态尺寸。报告中的逻辑字节和可能重复计算别名，也未覆盖全部中间值；它不是完整 IR 的实际峰值内存或机器内存需求。上述 139.71 MiB 仅为读取元数据、校验文件的进程峰值。 - -## 回归与代码复核 - -最终相关测试为 **403 passed，0 failed，0 skipped**，耗时 25.56 秒；范围是新增 W3 测试、IR 解释器及受影响的 W2 small/runtime-model 测试，不代表整仓库测试全部运行。证据位于 `output/w3-regression-final-02.xml` 和同名 `.log`。 - -另外实跑了已有两层 small 的兼容性检查：7 组输入、29 个检查点的 host 检查通过；普通/诊断图 × none/all × 7 输入，共 **28/28 次 QEMU 通过**，最大误差约 **1.66893005e-6**。其 QEMU 累计时长为 130.554 秒，整体流水线耗时 141.150 秒。报告分别位于 `output/w3-compat-small-ir/report.json` 和 `output/w3-compat-small-qemu/report.json`。这些执行结果独立于 403 项单元/回归测试计数。 - -本轮复核发现并修复了以下问题，最终五项复跑在修复后完成： +| 六层 medium | 7 输入、81 检查点、18 不变量；普通/诊断图；IR none/basic/all | 全部位置含 padding，`max_abs < 1e-5`、rtol=0 | PASS；全部比较最大 `2.771615982055664e-6` | +| 真实维度子图 | 第一层真实权重的 14 个 L256 子图、10 不变量；Torch/ORT/IR | `max_abs < 1e-4`、rtol=0 | PASS；全部比较最大 `3.0517578125e-5` | +| 组合 Attention 后端 | 6 图 × none/all；12 次真实 QEMU | `max_abs < 1e-4`、rtol=0 | PASS；QEMU/ORT 最大 `3.5762786865234375e-7` | +| 完整资产预检 | 固定哈希与 7847 节点 ONNX 契约 | manifest 与图契约一致 | PASS；不执行完整模型数值 | +| layer-diff | medium short_17，IR all 与 ORT 诊断数组，81/81 点 | `max_abs < 1e-5`、rtol=0 | PASS；最大 `2.205371856689453e-6` | -- Attention 参考实现曾复用被测图的 mask，可能让错误掩码同时污染双方结果；改为独立枚举可见 key，并加入故意删掉合法注意力边的反例。 -- medium 和统一入口补齐精确 case、后端、检查点、不变量矩阵及产物哈希校验，防止缺测或空集合产生 PASS。 -- 报告写入失败时清理已有 PASS 视图并保留 FAIL；源码采集异常也进入失败报告。 -- QEMU 超时或执行失败保留实际时长、命令和日志；用户中断统一入口时清理子进程树，退出 130 并记录 FAIL。 -- 逐层工具拒绝不完整/歧义 NPZ、无效形状、非有限数值和超额数据；选择子集始终标记 PARTIAL。 -- CI 增加实际/参考张量、schema、QEMU 原始输出与失败数组留存，避免只有汇总报告而无法复核。 -- 完整预检明确列出缺失静态尺寸的值，未知 rank 不再被当作标量计数。 +medium 是六层随机小配置：hidden64、FFN192、Q4/KV2、head_dim16、vocab128、L256，共 304128 个随机参数；其宽度是待团队确认的候选。真实子图采用固定 checkpoint 第一层的预训练权重、受控合成激活，验证真实尺寸的局部计算，不等于完整预训练 decoder 层执行。W3 的阈值不修改 W2 的 `<1e-5` 门槛。 -最终执行前后及五个子入口的 147 项生产源码、依赖文件和资产 manifest 指纹一致；文档、测试和 workflow 不在这组生产源码指纹中。改动及证据目前未发现阻塞上述预备范围的剩余问题，完整尺寸执行仍需另行验证。 +## 耗时、回归与证据 -## 独立复现和剩余工作 +- preparation 总墙钟时长 `132.923089 s`；medium `97.368098 s`、真实子图 `5.775279 s`、Attention 子入口 `22.837814 s`、预检 `2.164485 s`。 +- 12 次 QEMU 进程墙钟合计 `0.913058207 s`，包含启动、guest 计算、UART 和退出，不能当作目标硬件纯推理性能。 +- 同次 preparation 回归：1297 passed、1 项非 Linux 平台进程所有权兼容测试 skipped；不得表述为零跳过。 +- 诊断收集清单中的 63 个证据文件共 `21,516,508` bytes，下载后大小与 SHA256 全部匹配、无遗漏；选定 81 点数组离线重算通过。离线重算没有执行模型，不是第二人复现。 -复现步骤、工具路径和输出说明见 [W3 预备工作入口](README.md)。本次环境为 Windows、Python 3.12.13、Torch 2.7.1+cpu、Transformers 4.51.3、NumPy 2.2.6、ONNX 1.18.0、ORT 1.22.1；固定 CPU 依赖随项目维护。所有复跑使用新的输出目录，已有历史运行结果不覆盖。 +原始报告分别保存进程全生命周期 RSS、解释器保留张量逻辑字节、去重底层存储字节和采样值；不同口径不可混用。只读权重映射和最后使用后释放服务于完整模型容量，不构成后续性能优化验收。 -手动 CI 配置已完成本地 YAML 与 Bash 语法检查，但尚未提交和在 GitHub 上执行，也未启用 Nightly。因此目前结论是本地预备验收通过。 +## 结论与历史记录 -| 下一步 | 当前状态与原因 | -|---|---| -| W1 人工验收及 W3 契约确认 | 仍由团队推进；需确认 medium 候选配置和输入/比较范围 | -| 完整 28 层 ORT 与 IR 执行入口 | 尚未实现本轮完整数值执行；应先分进程测量真实内存，再决定权重复用及中间值回收方案 | -| 完整模型误差定位 | 必须比较含 padding 的完整输出，保持 `<1e-4`；此前完整导出 PyTorch/ORT padding 差异也需专门定位 | -| E1/E2/E5 独立复现和 Nightly | 本地重复运行不能代替其他人的独立复现，也不能代替远端 CI | +本次 preparation 报告是 `full_ir_executed=false`、`w3_exit_accepted=false`。完整七组是同提交的另一项执行，见 [完整模型 Linux 记录](W3-完整模型执行与数值诊断报告.md)。团队契约确认、E1/E2/E5 各自 Linux 执行和实际定时 Nightly 仍待完成。 -完成这些项目后才能讨论 W3 收尾。完整模型的权重独立装载及 QEMU 前向属于后续 W4 工作，生成循环属于 W5。 +早期个人开发机报告保留在 [固定提交历史原文](https://github.com/ScratchV-Compiler/ScratchV/blob/77c3aa86e9794e5bfe3c81fd873a1506d14bc753/docs/llm-deploy-v1.0/W3/W3-预备工作本地验收报告.md)。其中路径、环境、失败与耗时保持历史含义，不用于 Linux 验收，也没有把原报告改写成 Linux 执行结果。 diff --git "a/docs/llm-deploy-v1.0/W3/\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\351\252\214\346\224\266\346\270\205\345\215\225.md" "b/docs/llm-deploy-v1.0/W3/\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\351\252\214\346\224\266\346\270\205\345\215\225.md" index 80652ce..bdb1091 100644 --- "a/docs/llm-deploy-v1.0/W3/\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\351\252\214\346\224\266\346\270\205\345\215\225.md" +++ "b/docs/llm-deploy-v1.0/W3/\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\351\252\214\346\224\266\346\270\205\345\215\225.md" @@ -1,5 +1,7 @@ # W3 独立复现与验收清单 +对外复现统一使用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先按第 2.1 节固定源码，再完成 [Linux 统一环境与资产准备](../LINUX_REPRODUCTION.md)，并在同一个 Bash 会话、仓库根目录执行探测命令；`SCRATCHV_PYTHON`、`SCRATCHV_CC`、`SCRATCHV_QEMU` 由该指南设置。 + 本文件把 W3 的本地交付、待确认口径和团队验收记录放在一起。它是复现说明和待填写模板，不是验收签字。开发计划要求完整 Qwen3 IR 数值硬门槛通过、E2/E5/E1 三人独立复现通过、Nightly 绿；本地通过、读取作者的报告或校验作者的数组，都不能替代三人的独立执行。 W3 已提交至 [PR #96](https://github.com/ScratchV-Compiler/ScratchV/pull/96)，本地工作树为 `codex/w3-preparation`，原基线为 `65616a8cde6a581661ac3734378da78a9fcbe5ed`。提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在同提交通过 preparation、完整七组 full 和汇总；preparation 下载已复核，完整 raw 已下载验真、audit 通过。历史 v2/v3 和前三轮失败报告原样保留。定时 schedule 未启用，E1/E2/E5 各自执行仍待登记。详见 [Linux 实测及产物核对](Linux-CI与Nightly.md)。PR 继续更新时，每位复现者必须固定并记录自己的实际 SHA；旧基线不能单独标识当前 W3 代码。 @@ -31,30 +33,40 @@ v4 的完整 profile 还记录 MatMul 的 K 分块、行分块、尾行/单列 ### 2.1 固定 PR 提交；源码快照作为补充 -接收者可在新目录获取 PR #96 的 head，避免将其他分支或既有工作区修改混入复现： +先在 Linux 主机准备 Git 和 Git LFS（Ubuntu 可执行 `sudo apt-get update` 和 `sudo apt-get install -y git git-lfs`）。接收者在新目录获取 PR #96 的 head，避免将其他分支或既有工作区修改混入复现；签出后显式拉取 LFS 实体文件： ```bash +set -euo pipefail git clone --no-checkout https://github.com/ScratchV-Compiler/ScratchV.git ScratchV-w3-repro git -C ScratchV-w3-repro fetch origin refs/pull/96/head git -C ScratchV-w3-repro switch --detach FETCH_HEAD +git -C ScratchV-w3-repro lfs install --local +git -C ScratchV-w3-repro lfs pull git -C ScratchV-w3-repro rev-parse HEAD git -C ScratchV-w3-repro status --short +cd ScratchV-w3-repro ``` -上述命令仅用于新建的 `ScratchV-w3-repro` 目录；不要对包含自己未提交工作的目录执行切换。记录 `rev-parse HEAD` 返回的完整 SHA，之后进入该目录运行测试。PR 后续更新不改变这份已固定的 checkout；若要复现更新，应另存新记录，不能继续引用旧 SHA 的通过结论。本文登记的 `2e5f9f4` 是一次具体运行的提交，不是所有后续修改的验收标签。 +上述命令仅用于新建的 `ScratchV-w3-repro` 目录；不要对包含自己未提交工作的目录执行切换。记录 `rev-parse HEAD` 返回的完整 SHA，之后进入该目录运行测试。PR 后续更新不改变这份已固定的 checkout；若要复现更新，应另存新记录，不能继续引用旧 SHA 的通过结论。本文登记的 `1c49e8a` 是一次具体运行的提交，不是所有后续修改的验收标签。 如需交付未提交修改或离线源码，可从所需 W3 checkout 根目录生成本地源码包： -```powershell -python -B -X utf8 scripts/package_w3_repro.py create --output output/w3-local-handoff/w3-source.zip +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 scripts/package_w3_repro.py create --output output/w3-local-handoff/w3-source.zip ``` 包内清单记录 `snapshot_id` 及文件 SHA256，旁边的 `.sha256` 校验压缩包本身。包包含本轮源码、测试、工作流、依赖清单、文档和许可证，不包含 Git 历史、模型权重或运行产物。最终身份以实际生成的清单为准，不能只记录分支名或基线提交号。 +当前源码包清单为 schema 2，声明 Linux/Bash 复现契约，并要求包含 Linux 环境指南和 W3 入口。 +该声明是交付目标，不是模型已执行的证明。历史 schema 1 包应使用随其原版本提供的验证器； +不能修改旧清单字段来冒充新包。打包保留源码原始字节，接收者安装自己的 Linux 环境。 + 接收者解压到新目录后，使用发件人提供的清单 ID 核对： -```powershell -python -B -X utf8 scripts/package_w3_repro.py verify --root D:/path/to/extracted-w3 --expected-snapshot-id '填写实际snapshot_id' +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 scripts/package_w3_repro.py verify --root /absolute/path/to/extracted-w3 --expected-snapshot-id '填写实际snapshot_id' ``` 请把示例中的路径和 ID 换成实际值。执行工具前先核对收到的压缩包 SHA256；该校验确定收到的文件一致性，不提供发送者身份认证。交付文件变化后需要新快照；数值执行代码变化后需要重跑受影响的验收并生成新记录。历史报告保存的是其生成时的 `source_sha256`，不应人为改成当前快照的哈希；工具或文档的后续变化需要单独说明。 @@ -69,7 +81,7 @@ python -B -X utf8 scripts/package_w3_repro.py verify --root D:/path/to/extracted 报告的 `environment.numeric_runtime` 记录 CPU 架构、处理器、线程环境及 NumPy 构建/运行诊断。还应保留 `blas_environment.OPENBLAS_CORETYPE` 原值（未设置为 `null`）以及 `numpy_openblas.status`、`libraries`；可用库项中的 `path/status/core_name/config` 和不可用时的 `reason/error` 都按原样保存。环境变量设置不证明实际生效，CPU SIMD 能力也不等于 BLAS 正在使用的内核。此诊断只覆盖已装载的 NumPy 私有 OpenBLAS 库，不证明 ORT 内核。 -可选诊断不可用或旧证据缺少该字段时保留缺失说明，不能推断未记录的内核，更不能掩盖任何数值失败。跨操作系统结果仍需实际运行验证。 +可选诊断不可用或旧证据缺少该字段时保留缺失说明，不能推断未记录的内核，更不能掩盖任何数值失败。不同 Linux CPU、计算库或代码版本仍需实际运行验证。 ### 2.3 固定资产 @@ -82,36 +94,60 @@ python -B -X utf8 scripts/package_w3_repro.py verify --root D:/path/to/extracted | 外置权重 | `weights-00001.data`、`weights-00002.data`、`weights-00003.data`，合计 2384201728 bytes；逐文件尺寸和哈希以 manifest 为准 | | 配置及 tokenizer 来源 | 使用 manifest 中固定 `source_files`，保存入口实际核验结果 | -当前本机源 checkpoint 目录为 `D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca`，ONNX 目录为 `D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx`。这些是作者机器路径，复现者必须填写自己的资产路径。运行入口只校验和读取已有资产，不负责安装依赖或下载模型。 +先按 [Linux 统一指南](../LINUX_REPRODUCTION.md) 完成 Python 依赖、工具链、完整 ONNX 和 tokenizer 准备。W3 的真实权重子图还需要原始 HF checkpoint；它不是 ONNX 外置权重文件，必须另行准备。 + +以下命令**仅在含 W3 实现的固定 checkout 中**执行，因为它调用 `probes.w3_qwen3_subgraphs.assets.verify_source`。下载步骤与 [.github/workflows/w3-preparation.yml](../../../.github/workflows/w3-preparation.yml) 相同：从仓库 manifest 读取模型 ID、固定 revision 和允许文件列表，下载到 `output/assets/source`，随即流式核验 checkpoint、全部 source 文件 SHA256 及配置。此准备步骤需要联网，后续探测入口只读、离线校验已有资产。 + +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 - <<'PY' +from pathlib import Path +from huggingface_hub import snapshot_download +from probes.w1_qwen3_export.run import MANIFEST +from probes.w3_qwen3_subgraphs.assets import verify_source + +source = Path("output/assets/source") +snapshot_download( + repo_id=MANIFEST["model_id"], + revision=MANIFEST["revision"], + local_dir=source, + allow_patterns=["model.safetensors"] + [item["name"] for item in MANIFEST["source_files"]], +) +verified = verify_source(source) +print(f"Verified {verified['model_id']} @ {verified['revision']}: {source.resolve()}") +PY +W3_SOURCE="$PWD/output/assets/source" +``` + +下载或校验失败时先保留错误并解决资产问题，不得绕过 SHA256 检查。已有另一份固定 checkpoint 可跳过下载，将 `W3_SOURCE` 设为其 Linux 绝对路径；preparation 仍会执行相同的离线验证。`W3_MODEL` 指向统一指南准备的完整 ONNX 目录，两个目录用途不同。 ## 3. 独立执行步骤 在已核对的快照或 checkout 根目录运行。每次重试使用新的输出目录；保留失败目录，不能将以前的成功产物覆盖到新结果中。 -```powershell -$w3Python = 'D:/path/to/python.exe' -$w3Source = 'D:/path/to/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca' -$w3Model = 'D:/path/to/qwen3-0.6b-onnx' -$w3Cc = 'D:/path/to/zig.exe' -$w3Qemu = 'D:/path/to/qemu-system-riscv64.exe' -$env:OMP_NUM_THREADS = '1' -$env:OPENBLAS_NUM_THREADS = '1' -$env:MKL_NUM_THREADS = '1' -$env:PYTHONIOENCODING = 'utf-8' -$env:SCRATCHV_FP32_REFERENCE_CPU = 'auto' - -& $w3Python -B -X utf8 scripts/run_w3_preparation.py ` - --source-dir $w3Source --model-dir $w3Model ` - --output-dir output/w3-repro-preparation-01 --cc $w3Cc --qemu $w3Qemu -if ($LASTEXITCODE -ne 0) { throw 'W3 preparation 未通过；先保留并检查报告。' } - -& $w3Python -B -X utf8 probes/w3_qwen3_full/run.py ` - --model-dir $w3Model --output-dir output/w3-repro-full-01 ` +```bash +set -euo pipefail +W3_SOURCE="${W3_SOURCE:-$PWD/output/assets/source}" +W3_MODEL=/absolute/path/to/qwen3-0.6b-onnx +export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 +export PYTHONIOENCODING=utf-8 SCRATCHV_FP32_REFERENCE_CPU=auto + +if [[ -e .git ]]; then + git rev-parse HEAD + git status --short +else + "$SCRATCHV_PYTHON" -B -X utf8 scripts/package_w3_repro.py verify --root "$PWD" +fi +"$SCRATCHV_PYTHON" -B -X utf8 scripts/run_w3_preparation.py \ + --source-dir "$W3_SOURCE" --model-dir "$W3_MODEL" \ + --output-dir output/w3-repro-preparation-01 \ + --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" +"$SCRATCHV_PYTHON" -B -X utf8 probes/w3_qwen3_full/run.py \ + --model-dir "$W3_MODEL" --output-dir output/w3-repro-full-01 \ --fp32-mode reference --worker-timeout 1800 --max-worker-memory-gib 10 -if ($LASTEXITCODE -ne 0) { throw 'W3 完整数值未通过；先保留并检查报告。' } ``` -Linux 采用相同 Python 参数，将变量和续行写成 Bash 语法；参照 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。两套工作流支持手动运行，Nightly 编排准备项与完整七组。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在同提交通过两套 gate 和汇总，完整 raw audit 已通过；应按每次实际 SHA、profile 和各套结果分别登记，不能拼接不同提交的通过项。配置存在、手动调用成功和定时 schedule 成功是三种不同状态；作者运行不能替代复现者在自己环境执行。 +上面的 Bash 命令使用 `set -euo pipefail`，任何入口返回非零都会终止后续步骤，失败目录须保留。远端执行参照 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。两套工作流支持手动运行，Nightly 编排准备项与完整七组。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在同提交通过两套 gate 和汇总，完整 raw audit 已通过；应按每次实际 SHA、profile 和各套结果分别登记，不能拼接不同提交的通过项。配置存在、手动调用成功和定时 schedule 成功是三种不同状态；作者运行不能替代复现者在自己环境执行。 `--max-worker-memory-gib 10` 是 worker 进程树的采样终止阈值，不是总机器内存预算或操作系统硬配额。超时和内存不足属于运行失败；可以在记录机器资源后调整资源预算，但不能用改模型、缩短覆盖、放宽数值误差来取得全量 PASS。解释器张量存储统计、进程全生命周期 RSS 和父进程采样值口径不同，原始字段应分别保留。 @@ -144,17 +180,19 @@ Linux 采用相同 Python 参数，将变量和续行写成 Bash 语法；参照 失败先保存日志、输入、两后端报告、完整 logits、检查点和 schema。检查 `cases[].comparison.logits` 与 `first_divergence`，再按 [LOCALIZE.md](../../../probes/w3_qwen3_full/LOCALIZE.md) 定位。不得删除不通过的 padding 位置、换用不同参考、将局部成功写成全量成功。 -## 4. 历史本地证据与接收时的核对 +## 4. 固定 Linux 证据与接收时的核对 -以下是作者 Windows 环境的现有记录，保留其原始源码指纹，不作为接收者的执行记录。目录均相对 W3 工作树根目录。 +作者已在 Ubuntu 24.04.5、Python 3.12.14、AMD EPYC 7763 / OpenBLAS Haswell 环境运行以下同提交证据，代码 SHA 为 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a`，来源为 [run 37299235103](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)。这些是已有运行记录，本次文档迁移没有重新执行，也不能代替接收者执行。 -| 记录 | 路径 | 已保存的结果 | -|---|---|---| -| 五项 preparation | `output/w3-preparation-review-v2-01/report.json` | PASS；medium 7 输入、subgraphs 14 图、Attention 12 次 QEMU、资产预检、layer-diff 81 检查点 | -| 完整模型 | `output/w3-full-review-v2-01/report.json` | PASS；7 输入、每组 30 检查点、4 项不变量；普通/诊断 logits 及检查点最大绝对误差均为 0；总耗时约 664.26 秒 | -| 扩展回归 | `output/w3-review-regression-final.xml` | 1200 passed、1 skipped；跳过项为 Windows 无权限创建符号链接的收集器测试 | +| 记录 | 已保存的结果 | +|---|---| +| 五项 preparation | 5/5 PASS；132.923089 秒；相关回归 1297 passed、1 项非 Linux 平台兼容性测试 skipped | +| 完整模型 | 7/7 输入、30 检查点/组、4 不变量；v4/avx2-fma3、IR none；普通/诊断完整 logits 最大绝对误差 0；1477.314971525 秒 | +| 完整原始数组离线复核 | 7 组、4 不变量通过；未重新执行模型，`independent_reproduction=false` | + +完整报告 SHA256 为 `0ddb97d13b6bab44442fb0b453d1e0132ef7a02c96be9c8caf70ae2862e31746`。raw artifact ID 为 `11341924764`；其保存期限有限，应使用自己已留存的对应产物或重新运行。详细 artifact 身份与来源见 [Linux CI 与 Nightly](Linux-CI与Nightly.md)。不同环境只要求满足固定误差门槛，不要求重复零误差。 -完整模型报告 SHA256 为 `822479ef25749e547692b0367b1797cf25b499771b1869926c08d86be7e8b0be`。本机误差为 0 是该次环境的实测值，不是跨平台预期值，也不是所有未来改动的验收承诺。后续运行应保留新目录和新报告，不覆写本表所指的历史记录。 +早期个人开发机证据只在 [固定提交历史原文](https://github.com/ScratchV-Compiler/ScratchV/blob/77c3aa86e9794e5bfe3c81fd873a1506d14bc753/docs/llm-deploy-v1.0/W3/独立复现与验收清单.md) 留存，不计入 Linux 验收。 接收证据时需同时保留并核对： @@ -166,27 +204,28 @@ Linux 采用相同 Python 参数，将变量和续行写成 Bash 语法；参照 摘要、截图和 SHA256 能辅助交接，但不能代替原始数组；离线复核原始数组也不等于在另一台机器上执行模型。CI 报告和 raw 产物保留期分别管理，接收者应在过期前保留完整证据。CI raw 解压后模型证据位于 `下载目录/w3-full`；完整 `gh run download`、报告 SHA256 获取和 audit 参数模板见 [Linux CI 的产物与失败定位](Linux-CI与Nightly.md#产物与失败定位)。精度问题的排查过程不在本清单中记录。 -拿到完整原始目录后，可使用对应版本的离线工具重新读取数组和重算比较，无需下载模型或执行 ORT/IR。下面的目录和 hash 属于历史 v2 报告，必须在该报告对应的历史源码中执行；当前 v4 工具不将 v2/v3 profile 自动转换成新契约： +拿到完整原始目录后，可使用与报告契约兼容的离线工具重新读取数组和重算比较。以下命令对应上述 Linux v4 报告；证据目录必须包含已解压的 `w3-full/` 内容，不能只有摘要报告： -```powershell -python -B -X utf8 scripts/verify_w3_evidence.py ` - --evidence-dir output/w3-full-review-v2-01 ` - --output-dir output/w3-saved-evidence-audit-01 ` - --expected-report-sha256 822479ef25749e547692b0367b1797cf25b499771b1869926c08d86be7e8b0be +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 scripts/verify_w3_evidence.py \ + --evidence-dir output/downloaded-w3-37299235103/raw/w3-full \ + --output-dir output/w3-saved-evidence-audit-01 \ + --expected-report-sha256 0ddb97d13b6bab44442fb0b453d1e0132ef7a02c96be9c8caf70ae2862e31746 ``` -上述 hash 只适用于本表指定的作者报告。复核本人新报告时，换成本人另行保存的报告 SHA256。v4 离线工具依据保存的 `cpu_strategy` 校验整个规范 profile（包括 MatMul 布局字段），不根据审计机器 CPU 或当前环境变量猜测生产策略；跨机器审计不执行 ORT/IR。`audit:w3-full-saved-evidence` PASS 只表示保存证据完整、自洽并满足记录的数值契约；源码差异在 `source_comparison` 中单独列出，不能据此声称当前源码刚刚通过全量运行。详见 [离线证据复核说明](../../../probes/w3_qwen3_full/EVIDENCE.md)。 +上述 hash 只适用于指定的作者 Linux 报告。复核本人新报告时，换成本人另行保存的报告 SHA256。v4 离线工具依据保存的 `cpu_strategy` 校验整个规范 profile（包括 MatMul 布局字段），不根据审计机器 CPU 或当前环境变量猜测生产策略；跨机器审计不执行 ORT/IR。`audit:w3-full-saved-evidence` PASS 只表示保存证据完整、自洽并满足记录的数值契约；源码差异在 `source_comparison` 中单独列出，不能据此声称当前源码刚刚通过全量运行。详见 [离线证据复核说明](../../../probes/w3_qwen3_full/EVIDENCE.md)。 ## 5. E1 / E2 / E5 独立复现记录模板 -每位复现者各复制一份下表，在自己的环境实际执行后填写。三人均应执行完整硬门槛；角色分工只表示额外重点，不允许只复核他人报告代替运行。 +每位复现者各复制一份下表，在自己的 Linux 环境实际执行后填写。三人均应执行完整硬门槛；角色分工只表示额外重点，不允许只复核他人报告代替运行。 | 字段 | 复现者填写 | |---|---| | 姓名 / 角色 / 日期 | 待填写 | | 代码 snapshot ID / 压缩包 SHA256 / 实际提交 SHA（如有） | 待填写；未提交快照不可仅填基线提交号 | | 源码校验结果 / 本机解压或 checkout 绝对路径 | 待填写 | -| OS / CPU / RAM / 可用磁盘 / Python 绝对路径 | 待填写 | +| Linux 发行版 / 内核 / CPU / RAM / 可用磁盘 / Python 绝对路径 | 待填写；Ubuntu 24.04 x86_64、Python 3.12 | | Python / pip freeze / NumPy-CPU-BLAS 诊断原始文件路径 | 待填写；包括 `OPENBLAS_CORETYPE` 原值、`numpy_openblas.status/libraries` 及每库 `core_name/config` 或不可用原因 | | checkpoint 绝对路径 / revision / 实测 SHA256 | 待填写 | | ONNX 绝对路径 / manifest 校验结果 | 待填写 | diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index 57536a7..1e4d82a 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -17,7 +17,7 @@ ### 1.1 背景 -ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译器，核心管线为 ONNX 解析 → IR 构建 → 优化 Pass → RISC-V 代码生成。项目最初主要验证 CNN 类静态计算图；当前已沿用共享 IR 和 CompilerDriver，补入 NumPy IR 解释器、张量 C 后端和 RV64 裸机运行器，跑通真实 Qwen3 结构的两层缩小模型。完整 Qwen3-0.6B 已在作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)的提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 下通过全部七组 IR/ORT 数值，preparation 和同提交汇总也通过；完整 raw 数组下载验真及 audit 通过，团队独立复现及正式验收尚待完成。历史 Windows v2 与 Linux v3 证据分别保留，详情见 [W3 工作总结](W3/W3-工作总结与验收进度.md)。大权重装载、目标容量和完整 QEMU 前向按 W4 推进。 +ScratchV 是一个面向计算机体系结构与编译原理学习的 AI 编译器，核心管线为 ONNX 解析 → IR 构建 → 优化 Pass → RISC-V 代码生成。项目最初主要验证 CNN 类静态计算图；当前已沿用共享 IR 和 CompilerDriver，补入 NumPy IR 解释器、张量 C 后端和 RV64 裸机运行器，跑通真实 Qwen3 结构的两层缩小模型。完整 Qwen3-0.6B 已在作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)的提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 下通过全部七组 IR/ORT 数值，preparation 和同提交汇总也通过；完整 raw 数组下载验真及 audit 通过，团队独立复现及正式验收尚待完成。早期 v2 记录保留在固定历史提交，Linux v3 证据按原身份保留，详情见 [W3 工作总结](W3/W3-工作总结与验收进度.md)。大权重装载、目标容量和完整 QEMU 前向按 W4 推进。 ### 1.2 目标 @@ -204,7 +204,7 @@ Host 端循环： > ⚠️ 若 W3 硬门槛不通过，W4 暂停后端集成，E2+E5 主导修复，E1/E3/E4 支援。 -**W3 实施补记（2026-10-05）**：提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)中，preparation 五项、v4/avx2-fma3 完整七组和汇总均通过。普通/诊断完整 logits、每组 30 检查点及四项不变量最大误差均为 0，完整 runner 用时约 24.6 分钟；同提交通用 CI 为 3821 passed、5 skipped，W1/W2 和 Topic06 通过。preparation 下载已复核，完整 raw 已下载验真、audit 通过。此前 Windows v2、Linux v3 以及前三轮失败记录保留，不能跨提交替代本轮证据。定时 schedule 未启用，团队口径确认及 E1/E2/E5 独立复现仍待完成。作者远端执行、数组离线复核与团队正式验收分别记录，详见 [W3 工作总结与验收进度](W3/W3-工作总结与验收进度.md)。 +**W3 实施补记（2026-10-05）**：提交 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 的[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)中，preparation 五项、v4/avx2-fma3 完整七组和汇总均通过。普通/诊断完整 logits、每组 30 检查点及四项不变量最大误差均为 0，完整 runner 用时约 24.6 分钟；同提交通用 CI 为 3821 passed、5 skipped，W1/W2 和 Topic06 通过。preparation 下载已复核，完整 raw 已下载验真、audit 通过。早期个人环境记录和 Linux 前三轮失败记录按原身份保留，不能跨提交替代本轮证据。定时 schedule 未启用，团队口径确认及 E1/E2/E5 独立复现仍待完成。作者远端执行、数组离线复核与团队正式验收分别记录，详见 [W3 工作总结与验收进度](W3/W3-工作总结与验收进度.md)。 当前已提交的完整模型接口使用 `numpy-fp32-reference-v4`，通过 `SCRATCHV_FP32_REFERENCE_CPU=auto|avx2-fma3|avx512` 选择 IR 参考算术策略，默认 `auto`；该设置不控制 ORT 的 CPU dispatch。报告必须记录实际 `cpu_strategy`，以及 `environment.numeric_runtime` 中 `blas_environment.OPENBLAS_CORETYPE` 和 `numpy_openblas` 实际库诊断；CPU SIMD 与 BLAS 内核分别记录，可选诊断不可用不豁免数值门槛。完整七组按原门槛重新验收。历史 v2/v3 报告与其源码版本保持对应，不自动升级为 v4 的通过证据。 diff --git a/probes/w3_attention/README.md b/probes/w3_attention/README.md index 9a0a11b..06c12ae 100644 --- a/probes/w3_attention/README.md +++ b/probes/w3_attention/README.md @@ -1,9 +1,12 @@ # W3 组合 Attention 后端 +复现统一采用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 环境与资产准备](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)，再在同一个 Bash 会话、仓库根目录执行命令；该指南设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC` 和 `SCRATCHV_QEMU`。 + 入口 run.py 将 Q/K RMSNorm、全 head_dim RoPE、GQA 连续 KV 重复、因果/key-padding mask、Softmax 和 context MatMul 连成一个 ONNX 图。4 个 Q head、2 个 KV head、head_dim8，包含 L17 和 L256 边界；它是小尺寸组合验证，不是完整 Qwen3 前向。 -~~~text -python -B -X utf8 probes/w3_attention/run.py --output-dir output/w3-attention-new --cc /path/to/zig --qemu /path/to/qemu-system-riscv64 +~~~bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B -X utf8 probes/w3_attention/run.py --output-dir output/w3-attention-new --cc "$SCRATCHV_CC" --qemu "$SCRATCHV_QEMU" ~~~ 6 图分别执行 none/all，要求 12 次真实 RV64 QEMU、每次 IR/QEMU 同时对照 ORT 和独立 FP64 NumPy 公式，全部 max_abs <1e-4、rtol=0。未来 token 和 padding key/value 扰动产生 10 项跨 ORT/IR/QEMU 的隔离检查。NumPy 按 query 的可见 key 范围独立计算，不复用导出 mask；单测注入“额外屏蔽合法 key”确认参考能捕获错误。 diff --git a/probes/w3_full_preflight/README.md b/probes/w3_full_preflight/README.md index 657d98e..02661f8 100644 --- a/probes/w3_full_preflight/README.md +++ b/probes/w3_full_preflight/README.md @@ -1,7 +1,11 @@ # W3 完整资产与静态尺寸预检 -~~~text -python -B -X utf8 probes/w3_full_preflight/run.py --model-dir /path/to/qwen3-0.6b-onnx --output-dir output/w3-full-preflight-new +复现统一采用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 环境与资产准备](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)，再在同一个 Bash 会话、仓库根目录执行命令；该指南设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC` 和 `SCRATCHV_QEMU`。 + +~~~bash +set -euo pipefail +W3_MODEL=/absolute/path/to/qwen3-0.6b-onnx +"$SCRATCHV_PYTHON" -B -X utf8 probes/w3_full_preflight/run.py --model-dir "$W3_MODEL" --output-dir output/w3-full-preflight-new ~~~ 离线流式校验 W1 固定 manifest 的模型和三个 external-data 文件，复用 ONNX 静态契约/范围检查，统计 initializer、嵌入/外部 tensor、具名 value 与输出的逻辑字节。不加载外部数组进行 IR 运算，也不执行完整 ORT。 diff --git a/probes/w3_layer_diff/README.md b/probes/w3_layer_diff/README.md index 20c244c..94c18dd 100644 --- a/probes/w3_layer_diff/README.md +++ b/probes/w3_layer_diff/README.md @@ -1,11 +1,14 @@ # W3 offline checkpoint comparison +Reproduction targets **Ubuntu 24.04 x86_64, Bash and Python 3.12**. Complete the [Linux environment and asset setup](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) first. Run commands from the repository root in the same Bash session; the setup exports `SCRATCHV_PYTHON`, `SCRATCHV_CC` and `SCRATCHV_QEMU`. + This tool compares recorded NPZ tensors without executing a model. It reuses `probes.w2_qwen3_small.diagnostics.compare_outputs`: exact shape and dtype, finite numeric values, and strict `max_abs < atol` (default `1e-5`). -```powershell -python probes/w3_layer_diff/run.py --actual output/w3/medium/traces/full_seed_0/ir_diagnostic_none.npz --reference output/w3/medium/traces/full_seed_0/reference.npz --schema output/w3/medium/trace_schema.json --out output/w3/layerdiff-full +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" probes/w3_layer_diff/run.py --actual output/w3/medium/traces/full_seed_0/ir_diagnostic_none.npz --reference output/w3/medium/traces/full_seed_0/reference.npz --schema output/w3/medium/trace_schema.json --out output/w3/layerdiff-full ``` `--out` must be new. The shared W3 publisher writes `report.json`, `report.md` diff --git a/probes/w3_qwen3_full/EVIDENCE.md b/probes/w3_qwen3_full/EVIDENCE.md index b3ff20f..83e78aa 100644 --- a/probes/w3_qwen3_full/EVIDENCE.md +++ b/probes/w3_qwen3_full/EVIDENCE.md @@ -1,17 +1,20 @@ # 完整 W3 结果的离线复核 +复现统一采用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 环境与资产准备](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)，再在同一个 Bash 会话、仓库根目录执行命令；该指南设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC` 和 `SCRATCHV_QEMU`。 + `scripts/verify_w3_evidence.py` 对**已有七案例原始产物**重新计算误差，不执行模型、不下载资产。 它复用完整模型 runner 的 `validate_worker`、`compare_case`、`invariants`，以及安全 NPZ 读取器。 在仓库根目录，使用已有 Python 3.12 和项目锁定依赖运行： -```sh -python -B scripts/verify_w3_evidence.py \ +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B scripts/verify_w3_evidence.py \ --evidence-dir output/downloaded-w3-run/raw/w3-full \ --output-dir output/w3-evidence-audit-01 ``` -PowerShell 可将命令写成一行。输出目录必须不存在；不会覆盖被审核的原报告。 +输出目录必须不存在；不会覆盖被审核的原报告。 输出为 `report.json`、`report.md`、`report.html`，退出码 `0` 表示复核通过，`1` 表示证据无效或不足。 非法参数、已有输出目录等启动错误同样返回非零。 @@ -43,8 +46,9 @@ v4 复核读取生产报告保存的 `cpu_strategy`，不从审计机器 CPU 或 默认 SHA-256 检查发现文件与报告不一致；若报告和数据同时被替换，它不能充当数字签名。 可从可信渠道预先独立保留原始顶层报告哈希，然后传入： -```sh -python -B scripts/verify_w3_evidence.py --evidence-dir output/downloaded-w3-run/raw/w3-full --output-dir output/w3-evidence-audit-02 --expected-report-sha256 ORIGINAL_REPORT_SHA256 +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -B scripts/verify_w3_evidence.py --evidence-dir output/downloaded-w3-run/raw/w3-full --output-dir output/w3-evidence-audit-02 --expected-report-sha256 ORIGINAL_REPORT_SHA256 ``` `source_comparison` 明确列出原报告中生产源码与当前审计源码的更改、新增、缺失。 diff --git a/probes/w3_qwen3_full/LOCALIZE.md b/probes/w3_qwen3_full/LOCALIZE.md index 713551e..0af0093 100644 --- a/probes/w3_qwen3_full/LOCALIZE.md +++ b/probes/w3_qwen3_full/LOCALIZE.md @@ -1,19 +1,23 @@ # 完整模型的局部数值定位 +复现统一采用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 环境与资产准备](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)，再在同一个 Bash 会话、仓库根目录执行命令；该指南设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC` 和 `SCRATCHV_QEMU`。 + `localize.py` 将同一个完整 ORT 层输入送入原图切片的 ORT 和 ScratchV IR，帮助区分当前层局部误差与上游累计误差。它保留完整 FP32 运算、真实权重和所有 padding query。不会接受 W3 完整数值 gate，也不会修改固定模型或门槛。 先用本目录的 `run.py` 生成完整模型证据，再运行： -```powershell -$env:OMP_NUM_THREADS = '1' -$env:OPENBLAS_NUM_THREADS = '1' -$env:MKL_NUM_THREADS = '1' -$env:PYTHONIOENCODING = 'utf-8' -$env:SCRATCHV_FP32_REFERENCE_CPU = 'auto' -python -B -X utf8 probes/w3_qwen3_full/localize.py ` - --model-dir /path/to/qwen3-0.6b-onnx ` - --case-dir output/full-run/short_17 ` - --output-dir output/full-localize-new ` +```bash +set -euo pipefail +W3_MODEL=/absolute/path/to/qwen3-0.6b-onnx +export OMP_NUM_THREADS='1' +export OPENBLAS_NUM_THREADS='1' +export MKL_NUM_THREADS='1' +export PYTHONIOENCODING='utf-8' +export SCRATCHV_FP32_REFERENCE_CPU='auto' +"$SCRATCHV_PYTHON" -B -X utf8 probes/w3_qwen3_full/localize.py \ + --model-dir "$W3_MODEL" \ + --case-dir output/full-run/short_17 \ + --output-dir output/full-localize-new \ --layer 0 --layer 21 ``` diff --git a/probes/w3_qwen3_full/README.md b/probes/w3_qwen3_full/README.md index 9493902..17ae001 100644 --- a/probes/w3_qwen3_full/README.md +++ b/probes/w3_qwen3_full/README.md @@ -1,5 +1,7 @@ # 完整 Qwen3 IR 数值验证 +复现统一采用 **Ubuntu 24.04 x86_64、Bash、Python 3.12**。先完成 [Linux 环境与资产准备](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md)，再在同一个 Bash 会话、仓库根目录执行命令；该指南设置 `SCRATCHV_PYTHON`、`SCRATCHV_CC` 和 `SCRATCHV_QEMU`。 + 该入口实际执行固定发布的 28 层 Qwen3-0.6B FP32 ONNX，比较 ScratchV IR 与 ORT 的完整 logits，包括右侧 padding query。它使用优化前 IR（none），不会把资产检查、两层模型或部分输入通过视为完整模型数值验收。 ## 验证范围 @@ -39,20 +41,18 @@ v4 把矩阵乘的分块与内存布局纳入版本化 profile；数值策略名 AVX2 矩阵路径按两个输出行计算，尾行和单列临时补零后裁回原输出形状，每个 K 分块的两个矩阵操作数使用 C 连续切片；向量提升继续遵循 NumPy。以上是参考计算契约，不改模型图、权重、输出形状或误差门槛，也不是性能优化验收。本节描述已提交的 v4 实现；第四轮 Linux 已取得同提交七组完整运行 PASS，普通/诊断完整 logits 及四项不变量最大误差均为 0；这是 v4 的新实测证据，历史 v3 或单例结果不能替代它。 -~~~powershell -$w3Python = 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -$env:SCRATCHV_FP32_REFERENCE_CPU = 'auto' -& $w3Python -B -X utf8 probes/w3_qwen3_full/run.py ` - --model-dir D:/cyq/code/ScratchV/models/qwen3-0.6b-onnx ` - --output-dir output/w3-full-new ` - --fp32-mode reference ` - --worker-timeout 1800 --max-worker-memory-gib 10 -$LASTEXITCODE +~~~bash +set -euo pipefail +W3_MODEL=/absolute/path/to/qwen3-0.6b-onnx +export SCRATCHV_FP32_REFERENCE_CPU=auto +"$SCRATCHV_PYTHON" -B -X utf8 probes/w3_qwen3_full/run.py \ + --model-dir "$W3_MODEL" --output-dir output/w3-full-new \ + --fp32-mode reference --worker-timeout 1800 --max-worker-memory-gib 10 ~~~ 需要复现单个失败时追加 `--case short_17`。成功但仅覆盖部分 case 返回 PARTIAL/2，完整七组和不变量全部通过才返回 PASS/0；实际执行失败或数值超标返回 FAIL/1。中断返回 130。输出目录必须是新目录。 -每个 worker 的超时和采样内存上限覆盖本次拥有的整棵进程树，包括 Windows venv Python 启动器创建的实际计算进程。Windows 在恢复启动器执行前将其加入 Job Object，累计该 Job 内的 RSS 与私有提交内存；Linux 使用独立进程组并检查 RSS。采样间隔为 0.2 秒，共享内存页可能按进程重复计数。它是超限终止措施，不是操作系统的硬内存配额，短暂峰值仍可能超过界限。失败保留阶段进度、日志、已完成产物和报告；中断、超时及超限会清理本次拥有的进程树。 +每个 worker 的超时和采样内存上限覆盖本次拥有的整棵进程树。Linux 使用独立进程组，采样整个进程树的 RSS，并在超时、超限或中断时清理本次拥有的进程。采样间隔为 0.2 秒，共享内存页可能按进程重复计数。它是超限终止措施，不是操作系统的硬内存配额，短暂峰值仍可能超过界限。失败保留阶段进度、日志、已完成产物和报告；中断、超时及超限会清理本次拥有的进程树。 ## 证据与时长 @@ -80,4 +80,4 @@ CPU SIMD 能力、IR 的 `cpu_strategy` 与实际 BLAS 内核须分别看待； CI 将报告、哈希和 schema 放入 `w3-full-numeric-reports`（保留 30 天），完整原始输入、logits 和检查点另放入 `w3-full-numeric-raw`（保留 3 天，也包含报告）。需要独立复核原始数值时，应在过期前下载 raw 产物；其解压目录中保留 `w3-full/` 前缀，所以 `--evidence-dir` 应为 `下载目录/w3-full`。下载命令、报告 SHA256 获取与完整 audit 示例见 [产物与失败定位](../../docs/llm-deploy-v1.0/W3/Linux-CI与Nightly.md#产物与失败定位)。仅有摘要和哈希不足以重新比较数组。失败或中断运行的产物可能不完整，须先核对报告状态与覆盖范围。 -即使本地数值 PASS，`w3_exit_accepted` 仍为 false：W1 人工前置验收、团队确认、E1/E2/E5 独立复现及 Nightly 状态由团队单独核对。[历史本地验收报告](../../docs/llm-deploy-v1.0/W3/W3-完整模型执行与数值诊断报告.md) 保留当时的模式和数据，不代表之后版本的最新状态。当前结论以本次完整运行生成的 `report.json`、实际 profile 和源码指纹为准。 +即使本地数值 PASS，`w3_exit_accepted` 仍为 false：W1 人工前置验收、团队确认、E1/E2/E5 独立复现及 Nightly 状态由团队单独核对。[Linux 完整执行记录](../../docs/llm-deploy-v1.0/W3/W3-完整模型执行与数值诊断报告.md) 保留固定提交、模式和数据，不代表之后版本已经重跑。当前结论以本次完整运行生成的 `report.json`、实际 profile 和源码指纹为准。 diff --git a/probes/w3_qwen3_medium/README.md b/probes/w3_qwen3_medium/README.md index 943eeb6..c06072d 100644 --- a/probes/w3_qwen3_medium/README.md +++ b/probes/w3_qwen3_medium/README.md @@ -1,5 +1,7 @@ # W3 medium Qwen3 host numerical probe +Reproduction targets **Ubuntu 24.04 x86_64, Bash and Python 3.12**. Complete the [Linux environment and asset setup](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) first. Run commands from the repository root in the same Bash session; the setup exports `SCRATCHV_PYTHON`, `SCRATCHV_CC` and `SCRATCHV_QEMU`. + This is a fixed candidate for the W3 six-layer experiment. The plan fixes six layers; the dimensions below are the candidate chosen for this experiment, not a claim that the plan mandated a specific hidden or feed-forward width. @@ -27,11 +29,12 @@ there is no KV cache. All numerical backends use one CPU thread. Use Python 3.12 and the existing pinned `requirements/qwen3-small-probe.txt` environment. From the W3 worktree: -```powershell -$env:OMP_NUM_THREADS = '1' -$env:OPENBLAS_NUM_THREADS = '1' -$env:MKL_NUM_THREADS = '1' -D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe -m probes.w3_qwen3_medium.run --output-dir output/w3-medium --model-seed 0 +```bash +set -euo pipefail +export OMP_NUM_THREADS='1' +export OPENBLAS_NUM_THREADS='1' +export MKL_NUM_THREADS='1' +"$SCRATCHV_PYTHON" -m probes.w3_qwen3_medium.run --output-dir output/w3-medium --model-seed 0 ``` The output directory must not already exist, even if it is empty. Preserve a @@ -95,8 +98,9 @@ mark, including export and every backend; it is not an IR memory measurement. ## Regression tests -```powershell -D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe -m pytest tests/test_qwen3_medium_model.py tests/test_qwen3_medium_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_probe.py tests/test_qwen3_small_gate.py -q +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -m pytest tests/test_qwen3_medium_model.py tests/test_qwen3_medium_probe.py tests/test_qwen3_small_model.py tests/test_qwen3_small_probe.py tests/test_qwen3_small_gate.py -q ``` Tests exercise all 81 medium shapes, a second configuration changing every diff --git a/probes/w3_qwen3_subgraphs/README.md b/probes/w3_qwen3_subgraphs/README.md index 0405955..5db8c48 100644 --- a/probes/w3_qwen3_subgraphs/README.md +++ b/probes/w3_qwen3_subgraphs/README.md @@ -1,5 +1,7 @@ # W3 authenticated Qwen3 pretrained subgraphs +Reproduction targets **Ubuntu 24.04 x86_64, Bash and Python 3.12**. Complete the [Linux environment and asset setup](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) first. Run commands from the repository root in the same Bash session; the setup exports `SCRATCHV_PYTHON`, `SCRATCHV_CC` and `SCRATCHV_QEMU`. + This offline gate exercises selected first-layer subgraphs of the pinned `Qwen/Qwen3-0.6B` checkpoint at revision `c1899de289a04d12100db370d81485cdf75e47ca`. It loads individual tensors with @@ -15,11 +17,13 @@ FP32 hash and byte count. No generated or random weights enter this gate. Run from this checkout with the pinned CPU export environment: -```powershell -& 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -X utf8 -B ` - probes/w3_qwen3_subgraphs/run.py ` - --source-dir D:/cyq/code/ScratchV/models/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca ` - --output-dir output/w3-qwen3-subgraphs-l256 ` +```bash +set -euo pipefail +W3_SOURCE=/absolute/path/to/qwen3-source/c1899de289a04d12100db370d81485cdf75e47ca +"$SCRATCHV_PYTHON" -X utf8 -B \ + probes/w3_qwen3_subgraphs/run.py \ + --source-dir "$W3_SOURCE" \ + --output-dir output/w3-qwen3-subgraphs-l256 \ --seq-len 256 ``` @@ -92,8 +96,9 @@ adds output storage. No peak-memory improvement is claimed. Unit checks: -```powershell -& 'D:/cyq/code/ScratchV/.venv-qwen-export/Scripts/python.exe' -m pytest ` +```bash +set -euo pipefail +"$SCRATCHV_PYTHON" -m pytest \ tests/test_w3_qwen3_subgraphs.py -q ``` diff --git a/scripts/package_w3_repro.py b/scripts/package_w3_repro.py index 8d637cb..3573c65 100644 --- a/scripts/package_w3_repro.py +++ b/scripts/package_w3_repro.py @@ -16,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] MANIFEST = "W3_SOURCE_MANIFEST.json" +SCHEMA_VERSION = 2 ROOT_FILES = {"pyproject.toml", "README.md", "LICENSE", "THIRD_PARTY_NOTICES.txt", ".gitignore", ".gitattributes"} TREES = {"scratchv", "scratchv_dag", "benchmarks", "examples", "probes", "scripts", "requirements", @@ -28,7 +29,12 @@ MAX_TOTAL_BYTES = 64 * 1024**2 REQUIRED = {"pyproject.toml", "LICENSE", "scripts/package_w3_repro.py", "probes/w3_qwen3_full/run.py", "requirements/qwen3-small-probe.txt", - "examples/run_ir_interpreter.py"} + "examples/run_ir_interpreter.py", + "docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md", + "docs/llm-deploy-v1.0/W3/README.md"} +REPRODUCTION = {"platform": "linux", "shell": "bash", + "guide": "docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md", + "stage_guide": "docs/llm-deploy-v1.0/W3/README.md"} def digest(data): @@ -110,10 +116,11 @@ def create(root, output): raise ValueError(f"Missing required source: {required}") files = [{"path": name, "bytes": len(value), "sha256": digest(value)} for name, value in payload.items()] - manifest = {"schema_version": 1, "snapshot_id": snapshot_id(files), + manifest = {"schema_version": SCHEMA_VERSION, "snapshot_id": snapshot_id(files), "base_commit": git(root, "rev-parse", "HEAD").decode().strip(), "scope": "Selected current working-tree source bytes; includes uncommitted W3 files.", "excludes": "Git history, model weights/binaries, environments and run evidence.", + "reproduction": dict(REPRODUCTION), "is_commit": False, "numerical_acceptance": False, "files": files} if candidates(root) != list(payload): raise ValueError("Source inventory changed during packaging; retry after edits finish") @@ -139,9 +146,11 @@ def create(root, output): def verify(root, expected_snapshot_id=None): root = Path(root).resolve() manifest = json.loads(read_source(root, MANIFEST)) - if (not isinstance(manifest, dict) or manifest.get("schema_version") != 1 + if (not isinstance(manifest, dict) or manifest.get("schema_version") != SCHEMA_VERSION or not isinstance(manifest.get("files"), list)): raise ValueError("Unsupported source manifest") + if manifest.get("reproduction") != REPRODUCTION: + raise ValueError("Source manifest must declare the Linux reproduction contract") files = manifest["files"] if not files: raise ValueError("Empty source manifest") diff --git a/tests/test_w3_source_package.py b/tests/test_w3_source_package.py index 85f6bb4..8de0bb8 100644 --- a/tests/test_w3_source_package.py +++ b/tests/test_w3_source_package.py @@ -88,7 +88,8 @@ def test_duplicate_manifest_path_rejected(source, tmp_path): def test_empty_manifest_does_not_pass(tmp_path): (tmp_path / pack.MANIFEST).write_text(json.dumps( - {"schema_version": 1, "files": [], "snapshot_id": pack.snapshot_id([])})) + {"schema_version": pack.SCHEMA_VERSION, "reproduction": pack.REPRODUCTION, + "files": [], "snapshot_id": pack.snapshot_id([])})) with pytest.raises(ValueError, match="Empty"): pack.verify(tmp_path) @@ -118,3 +119,23 @@ def changing(root, name): with pytest.raises(ValueError, match="changed during packaging"): pack.create(source, tmp_path / "unstable.zip") assert not (tmp_path / "unstable.zip").exists() + + +def test_linux_handoff_requires_its_environment_guide(source, tmp_path): + (source / pack.REPRODUCTION["guide"]).unlink() + with pytest.raises(ValueError, match="Missing required source"): + pack.create(source, tmp_path / "incomplete.zip") + assert not (tmp_path / "incomplete.zip").exists() + + +def test_reproduction_contract_is_verified_and_does_not_claim_execution(source, tmp_path): + extracted, result = unpack(source, tmp_path) + path = extracted / pack.MANIFEST + manifest = json.loads(path.read_text(encoding="utf-8")) + assert manifest["reproduction"] == pack.REPRODUCTION + assert manifest["numerical_acceptance"] is False + assert pack.verify(extracted, result["snapshot_id"])["passed"] + manifest["reproduction"]["platform"] = "other" + path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="Linux reproduction contract"): + pack.verify(extracted) diff --git a/tests/test_w3_workflows.py b/tests/test_w3_workflows.py index abeeb5b..9d05af3 100644 --- a/tests/test_w3_workflows.py +++ b/tests/test_w3_workflows.py @@ -30,7 +30,7 @@ def step(name, label): @pytest.fixture(scope="module") def bash(): - candidates = [shutil.which("bash"), "C:/Program Files/Git/bin/bash.exe"] + candidates = [os.environ.get("SCRATCHV_BASH"), shutil.which("bash")] for candidate in candidates: if candidate and Path(candidate).is_file(): result = subprocess.run([candidate, "-c", "exit 0"], capture_output=True) From f7ebbfa745ca5be31aea857425b0e97e5ce614c2 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Tue, 6 Oct 2026 17:27:11 +0800 Subject: [PATCH 15/30] docs: align shared setup guide with Linux reproduction --- ...55\345\273\272\346\214\207\345\215\227.md" | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" index 952e2bd..868d42c 100644 --- "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" +++ "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" @@ -4,6 +4,10 @@ > **预计时间**: 15-30 分钟 > **前置要求**: 会用终端（命令行），电脑能上网 +本指南面向 Linux/Bash，推荐 Ubuntu 24.04。Qwen3 部署和 W1 及后续阶段的独立复现 +须使用 [Linux 固定环境指南](../llm-deploy-v1.0/LINUX_REPRODUCTION.md)，不要用本页的 +通用可选依赖替代该项目固定版本。个人开发环境不作为 Linux 验收依据。 + --- ## 目录 @@ -29,23 +33,14 @@ ### 如果你还没有安装 -**Ubuntu / Debian / WSL**: +**Ubuntu / Debian**: ```bash sudo apt update sudo apt install python3 python3-pip python3-venv python3-dev git build-essential ``` -**macOS**: -```bash -# 先安装 Homebrew（如果还没有）: https://brew.sh -brew install python@3.12 git -``` - -**Windows**: -- 推荐安装 [WSL2](https://learn.microsoft.com/zh-cn/windows/wsl/install)（Windows Subsystem for Linux），然后在 WSL 里按 Ubuntu 的方式操作 -- 或者从 [python.org](https://www.python.org/downloads/) 下载 Python 安装包 - -> 💡 **为什么推荐 WSL？** ScratchV 的 RISC-V 仿真工具（TinyFive、Spike）主要在 Linux 上测试过。WSL 让你在 Windows 上拥有一个完整的 Linux 环境，体验最好。 +在 Linux 主机上安装和验证编译器及仿真工具。复现记录须保存发行版、CPU、 +Python 和工具版本；其他个人环境的运行结果不能直接记作 Linux 通过。 --- @@ -87,7 +82,7 @@ remote: Counting objects: 100% (1234/1234), done. python3 -m venv .venv # 激活虚拟环境 -source .venv/bin/activate # Linux / macOS / WSL +source .venv/bin/activate ``` 激活成功后，你的终端提示符前面会出现 `(.venv)`： From 77b904527de54f695ff0d2cd312b1a4d456dda71 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Tue, 6 Oct 2026 17:27:11 +0800 Subject: [PATCH 16/30] docs: align shared setup guide with Linux reproduction --- ...55\345\273\272\346\214\207\345\215\227.md" | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" index 952e2bd..868d42c 100644 --- "a/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" +++ "b/docs/guide/00-\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\215\227.md" @@ -4,6 +4,10 @@ > **预计时间**: 15-30 分钟 > **前置要求**: 会用终端（命令行），电脑能上网 +本指南面向 Linux/Bash，推荐 Ubuntu 24.04。Qwen3 部署和 W1 及后续阶段的独立复现 +须使用 [Linux 固定环境指南](../llm-deploy-v1.0/LINUX_REPRODUCTION.md)，不要用本页的 +通用可选依赖替代该项目固定版本。个人开发环境不作为 Linux 验收依据。 + --- ## 目录 @@ -29,23 +33,14 @@ ### 如果你还没有安装 -**Ubuntu / Debian / WSL**: +**Ubuntu / Debian**: ```bash sudo apt update sudo apt install python3 python3-pip python3-venv python3-dev git build-essential ``` -**macOS**: -```bash -# 先安装 Homebrew（如果还没有）: https://brew.sh -brew install python@3.12 git -``` - -**Windows**: -- 推荐安装 [WSL2](https://learn.microsoft.com/zh-cn/windows/wsl/install)（Windows Subsystem for Linux），然后在 WSL 里按 Ubuntu 的方式操作 -- 或者从 [python.org](https://www.python.org/downloads/) 下载 Python 安装包 - -> 💡 **为什么推荐 WSL？** ScratchV 的 RISC-V 仿真工具（TinyFive、Spike）主要在 Linux 上测试过。WSL 让你在 Windows 上拥有一个完整的 Linux 环境，体验最好。 +在 Linux 主机上安装和验证编译器及仿真工具。复现记录须保存发行版、CPU、 +Python 和工具版本；其他个人环境的运行结果不能直接记作 Linux 通过。 --- @@ -87,7 +82,7 @@ remote: Counting objects: 100% (1234/1234), done. python3 -m venv .venv # 激活虚拟环境 -source .venv/bin/activate # Linux / macOS / WSL +source .venv/bin/activate ``` 激活成功后，你的终端提示符前面会出现 `(.venv)`： From 9397028031b875e37c4b3f4467cbed2fd6c021cc Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Thu, 8 Oct 2026 10:17:13 +0800 Subject: [PATCH 17/30] feat(reports): add alignment metrics and simplify probe summaries --- docs/llm-deploy-v1.0/W2/README.md | 8 + probes/numeric_summary.py | 119 +++++++++++ probes/w2_backend_ops/run.py | 22 +- probes/w2_qwen3_small/diagnostics.py | 8 + probes/w2_qwen3_small/riscv.py | 207 ++++++++++--------- probes/w2_qwen3_small/run.py | 195 ++++++++++-------- probes/w2_runtime/run.py | 9 +- probes/w2_runtime_model/run.py | 39 +++- probes/w3_common.py | 13 +- probes/w3_qwen3_full/README.md | 4 + probes/w3_qwen3_full/comparison.py | 7 + probes/w3_qwen3_subgraphs/run.py | 4 + probes/w3_summary.py | 226 +++++++++++++++++++++ scratchv/verification/numeric_metrics.py | 140 +++++++++++++ scripts/verify_w3_evidence.py | 42 +++- tests/test_numeric_metrics.py | 154 ++++++++++++++ tests/test_numeric_summary.py | 123 +++++++++++ tests/test_qwen3_small_reports.py | 119 +++++++++++ tests/test_w2_runtime_model.py | 2 +- tests/test_w3_evidence_verifier.py | 69 +++++++ tests/test_w3_qwen3_subgraphs.py | 36 ++++ tests/test_w3_summary.py | 247 +++++++++++++++++++++++ 22 files changed, 1604 insertions(+), 189 deletions(-) create mode 100644 probes/numeric_summary.py create mode 100644 probes/w3_summary.py create mode 100644 scratchv/verification/numeric_metrics.py create mode 100644 tests/test_numeric_metrics.py create mode 100644 tests/test_numeric_summary.py create mode 100644 tests/test_qwen3_small_reports.py create mode 100644 tests/test_w3_summary.py diff --git a/docs/llm-deploy-v1.0/W2/README.md b/docs/llm-deploy-v1.0/W2/README.md index 0db79de..76055ee 100644 --- a/docs/llm-deploy-v1.0/W2/README.md +++ b/docs/llm-deploy-v1.0/W2/README.md @@ -56,6 +56,14 @@ git status --short 总报告位于本次 `--output-dir` 下的 `report.json`、`report.md`、`report.html`；子目录保存各门禁原始报告、命令、日志、数组和模型/ELF 身份。源码指纹核对的是实际执行文件，不能把别处安装的模块误当当前 checkout。 +数值报告首页按链路展示普通图的最终 logits（模型输出）：ORT 对 PyTorch、原生 NumPy IR 对 ORT，以及生成 C 后经 RISC-V/QEMU 执行的输出对 ORT。最后一项同时验证 C 生成和目标端执行，并不是独立的宿主 C Transformer 测试，也不验证旧标量汇编后端。中间检查点用于定位误差，单独折叠，不混入最终输出的精度。 + +每次实际数组比较新增余弦相似度与相对 L2 误差：前者为两输出向量夹角的余弦，越接近 1 越一致；后者为 `||实际输出−参考输出||₂ / ||参考输出||₂`，越接近 0 越一致。汇总取各次比较中最小余弦、最大相对 L2 和最大绝对误差，它们可能来自不同用例；原有最大绝对误差门槛保持不变。零范数、非有限值等无法定义的指标明确标注原因，不填成 0；旧报告缺少新增指标时显示未记录。 + +逐用例、检查点、计时和完整 JSON 默认折叠。对应表下解释普通图/诊断图及 IR 优化等级；QEMU 秒数仅为进程墙钟时长。报告精简不减少实际测试：七组输入包含两组满长随机输入与五组边界/扰动用例，不是七个随机种子。 + +**PPL / zero-shot 暂不纳入当前验收。** PPL（困惑度）衡量模型预测真实文本下一个 token 的能力；zero-shot 衡量不提供示例时完成指定任务的能力。W2 使用随机权重小模型，这两项不能说明其语言质量。后续应在真实预训练权重上固定语料、Tokenizer、上下文窗口和任务评分方式，再比较编译前后质量；本轮明确标记未评测。 + - 保存 checkout SHA、dirty/diff、Linux 发行版、CPU、Python/NumPy/ORT/Torch、Zig/QEMU 版本。 - 保存每项实际退出码、PASS/FAIL/PARTIAL/未执行、原始误差、首次失败阶段及报告位置。 - QEMU 时长是进程墙钟，包含启动、guest 执行、UART 和退出；不是目标硬件性能或纯 forward 时间。 diff --git a/probes/numeric_summary.py b/probes/numeric_summary.py new file mode 100644 index 0000000..3410d83 --- /dev/null +++ b/probes/numeric_summary.py @@ -0,0 +1,119 @@ +"""Compact numerical evidence tables; never invent metrics for older reports. + +The ``passed`` flag remains the existing numerical gate's result. The two +additional metrics are observations, not new pass/fail thresholds. A table row +is an aggregation of tensor comparisons, not a comparison of concatenated data. +""" +from __future__ import annotations + +from collections import Counter +from html import escape +import math +from numbers import Real + + +METRIC_EXPLANATION = ( + "最大绝对误差是单个元素最坏的偏差；相对 L2 = ||实际−参考||₂ / ||参考||₂，越小越好；" + "余弦相似度衡量两个输出向量方向的一致性，越接近 1 越好。" + "先逐次比较，再取最大绝对误差、最大相对 L2、最小余弦（可能来自不同用例，不拼接计算）。" + "原有通过门槛不变，其他检查仍生效；新增两项仅供观察，缺失或未定义明确标注。" +) + +_METRICS = ("max_abs", "relative_l2", "cosine_similarity") +_HEADERS = ("比较", "最大绝对误差", "相对 L2 误差", "余弦相似度", "结果") + + +def _metric_summary(rows, key): + values, missing, reasons = [], 0, Counter() + for row in rows: + if key not in row: + missing += 1 + continue + value = row[key] + reason = row.get(key + "_reason") or row.get("reason") + valid = isinstance(value, Real) and not isinstance(value, bool) + try: + number = float(value) if valid else None + valid = valid and math.isfinite(number) + except (OverflowError, ValueError): + valid = False + if valid and (number < 0 if key != "cosine_similarity" else not -1 <= number <= 1): + valid = False + if valid: + values.append(number) + else: + reasons[str(reason or ("未提供原因" if value is None else "数值无效"))] += 1 + worst = (min(values) if key == "cosine_similarity" else max(values)) if values else None + return dict(value=worst, valid_count=len(values), total=len(rows), + missing=missing, undefined=sum(reasons.values()), reasons=reasons) + + +def _metric_text(summary, key): + parts = [] + if summary["value"] is not None: + # 17 significant digits distinguish adjacent binary64 cosine values. + # Scientific notation preserves even the smallest nonzero errors. + number = summary["value"] + parts.append(format(number, ".17g") if key == "cosine_similarity" else format(number, ".16e")) + if summary["missing"]: + parts.append(f"未记录 {summary['missing']}/{summary['total']}") + if summary["undefined"]: + reasons = "；".join(f"{reason} ×{count}" for reason, count in summary["reasons"].items()) + parts.append(f"未定义 {summary['undefined']}/{summary['total']}（{reasons}）") + if summary["value"] is not None and (summary["missing"] or summary["undefined"]): + parts[0] += f"（已记录 {summary['valid_count']}/{summary['total']}）" + return "；".join(parts) or "未记录（无比较）" + + +def _table_rows(comparisons): + """Preserve caller ordering and distinguish gate results from coverage.""" + result = [] + for label, records in comparisons: + rows = list(records) + if not all(isinstance(row, dict) for row in rows): + raise TypeError("comparison rows must be dictionaries") + metrics = [_metric_summary(rows, key) for key in _METRICS] + passed = sum(row.get("passed") is True for row in rows) + failed = sum(row.get("passed") is False for row in rows) + unknown = len(rows) - passed - failed + if not rows: + status = "未测（0 条比较）" + elif passed == len(rows): + status = f"门槛通过（{passed}/{len(rows)}）" + else: + status = f"门槛未通过（通过 {passed}/{len(rows)}，失败 {failed}，未确认 {unknown}）" + if rows and any(item["missing"] or item["undefined"] for item in metrics): + status += "；指标不完整" + result.append([str(label), *(_metric_text(item, key) for item, key in zip(metrics, _METRICS)), status]) + return result + + +def _markdown_cell(value): + # Escape HTML as well: GitHub renders HTML inside Markdown table cells. + return escape(value).replace("|", "|").replace("\r", " ").replace("\n", " ") + + +def compact_metric_table(comparisons): + """Render ``[(label, [tensor_comparison, ...]), ...]`` as Markdown. + + Only individual comparison flags determine this table's gate status. The + caller must report its overall probe status (including execution failures) + separately; this table is not a replacement for that status. + """ + rows = _table_rows(comparisons) + if not rows: + return "暂无数值比较记录。" + def render(row): + return "| " + " | ".join(_markdown_cell(value) for value in row) + " |" + + return "\n".join([render(_HEADERS), render(["---"] * len(_HEADERS)), *(render(row) for row in rows)]) + + +def compact_metric_table_html(comparisons): + """HTML equivalent of :func:`compact_metric_table`, using identical rules.""" + rows = _table_rows(comparisons) + if not rows: + return "

暂无数值比较记录。

" + header = "".join(f"{escape(value)}" for value in _HEADERS) + body = "".join("" + "".join(f"{escape(value)}" for value in row) + "" for row in rows) + return f"{header}{body}
" diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py index c801a31..53fc22f 100644 --- a/probes/w2_backend_ops/run.py +++ b/probes/w2_backend_ops/run.py @@ -27,6 +27,7 @@ from probes.w2_backend_ops.cases import build_cases from probes.w2_qwen3_small.diagnostics import tensor_diff +from probes.numeric_summary import compact_metric_table, compact_metric_table_html, METRIC_EXPLANATION from scratchv.analysis.ir_verifier import verify_ir from scratchv.compiler import CompilerConfig, CompilerDriver from scratchv.frontend.onnx_parser import ONNXParser @@ -68,7 +69,8 @@ def interpret(program, bindings, feed, level): def evidence(): sources = [Path(__file__), ROOT / "probes/w2_backend_ops/cases.py", - ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "scratchv/compiler.py", + ROOT / "probes/w2_qwen3_small/diagnostics.py", ROOT / "probes/numeric_summary.py", + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "scratchv/compiler.py", ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/ir/types.py", ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/pass_manager.py", ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/verification/ir_interpreter.py", @@ -177,7 +179,8 @@ def _report_markdown(report): for item in case["executions"]: lines.append(f"| {case['name']} | {item['optimization']} | {'PASS' if item['passed'] else 'FAIL'} | " f"{item.get('qemu_vs_ort', {}).get('max_abs')} | {item['qemu_process_wall_seconds']} |") - lines.extend(["", "Timing includes QEMU startup, guest execution, UART transfer and exit; not guest cycles.", + lines.extend(["", "Level 的 none/all 表示关闭/开启全部 ScratchV IR 优化；C 交叉编译均使用 -O2。" + "QEMU process seconds 为进程墙钟秒数，包含启动、目标程序计算、输出传输和退出，不是纯计算耗时。", "", "Full diagnostics, commands, environment and source hashes:", "", "```json", payload.rstrip(), "```", ""]) return "\n".join(lines) @@ -209,9 +212,20 @@ def record_failure(): def views(): markdown = _report_markdown(report) - write("report.md", markdown) + summary = f"W2 基础后端算子检查：{'PASS' if report['passed'] else 'FAIL'}（展开详情）" + comparisons = [("基础算子 C→RISC-V/QEMU 对 ORT", [ + next((execution.get("qemu_vs_ort", {}) for execution in case.get("executions", []) + if execution.get("optimization") == level), {}) + for case in report.get("cases", []) for level in LEVELS])] + description = ("基础后端检查 MatMul、逐元素、Softmax、RMSNorm、RoPE、SwiGLU、GQA 七类算子或组合；" + "16 个用例分别关闭/开启 IR 优化，计划 32 次 QEMU 执行。不是完整 Transformer 模型。") + write("report.md", f"
\n{summary}\n\n{description}\n\n" + + compact_metric_table(comparisons) + "\n\n" + METRIC_EXPLANATION + + f"\n\n{markdown}\n
\n\n") write("report.html", "W2 backend operators" - "

W2 backend operators

" + escape(markdown) + "
") + "
" + escape(summary) + "

" + escape(description) + "

" + + compact_metric_table_html(comparisons) + "

" + escape(METRIC_EXPLANATION) + "

" + escape(markdown)
+              + "
") views() if failures: diff --git a/probes/w2_qwen3_small/diagnostics.py b/probes/w2_qwen3_small/diagnostics.py index 5766f66..dbb8b83 100644 --- a/probes/w2_qwen3_small/diagnostics.py +++ b/probes/w2_qwen3_small/diagnostics.py @@ -15,6 +15,8 @@ import onnx from onnx import TensorProto, helper, numpy_helper +from scratchv.verification.numeric_metrics import numeric_metrics, undefined_numeric_metrics + def _names(names, label): names = list(names) @@ -154,14 +156,18 @@ def tensor_diff(actual, expected, atol=1e-5) -> dict: actual_finite=finite[0], expected_finite=finite[1], finite=all(finite), empty=bool(comparable and actual.size == expected.size == 0), worst_index=None, actual_value=None, expected_value=None, reason=None, atol=atol) + report.update(undefined_numeric_metrics("comparison is invalid")) if not comparable: report["reason"] = "outputs must be NumPy arrays" + report.update(undefined_numeric_metrics(report["reason"])) return report if not shape_matches or not dtype_matches: report["reason"] = "shape mismatch" if not shape_matches else "dtype mismatch" + report.update(undefined_numeric_metrics(report["reason"])) return report if actual.dtype.kind not in "biuf": report["reason"] = "outputs must have numeric dtypes" + report.update(undefined_numeric_metrics(report["reason"])) return report if not all(finite): invalid = ~np.isfinite(actual) | ~np.isfinite(expected) @@ -169,7 +175,9 @@ def tensor_diff(actual, expected, atol=1e-5) -> dict: report.update(worst_index=[int(dim) for dim in index], actual_value=_json_scalar(actual[index]), expected_value=_json_scalar(expected[index]), reason="nonfinite output or reference") + report.update(undefined_numeric_metrics(report["reason"])) return report + report.update(numeric_metrics(actual, expected)) if not actual.size: report.update(passed=True, max_abs=0.0) return report diff --git a/probes/w2_qwen3_small/riscv.py b/probes/w2_qwen3_small/riscv.py index 71a7cc1..d71b5e8 100644 --- a/probes/w2_qwen3_small/riscv.py +++ b/probes/w2_qwen3_small/riscv.py @@ -26,7 +26,10 @@ import onnxruntime as ort from probes.w2_qwen3_small.diagnostics import compare_outputs, tensor_diff, unpack_trace -from probes.w2_qwen3_small.run import ATOL, arrays_sha256, input_cases +from probes.w2_qwen3_small.run import ( + ATOL, _details_views, _page_html, _table_views, arrays_sha256, input_cases, +) +from probes.numeric_summary import METRIC_EXPLANATION, compact_metric_table, compact_metric_table_html from scratchv.analysis.ir_verifier import verify_ir from scratchv.compiler import CompilerConfig, CompilerDriver from scratchv.frontend.onnx_parser import ONNXParser @@ -98,6 +101,7 @@ def execution_evidence(): ROOT / "scratchv/ir/builder.py", ROOT / "scratchv/pass_manager.py", ROOT / "scratchv/pass_interface.py", ROOT / "scratchv/verification/ir_interpreter.py", ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "probes/numeric_summary.py", *sorted((ROOT / "scratchv/analysis").glob("*.py")), *sorted((ROOT / "scratchv/optimizer").glob("*.py")), ROOT / "scratchv/backend/tensor_c_codegen.py", ROOT / "scratchv/runtime/riscv_tensor.py"] @@ -316,98 +320,121 @@ def run_probe(model_dir, out, report, *, cc=None, qemu=None, timeout=180): def _report_views(report): timing = report["timing"] counts = timing["status_counts"] - lines = ["# Two-layer Qwen3: ScratchV IR → C tensor kernels → RV64GC → QEMU", "", - f"Result: **{'PASS' if report['passed'] else 'FAIL'}**; stage: {report['stage']}", "", - "## 仿真耗时", "", - f"- 探测流程总耗时：**{format_seconds(timing['pipeline_seconds'])} 秒**（不含报告渲染）。", - f"- 已记录 QEMU 进程耗时合计：**{format_seconds(timing['qemu_process_wall_seconds_total'])} 秒**。", - f"- 成功交叉编译记录合计：**{format_seconds(timing['cross_compile_seconds_total'])} 秒**；" - f"已记录 {timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次构建，" - "不含 ScratchV 解析/IR 优化/代码生成或失败构建的耗时。", - f"- 计划 {timing['planned_count']} 次，已尝试 {timing['attempted_count']} 次；" - f"通过 {counts['success']}，数值失败 {counts['numeric_failed']}，" - f"运行失败 {counts['runtime_error']}，超时 {counts['timeout']}，" - f"未启动 {counts['not_started']}，未尝试 {timing['not_attempted_count']}。", - f"- 记录范围：{'部分结果，尚未完成全部执行/数值核验' if timing['incomplete'] else '全部执行/数值核验已有记录'}；" - "失败样本与通过样本分别统计，未测得的时长不按零计。", "", - "QEMU 进程墙钟时长包含启动、guest 计算、输出打包/UART 和退出；超时还包括进程清理。" - "不包含主机输入准备或输出解码。diagnostic 包含检查点开销，须与 normal 分开。" - "这些数据仅用于监控，不代表纯前向时长或目标硬件性能，也不作为性能通过阈值。", "", - "| 图/优化级别 | 已尝试/计划 | 通过/失败 | 未尝试 | 通过平均/最大(s) | 失败平均/最大(s) | 已测合计(s) |", - "|---|---:|---:|---:|---:|---:|---:|"] - group_rows = [] + cases = report.get("cases", []) + comparisons = [] + for field, levels, label in ( + ("optimized_ir", ("none", "basic", "all"), "IR 原生 NumPy（优化回归）/ ORT"), + ("qemu", ("none", "all"), "生成 C → RV64/QEMU / ORT"), + ): + rows = [] + for case in cases + [{}] * max(0, QEMU_CASE_COUNT - len(cases)): + for level in levels: + rows.append(next((row for row in case.get(field, []) + if row.get("graph") == "normal" and row.get("optimization") == level), {})) + comparisons.append((label, rows)) + title = "两层 Qwen3：编译到 RISC-V 的数值精度" + status = "PASS" if report.get("passed") else "FAIL" + description = ("真实 Qwen3 结构的两层缩小模型，固定随机权重，FP32、L=256、无 KV cache。" + "同一 ONNX 模型、权重和输入分别经 ORT、原生 NumPy IR，以及 ScratchV IR → C 张量代码 → " + "C 交叉编译器 → RV64GC → QEMU 执行。C 行测的是 RISC-V 上的实际输出，不是单独的主机 C 测试。") + scope = ("核心表只汇总普通图最终 logits 的最坏指标（全部位置含 padding），比较按“被测 / 参考”排列。" + "七个输入场景覆盖两份满长输入、1/17/255 token 边界、因果与补齐隔离。" + "普通/诊断图、29 个检查点及隔离检查仍须全部通过严格 MaxAbs < 1e-5（rtol=0）。") + timing_line = (f"仿真耗时：探测流程 {format_seconds(timing['pipeline_seconds'])} 秒；" + f"已记录 QEMU 进程累计 {format_seconds(timing['qemu_process_wall_seconds_total'])} 秒" + "（含启动和传输，不代表纯前向或目标硬件性能）。") + coverage = (f"执行覆盖：{timing['attempted_count']}/{timing['planned_count']} 次已尝试，" + f"{counts['success']} 次通过；" + + ("部分结果，尚未完成全部执行。" if timing['incomplete'] else "全部执行完成。")) + markdown = [f"# {title}", "", f"Result: **{status}**; stage: {report.get('stage')}", "", + description, "", scope, "", compact_metric_table(comparisons), "", METRIC_EXPLANATION, + "", coverage, "", timing_line] + html = [f"

{escape(title)}

Result: {status}; stage: {escape(str(report.get('stage')))}

", + f"

{escape(description)}

{escape(scope)}

", compact_metric_table_html(comparisons), + f"

{escape(METRIC_EXPLANATION)}

{escape(coverage)}

{escape(timing_line)}

"] + failures = [] + if report.get("error"): + failures.append(f"阶段 {report.get('stage')}: {report['error']}") + for case in cases: + if case.get("passed") is not True: + failures.append(f"{case.get('name')}: FAIL；有检查失败或未完成") + for row in case.get("qemu", []): + if row.get("passed") is not True: + reason = row.get("error") or row.get("reason") or "尚未完成数值核验" + first = row.get("checkpoints", {}).get("first_divergence") + failures.append(f"{case.get('name')} / {row.get('graph')}/{row.get('optimization')}: " + f"{row.get('status')}; {reason}" + (f"；首次超限检查点 {first}" if first else "")) + for row in case.get("optimized_ir", []): + if row.get("passed") is not True: + failures.append(f"{case.get('name')} IR {row.get('graph')}/{row.get('optimization')}: " + f"{row.get('reason') or 'FAIL'}") + for row in report.get("invariants", []): + if row.get("passed") is not True: + failures.append(f"{row.get('name')} / {row.get('graph')}/{row.get('optimization')}: 隔离检查 FAIL") + if not report.get("passed") and not failures: + failures.append("尚未完成全部验收，请检查 report.json。") + if failures: + markdown += ["", "**失败／未完成项**", "", *[f"- {escape(item)}" for item in failures]] + html.append("

失败／未完成项

    " + + "".join(f"
  • {escape(item)}
    • " for item in failures) + "
") + + def append_detail(title, headers, rows, note): + md, body = _table_views(headers, rows) + md, body = _details_views(title, md + "\n\n" + note, body + f"

{escape(note)}

") + markdown.extend(["", md]) + html.append(body) + + graph_note = ("图指模型计算步骤及其连接关系：normal 只输出最终 logits；diagnostic 增加中间检查点以定位误差，含额外打包与传输开销。" + "none/basic/all 指 ScratchV IR 优化：none 不优化；basic 提前计算常量并删除不影响输出的计算；" + "all 再加入局部指令简化、乘加融合和循环内重复计算外提。它们不是 C 编译优化等级，本探测的 C 编译固定使用 -O2。") + time_note = (graph_note + " QEMU 墙钟时间包含启动、目标程序计算、输出打包/串口传输和退出；超时还含清理，" + "不含主机输入准备或输出解码。未测得的时长不按零计；平均/最大值仅统计有测量值的样本。" + "七种输入不是重复采样，不能据此推导稳定加速比，也不设置性能通过阈值。" + f"成功交叉编译记录累计 {format_seconds(timing['cross_compile_seconds_total'])} 秒" + f"（{timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次），" + "不含 ScratchV 解析、IR 优化、代码生成及失败构建；流程时间不含报告渲染。") + groups = [] for group in timing["groups"]: successful, failed = group["successful"], group["failed"] - failed_count = group["attempted_count"] - group["status_counts"]["success"] - cells = [f"{group['graph']}/{group['optimization']}", - f"{group['attempted_count']}/{group['planned_count']}", - f"{group['status_counts']['success']}/{failed_count}", str(group['not_attempted_count']), - f"{format_seconds(successful['mean_seconds'])}/{format_seconds(successful['max_seconds'])}", - f"{format_seconds(failed['mean_seconds'])}/{format_seconds(failed['max_seconds'])}", - format_seconds(group['all_measured']['total_seconds'])] - group_rows.append(cells) - lines.append("| " + " | ".join(cells) + " |") - lines += ["", "平均/最大值只统计有实测时长的样本；失败数包含尚未启动但已尝试的条目。" - "七种输入不是重复采样，不据此宣称稳定加速比。", "", - "FP32, L=256, random weights, 29 checkpoints, strict max absolute error < 1e-5.", - "The C cross-compiler is Zig/LLVM; this does not validate the legacy scalar assembly selector.", - "IR optimization levels: none/basic/all. QEMU: none/all, normal and diagnostic graphs.", ""] - if report.get("error"): - lines += [f"Error: {report['error']}", ""] - lines += ["| Case | Graph | IR optimization | QEMU进程(s) | 超时设定(s) | Max abs | First divergent checkpoint | Status |", - "|---|---|---|---:|---:|---:|---|---|"] - execution_rows = [] - for case in report.get("cases", []): + groups.append([f"{group['graph']}/{group['optimization']}", + f"{group['attempted_count']}/{group['planned_count']}", + f"{group['status_counts']['success']}/{group['attempted_count'] - group['status_counts']['success']}", + group['not_attempted_count'], + f"{format_seconds(successful['mean_seconds'])}/{format_seconds(successful['max_seconds'])}", + f"{format_seconds(failed['mean_seconds'])}/{format_seconds(failed['max_seconds'])}", + format_seconds(group['all_measured']['total_seconds'])]) + append_detail("图与优化级别：仿真耗时明细", + ["图/优化级别", "已尝试/计划", "通过/失败", "未尝试", "通过平均/最大(s)", "失败平均/最大(s)", "已测合计(s)"], + groups, time_note) + executions = [] + for case in cases: for row in case.get("qemu", []): - cells = [case['name'], row['graph'], row['optimization'], - format_seconds(row.get('qemu_process_wall_seconds')), - format_seconds(row.get('timeout_seconds')), str(row.get('max_abs')), - row.get('checkpoints', {}).get('first_divergence') or 'none', row['status']] - execution_rows.append(cells) - lines.append("| " + " | ".join(cells) + " |") - markdown = "\n".join(lines) + "\n" - details = [] - for case in report.get("cases", []): - details.append(f"
{escape(case['name'])}: " - f"{'PASS' if case['passed'] else 'FAIL'}
"
-                       f"{escape(json.dumps(case, indent=2))}
") - - def html_table(headers, rows): - header = "".join(f"{escape(value)}" for value in headers) - body = "".join("" + "".join(f"{escape(value)}" for value in row) + "" - for row in rows) - return f"
{header}{body}
" - - overview = ("

仿真耗时

" - f"

探测流程总耗时：{format_seconds(timing['pipeline_seconds'])} 秒；" - f"已记录 QEMU 进程合计：{format_seconds(timing['qemu_process_wall_seconds_total'])} 秒；" - f"成功交叉编译记录合计：{format_seconds(timing['cross_compile_seconds_total'])} 秒" - f"（{timing['cross_compile_completed_count']}/{timing['cross_compile_planned_count']} 次）。

" - f"

计划 {timing['planned_count']} 次，已尝试 {timing['attempted_count']} 次；" - f"通过 {counts['success']}，数值失败 {counts['numeric_failed']}，" - f"运行失败 {counts['runtime_error']}，超时 {counts['timeout']}，" - f"未启动 {counts['not_started']}，未尝试 {timing['not_attempted_count']}。" - f"{'部分结果' if timing['incomplete'] else '完整执行记录'}。

" - "

QEMU 时长包含进程启动、guest 计算、输出打包/UART 和退出，超时包含进程清理；" - "不含主机输入准备或输出解码。diagnostic 含检查点开销，与 normal 分开统计。" - "流程总耗时不含报告渲染；交叉编译仅统计成功记录，不含 ScratchV 解析/IR 优化/代码生成。" - "未测得不按零计，失败与通过样本分别计算平均/最大值。七种输入不是重复采样，" - "仅用于监控，不代表纯前向时长、稳定加速比或目标硬件性能，无性能通过阈值。

") - overview += html_table(["图/优化", "已尝试/计划", "通过/失败", "未尝试", "通过平均/最大(s)", - "失败平均/最大(s)", "已测合计(s)"], group_rows) - overview += "

逐次执行

" + html_table( - ["Case", "Graph", "IR优化", "QEMU进程(s)", "超时设定(s)", "Max abs", "首次偏差", "Status"], - execution_rows) - if report.get("error"): - overview += f"

Error: {escape(report['error'])}

" - html = ( - "Qwen3 RV64 probe" - "" - f"

Qwen3 RV64 QEMU: {'PASS' if report['passed'] else 'FAIL'}

" - + overview + "".join(details) + "") - return {"report.md": markdown, "report.html": html} + executions.append([case.get('name'), row.get('graph'), row.get('optimization'), + format_seconds(row.get('qemu_process_wall_seconds')), + format_seconds(row.get('timeout_seconds')), row.get('max_abs', '未统计'), + row.get('checkpoints', {}).get('first_divergence') or 'none', row.get('status')]) + append_detail("逐场景 QEMU 执行明细", + ["场景", "图", "IR 优化", "QEMU 进程(s)", "超时设定(s)", "本图输出 MaxAbs", "首次超限检查点", "结果"], + executions, graph_note + " 此明细中的 diagnostic MaxAbs 包含全部打包检查点；首页核心表只使用 normal logits。" + "超时设定是限制，QEMU 进程(s) 才是实测墙钟时间。") + checks = [] + for case in cases: + for row in case.get('optimized_ir', []): + checks.append([case.get('name'), f"IR {row.get('graph')}/{row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + for row in case.get('normal_vs_trace', []): + checks.append([case.get('name'), f"normal/diagnostic logits {row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + for row in report.get('invariants', []): + checks.append([row.get('name'), f"{row.get('graph')}/{row.get('optimization')}", + row.get('max_abs'), 'PASS' if row.get('passed') else row.get('reason') or 'FAIL']) + append_detail("优化 IR、普通/诊断图一致性与隔离检查", ["场景/检查", "路径", "MaxAbs", "结果"], checks, + "所有这些检查继续参与验收，完整逐检查点误差保存在 report.json。") + footer = ("完整数值明细、命令、环境、源码指纹、C/ELF 与原始数组随 CI artifact 提供；结构化报告为 report.json。" + "这条路径不验证旧的标量汇编选择器，也不声称完整预训练模型已在 QEMU 运行。") + markdown.extend(["", footer]) + html.append(f"

{escape(footer)}

") + return {"report.md": "\n".join(markdown) + "\n", "report.html": _page_html(title, "".join(html))} def write_reports(out, report): diff --git a/probes/w2_qwen3_small/run.py b/probes/w2_qwen3_small/run.py index 9aa4f38..dbcea9f 100644 --- a/probes/w2_qwen3_small/run.py +++ b/probes/w2_qwen3_small/run.py @@ -30,6 +30,9 @@ from probes.w2_qwen3_small.diagnostics import ( # noqa: E402 build_diagnostic_model, compare_outputs, tensor_diff, unpack_trace, ) +from probes.numeric_summary import ( # noqa: E402 + METRIC_EXPLANATION, compact_metric_table, compact_metric_table_html, +) ATOL = 1e-5 SEQ = 256 @@ -165,7 +168,9 @@ def provenance() -> dict: for path in [*sorted(Path(__file__).parent.glob("*.py")), ROOT / "scratchv/frontend/onnx_parser.py", ROOT / "scratchv/verification/ir_interpreter.py", - ROOT / "scratchv/verification/ir_numpy_ops.py"]} + ROOT / "scratchv/verification/ir_numpy_ops.py", + ROOT / "scratchv/verification/numeric_metrics.py", + ROOT / "probes/numeric_summary.py"]} return result @@ -311,90 +316,122 @@ def reference(feed): report.pop("current_case", None) -def _report_markdown(report: dict) -> str: - lines = ["# Real two-layer Qwen3 numerical probe", "", - f"Result: **{'PASS' if report['passed'] else 'FAIL'}**", "", - "Official transformers 4.51.3 Qwen3, seeded random weights, FP32, L=256, no KV cache.", - "All checkpoint positions must pass max absolute error < 1e-5 (rtol=0).", - "Padding queries are also checked and are reported separately in JSON.", - "Diagnostic outputs observe the same forward; ordinary logits-only IR is checked separately.", - "This probe does not validate pretrained language quality or RISC-V execution.", ""] +def _table_views(headers, rows): + """Render the same small, escaped table in both report formats.""" + def markdown_cell(value): + return escape(str(value)).replace("|", "|").replace("\n", "
") + + markdown = ["| " + " | ".join(map(markdown_cell, headers)) + " |", + "|" + "---|" * len(headers)] + markdown += ["| " + " | ".join(map(markdown_cell, row)) + " |" for row in rows] + header = "".join(f"{escape(str(value))}" for value in headers) + body = "".join("" + "".join(f"{escape(str(value))}" for value in row) + + "" for row in rows) + return "\n".join(markdown), f"{header}{body}
" + + +def _details_views(title, markdown, html, *, opened=False): + attribute = " open" if opened else "" + heading = f"{escape(title)}" + return f"{heading}\n\n{markdown}\n\n", f"{heading}{html}" + + +def _page_html(title, content): + css = ("body{font:16px/1.5 system-ui,sans-serif;max-width:1100px;margin:32px auto;padding:0 20px;color:#182536}" + "details{margin:16px 0;padding:12px;border:1px solid #dce3eb;overflow:auto}" + "summary{cursor:pointer;font-weight:650}table{border-collapse:collapse;width:100%;font-size:14px}" + "th,td{text-align:left;padding:8px;border:1px solid #dce3eb}" + "pre{white-space:pre-wrap;overflow-wrap:anywhere}.failure{color:#a42020;font-weight:bold}") + return ("" + "" + f"{escape(title)}{content}") + + +def _host_report_views(report): + cases = report.get("cases", []) + comparisons = [] + for name, label in (("pytorch_vs_ort", "ORT / PyTorch（Transformer 导出）"), + ("ir_vs_ort", "IR 原生 NumPy / ORT")): + rows = [case.get("ordinary_logits", {}).get(name, {}) for case in cases] + # Missing cases or old metric fields must not look like full coverage. + rows += [{}] * max(0, 7 - len(cases)) + comparisons.append((label, rows)) + description = ("官方 Qwen3 结构的两层缩小模型，固定随机权重，FP32、L=256、无 KV cache。" + "同一模型、同一权重和输入分别执行 PyTorch、ONNX Runtime（ORT）与 ScratchV IR 解释器。") + scope = ("核心表仅汇总普通图的最终 logits，覆盖全部位置（含 padding）。" + "比较名称按“被测 / 参考”排列；七个输入场景包括两份满长输入、1/17/255 token 边界、因果与补齐隔离检查。" + "验收仍要求所有检查点及普通/诊断图一致性通过严格 MaxAbs < 1e-5（rtol=0）。") + limitation = "本报告验证数值计算，不运行 C/RISC-V，也不评估随机权重模型的语言能力（PPL、zero-shot 未启用）。" + status = "PASS" if report.get("passed") else "FAIL" + title = "两层 Qwen3：Transformer 导出与 IR 精度" + markdown = [f"# {title}", "", f"Result: **{status}**", "", description, "", scope, "", + compact_metric_table(comparisons), "", METRIC_EXPLANATION, "", limitation] + html = [f"

{escape(title)}

Result: {status}

", + f"

{escape(description)}

{escape(scope)}

", compact_metric_table_html(comparisons), + f"

{escape(METRIC_EXPLANATION)}

{escape(limitation)}

"] + failures = [] if report.get("error"): - lines += [f"Failed stage: `{report['stage']}`", "", "```text", report["error"], "```", ""] - lines += ["| Case | Valid tokens | PyTorch / ORT max error | IR / ORT max error | First divergence | Result |", - "|---|---:|---:|---:|---|---|"] - for case in report.get("cases", []): - pairs = [case.get(name, {}) for name in ("pytorch_vs_ort", "ir_vs_ort")] - maxima = [max((row["max_abs"] for row in pair.get("checkpoints", []) - if row.get("max_abs") is not None), default=None) for pair in pairs] - values = [f"{value:.3e}" if value is not None else "n/a" for value in maxima] - first = next((pair.get("first_divergence") for pair in pairs if pair.get("first_divergence")), "none") - lines.append(f"| {case['name']} | {case['valid_length']} | {values[0]} | {values[1]} | " - f"{first} | {'PASS' if case['passed'] else 'FAIL'} |") - for case in report.get("cases", []): - lines += ["", f"
{case['name']}: checkpoint comparisons", "", - "| Comparison | Checkpoint | Max absolute error | Result |", "|---|---|---:|---|"] + failures.append(f"阶段 {report.get('stage', 'unknown')}: {report['error']}") + for case in cases: + if case.get("passed") is not True: + first = next((case.get(pair, {}).get("first_divergence") for pair in + ("pytorch_vs_ort", "ir_vs_ort") if case.get(pair, {}).get("first_divergence")), None) + failures.append(f"{case.get('name', 'unknown')}: FAIL；" + + (f"首次超限检查点 {first}" if first else "有检查失败或尚未完成，请展开明细")) + for check in report.get("invariants", []): + if check.get("passed") is not True: + failures.append(f"{check.get('name')} / {check.get('backend')}: 隔离检查 FAIL") + if not report.get("passed") and not failures: + failures.append(f"阶段 {report.get('stage', 'unknown')}: 尚未完成全部验收") + if failures: + markdown += ["", "**失败／未完成项**", "", *[f"- {escape(item)}" for item in failures]] + html.append("

失败／未完成项

    " + + "".join(f"
  • {escape(item)}
    • " for item in failures) + "
") + + def append_detail(title, headers, rows, note="", *, opened=False): + md, body = _table_views(headers, rows) + if note: + md += "\n\n" + note + body += f"

{escape(note)}

" + md, body = _details_views(title, md, body, opened=opened) + markdown.extend(["", md]) + html.append(body) + + case_rows = [] + for case in cases: + ordinary = case.get("ordinary_logits", {}) + case_rows.append([case.get("name"), case.get("valid_length"), + ordinary.get("pytorch_vs_ort", {}).get("max_abs", "未统计"), + ordinary.get("ir_vs_ort", {}).get("max_abs", "未统计"), + "PASS" if case.get("passed") else "FAIL"]) + append_detail("输入场景明细", ["场景", "有效 token", "ORT / PyTorch MaxAbs", "IR / ORT MaxAbs", "全部检查"], case_rows) + checkpoint_rows = [] + for case in cases: for pair in ("pytorch_vs_ort", "ir_vs_ort"): for row in case.get(pair, {}).get("checkpoints", []): - lines.append(f"| {pair} | {row['name']} | {row.get('max_abs')} | " - f"{'PASS' if row['passed'] else 'FAIL'} |") - lines += ["", "
"] - lines += ["", "| Invariant | Backend | Max absolute error | Result |", "|---|---|---:|---|"] - for check in report.get("invariants", []): - lines.append(f"| {check['name']} | {check['backend']} | {check.get('max_abs')} | " - f"{'PASS' if check['passed'] else 'FAIL'} |") - return "\n".join(lines) + "\n" + checkpoint_rows.append([case.get("name"), pair, row.get("name"), row.get("max_abs"), + "PASS" if row.get("passed") else row.get("reason") or "FAIL"]) + for name, row in case.get("ordinary_logits", {}).items(): + if "diagnostic_vs_ordinary" in name: + checkpoint_rows.append([case.get("name"), name, "logits", row.get("max_abs"), + "PASS" if row.get("passed") else row.get("reason") or "FAIL"]) + append_detail("检查点与普通/诊断图一致性", ["场景", "比较", "检查点", "MaxAbs", "结果"], checkpoint_rows, + "normal 为只输出最终 logits 的普通图；diagnostic 额外输出中间检查点以定位误差。两种图都参与验收。") + append_detail("因果与补齐隔离检查", ["检查", "执行路径", "MaxAbs", "结果"], + [[row.get("name"), row.get("backend"), row.get("max_abs"), + "PASS" if row.get("passed") else "FAIL"] for row in report.get("invariants", [])]) + footer = "完整诊断、位置明细、命令、环境与源码指纹保存在 report.json；原始数组随 CI artifact 提供。" + markdown.extend(["", footer]) + html.append(f"

{escape(footer)}

") + return "\n".join(markdown) + "\n", _page_html(title, "".join(html)) + + +def _report_markdown(report: dict) -> str: + return _host_report_views(report)[0] def _report_html(report: dict) -> str: - """A portable, dependency-free view of the same numeric evidence.""" - def cell(value): - return escape(str(value)) - - def table(rows, columns): - header = "".join(f"{cell(column)}" for column in columns) - body = "".join("" + "".join(f"{cell(value)}" for value in row) + "" - for row in rows) - return f"{header}{body}
" - - status = "PASS" if report["passed"] else "FAIL" - sections = [f"

Real two-layer Qwen3 probe

{status}

", - "

FP32 · L=256 · 29 checkpoints · max absolute error < 1e-5

", - "

Official Qwen3 with random weights. All positions, including padding queries, are checked. " - "Ordinary logits and diagnostic graphs must both pass. This is not a RISC-V or pretrained quality test.

"] - if report.get("error"): - sections.append(f"
{cell(report['stage'])}: {cell(report['error'])}
") - for case in report.get("cases", []): - case_status = "PASS" if case["passed"] else "FAIL" - details = [] - for pair in ("pytorch_vs_ort", "ir_vs_ort"): - comparison = case.get(pair, {}) - details.append(f"

{cell(pair)}

First divergence: " - f"{cell(comparison.get('first_divergence') or 'none')}

") - rows = [[row["name"], row.get("max_abs"), row.get("worst_index"), - row.get("actual_value"), row.get("expected_value"), - "PASS" if row["passed"] else row.get("reason")] - for row in comparison.get("checkpoints", [])] - details.append(table(rows, ["Checkpoint", "Max abs", "Worst index", "Actual", "Reference", "Result"])) - auxiliary = {"capture_preserves_logits": case.get("capture_preserves_logits"), - "ordinary_logits": case.get("ordinary_logits"), - "attention_checks": case.get("attention_checks")} - details.append(f"

Other checks

{cell(json.dumps(auxiliary, indent=2))}
") - opened = " open" if not case["passed"] else "" - sections.append(f"{cell(case['name'])} · " - f"{case['valid_length']} valid tokens · {case_status}{''.join(details)}") - rows = [[check["name"], check["backend"], check.get("max_abs"), - "PASS" if check["passed"] else "FAIL"] for check in report.get("invariants", [])] - sections.append("

Metamorphic checks

" + table(rows, ["Invariant", "Backend", "Max abs", "Result"])) - sections.append("

Full environment, schemas, hashes and valid/padding breakdowns are in report.json.

") - css = ("body{font:16px/1.5 system-ui,sans-serif;max-width:1200px;margin:40px auto;padding:0 24px;color:#182536;background:#f7f9fc}" - "h1{font-size:30px}h3{font-size:18px}details{background:white;margin:16px 0;padding:16px;border:1px solid #dce3eb;border-radius:8px;overflow:auto}" - "summary{cursor:pointer;font-weight:650}table{border-collapse:collapse;width:100%;font-size:13px;background:white}" - "th,td{text-align:left;padding:8px;border-bottom:1px solid #dce3eb}pre{white-space:pre-wrap;overflow-wrap:anywhere;font-size:12px}" - ".pass{color:#126538;font-weight:bold}.fail{color:#a42020;font-weight:bold}") - document = ("" - f"Qwen3 small probe · {status}{''.join(sections)}") - return document + return _host_report_views(report)[1] def write_reports(out: Path, report: dict) -> None: diff --git a/probes/w2_runtime/run.py b/probes/w2_runtime/run.py index 7ec2e08..f6b93ec 100644 --- a/probes/w2_runtime/run.py +++ b/probes/w2_runtime/run.py @@ -280,17 +280,24 @@ def write(name, data): "Fixed official tokenizer assets; encode/decode vs Transformers fast + slow and golden corpus.", "NFC normalization is official behavior; equivalence does not mean arbitrary raw-text byte preservation.", "No pretrained weights, full-model inference or text generation is executed.", "", + "
Tokenizer 与输入准备逐项检查（展开详情）", "", "| Case | Checks | Passed |", "|---|---:|---|"] for row in report["tokenizer_cases"] + report["runtime_cases"]: lines.append(f"| {row['name']} | {row.get('checks', 1)} | {row['passed']} |") + lines.extend(["", "
", ""]) if "error" in report: lines.extend(["", "Error: " + report["error"]]) markdown = "\n".join(lines) + "\n" + intro, detail = markdown.split("
", 1) + detail, suffix = detail.split("
", 1) + caption, detail = detail.split("", 1) + caption = caption.removeprefix("") failures = [] documents = { "report.md": markdown, "report.html": 'W2 runtime gate
'
-        + html.escape(markdown) + "
", + + html.escape(intro) + '
' + html.escape(caption) + + '
' + html.escape(detail) + '
' + html.escape(suffix) + '
', } for name, content in documents.items(): if error := write(name, content): diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py index 05f6f29..ae8d091 100644 --- a/probes/w2_runtime_model/run.py +++ b/probes/w2_runtime_model/run.py @@ -24,7 +24,9 @@ from probes.w2_runtime.run import MANIFEST, checkout_evidence, require_environment, sha256, verify_assets from probes.w2_qwen3_small.run import arrays_sha256, environment +from probes.numeric_summary import compact_metric_table, compact_metric_table_html, METRIC_EXPLANATION from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token, prepare_inputs +from scratchv.verification.numeric_metrics import numeric_metrics SEQ, VOCAB, ATOL = 256, 151936, 1e-5 LEVELS = ("none", "all") @@ -79,7 +81,7 @@ def compare_logits(actual, expected, valid_length, *, vocab_size=VOCAB): maxima["valid_max_abs"] = max(maxima["valid_max_abs"], float(difference[:boundary].max())) if boundary < stop - start: maxima["padding_max_abs"] = max(maxima["padding_max_abs"], float(difference[boundary:].max())) - return {**maxima, "worst_index": worst, "elements": int(actual.size), + return {**maxima, **numeric_metrics(actual, expected), "worst_index": worst, "elements": int(actual.size), "passed": maxima["max_abs"] < ATOL, "atol": ATOL, "rtol": 0} @@ -154,7 +156,8 @@ def check_onnx_contract(model): def source_evidence(): - paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", + paths = [Path(__file__), ROOT / "probes/w2_runtime/run.py", ROOT / "probes/numeric_summary.py", + ROOT / "scratchv/verification/numeric_metrics.py", ROOT / "probes/w2_qwen3_small/model.py", ROOT / "probes/w2_qwen3_small/run.py", ROOT / "probes/w1_qwen3_export/manifest.json", ROOT / "scratchv/runtime/qwen3_tokenizer.py", ROOT / "scratchv/runtime/llm_inputs.py", @@ -334,11 +337,33 @@ def save_reports(out, report): failures = [] def views(): - text = ("# W2 runtime/model integration\n\nResult: **" + ("PASS" if report["passed"] else "FAIL") - + "**\n\nOfficial two-layer Qwen3, random weights, full vocabulary; one forward and greedy token.\n" - + "Not pretrained inference or a generation loop. All positions use strict max_abs < 1e-5.\n\n```json\n" - + json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n```\n") - return {"report.md": text, "report.html": '
' + html.escape(text) + "
"} + status = "PASS" if report["passed"] else "FAIL" + description = ("两层随机权重 Qwen3，FP32、L=256、完整词表；两个文本场景使用同一权重和输入分别运行 " + "PyTorch、ORT、原生 NumPy IR，比较全部最终 logits（最大绝对误差 < 1e-5，rtol=0），" + "并核对最后有效位置选择分数最高 token 的结果。不是预训练模型或多步生成。") + comparisons = [] + for engine, label, reference in (("ort", "ORT 对 PyTorch", "vs_torch"), + ("ir-none", "IR 原生 NumPy（无优化）对 ORT", "vs_ort"), + ("ir-all", "IR 原生 NumPy（全部优化）对 ORT", "vs_ort")): + rows = [next((r.get(reference, {}) for r in case.get("executions", []) + if r.get("engine") == engine), {}) for case in report.get("cases", [])] + rows += [{}] * max(0, len(CASES) - len(rows)) + comparisons.append((label, rows)) + note = "PPL / zero-shot：未评测；随机权重模型仅用于验证计算和接口，不评价语言能力。" + error = str(report.get("error", "")) + payload = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + text = (f"# W2 runtime/model integration\n\nResult: **{status}**\n\n{description}\n\n" + + compact_metric_table(comparisons) + f"\n\n{METRIC_EXPLANATION}\n\n{note}\n\n" + + (f"失败原因：{error}\n\n" if error else "") + + "
文本、采样及完整诊断（展开详情）\n\n```json\n" + + payload + "\n```\n\n
\n\n") + page = ('

W2 runtime/model integration

' + + f'

Result: {status}

{html.escape(description)}

' + + compact_metric_table_html(comparisons) + f'

{html.escape(METRIC_EXPLANATION)}

' + + f'

{html.escape(note)}

' + (f'

失败原因：{html.escape(error)}

' if error else "") + + '
文本、采样及完整诊断（展开详情）
'
+                + html.escape(payload) + '
') + return {"report.md": text, "report.html": page} def failed(): report["report_write_errors"] = failures diff --git a/probes/w3_common.py b/probes/w3_common.py index 6570a51..eacf19a 100644 --- a/probes/w3_common.py +++ b/probes/w3_common.py @@ -203,14 +203,17 @@ def write_reports(out, report): title = str(report.get("gate", "W3 preparation")) status = str(report.get("status", "PASS" if report.get("passed") else "FAIL")) markdown = f"# {title}\n\nResult: **{status}**\n\n" - markdown += "See the structured evidence below for scope, source identity, cases, errors and measurements.\n\n" - fence = chr(96) * 3 - markdown += fence + "json\n" + payload + fence + "\n" + from probes.w3_summary import evidence_index_views, summary_views + summary_md, summary_html = summary_views(report) + index_md, index_html = evidence_index_views(report) + markdown += summary_md + index_md html = ('' + escape(title) + '' '' + 'table{width:100%;border-collapse:collapse;margin:16px 0}' + 'th,td{border:1px solid #ccc;padding:8px 12px;text-align:left;overflow-wrap:anywhere}' + 'th{background:#f3f5f7}summary{cursor:pointer}details{margin-top:20px}' '

' + escape(title) + '

Result: ' + escape(status) + - '

' + escape(payload) + '
') + '

' + summary_html + index_html) atomic_text(out / "report.md", markdown) atomic_text(out / "report.html", html) atomic_text(out / "report.json", payload) diff --git a/probes/w3_qwen3_full/README.md b/probes/w3_qwen3_full/README.md index 17ae001..5b51e48 100644 --- a/probes/w3_qwen3_full/README.md +++ b/probes/w3_qwen3_full/README.md @@ -58,6 +58,10 @@ export SCRATCHV_FP32_REFERENCE_CPU=auto 统一 `report.json`、`report.md` 和 `report.html` 包含每组输入、每个 worker、完整比较和四项不变量。`cases[].comparison.logits` 是最终精度验收数据，`checkpoints` 及 `first_divergence` 定位最早超过诊断阈值的检查点。 +Summary 将普通图与诊断图的最终 logits 分开，明确实际 `native/reference` 模式，并报告最大绝对误差、余弦相似度和相对 L2 误差；逐用例索引默认折叠，完整检查点和机器证据仅保存在 `report.json`，不重复嵌入摘要。后两项由实际输出和参考数组分块计算，使用 FP64 累积，不改变原有通过门槛。不能把参考兼容模式的通过结果归给默认 NumPy 模式。旧报告不补写或重新命名为本轮运行；离线重算保存数组也不代表重新执行了模型。 + +PPL（真实文本预测的困惑度）和 zero-shot（无示例任务评分）本轮未评测。这里的随机 token 输入用于数值对齐，不能作为语言质量语料；加入这类评测前需要固定真实文本、Tokenizer、L=256 的分窗及计分方式。 + 每组目录内有 `inputs.npz`、ORT/IR 日志，以及两个后端各自的完整 logits、诊断 logits、检查点 NPZ、schema、报告和阶段进度。报告包含固定模型、输入、输出及源码哈希；统一入口重新校验原始数组与报告一致性。 `cases[].workers[].stages_seconds` 可查看模型解析、普通执行、诊断执行和写盘时间；统一 `elapsed_seconds` 包括所有 worker 及文件核对。`process_peak_rss_bytes` 为该 worker 的全生命周期峰值，`resource_monitor` 是父进程采样观测。两者测量口径不同。 diff --git a/probes/w3_qwen3_full/comparison.py b/probes/w3_qwen3_full/comparison.py index e81d56a..478b864 100644 --- a/probes/w3_qwen3_full/comparison.py +++ b/probes/w3_qwen3_full/comparison.py @@ -4,6 +4,8 @@ import math import numpy as np +from scratchv.verification.numeric_metrics import numeric_metrics, undefined_numeric_metrics + ATOL = 1e-4 @@ -16,11 +18,14 @@ def compare_tensor(actual, reference, *, atol=ATOL, chunk_elements=262144): "shape": list(actual.shape), "reference_shape": list(reference.shape), "dtype": str(actual.dtype), "reference_dtype": str(reference.dtype), "elements": int(actual.size), "max_abs": None, "finite": False} + row.update(undefined_numeric_metrics("comparison is invalid")) if actual.shape != reference.shape or actual.dtype != np.float32 or reference.dtype != np.float32: row["reason"] = "shape or FP32 dtype mismatch" + row.update(undefined_numeric_metrics(row["reason"])) return row if not actual.size: row["reason"] = "empty comparison is not numerical evidence" + row.update(undefined_numeric_metrics(row["reason"])) return row # nditer handles strided slices without a full-size ravel/copy allocation. chunks = np.nditer([actual, reference], flags=["external_loop", "buffered"], @@ -32,6 +37,7 @@ def compare_tensor(actual, reference, *, atol=ATOL, chunk_elements=262144): index = offset + int(np.flatnonzero(~finite)[0]) row.update(reason="nonfinite value", worst_index=list(np.unravel_index(index, actual.shape))) row["worst_index"] = [int(x) for x in row["worst_index"]] + row.update(undefined_numeric_metrics(row["reason"])) return row difference = np.abs(a.astype(np.float64) - b.astype(np.float64)) local = int(np.argmax(difference)) @@ -42,6 +48,7 @@ def compare_tensor(actual, reference, *, atol=ATOL, chunk_elements=262144): row.update(passed=maximum < atol, finite=True, max_abs=maximum, worst_index=[int(x) for x in np.unravel_index(worst[0], actual.shape)], actual_value=worst[1], reference_value=worst[2]) + row.update(numeric_metrics(actual, reference, block_elements=chunk_elements)) return row diff --git a/probes/w3_qwen3_subgraphs/run.py b/probes/w3_qwen3_subgraphs/run.py index 78e0fe2..7d18753 100644 --- a/probes/w3_qwen3_subgraphs/run.py +++ b/probes/w3_qwen3_subgraphs/run.py @@ -38,6 +38,7 @@ from scratchv.frontend.onnx_parser import ONNXParser from scratchv.pass_manager import create_optimization_pass_manager from scratchv.verification.ir_interpreter import IRInterpreter +from scratchv.verification.numeric_metrics import numeric_metrics, undefined_numeric_metrics ATOL = 1e-4 LEVELS = ("none", "basic", "all") @@ -65,10 +66,13 @@ def tensor_diff(actual, expected, atol=ATOL): "max_abs_error": None, "firstdiff": None, "passed": False} if not row["shape_matches"]: row["firstdiff"] = {"reason": "shape mismatch"} + row.update(undefined_numeric_metrics("shape mismatch")) return row if actual.dtype != np.float32 or expected.dtype != np.float32: row["firstdiff"] = {"reason": "FP32 dtype mismatch"} + row.update(undefined_numeric_metrics("FP32 dtype mismatch")) return row + row.update(numeric_metrics(actual, expected)) with np.errstate(invalid="ignore", over="ignore"): error = np.abs(actual.astype(np.float64) - expected.astype(np.float64)) bad = ~np.isfinite(actual) | ~np.isfinite(expected) | (error >= atol) diff --git a/probes/w3_summary.py b/probes/w3_summary.py new file mode 100644 index 0000000..63679c1 --- /dev/null +++ b/probes/w3_summary.py @@ -0,0 +1,226 @@ +"""Small human summaries; complete machine evidence remains in report.json.""" +from __future__ import annotations + +from html import escape +import math + +from probes.numeric_summary import compact_metric_table, compact_metric_table_html, METRIC_EXPLANATION + +# The medium gate uses the seven fixed input_cases from w2_qwen3_small.run. +_MEDIUM_CASE_COUNT = 7 +_INDEX_LIMIT = 20 + + +def _short(value, limit=240): + """Bound human-facing text without truncating the machine evidence.""" + text = " ".join(str(value).split()) + return text if len(text) <= limit else text[:limit] + "…（完整内容见 report.json）" + + +def _first_issue(row, depth=0): + """Find one failed check, without printing its entire nested result.""" + if depth > 6: + return "" + if isinstance(row, list): + return next((text for item in row if (text := _first_issue(item, depth + 1))), "") + if not isinstance(row, dict) or row.get("passed") is True: + return "" + for key in ("error", "reason"): + if row.get(key): + return _short(row[key]) + first = row.get("first_divergence") + if first: + if isinstance(first, dict): + first = " / ".join(str(first[key]) for key in ("comparison", "name", "checkpoint", "reason") + if first.get(key)) or "结构化定位数据见 report.json" + return "首个差异：" + _short(first) + # Visit execution/check result structures, never provenance or array data. + for key in ("comparison", "comparisons", "ort", "ir", "normal", "diagnostic", + "none", "basic", "all", "executions", "workers", "pytorch_comparison", + "ort_comparison", "capture_preserves_logits", "pack_layout", "checkpoints"): + child = row.get(key) + if key == "comparisons" and isinstance(child, dict): + child = list(child.values()) + issue = _first_issue(child, depth + 1) + if issue: + return issue + return "" + + +def _index_status(row): + status = row.get("status") + if status in ("PARTIAL", "COMPLETE"): + return status + if row.get("passed") is True: + return "PASS" if status in (None, "PASS", "success") else "状态冲突（见 JSON）" + if row.get("passed") is False: + return _short(status) if status and status != "PASS" else "FAIL" + return _short(status) if status else "未确认" + + +def evidence_index_views(report): + """A bounded index of recorded results, with the JSON as source of detail.""" + entries = [] + for key, label in (("cases", "用例"), ("gates", "子门禁"), ("invariants", "隔离检查")): + for index, row in enumerate(report.get(key, []), 1): + name = row.get("name", row.get("case", f"{label} {index}")) + entries.append((f"{label}：{_short(name, 100)}", row)) + if report.get("gate") == "w3_layer_diff": + entries.extend(("检查点：" + _short(row.get("name", "未命名"), 100), row) + for row in report.get("checkpoints", [])) + if not entries: + entries.append(("流程阶段：" + _short(report.get("stage", "未记录"), 100), report)) + # A late failure must not disappear behind the first twenty successful rows. + entries.sort(key=lambda item: item[1].get("passed") is True) + selected = entries[:_INDEX_LIMIT] + rows = [(name, _index_status(row), _first_issue(row) or "—") for name, row in selected] + coverage = f"结果索引：共 {len(entries)} 项，展示 {len(selected)} 项（失败/未确认优先）。" + if len(entries) > len(selected): + coverage += f" 另 {len(entries) - len(selected)} 项见 report.json。" + explanation = ("完整逐用例、检查点、命令、环境和源码指纹保留在同目录 report.json。" + "在 GitHub Actions Summary 中，请到本次运行的 Artifacts 下载对应报告；" + "下面的相对链接用于下载后同目录查看。") + markdown = "
简短结果索引与完整证据入口（展开详情）\n\n" + markdown += coverage + "\n\n| 项目 | 结果 | 首个问题 |\n| --- | --- | --- |\n" + for row in rows: + markdown += "| " + " | ".join(escape(value).replace("|", "|") for value in row) + " |\n" + markdown += "\n" + explanation + "\n\n[完整机器报告](report.json)\n\n
\n\n" + header = "项目结果首个问题" + body = "".join("" + "".join("" + escape(value) + "" for value in row) + "" for row in rows) + page = ('
简短结果索引与完整证据入口（展开详情）

' + + escape(coverage) + '

' + header + '' + body + + '

' + escape(explanation) + + '

完整机器报告

') + return markdown, page + + +def _checkpoint(comparison, name="logits"): + return _named(comparison.get("checkpoints", []), name) + + +def _named(rows, name): + matches = [row for row in rows if row.get("name") == name] + if len(matches) > 1: + return {"passed": False, "reason": f"重复的输出比较：{name}"} + return matches[0] if matches else {} + + +def _subgraph_output(row): + """Adapt the existing subgraph schema without rewriting saved evidence.""" + result = dict(row) + if "max_abs_error" in row: + result["max_abs"] = row["max_abs_error"] + first = row.get("firstdiff") + if isinstance(first, dict) and first.get("reason"): + result.setdefault("reason", first["reason"]) + return result + + +def numerical_summary(report): + """Select explicit output comparisons, never a recursive max over checkpoints.""" + gate, cases = report.get("gate"), report.get("cases", []) + comparisons, notes = [], [] + description = _short(report.get("scope", "详细范围见完整报告。"), 600) + if gate in ("numeric:ir-full-qwen3", "audit:w3-full-saved-evidence"): + mode = report.get("fp32_mode", "未记录") + description = (f"完整预训练 28 层 Qwen3，FP32、L=256、无 KV cache；同一模型、权重和输入，" + f"IR（{mode} 模式）对照 ORT，比较全部位置，包括补齐位置。") + for filename, label in (("logits.npy", "普通最终 logits"), ("diagnostic_logits.npy", "诊断图最终 logits")): + comparisons.append((f"IR {mode} 对 ORT：{label}", [ + _named(case.get("comparison", {}).get("logits", []), filename) for case in cases])) + notes.append("native 是默认 NumPy 算术；reference 是对齐 ORT 浮点行为的兼容模式，两者的通过结果不能互相代替。") + notes.append("中间检查点用于定位，不混入最终 logits 指标。完整 RISC-V/QEMU 模型执行不在此项范围内。") + if gate.startswith("audit:"): + notes.append("本项只重算保存数组，没有重新执行模型，也不算第二人独立运行。") + notes.append("PPL / zero-shot：未评测；需另行固定真实语料、Tokenizer、上下文窗口和评分方式。") + elif gate == "w3-medium-host": + description = "六层随机权重 Qwen3，FP32、L=256；同权重同输入，对比 PyTorch、ORT 和原生 NumPy IR 的最终 logits。" + recorded_cases = cases + cases = cases + [{}] * max(0, _MEDIUM_CASE_COUNT - len(cases)) + comparisons.append(("ORT 对 PyTorch", [_checkpoint(c.get("ort", {}).get("normal", {}).get("pytorch_comparison", {})) for c in cases])) + for level in ("none", "basic", "all"): + comparisons.append((f"IR 原生 NumPy（{level}）对 ORT", [ + _checkpoint(c.get("ir", {}).get(level, {}).get("normal", {}).get("ort_comparison", {})) for c in cases])) + notes.extend(("none/basic/all 分别表示关闭、基础、全部 ScratchV IR 优化，不是 C 编译器的优化等级。", + "中间检查点和因果/补齐隔离检查仍参与原验收，逐项明细保留在 report.json。", + "PPL / zero-shot：未评测；随机权重小模型不用于评价语言能力。")) + notes.append(f"固定场景覆盖：已记录 {len(recorded_cases)}/{_MEDIUM_CASE_COUNT} 个；" + "尚未执行的场景或比较按未记录展示，不作为通过。") + elif gate == "w3_qwen3_subgraphs": + length = report.get("sequence_length", "未记录") + description = (f"预训练第一层的真实维度子图，FP32、L={length}；分别检查 ORT、PyTorch" + "与原生 NumPy IR 的子图输出，不是完整模型执行。") + if length != 256: + notes.append("本次未记录完整 L=256 验收范围；短序列调试结果不能替代正式验收。") + comparisons.append(("ORT 对 PyTorch：子图输出", [ + _subgraph_output(c.get("comparisons", {}).get("ordinary_ort_vs_torch", {})) for c in cases])) + for level in ("none", "basic", "all"): + comparisons.append((f"IR 原生 NumPy（{level}）对 ORT：子图输出", [ + _subgraph_output(c.get("comparisons", {}).get(f"ordinary_ir_{level}_vs_ort", {})) for c in cases])) + elif gate == "unit:attention-backend": + description = "小 Attention 组合图：IR→C→RISC-V→QEMU，同一输入对照 ORT 和独立 NumPy 公式；不是完整 Qwen3。" + for reference, label in (("ir_vs_ort", "Attention IR 原生 NumPy 对 ORT"), + ("qemu_vs_ort", "Attention C→RISC-V/QEMU 对 ORT"), + ("qemu_vs_numpy", "Attention C→RISC-V/QEMU 对独立 NumPy")): + comparisons.append((label, [ + next((e.get(reference, {}) for e in c.get("executions", []) if e.get("optimization") == level), {}) + for c in cases for level in ("none", "all")])) + notes.append("每项汇总关闭/开启 ScratchV IR 优化的结果；C 交叉编译均使用 -O2。") + elif gate == "preparation:w3": + notes.append("本项汇总预备检查；子门禁结果见折叠索引，不代表完整模型或 W3 团队验收。") + elif gate == "w3_layer_diff": + coverage = report.get("coverage", {}) + notes.append(f"检查点覆盖：{coverage.get('compared_checkpoints', '未记录')}/" + f"{coverage.get('total_checkpoints', '未记录')}。只核对保存的检查点，不代表重新执行完整模型。") + if report.get("partial") is True or coverage.get("complete") is False: + notes.append("本次仅比较选定检查点（PARTIAL），不能作为完整覆盖通过。") + if report.get("first_divergence"): + notes.append("首个差异检查点：" + _short(report["first_divergence"])) + elif gate == "numeric:w3-full-worker": + notes.append("本项只执行单个后端并检查普通/诊断图一致性；IR 对 ORT 精度由上层门禁核验。") + elif gate == "preflight:w3-full-assets": + notes.append("本项只核验模型资产和静态规模，没有执行完整模型或数值验收。") + elif gate == "diagnostic:w3-full-localize": + notes.append("COMPLETE 只表示诊断完成，不表示数值门槛通过；numeric_gate_passed 和团队验收仍独立。") + return description, comparisons, notes + + +def summary_views(report): + description, comparisons, notes = numerical_summary(report) + if comparisons: + threshold = report.get("atol") + if isinstance(threshold, (int, float)) and math.isfinite(threshold): + notes.insert(0, f"原验收门槛：最大绝对误差 < {threshold:g}，rtol=0；新增两项指标用于观察，不放宽门槛。") + notes.insert(0, METRIC_EXPLANATION) + notes.append("表内结果只表示对应输出比较；总体结论仍包含执行、覆盖率和原有诊断检查。") + if report.get("coverage_complete") is False: + notes.append("验收覆盖不完整，已测输出通过不能替代全部用例验收。") + required, selected = report.get("required_cases"), report.get("selected_cases") + if isinstance(required, list) and isinstance(selected, list): + notes.append(f"本次选择 {len(selected)}/{len(required)} 个规定用例；已记录 {len(report.get('cases', []))} 个用例。") + seconds = report.get("elapsed_seconds", report.get("seconds")) + if isinstance(seconds, (int, float)) and math.isfinite(seconds): + notes.append(f"本次流程耗时 {seconds:.3f} 秒，包含准备和核验，不代表纯推理速度。") + if report.get("error"): + notes.insert(0, "失败原因：" + _short(report["error"], 600)) + if report.get("passed") is not True and report.get("stage"): + notes.insert(0, "当前/失败阶段：" + _short(report["stage"])) + failed_cases = [_short(c.get("name", "未命名"), 100) for c in report.get("cases", []) if c.get("passed") is False] + if failed_cases: + names = "、".join(failed_cases[:_INDEX_LIMIT]) + remainder = f"，另 {len(failed_cases) - _INDEX_LIMIT} 项" if len(failed_cases) > _INDEX_LIMIT else "" + notes.insert(0, "未通过的用例：" + names + remainder + "。首个问题见折叠索引，完整定位数据见 report.json。") + failed_gates = [row for row in report.get("gates", []) if row.get("passed") is not True] + if failed_gates: + notes.insert(0, f"子门禁未通过/未确认 {len(failed_gates)} 项，名称与首个错误见折叠索引。") + failed_invariants = [row for row in report.get("invariants", []) if row.get("passed") is False] + if failed_invariants: + notes.insert(0, f"因果/补齐隔离检查未通过 {len(failed_invariants)} 项，见折叠索引。") + md = escape(description) + "\n\n" + page = "

" + escape(description) + "

" + if comparisons: + md += compact_metric_table(comparisons) + "\n\n" + page += compact_metric_table_html(comparisons) + md += "\n\n".join(escape(note) for note in notes) + "\n\n" + page += "".join("

" + escape(note) + "

" for note in notes) + return md, page diff --git a/scratchv/verification/numeric_metrics.py b/scratchv/verification/numeric_metrics.py new file mode 100644 index 0000000..5a47965 --- /dev/null +++ b/scratchv/verification/numeric_metrics.py @@ -0,0 +1,140 @@ +"""Descriptive tensor metrics; these never decide numerical acceptance. + +Norms are accumulated in float64 relative to a power-of-two scale. Iteration +is bounded even for noncontiguous arrays and memory-mapped model outputs. +""" +from __future__ import annotations + +import math + +import numpy as np + + +def undefined_numeric_metrics(reason): + """JSON-safe fields for a comparison that has no numerical meaning.""" + return dict(relative_l2=None, cosine_similarity=None, + relative_l2_reason=reason, cosine_similarity_reason=reason) + + +class _ScaledSquares: + def __init__(self): + self.exponent = 0 + self.squares = 0.0 + + def add(self, values, exponent_shift=0): + maximum = float(np.max(np.abs(values), initial=0.0)) + if maximum == 0: + return + _, exponent = math.frexp(maximum) + normalized = np.ldexp(values, -exponent) + squares = float(np.sum(normalized * normalized, dtype=np.float64)) + exponent += exponent_shift + if self.squares == 0: + self.exponent, self.squares = exponent, squares + elif exponent > self.exponent: + self.squares = math.ldexp(self.squares, 2 * (self.exponent - exponent)) + squares + self.exponent = exponent + else: + self.squares += math.ldexp(squares, 2 * (exponent - self.exponent)) + + +def _chunks(actual, expected, block_elements): + return np.nditer([actual, expected], flags=["external_loop", "buffered"], + op_flags=[["readonly"], ["readonly"]], order="C", + buffersize=block_elements) + + +def numeric_metrics(actual, expected, *, block_elements=262144): + """Return cosine and ||actual-expected||_2 / ||expected||_2. + + Shapes and dtypes must match, with real numeric (at most FP64) data. Empty + and nonfinite tensors have undefined metrics. Cosine is undefined if either + norm is zero. A zero reference gives relative L2 zero only when actual is + also zero; otherwise it is undefined. Unrepresentable relative L2 values + are reported as null with a reason, never as infinity or an invented zero. + """ + if type(block_elements) is not int or block_elements < 1: + raise ValueError("block_elements must be a positive integer") + if not isinstance(actual, np.ndarray) or not isinstance(expected, np.ndarray): + return undefined_numeric_metrics("outputs must be NumPy arrays") + if actual.shape != expected.shape: + return undefined_numeric_metrics("shape mismatch") + if actual.dtype != expected.dtype: + return undefined_numeric_metrics("dtype mismatch") + if actual.dtype.kind not in "biuf" or (actual.dtype.kind == "f" and actual.dtype.itemsize > 8): + return undefined_numeric_metrics("metrics require real numeric dtypes up to float64") + if not actual.size: + return undefined_numeric_metrics("empty tensor") + + actual_norm, expected_norm, difference_norm = (_ScaledSquares() for _ in range(3)) + identical = True + for a, b in _chunks(actual, expected, block_elements): + if not np.isfinite(a).all() or not np.isfinite(b).all(): + return undefined_numeric_metrics("nonfinite output or reference") + equal = bool(np.array_equal(a, b)) + identical = identical and equal + a64, b64 = a.astype(np.float64), b.astype(np.float64) + actual_norm.add(a64) + expected_norm.add(b64) + if equal: + continue + if actual.dtype.kind in "biu": + # Subtract before rounding to FP64: int64/uint64 neighbours above + # 2**53 must retain their nonzero difference. Copies stay bounded. + difference = (a.astype(object) - b.astype(object)).astype(np.float64) + difference_norm.add(difference) + else: + with np.errstate(over="ignore", invalid="ignore"): + difference = a64 - b64 + overflow = ~np.isfinite(difference) + if overflow.any(): + difference_norm.add(difference[~overflow]) + # Only overflowing differences are halved, so unrelated tiny + # values are not lost to a premature division by two. + difference_norm.add(a64[overflow] * 0.5 - b64[overflow] * 0.5, 1) + else: + difference_norm.add(difference) + + result = dict(relative_l2=None, cosine_similarity=None, + relative_l2_reason=None, cosine_similarity_reason=None) + if expected_norm.squares == 0: + if actual_norm.squares == 0: + result["relative_l2"] = 0.0 + else: + result["relative_l2_reason"] = "zero reference norm with nonzero actual norm" + elif difference_norm.squares == 0: + result["relative_l2"] = 0.0 + else: + factor = math.sqrt(difference_norm.squares / expected_norm.squares) + try: + relative = math.ldexp(factor, difference_norm.exponent - expected_norm.exponent) + except OverflowError: + relative = math.inf + if not math.isfinite(relative): + result["relative_l2_reason"] = "relative L2 exceeds float64 range" + elif relative == 0: + result["relative_l2_reason"] = "nonzero relative L2 is below float64 range" + else: + result["relative_l2"] = relative + + if actual_norm.squares == 0 or expected_norm.squares == 0: + result["cosine_similarity_reason"] = "cosine is undefined for a zero norm" + elif identical: + result["cosine_similarity"] = 1.0 + else: + # Scale each vector independently; squaring unscaled FP64 values can + # overflow, and using one shared scale can erase the smaller vector. + dot, correction = 0.0, 0.0 + for a, b in _chunks(actual, expected, block_elements): + a64 = np.ldexp(a.astype(np.float64), -actual_norm.exponent) + b64 = np.ldexp(b.astype(np.float64), -expected_norm.exponent) + term = float(np.sum(a64 * b64, dtype=np.float64)) + total = dot + term + correction += ((dot - total) + term if abs(dot) >= abs(term) + else (term - total) + dot) + dot = total + cosine = ((dot + correction) / math.sqrt(actual_norm.squares) + / math.sqrt(expected_norm.squares)) + # Cauchy-Schwarz bounds the exact result; trim only FP64 roundoff. + result["cosine_similarity"] = min(1.0, max(-1.0, cosine)) + return result diff --git a/scripts/verify_w3_evidence.py b/scripts/verify_w3_evidence.py index 56be3c0..e4bb4fd 100644 --- a/scripts/verify_w3_evidence.py +++ b/scripts/verify_w3_evidence.py @@ -24,6 +24,8 @@ MAX_JSON_BYTES = 8 * 1024**2 HASH = re.compile(r"[0-9a-f]{64}\Z") +OBSERVATIONAL_FIELDS = frozenset(("relative_l2", "cosine_similarity", + "relative_l2_reason", "cosine_similarity_reason")) SCOPE = ("Offline consistency and numerical recheck of saved seven-case arrays. " "No model execution, independent-person reproduction, current-source execution, " "Linux execution, Nightly run, or team acceptance is established. " @@ -80,6 +82,37 @@ def source_difference(recorded, current): "This audit recomputes stored-array comparisons only."} +def comparison_matches_saved(recomputed, saved): + """Permit only the absent four-field observation group in legacy rows. + + Old reports predate these descriptive metrics. Existing fields and any + recorded metrics must still match the arrays; a partial group is invalid. + This does not relax profiles, source evidence, thresholds or PASS checks. + """ + if isinstance(recomputed, dict): + if not isinstance(saved, dict): + return False + wanted = set(recomputed) + observed = OBSERVATIONAL_FIELDS & saved.keys() + if OBSERVATIONAL_FIELDS <= wanted: + if not observed: + wanted -= OBSERVATIONAL_FIELDS + elif observed != OBSERVATIONAL_FIELDS: + return False + else: + # JSON booleans must not impersonate numeric 0/1 observations. + if any(type(saved[key]) is not type(recomputed[key]) + for key in OBSERVATIONAL_FIELDS): + return False + if set(saved) != wanted: + return False + return all(comparison_matches_saved(recomputed[key], saved[key]) for key in wanted) + if isinstance(recomputed, list): + return (isinstance(saved, list) and len(recomputed) == len(saved) + and all(comparison_matches_saved(a, b) for a, b in zip(recomputed, saved))) + return recomputed == saved + + def verify(directory, *, model_dir=None, expected_report_sha256=None, progress=None): """Reuse the runner's numerical validators; never call its worker launcher.""" directory = Path(directory).resolve() @@ -149,7 +182,8 @@ def verify(directory, *, model_dir=None, expected_report_sha256=None, progress=N profiles.append(evidence["ir"]["fp32_profile"]) comparison = full.compare_case(folder, valid) require(comparison["passed"], f"Recomputed logits fail threshold: {name}") - require(comparison == row.get("comparison"), f"Stored comparison differs from arrays: {name}") + require(comparison_matches_saved(comparison, row.get("comparison")), + f"Stored comparison differs from arrays: {name}") verified.append({"name": name, "passed": True, "valid_length": valid, "input_sha256": row["input_sha256"], "comparison": comparison}) if progress: @@ -158,7 +192,7 @@ def verify(directory, *, model_dir=None, expected_report_sha256=None, progress=N "Inconsistent numerical profiles") invariants = full.invariants(directory, set(CASE_NAMES)) require(len(invariants) == 4 and all(row["passed"] for row in invariants), "Recomputed invariants fail") - require(invariants == top.get("invariants"), "Stored invariants differ from arrays") + require(comparison_matches_saved(invariants, top.get("invariants")), "Stored invariants differ from arrays") require(sha256_file(top_path) == digest, "Producer report changed during audit") return {"gate": "audit:w3-full-saved-evidence", "schema_version": 1, "status": "PASS", "passed": True, "scope": SCOPE, "model_executed": False, "independent_reproduction": False, @@ -168,6 +202,10 @@ def verify(directory, *, model_dir=None, expected_report_sha256=None, progress=N "producer_git": top.get("git"), "auditor": current, "source_comparison": differences, "fp32_mode": top["fp32_mode"], "fp32_profile": profiles[0], "cases": verified, "invariants": invariants, "coverage_complete": True, + "comparison_compatibility": { + "allowed_legacy_missing_fields": sorted(OBSERVATIONAL_FIELDS), + "scope": "Only a wholly absent observation group is accepted for legacy tensor rows. " + "All observations in this audit are recomputed from saved arrays, not a new model run."}, "checkpoint_threshold_scope": "Checkpoints are finite FP32 diagnostics; the strict numerical gate " "applies to every ordinary/diagnostic output logit, including padding."} diff --git a/tests/test_numeric_metrics.py b/tests/test_numeric_metrics.py new file mode 100644 index 0000000..6f4aee8 --- /dev/null +++ b/tests/test_numeric_metrics.py @@ -0,0 +1,154 @@ +"""Independent metric formulas, extreme ranges and unchanged gate decisions.""" +import json +import math + +import numpy as np +import pytest + +from scratchv.verification.numeric_metrics import numeric_metrics + + +@pytest.mark.parametrize("actual,reference,cosine,relative", [ + ([3, 4], [3, 4], 1, 0), + ([6, 8], [3, 4], 1, 1), + ([-3, -4], [3, 4], -1, 2), + ([1, 0], [0, 1], 0, math.sqrt(2)), + ([0, 0], [3, 4], None, 1), + ([0, 0], [0, 0], None, 0), + ([3, 4], [0, 0], None, None), +]) +def test_analytic_vectors(actual, reference, cosine, relative): + result = numeric_metrics(np.array(actual, dtype=np.float64), + np.array(reference, dtype=np.float64), block_elements=1) + for key, value in (("cosine_similarity", cosine), ("relative_l2", relative)): + if value is None: + assert result[key] is None and result[key + "_reason"] + else: + assert result[key] == pytest.approx(value, rel=2e-15, abs=1e-15) + assert result[key + "_reason"] is None + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("scale", [np.finfo(np.float64).max, 1e-300, + np.nextafter(np.float64(0), np.float64(1))]) +def test_squares_and_differences_do_not_overflow_or_erase_tiny_values(scale): + reference = np.array([scale, scale], dtype=np.float64) + opposite = numeric_metrics(-reference, reference, block_elements=1) + assert opposite["relative_l2"] == pytest.approx(2) + assert opposite["cosine_similarity"] == pytest.approx(-1) + orthogonal = numeric_metrics(np.array([scale, -scale]), reference) + assert orthogonal["relative_l2"] == pytest.approx(math.sqrt(2)) + assert orthogonal["cosine_similarity"] == 0 + + +def test_each_vector_has_its_own_scale_for_cosine(): + maximum = np.finfo(np.float64).max + minimum = np.nextafter(np.float64(0), np.float64(1)) + result = numeric_metrics(np.array([maximum]), np.array([minimum])) + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] is None + assert "exceeds" in result["relative_l2_reason"] + result = numeric_metrics(np.array([maximum, minimum]), np.array([maximum, 0])) + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] is None + assert "below" in result["relative_l2_reason"] + + +@pytest.mark.parametrize("dtype,base", [(np.int64, 2**63 - 2), (np.uint64, 2**64 - 2)]) +def test_integer_neighbours_keep_nonzero_difference_above_fp64_exact_range(dtype, base): + actual, expected = np.array([base + 1], dtype), np.array([base], dtype) + result = numeric_metrics(actual, expected) + assert result["relative_l2"] == pytest.approx(1 / base, rel=2e-15, abs=0) + assert result["relative_l2"] > 0 + assert result["cosine_similarity"] == 1 + + +def test_signed_integer_subtraction_cannot_wrap(): + limits = np.iinfo(np.int64) + result = numeric_metrics(np.array([limits.min]), np.array([limits.max])) + assert result["relative_l2"] == pytest.approx((2**64 - 1) / (2**63 - 1)) + assert result["cosine_similarity"] == -1 + + +@pytest.mark.parametrize("actual,expected,reason", [ + (None, np.ones(1), "NumPy arrays"), + (np.ones(2), np.ones(1), "shape mismatch"), + (np.ones(1, np.float32), np.ones(1, np.float64), "dtype mismatch"), + (np.ones(1, complex), np.ones(1, complex), "real numeric"), + (np.empty((2, 0)), np.empty((2, 0)), "empty"), + (np.array([1, np.nan]), np.ones(2), "nonfinite"), + (np.array([1, np.inf]), np.array([1, np.inf]), "nonfinite"), +]) +def test_undefined_inputs_have_explicit_reasons(actual, expected, reason): + result = numeric_metrics(actual, expected) + for key in ("relative_l2", "cosine_similarity"): + assert result[key] is None and reason in result[key + "_reason"] + json.dumps(result, allow_nan=False) + + +@pytest.mark.parametrize("block", [0, -1, True, 1.5]) +def test_bad_block_size_is_rejected(block): + with pytest.raises(ValueError, match="positive integer"): + numeric_metrics(np.ones(1), np.ones(1), block_elements=block) + + +@pytest.mark.parametrize("block", [1, 7, 262144]) +def test_strided_memmap_matches_independent_vector_formula(tmp_path, block): + source = np.lib.format.open_memmap(tmp_path / "arrays.npy", mode="w+", + dtype=np.float32, shape=(2, 23, 37)) + source[:] = np.random.default_rng(487).normal(size=source.shape) + actual, expected = source[0, ::2, ::-3].T, source[1, ::2, ::-3].T + assert not actual.flags.c_contiguous + a, b = actual.astype(np.float64).ravel(), expected.astype(np.float64).ravel() + result = numeric_metrics(actual, expected, block_elements=block) + assert result["relative_l2"] == pytest.approx(np.linalg.norm(a - b) / np.linalg.norm(b), rel=3e-15) + assert result["cosine_similarity"] == pytest.approx(np.dot(a, b) / np.linalg.norm(a) / np.linalg.norm(b), + rel=3e-14, abs=1e-16) + + +def test_iteration_never_exceeds_requested_buffer(monkeypatch): + import scratchv.verification.numeric_metrics as module + original = module._chunks + lengths = [] + + def observed(*args): + for a, b in original(*args): + lengths.append(a.size) + yield a, b + + monkeypatch.setattr(module, "_chunks", observed) + array = np.arange(20000, dtype=np.float32).reshape(100, 200).T[:, ::2] + result = numeric_metrics(array, array + 1, block_elements=127) + assert result["relative_l2"] > 0 + assert len(lengths) > 2 and max(lengths) <= 127 + + +@pytest.mark.parametrize("gate", ["w2", "w3"]) +def test_new_metrics_do_not_weaken_absolute_gate(gate): + from probes.w2_qwen3_small.diagnostics import tensor_diff + from probes.w3_qwen3_full.comparison import compare_tensor + compare = tensor_diff if gate == "w2" else compare_tensor + actual = np.array([2, 4], dtype=np.float32) + expected = np.array([1, 2], dtype=np.float32) + result = compare(actual, expected) + assert not result["passed"] and result["max_abs"] == 2 + assert result["cosine_similarity"] == 1 + assert result["relative_l2"] == 1 + passed = compare(actual, actual) + assert passed["passed"] and passed["max_abs"] == 0 + assert passed["cosine_similarity"] == 1 and passed["relative_l2"] == 0 + invalid = compare(actual, expected.astype(np.float64)) + assert not invalid["passed"] + for key in ("relative_l2", "cosine_similarity"): + assert invalid[key] is None and invalid[key + "_reason"] + + +def test_empty_gate_policies_are_preserved(): + from probes.w2_qwen3_small.diagnostics import tensor_diff + from probes.w3_qwen3_full.comparison import compare_tensor + array = np.empty(0, dtype=np.float32) + w2, w3 = tensor_diff(array, array), compare_tensor(array, array) + assert w2["passed"] and not w3["passed"] + for row in (w2, w3): + assert row["relative_l2"] is None and row["cosine_similarity"] is None + assert row["relative_l2_reason"] and row["cosine_similarity_reason"] diff --git a/tests/test_numeric_summary.py b/tests/test_numeric_summary.py new file mode 100644 index 0000000..432055f --- /dev/null +++ b/tests/test_numeric_summary.py @@ -0,0 +1,123 @@ +"""Keep concise reports truthful when evidence is missing or undefined.""" +from html import unescape +import re + +import pytest + +from probes.numeric_summary import ( + METRIC_EXPLANATION, + compact_metric_table, + compact_metric_table_html, +) + + +def record(**updates): + row = dict(passed=True, max_abs=1e-6, relative_l2=2e-7, cosine_similarity=0.999) + row.update(updates) + return row + + +def test_worst_metrics_are_aggregated_independently_without_mutating_evidence(): + rows = [record(), {"passed": True, "max_abs": 3e-6, "relative_l2": 1e-8, "cosine_similarity": 0.9}] + original = [dict(row) for row in rows] + result = compact_metric_table([("IR vs ORT", rows)]) + assert format(3e-6, ".16e") in result + assert format(2e-7, ".16e") in result + assert format(0.9, ".17g") in result + assert "门槛通过（2/2）" in result + assert rows == original + + +@pytest.mark.parametrize("render", [compact_metric_table, compact_metric_table_html]) +def test_legacy_pass_does_not_claim_complete_metric_evidence(render): + result = render([("legacy", [{"passed": True, "max_abs": 0.0}])]) + assert "门槛通过（1/1）；指标不完整" in result + assert result.count("未记录 1/1") == 2 + + +@pytest.mark.parametrize("rows", [[], [{}], [{"passed": "true"}], [{"passed": 1}]]) +def test_empty_missing_and_nonboolean_status_never_pass(rows): + result = compact_metric_table([("unexecuted", rows)]) + assert "门槛通过" not in result + assert "未测" in result or "未确认 1" in result + + +def test_failures_are_not_hidden_by_passing_rows(): + failed = record() + failed["passed"] = False + result = compact_metric_table([("mixed", [record(), failed, {}])]) + assert "门槛未通过（通过 1/3，失败 1，未确认 1）" in result + assert "已记录 2/3" in result + assert "未记录 1/3" in result + + +def test_undefined_values_show_reason_and_coverage_not_zero(): + undefined = dict(passed=True, max_abs=0.0, relative_l2=None, cosine_similarity=None, + relative_l2_reason="zero reference norm", cosine_similarity_reason="zero norm") + result = compact_metric_table([("partial", [record(), undefined, {}])]) + assert "未定义 1/3（zero reference norm ×1）" in result + assert "未定义 1/3（zero norm ×1）" in result + assert "已记录 1/3" in result + assert "未记录 1/3" in result + + +def test_undefined_general_reason_is_preserved(): + result = compact_metric_table([("bad", [{"passed": False, "max_abs": None, "reason": "shape mismatch"}])]) + assert "未定义 1/1（shape mismatch ×1）" in result + + +@pytest.mark.parametrize("invalid", [float("nan"), float("inf"), -1.0, True, "0"]) +def test_invalid_error_values_cannot_be_reported_as_numeric_evidence(invalid): + row = record() + row["relative_l2"] = invalid + result = compact_metric_table([("bad", [row])]) + assert "未定义 1/1（数值无效 ×1）" in result + assert "指标不完整" in result + + +def test_small_nonzero_errors_and_near_one_cosines_remain_distinguishable(): + row = dict(passed=True, max_abs=5e-324, relative_l2=1e-100, cosine_similarity=0.9999999999999999) + result = compact_metric_table([("tiny", [row])]) + assert format(5e-324, ".16e") in result + assert format(1e-100, ".16e") in result + assert "0.99999999999999989" in result + + +def test_cosine_domain_and_existing_gate_flags_are_not_reinterpreted(): + # Cosine observes directional similarity; it does not impose a new gate. + result = compact_metric_table([("opposite", [record(cosine_similarity=-1.0)])]) + assert "门槛通过（1/1）" in result and "未定义" not in result + invalid = compact_metric_table([("invalid", [record(cosine_similarity=1.1)])]) + assert "未定义 1/1（数值无效 ×1）" in invalid + + +def test_entirely_undefined_metrics_do_not_acquire_a_numeric_value(): + row = dict(passed=False, max_abs=None, relative_l2=None, cosine_similarity=None) + result = compact_metric_table([("undefined", [row])]) + assert result.count("未定义 1/1（未提供原因 ×1）") == 3 + assert "已记录" not in result + + +def test_html_and_markdown_have_same_values_order_and_safe_untrusted_text(): + labels = ["second " + front = riscv._report_views(report)[filename].split("" not in front + assert "<script>" in front + + +def test_report_rendering_does_not_mutate_numeric_evidence(): + for report, render in ((host_report(), run._host_report_views), (qemu_report(), riscv._report_views)): + before = deepcopy(report) + render(report) + assert report == before diff --git a/tests/test_w2_runtime_model.py b/tests/test_w2_runtime_model.py index 99969f4..333147b 100644 --- a/tests/test_w2_runtime_model.py +++ b/tests/test_w2_runtime_model.py @@ -169,7 +169,7 @@ def replace(path, dest): assert not (tmp_path / ".report.json.tmp").exists() for name in ("report.md", "report.html"): if name != failure: - assert "FAIL" in (tmp_path / name).read_text() + assert "FAIL" in (tmp_path / name).read_text(encoding="utf-8") def test_main_preserves_existing_evidence(tmp_path): diff --git a/tests/test_w3_evidence_verifier.py b/tests/test_w3_evidence_verifier.py index f054d4e..8325226 100644 --- a/tests/test_w3_evidence_verifier.py +++ b/tests/test_w3_evidence_verifier.py @@ -237,6 +237,75 @@ def test_reported_error_must_match_actual_arrays(evidence): audit.verify(root) +def remove_observations(value): + if isinstance(value, dict): + for key in audit.OBSERVATIONAL_FIELDS: + value.pop(key, None) + for child in value.values(): + remove_observations(child) + elif isinstance(value, list): + for child in value: + remove_observations(child) + + +def test_legacy_observation_absence_recomputes_metrics_without_rewriting_evidence(evidence, monkeypatch): + root, top = evidence + remove_observations(top) + write_json(root / "report.json", top) + original = (root / "report.json").read_bytes() + monkeypatch.setattr(full, "run_worker", lambda *a, **k: pytest.fail("Audit executed a model")) + result = audit.verify(root, expected_report_sha256=sha256_file(root / "report.json")) + assert result["passed"] and result["trusted_report_hash_checked"] + row = result["cases"][0]["comparison"]["logits"][0] + assert row["relative_l2"] == 0 and row["cosine_similarity"] is None + assert "zero norm" in row["cosine_similarity_reason"] + assert "relative_l2" in result["invariants"][0] + assert set(result["comparison_compatibility"]["allowed_legacy_missing_fields"]) == audit.OBSERVATIONAL_FIELDS + assert result["model_executed"] is False and result["independent_reproduction"] is False + assert result["w3_exit_accepted"] is False + assert (root / "report.json").read_bytes() == original + + +@pytest.mark.parametrize("target", ["logits", "padding", "checkpoint", "invariant"]) +@pytest.mark.parametrize("mutation", ["value", "reason", "partial", "boolean"]) +def test_recorded_observations_must_match_at_every_nested_location(evidence, target, mutation): + root, top = evidence + comparison = top["cases"][3]["comparison"] + rows = {"logits": comparison["logits"][0], + "padding": comparison["logits"][0]["padding_queries"], + "checkpoint": comparison["checkpoints"][0], "invariant": top["invariants"][0]} + row = rows[target] + if mutation == "value": + row["relative_l2"] = 0.125 + elif mutation == "reason": + row["cosine_similarity_reason"] = "invented" + elif mutation == "partial": + del row["relative_l2"] + else: + row["relative_l2"] = False + write_json(root / "report.json", top) + with pytest.raises(ValueError, match="Stored (comparison|invariants) differ"): + audit.verify(root) + + +@pytest.mark.parametrize("mutation", ["max_abs", "missing_atol", "extra_key", "wrong_order"]) +def test_legacy_compatibility_does_not_relax_original_comparison_fields(evidence, mutation): + root, top = evidence + remove_observations(top) + comparison = top["cases"][0]["comparison"] + if mutation == "max_abs": + comparison["logits"][0]["max_abs"] = 1e-6 + elif mutation == "missing_atol": + del comparison["logits"][0]["atol"] + elif mutation == "extra_key": + comparison["logits"][0]["relative_l2_typo"] = 0 + else: + comparison["logits"].reverse() + write_json(root / "report.json", top) + with pytest.raises(ValueError, match="Stored comparison differs"): + audit.verify(root) + + def test_invariants_recomputed_even_when_both_backends_agree(evidence): root, top = evidence def modify(report, folder): diff --git a/tests/test_w3_qwen3_subgraphs.py b/tests/test_w3_qwen3_subgraphs.py index caead92..cee9968 100644 --- a/tests/test_w3_qwen3_subgraphs.py +++ b/tests/test_w3_qwen3_subgraphs.py @@ -29,6 +29,42 @@ def test_strict_comparison_rejects_threshold_shape_dtype_and_nonfinite(): json.dumps(nonfinite, allow_nan=False) +def test_subgraph_metrics_keep_existing_error_field_and_absolute_gate(): + actual = np.array([1, 2], np.float32) + expected = np.array([1, 1], np.float32) + row = run.tensor_diff(actual, expected) + assert not row["passed"] and row["max_abs_error"] == 1 + assert "max_abs" not in row # Saved report schema remains compatible. + assert row["relative_l2"] == pytest.approx(1 / np.sqrt(2)) + assert row["cosine_similarity"] == pytest.approx(3 / np.sqrt(10)) + from probes.w3_summary import summary_views + report = {"gate": "w3_qwen3_subgraphs", "sequence_length": 256, "cases": [ + {"comparisons": {"ordinary_ort_vs_torch": row, + **{f"ordinary_ir_{level}_vs_ort": row for level in run.LEVELS}}}]} + markdown, page = summary_views(report) + assert format(row["max_abs_error"], ".16e") in markdown + assert format(row["relative_l2"], ".16e") in markdown + assert format(row["cosine_similarity"], ".17g") in page + assert "指标不完整" not in markdown + + +@pytest.mark.parametrize("invalid", ["shape", "dtype", "nonfinite"]) +def test_invalid_subgraph_comparisons_have_undefined_metric_reasons(invalid): + expected = np.array([1, 2], np.float32) + actual = expected.copy() + if invalid == "shape": + actual = actual.reshape(1, 2) + elif invalid == "dtype": + actual = actual.astype(np.float64) + else: + actual[0] = np.nan + row = run.tensor_diff(actual, expected) + assert not row["passed"] + for name in ("relative_l2", "cosine_similarity"): + assert row[name] is None and row[name + "_reason"] + json.dumps(row, allow_nan=False) + + def test_source_authentication_rejects_tampering_before_tensor_access(tmp_path): checkpoint = tmp_path / "model.safetensors" checkpoint.write_bytes(b"explicit authentication unit fixture, not a model") diff --git a/tests/test_w3_summary.py b/tests/test_w3_summary.py new file mode 100644 index 0000000..8df2126 --- /dev/null +++ b/tests/test_w3_summary.py @@ -0,0 +1,247 @@ +"""Human summaries select final outputs from the real W3 report schemas.""" +from copy import deepcopy +import json + +import pytest + +from probes.w3_common import write_reports +from probes.w3_summary import evidence_index_views, numerical_summary, summary_views + + +def metric(value=1e-6, *, name=None, passed=True): + row = dict(passed=passed, max_abs=value, relative_l2=2e-7, cosine_similarity=0.999, + atol=1e-4, rtol=0) + if name is not None: + row["name"] = name + return row + + +def test_full_logits_are_not_replaced_by_large_checkpoint_or_position_errors(): + ordinary = metric(name="logits.npy") + ordinary["valid_queries"] = metric(0.1, passed=False) + diagnostic = metric(3e-6, name="diagnostic_logits.npy") + report = {"gate": "numeric:ir-full-qwen3", "fp32_mode": "native", "cases": [ + {"name": "full_seed_0", "passed": True, "comparison": { + "logits": [ordinary, diagnostic], "checkpoints": [metric(0.5, name="layer_0.output")], + "worst": metric(0.5), "max_logits_abs": 0.5}}]} + before = deepcopy(report) + description, comparisons, notes = numerical_summary(report) + assert comparisons[0][1] == [ordinary] + assert comparisons[1][1] == [diagnostic] + assert "native" in description and "reference" in " ".join(notes) + markdown, _ = summary_views(report) + assert format(0.5, ".16e") not in markdown + assert format(0.1, ".16e") not in markdown + assert report == before + + +def test_full_missing_case_comparison_remains_an_explicit_placeholder(): + report = {"gate": "numeric:ir-full-qwen3", "fp32_mode": "reference", "cases": [ + {"name": "okay", "comparison": {"logits": [metric(name="logits.npy"), metric(name="diagnostic_logits.npy")]}}, + {"name": "failed_worker", "passed": False, "workers": [{"backend": "ir", "passed": False}]}]} + markdown, page = summary_views(report) + assert "未记录 1/2" in markdown and "未记录 1/2" in page + assert "门槛未通过" in markdown + assert "failed_worker" in markdown + + +def test_medium_schema_uses_normal_final_logits_not_diagnostic_checkpoints(): + ort_logits = metric(name="logits") + ir_logits = metric(2e-6, name="logits") + irrelevant = metric(0.9, name="layer_5.output", passed=False) + case = {"name": "short_17", "ort": {"normal": {"pytorch_comparison": {"checkpoints": [ort_logits]}}}, + "ir": {level: {"normal": {"ort_comparison": {"checkpoints": [ir_logits]}}, + "diagnostic": {"ort_comparison": {"checkpoints": [irrelevant]}}} + for level in ("none", "basic", "all")}} + _, comparisons, _ = numerical_summary({"gate": "w3-medium-host", "cases": [case]}) + assert comparisons[0][1] == [ort_logits] + [{}] * 6 + assert all(rows == [ir_logits] + [{}] * 6 for _, rows in comparisons[1:]) + markdown, page = summary_views({"gate": "w3-medium-host", "cases": [case]}) + assert "未记录 6/7" in markdown and "已记录 1/7" in page + assert "门槛通过" not in markdown + + +def test_missing_medium_logit_is_not_replaced_by_passing_intermediate(): + case = {"ort": {"normal": {"pytorch_comparison": {"checkpoints": [metric(name="embedding")]}}}} + _, comparisons, _ = numerical_summary({"gate": "w3-medium-host", "cases": [case]}) + assert comparisons[0][1] == [{}] * 7 + markdown, _ = summary_views({"gate": "w3-medium-host", "cases": [case]}) + assert "门槛通过" not in markdown + + +@pytest.mark.parametrize("length", [17, 256]) +def test_subgraphs_use_actual_length_and_ordinary_outputs(length): + def subgraph_metric(value): + row = metric(value) + row["max_abs_error"] = row.pop("max_abs") + return row + + comparisons = {"ordinary_ort_vs_torch": subgraph_metric(4e-6), + **{f"ordinary_ir_{level}_vs_ort": subgraph_metric(5e-6) for level in ("none", "basic", "all")}, + "ir_none_vs_ort": {"checkpoints": [metric(0.7)]}} + report = {"gate": "w3_qwen3_subgraphs", "sequence_length": length, + "cases": [{"name": "rmsnorm", "comparisons": comparisons}]} + description, rows, notes = numerical_summary(report) + assert f"L={length}" in description + assert rows[0][1] == [{**comparisons["ordinary_ort_vs_torch"], "max_abs": 4e-6}] + assert rows[1][1] == [{**comparisons["ordinary_ir_none_vs_ort"], "max_abs": 5e-6}] + assert "max_abs" not in comparisons["ordinary_ort_vs_torch"] + markdown, page = summary_views(report) + assert format(4e-6, ".16e") in markdown and format(5e-6, ".16e") in page + assert "指标不完整" not in markdown + assert ("短序列调试" in " ".join(notes)) == (length != 256) + + +def test_attention_missing_optimization_does_not_vanish_from_table(): + execution = {"optimization": "none", "ir_vs_ort": metric(), "qemu_vs_ort": metric(2e-6), + "qemu_vs_numpy": metric(3e-6)} + report = {"gate": "unit:attention-backend", "cases": [{"name": "full17", "executions": [execution]}]} + _, comparisons, _ = numerical_summary(report) + assert len(comparisons) == 3 + assert all(rows[1] == {} for _, rows in comparisons) + markdown, _ = summary_views(report) + assert "未记录 1/2" in markdown + assert "门槛通过" not in markdown + + +def test_audit_is_distinguished_from_execution_and_language_quality(): + description, _, notes = numerical_summary({"gate": "audit:w3-full-saved-evidence", "fp32_mode": "reference"}) + assert "reference" in description + text = " ".join(notes) + assert "没有重新执行模型" in text and "不算第二人独立运行" in text + assert "PPL / zero-shot：未评测" in text + + +def test_partial_coverage_and_failed_invariants_remain_visible(): + report = {"gate": "numeric:ir-full-qwen3", "coverage_complete": False, + "required_cases": ["one", "two"], "selected_cases": ["one"], "cases": [], + "invariants": [{"name": "changed_future", "passed": False}]} + markdown, _ = summary_views(report) + assert "覆盖不完整" in markdown + assert "选择 1/2" in markdown + assert "隔离检查未通过 1 项" in markdown + + +def test_duplicate_output_names_are_not_silently_reduced_to_first_pass(): + report = {"gate": "numeric:ir-full-qwen3", "cases": [{"comparison": {"logits": [ + metric(name="logits.npy"), metric(1.0, name="logits.npy", passed=False)]}}]} + _, rows, _ = numerical_summary(report) + assert rows[0][1][0]["passed"] is False + assert "重复" in rows[0][1][0]["reason"] + + +def test_views_fold_full_evidence_without_changing_overall_failed_status(tmp_path): + report = {"gate": "numeric:ir-full-qwen3", "passed": False, "status": "FAIL", "atol": 1e-4, + "cases": [{"passed": True, "comparison": {"logits": [metric(name="logits.npy"), + metric(name="diagnostic_logits.npy")], "checkpoints": [metric(0.8, name="layer_3.output")]}}], + "invariants": [{"passed": False}], "error": ""} + before = deepcopy(report) + write_reports(tmp_path, report) + assert json.loads((tmp_path / "report.json").read_text(encoding="utf-8")) == before + markdown = (tmp_path / "report.md").read_text(encoding="utf-8") + page = (tmp_path / "report.html").read_text(encoding="utf-8") + assert "Result: **FAIL**" in markdown + assert "门槛通过" in markdown and "总体结论" in markdown + assert "
" in markdown and "
" in page + assert markdown.index("最大绝对误差") < markdown.index("
") + assert '"checkpoints"' not in markdown + assert "[完整机器报告](report.json)" in markdown + assert 'href="report.json"' in page + assert "" + front = riscv._report_views(report)[filename].split("" not in front + assert "<script>" in front + + +def test_report_rendering_does_not_mutate_numeric_evidence(): + for report, render in ((host_report(), run._host_report_views), (qemu_report(), riscv._report_views)): + before = deepcopy(report) + render(report) + assert report == before diff --git a/tests/test_w2_actions_summary.py b/tests/test_w2_actions_summary.py new file mode 100644 index 0000000..040d68d --- /dev/null +++ b/tests/test_w2_actions_summary.py @@ -0,0 +1,69 @@ +"""Execute the workflow summary shell against completed, skipped and absent evidence.""" +import os +from pathlib import Path +import shutil +import subprocess +import textwrap + +import pytest + + +WORKFLOW = Path(__file__).parents[1] / ".github/workflows/llm-deploy.yml" +SUMMARIES = ( + ("Summarize W2 unit gates", ("w2-backend-ops", "w2-runtime", "w2-runtime-model")), + ("Summarize RISC-V numeric gate", ("qwen3-riscv",)), + ("Summarize Qwen3 small probe", ("qwen3-small",)), +) + + +def shell_block(name): + # Only read the literal run block: never evaluate Actions expressions. + source = WORKFLOW.read_text(encoding="utf-8") + step = source.split(" - name: " + name + "\n", 1)[1].split("\n - ", 1)[0] + return textwrap.dedent(step.split(" run: |\n", 1)[1]) + + +def execute(tmp_path, name, directories, outcome, have_report): + bash = os.environ.get("SCRATCHV_BASH") or shutil.which("bash") + if not bash: + pytest.skip("Bash is required to exercise GitHub Actions summary commands") + for directory in directories: + report = tmp_path / "output" / directory / "report.md" + report.parent.mkdir(parents=True) + if have_report: + report.write_text("SENTINEL measured report\n", encoding="utf-8") + summary = tmp_path / "summary.md" + env = dict(os.environ, GITHUB_STEP_SUMMARY=summary.as_posix()) + for key in ("FRONTEND_OUTCOME", "BACKEND_OUTCOME", "RUNTIME_OUTCOME", "RUNTIME_MODEL_OUTCOME", + "PROBE_OUTCOME", "TEST_OUTCOME", "ARTIFACT_TEST_OUTCOME"): + env[key] = outcome + result = subprocess.run([bash, "-eu", "-c", shell_block(name)], cwd=tmp_path, + env=env, capture_output=True, text=True, encoding="utf-8", timeout=10) + assert result.returncode == 0, result.stderr + return summary.read_text(encoding="utf-8") + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_ran_probe_keeps_available_report(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, True) + assert summary.count("SENTINEL measured report") == len(directories) + assert outcome in summary + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["skipped", ""]) +def test_unrun_probe_never_reuses_stale_report(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, True) + assert summary.count("Not run / 未运行") == len(directories) + assert "SENTINEL" not in summary + assert "No " in summary and "acceptance result" in summary + + +@pytest.mark.parametrize("name,directories", SUMMARIES) +@pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) +def test_absent_report_does_not_invent_acceptance(tmp_path, name, directories, outcome): + summary = execute(tmp_path, name, directories, outcome, False) + assert summary.count("Report unavailable / 未生成报告") == len(directories) + assert "No " in summary and "acceptance result" in summary + assert "SENTINEL" not in summary diff --git a/tests/test_w2_runtime_model.py b/tests/test_w2_runtime_model.py index 99969f4..333147b 100644 --- a/tests/test_w2_runtime_model.py +++ b/tests/test_w2_runtime_model.py @@ -169,7 +169,7 @@ def replace(path, dest): assert not (tmp_path / ".report.json.tmp").exists() for name in ("report.md", "report.html"): if name != failure: - assert "FAIL" in (tmp_path / name).read_text() + assert "FAIL" in (tmp_path / name).read_text(encoding="utf-8") def test_main_preserves_existing_evidence(tmp_path): From d31618b4e6f9d79a198c5f404f657acc6bec91f7 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 11 Oct 2026 17:48:51 +0800 Subject: [PATCH 19/30] feat(ci): publish compact W3 acceptance summaries --- .github/workflows/llm-deploy.yml | 34 +++++ .github/workflows/w3-full-numeric.yml | 8 +- .github/workflows/w3-preparation.yml | 6 + scripts/summarize_w3_ci.py | 179 ++++++++++++++++++++++++++ tests/test_w3_actions_summary.py | 142 ++++++++++++++++++++ tests/test_w3_workflows.py | 40 ++++++ 6 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 scripts/summarize_w3_ci.py create mode 100644 tests/test_w3_actions_summary.py diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index 6dc4a42..add0877 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -29,6 +29,7 @@ on: - 'scripts/check_w1_repro_env.py' - 'scripts/run_w2_acceptance.py' - 'scripts/check_w2_ci_results.py' + - 'scripts/summarize_w3_ci.py' - 'export_qwen3_onnx.py' - 'scratchv/**' - 'requirements/**' @@ -46,7 +47,10 @@ on: - 'tests/test_qwen3_tokenizer.py' - 'tests/test_llm_inputs.py' - 'tests/test_w2_*.py' + - 'tests/test_w3_actions_summary.py' + - 'tests/test_w3_workflows.py' - '.github/workflows/llm-deploy.yml' + - '.github/workflows/w3-*.yml' schedule: # Nightly, off the hour: 19:23 UTC = 03:23 北京时间. - cron: '23 19 * * *' @@ -521,3 +525,33 @@ jobs: env: W2_JOB_RESULTS: ${{ toJSON(needs) }} run: python3 scripts/check_w2_ci_results.py >> "$GITHUB_STEP_SUMMARY" + + w3-execution-summary: + name: "W3 / execution scope" + needs: [w3-validation] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: State whether W3 actually ran + env: + W3_RESULT: ${{ needs.w3-validation.result }} + run: | + python3 - <<'PY' + import os + result = os.environ.get("W3_RESULT", "") + labels = {"success": "PASS", "failure": "FAIL", "cancelled": "CANCELLED", "skipped": "NOT RUN"} + label = labels.get(result, "UNCONFIRMED") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream: + stream.write("## W3 execution scope\n\n") + stream.write(f"**{label}** — W3 preparation and full 28-layer IR numeric.\n\n") + if result == "skipped": + stream.write("W3 is opt-in and is not executed by default on pull requests. W1/W2 success is not W3 acceptance.\n\n") + elif result == "success": + stream.write("See the W3 preparation and W3 complete IR numeric summaries in this run for numerical data.\n\n") + else: + stream.write("No W3 acceptance claimed; inspect the called jobs and any partial evidence.\n\n") + stream.write("
Scope and evidence\n\nManual LLM Deploy runs can enable `run_w3_validation`. " + "Numerical reports are retained in `w3-preparation-reports` and `w3-full-numeric-reports` when generated. " + "Full-model QEMU is a separate W4 gate; independent reproduction and team sign-off remain separate.\n\n
\n") + PY diff --git a/.github/workflows/w3-full-numeric.yml b/.github/workflows/w3-full-numeric.yml index 8483602..2f99f0b 100644 --- a/.github/workflows/w3-full-numeric.yml +++ b/.github/workflows/w3-full-numeric.yml @@ -70,7 +70,7 @@ jobs: run: | set -euo pipefail mkdir -p output - python -B -X utf8 -m pytest tests/test_w3_full*.py tests/test_ir_interpreter*.py tests/test_onnx_initializers.py -q -p no:cacheprovider --basetemp output/pytest-w3-full --junitxml output/w3-full-tests.xml + python -B -X utf8 -m pytest tests/test_w3_full*.py tests/test_w3_actions_summary.py tests/test_w3_workflows.py tests/test_ir_interpreter*.py tests/test_onnx_initializers.py -q -p no:cacheprovider --basetemp output/pytest-w3-full --junitxml output/w3-full-tests.xml - name: Download fixed ONNX assets run: | python - <<'PY' @@ -79,12 +79,18 @@ jobs: acquire_model(Path("output/assets/onnx")) PY - name: Compare complete IR and ORT outputs + id: probe run: | set -euo pipefail args=() SELECTED_CASE="${SELECTED_CASE:-all}" if [[ "$SELECTED_CASE" != "all" ]]; then args=(--case "$SELECTED_CASE"); fi python -B -X utf8 probes/w3_qwen3_full/run.py --model-dir output/assets/onnx --output-dir output/w3-full --worker-timeout 1800 --max-worker-memory-gib 8 --fp32-mode reference "${args[@]}" + - name: Summarize W3 full numerical evidence + if: always() + env: + PROBE_OUTCOME: ${{ steps.probe.outcome }} + run: python -B -X utf8 scripts/summarize_w3_ci.py --kind full --report-dir output/w3-full - name: Record evidence retention scope if: always() run: | diff --git a/.github/workflows/w3-preparation.yml b/.github/workflows/w3-preparation.yml index 29aa934..97751c2 100644 --- a/.github/workflows/w3-preparation.yml +++ b/.github/workflows/w3-preparation.yml @@ -70,8 +70,14 @@ jobs: acquire_model(Path("output/assets/onnx")) PY - name: Five preparation checks (not full-model acceptance) + id: probe run: | python -B -X utf8 scripts/run_w3_preparation.py --source-dir output/assets/source --model-dir output/assets/onnx --output-dir output/w3-preparation --cc "$SCRATCHV_CC" --qemu qemu-system-riscv64 + - name: Summarize W3 preparation evidence + if: always() + env: + PROBE_OUTCOME: ${{ steps.probe.outcome }} + run: python -B -X utf8 scripts/summarize_w3_ci.py --kind preparation --report-dir output/w3-preparation - name: Retain bounded selected trace and failure evidence if: always() run: | diff --git a/scripts/summarize_w3_ci.py b/scripts/summarize_w3_ci.py new file mode 100644 index 0000000..2d01970 --- /dev/null +++ b/scripts/summarize_w3_ci.py @@ -0,0 +1,179 @@ +"""Publish bounded W3 numerical reports to GitHub Actions without changing gates.""" +from __future__ import annotations + +import argparse +from html import escape +import json +import math +import os +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.numeric_summary import compact_metric_table +from probes.w3_summary import evidence_index_views, numerical_summary + +GATES = ( + ("medium", "w3-medium-host", "6-layer random Qwen3 / host IR"), + ("subgraphs", "w3_qwen3_subgraphs", "Pretrained layer-0 subgraphs / host IR"), + ("attention", "unit:attention-backend", "Small Attention / RISC-V QEMU"), + ("full-preflight", "preflight:w3-full-assets", "Full assets / static checks only"), + ("layer-diff", "w3_layer_diff", "Saved medium-model checkpoints only"), +) + + +def cell(value, limit=180): + text = " ".join(str(value).split()) + if len(text) > limit: + text = text[:limit] + "…" + return escape(text).replace("|", "|") + + +def load_report(path, gate): + try: + report = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return None, "MISSING REPORT" + except (OSError, ValueError): + return None, "INVALID REPORT" + if not isinstance(report, dict) or report.get("gate") != gate: + return None, "INVALID REPORT (gate identity)" + # Fail closed on malformed structures before calling the shared renderer. + for key in ("cases", "gates", "invariants"): + rows = report.get(key, []) + if not isinstance(rows, list) or not all(isinstance(row, dict) for row in rows): + return None, "INVALID REPORT (rows)" + return report, "" + + +def status(report): + if report.get("status") == "PARTIAL": + return "PARTIAL (not acceptance)" + if report.get("status") == "PASS" and report.get("passed") is True: + if report.get("gate") == "numeric:ir-full-qwen3" and report.get("coverage_complete") is not True: + return "INCOMPLETE (not acceptance)" + return "PASS" + return "FAIL" if report.get("passed") is False else "UNCONFIRMED" + + +def seconds(report): + value = report.get("elapsed_seconds", report.get("seconds")) + if type(value) in (int, float) and math.isfinite(value) and value >= 0: + return f"{value:.3f} s" + return "not recorded" + + +def coverage(report): + gate = report.get("gate") + cases = report.get("cases", []) + if gate == "w3_layer_diff": + values = report.get("coverage", {}) + return f"{values.get('compared_checkpoints', '?')}/{values.get('total_checkpoints', '?')} checkpoints" + if gate == "preflight:w3-full-assets": + return "no model execution" + if gate == "unit:attention-backend": + count = sum(len(row.get("executions", [])) for row in cases) + return f"{len(cases)} cases / {count} target executions recorded" + required = report.get("required_cases") + expected = len(required) if isinstance(required, list) else {"w3-medium-host": 7, "w3_qwen3_subgraphs": 14}.get(gate, "?") + return f"{len(cases)}/{expected} cases recorded" + + +def details(report, title): + """Reuse bounded semantic selectors, never dump arbitrary report dictionaries.""" + description, comparisons, notes = numerical_summary(report) + index, _ = evidence_index_views(report) + text = f"
{cell(title)}\n\n{cell(description, 700)}\n\n" + if comparisons: + text += compact_metric_table(comparisons) + "\n\n" + if report.get("error"): + text += "Issue: " + cell(report["error"], 400) + "\n\n" + text += "\n\n".join(cell(note, 500) for note in notes) + "\n\n" + index + "
\n\n" + return text + + +def render(kind, directory, outcome): + title = "W3 preparation" if kind == "preparation" else "W3 complete IR numeric" + artifact = "w3-preparation-reports" if kind == "preparation" else "w3-full-numeric-reports" + lines = [f"## {title}\n\n"] + if outcome in ("skipped", "", "cancelled"): + label = "CANCELLED" if outcome == "cancelled" else "NOT RUN" + lines.append(f"**{label}** — probe step did not complete; no numerical acceptance claimed.\n\n") + lines.append("Existing files, if any, are not used as results for this step.\n") + return "".join(lines) + identity = "preparation:w3" if kind == "preparation" else "numeric:ir-full-qwen3" + report, problem = load_report(directory / "report.json", identity) + if report is None: + lines.append(f"**{problem}** — step outcome: **{cell(outcome)}**. No acceptance claimed; inspect step logs.\n") + return "".join(lines) + conclusion = status(report) + if outcome != "success" and conclusion == "PASS": + conclusion = "FAIL (step did not succeed)" + lines.append(f"**{conclusion}** · step: {cell(outcome)} · pipeline: {seconds(report)}\n\n") + comparisons, folded = [], [] + try: + if kind == "preparation": + lines.append("Preparation only; full 28-layer IR numeric and full-model QEMU are **NOT RUN by this job**.\n\n") + lines.append("| Check / scope | Result | Recorded coverage | Pipeline time |\n| --- | --- | --- | --- |\n") + for name, identity, scope in GATES: + rows = [row for row in report.get("gates", []) if row.get("name") == name] + row = rows[0] if len(rows) == 1 else None + child, problem = load_report(directory / name / "report.json", identity) if row else (None, "NOT RUN" if not rows else "INVALID REPORT (duplicate gate)") + child_status = problem or status(child) + if row and status(row) != "PASS" and child_status == "PASS": + child_status = "FAIL (parent validation)" + if child_status != "PASS" and conclusion == "PASS": + lines[1] = f"**INCOMPLETE REPORTS (not acceptance)** · step: {cell(outcome)} · pipeline: {seconds(report)}\n\n" + lines.append("| " + " | ".join(cell(x) for x in (scope, child_status, coverage(child) if child else "not recorded", seconds(row) if row else "not recorded")) + " |\n") + if child: + _, numeric, _ = numerical_summary(child) + selected = numeric[1:] if name in ("medium", "subgraphs") else numeric[1:2] if name == "attention" else [] + records = [item for _, items in selected for item in items] + if selected: + comparisons.append((scope + " vs ORT", records)) + folded.append(details(child, scope + " — details")) + folded.insert(0, details(report, "Preparation execution / failures")) + else: + lines.append("28-layer pretrained Qwen3, FP32 / L=256 / no KV cache; IR vs ORT. **No full-model QEMU result in this job.**\n\n") + lines.append(f"Coverage: **{cell(coverage(report))}** · FP32 mode: **{cell(report.get('fp32_mode', 'not recorded'))}** · maximum absolute error threshold: **{cell(report.get('atol', 'not recorded'))}** (strict <, rtol=0).\n\n") + display = dict(report) + recorded = {row.get("name") for row in report.get("cases", [])} + display["cases"] = list(report.get("cases", [])) + [ + {"name": name, "status": "NOT RUN"} for name in report.get("required_cases", []) + if name not in recorded] + _, comparisons, _ = numerical_summary(display) + folded.append(details(display, "Cases, diagnostics, invariants and failures")) + if comparisons: + lines.append("\n" + compact_metric_table(comparisons) + "\n\n") + lines.append("Metrics summarize final outputs only (worst MaxAbs / relative L2, lowest cosine across recorded comparisons); checkpoints still affect the original gate. Missing metrics remain explicit.\n\n") + if report.get("error"): + lines.append("Issue: " + cell(report["error"], 300) + "\n\n") + lines.extend(folded) + except (TypeError, KeyError, ValueError, AttributeError): + # Unexpected report schemas must not make an apparently passing page. + return f"## {title}\n\n**INVALID REPORT** — could not render numerical evidence; step: {cell(outcome)}. No acceptance claimed.\n" + lines.append(f"Artifacts: **{artifact}** (`report.json`, `report.md`, `report.html`). Raw arrays are separate where retained. This summary does not replace independent reproduction or team acceptance.\n") + return "".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--kind", choices=("preparation", "full"), required=True) + parser.add_argument("--report-dir", type=Path, required=True) + parser.add_argument("--outcome", default=os.environ.get("PROBE_OUTCOME", "")) + parser.add_argument("--summary", type=Path, default=os.environ.get("GITHUB_STEP_SUMMARY")) + args = parser.parse_args(argv) + text = render(args.kind, args.report_dir, args.outcome) + if args.summary: + with args.summary.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(text) + else: + print(text, end="") + return 0 # Reporting never substitutes for or changes the probe's exit code. + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_w3_actions_summary.py b/tests/test_w3_actions_summary.py new file mode 100644 index 0000000..bdc2ddb --- /dev/null +++ b/tests/test_w3_actions_summary.py @@ -0,0 +1,142 @@ +"""W3 Actions pages must distinguish measured results, partial coverage and skips.""" +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from scripts.summarize_w3_ci import GATES, main, render + +ROOT = Path(__file__).resolve().parents[1] + + +def save(folder, report): + folder.mkdir(parents=True, exist_ok=True) + (folder / "report.json").write_text(json.dumps(report), encoding="utf-8") + + +def metric(value, name): + return {"name": name, "passed": True, "max_abs": value, + "relative_l2": 2e-7, "cosine_similarity": 0.99999} + + +def full_report(**extra): + return {"gate": "numeric:ir-full-qwen3", "status": "PASS", "passed": True, + "coverage_complete": True, "fp32_mode": "reference", "atol": 1e-4, + "elapsed_seconds": 75.125, "required_cases": ["one", "two"], "selected_cases": ["one", "two"], + "cases": [{"name": name, "passed": True, "comparison": { + "logits": [metric(value, "logits.npy"), metric(value * 2, "diagnostic_logits.npy")], + "checkpoints": [metric(0.7, "layer_3.output")]}} + for name, value in [("one", 1e-6), ("two", 3e-6)]], **extra} + + +def test_full_key_metrics_are_visible_and_checkpoint_details_are_folded(tmp_path): + report = full_report() + before = deepcopy(report) + save(tmp_path, report) + text = render("full", tmp_path, "success") + visible = text.split("
", 1)[0] + assert "**PASS**" in visible and "75.125 s" in visible + assert "2/2 cases" in visible and "reference" in visible + assert format(3e-6, ".16e") in visible and format(6e-6, ".16e") in visible + assert format(0.7, ".16e") not in visible + assert "No full-model QEMU result" in visible + assert "w3-full-numeric-reports" in text and "
" in text + assert json.loads((tmp_path / "report.json").read_text()) == before + + +@pytest.mark.parametrize("outcome,expected", [("skipped", "NOT RUN"), ("", "NOT RUN"), ("cancelled", "CANCELLED")]) +def test_skipped_step_does_not_reuse_a_passing_report(tmp_path, outcome, expected): + save(tmp_path, full_report()) + text = render("full", tmp_path, outcome) + assert f"**{expected}**" in text and "PASS" not in text + assert "最大绝对误差" not in text + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_missing_report_does_not_claim_pass(tmp_path, outcome): + text = render("full", tmp_path, outcome) + assert "MISSING REPORT" in text and "PASS" not in text and outcome in text + + +@pytest.mark.parametrize("invalid", ["{", "[]", '{"gate":"wrong"}', '{"gate":"numeric:ir-full-qwen3","cases":null}']) +def test_invalid_reports_do_not_crash_summary(tmp_path, invalid): + (tmp_path / "report.json").write_text(invalid) + text = render("full", tmp_path, "failure") + assert "INVALID REPORT" in text and "PASS" not in text + + +def test_failed_step_overrides_passing_report(tmp_path): + save(tmp_path, full_report()) + text = render("full", tmp_path, "failure") + assert "**FAIL (step did not succeed)**" in text + assert "**PASS**" not in text + + +def test_partial_coverage_shows_unexecuted_case_and_missing_metrics(tmp_path): + report = full_report(status="PARTIAL", passed=False, coverage_complete=False, selected_cases=["one"]) + report["cases"] = report["cases"][:1] + save(tmp_path, report) + text = render("full", tmp_path, "failure") + assert "**PARTIAL (not acceptance)**" in text + assert "1/2 cases recorded" in text and "NOT RUN" in text and "two" in text + assert "未记录 1/2" in text and "**PASS**" not in text + + +def test_claimed_pass_with_incomplete_coverage_is_not_pass(tmp_path): + save(tmp_path, full_report(coverage_complete=False)) + assert "**INCOMPLETE (not acceptance)**" in render("full", tmp_path, "success") + + +def test_preparation_lists_all_checks_and_never_claims_full_model_execution(tmp_path): + report = {"gate": "preparation:w3", "status": "FAIL", "passed": False, "elapsed_seconds": 32, + "gates": [{"name": "attention", "passed": False, "status": "FAIL", "elapsed_seconds": 1.5}]} + save(tmp_path, report) + save(tmp_path / "attention", {"gate": "unit:attention-backend", "status": "PASS", "passed": True, + "cases": [{"name": "one", "executions": [{"optimization": level, "qemu_vs_ort": metric(3e-6, "logits")} + for level in ("none", "all")]}]}) + # A stale medium child without a corresponding parent invocation is ignored. + save(tmp_path / "medium", {"gate": "w3-medium-host", "status": "PASS", "passed": True}) + text = render("preparation", tmp_path, "failure") + visible = text.split("
", 1)[0] + assert "NOT RUN by this job" in visible + assert "FAIL (parent validation)" in visible and "1 cases / 2 target executions recorded" in visible + for _, _, scope in GATES: + assert scope in visible + assert visible.count("NOT RUN") == 5 + assert "w3-preparation-reports" in text + + +def test_missing_child_report_cannot_leave_overall_pass(tmp_path): + save(tmp_path, {"gate": "preparation:w3", "status": "PASS", "passed": True, + "gates": [{"name": "medium", "status": "PASS", "passed": True}]}) + text = render("preparation", tmp_path, "success") + assert "**INCOMPLETE REPORTS (not acceptance)**" in text and "MISSING REPORT" in text + assert "**PASS**" not in text + + +def test_errors_are_escaped_and_arbitrary_machine_evidence_is_not_dumped(tmp_path): + report = full_report(status="FAIL", passed=False, error="