From a714e11ddd3227f92f0f5fca443b021c5cba6bf5 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Mon, 12 Oct 2026 18:03:29 +0800 Subject: [PATCH] fix(ci): correct W2 summary results and operator coverage IR precision now uses ordinary final outputs. Keep missing comparisons in the planned counts and count frontend operators only after successful parsing and IR verification, including backend probe graphs. Preserve the original operator table labels and note, QEMU comparison scope, model calculations and accuracy thresholds. Add focused regression tests for missing results and parse failures. Validation: 78 summary tests and 182 probe tests passed; 54 summary tests passed again after restoring the original labels. Existing CI reports render with unchanged IR and QEMU precision values. --- probes/numeric_summary.py | 8 + probes/w2_backend_ops/run.py | 5 + probes/w2_qwen3_parse/run.py | 4 + probes/w2_qwen3_small/run.py | 7 + probes/w2_runtime_model/run.py | 6 +- scripts/summarize_w2_ci.py | 119 +++++++++--- tests/test_numeric_summary.py | 15 ++ tests/test_qwen3_full_parse.py | 10 + tests/test_qwen3_small_model.py | 82 ++++++++- tests/test_w2_acceptance_summary.py | 275 +++++++++++++++++++++++++--- tests/test_w2_backend_ops_gate.py | 25 +++ tests/test_w2_runtime_model.py | 17 ++ 12 files changed, 518 insertions(+), 55 deletions(-) diff --git a/probes/numeric_summary.py b/probes/numeric_summary.py index f4926a4..67a3d98 100644 --- a/probes/numeric_summary.py +++ b/probes/numeric_summary.py @@ -76,8 +76,16 @@ def _table_rows(comparisons): passed = sum(row.get("passed") is True for row in rows) failed = sum(row.get("passed") is False for row in rows) unknown = len(rows) - passed - failed + missing = Counter(str(row["missing_reason"]) for row in rows if row.get("missing_reason")) if not rows: status = "未测（0 条比较）" + elif missing: + reasons = "；".join(f"{reason} ×{count}" for reason, count in missing.items()) + failures = f"，数值比较失败 {failed}" if failed else "" + status = (f"未完成（通过 {passed}/{len(rows)}{failures}，" + f"未得到结果 {sum(missing.values())}：{reasons}）") + if unknown > sum(missing.values()): + status += f"；另有 {unknown - sum(missing.values())} 次结果未确认" elif passed == len(rows): status = f"通过（{passed}/{len(rows)}）" else: diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py index 53fc22f..2c01c7a 100644 --- a/probes/w2_backend_ops/run.py +++ b/probes/w2_backend_ops/run.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +from collections import Counter import copy import hashlib from html import escape @@ -104,6 +105,10 @@ def run_probe(out, report, *, cc=None, qemu=None, timeout=120): try: path = folder / "model.onnx" expected, parser, program = reference_case(case, path) + row["frontend_parse"] = { + "passed": True, + "operators": dict(sorted(Counter(node.op_type for node in case.model.graph.node).items())), + } np.savez(folder / "inputs.npz", **case.feed) np.save(folder / "numpy.npy", case.expected) np.save(folder / "ort.npy", expected) diff --git a/probes/w2_qwen3_parse/run.py b/probes/w2_qwen3_parse/run.py index 1cf2366..e7e7d22 100644 --- a/probes/w2_qwen3_parse/run.py +++ b/probes/w2_qwen3_parse/run.py @@ -140,6 +140,10 @@ def stage(name, operation): "issues": [str(issue) for issue in issues]} if not verified: raise ValueError("ScratchV IR verifier rejected the complete model; see ir_verifier.issues") + report["frontend_parse"] = { + "passed": True, + "operators": dict(sorted(Counter(node.op_type for node in model.graph.node).items())), + } try: audit = stage("ir-and-bindings", lambda: audit_parsed_model(model, parser, program, model_dir)) except Exception: diff --git a/probes/w2_qwen3_small/run.py b/probes/w2_qwen3_small/run.py index dbcea9f..d4b35bc 100644 --- a/probes/w2_qwen3_small/run.py +++ b/probes/w2_qwen3_small/run.py @@ -199,6 +199,7 @@ def run_probe(out: Path, report: dict, model_seed: int = 0) -> None: name: tensor.detach().cpu().numpy() for name, tensor in wrapper.model.state_dict().items()}) (out / "config.json").write_text(wrapper.model.config.to_json_string(use_diff=False), encoding="utf-8") cases = input_cases() + report["planned_cases"] = len(cases) def reference(feed): args = [torch.from_numpy(feed[name]) for name in ("input_ids", "attention_mask")] @@ -240,12 +241,18 @@ def reference(feed): providers=["CPUExecutionProvider"]) report["stage"] = "parse" parsers, programs = {}, {} + parsed_operators, parsed_graphs = Counter(), [] for kind, filename in [("normal", "model.onnx"), ("diagnostic", "diagnostics.onnx")]: parsers[kind] = ONNXParser() programs[kind] = parsers[kind].parse(str(out / filename)) valid, errors = verify_ir(programs[kind]) if not valid: raise ValueError(f"{kind} IR verification failed: {errors}") + graph = ordinary if kind == "normal" else diagnostic + parsed_operators.update(node.op_type for node in graph.graph.node) + parsed_graphs.append(kind) + report["frontend_parse"] = {"passed": True, "operators": dict(sorted(parsed_operators.items())), + "graphs": list(parsed_graphs)} report["stage"] = "numeric" report["cases"] = [] cached_logits = {} diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py index ae8d091..3ebe8f7 100644 --- a/probes/w2_runtime_model/run.py +++ b/probes/w2_runtime_model/run.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +from collections import Counter import copy from datetime import datetime, timezone import gc @@ -171,7 +172,8 @@ def source_evidence(): def run_probe(directory, out, report): - report.update(stage="environment", environment=environment(), tokenizer_environment=require_environment()) + report.update(stage="environment", planned_cases=len(CASES), levels=list(LEVELS), + environment=environment(), tokenizer_environment=require_environment()) import onnx import onnxruntime as ort import torch @@ -258,6 +260,7 @@ def require_valid(program): check_onnx_contract(exported) report["onnx"] = {"sha256": sha256(out / "model.onnx"), "bytes": (out / "model.onnx").stat().st_size, "nodes": len(exported.graph.node), "opset": 18, + "operators": dict(sorted(Counter(node.op_type for node in exported.graph.node).items())), "exporter": "torch.onnx.export(dynamo=True, optimize=False, fallback=False)", "inputs": {"input_ids": "INT64[1,256]", "attention_mask": "FP32[1,1,256,256]"}, "output": "FP32[1,256,151936]", "output_bytes": SEQ * VOCAB * 4} @@ -292,6 +295,7 @@ def require_valid(program): parser = ONNXParser() program = parser.parse(str(out / "model.onnx")) require_valid(program) + report["frontend_parse"] = {"passed": True, "operators": dict(report["onnx"]["operators"])} report["parse_seconds"] = time.perf_counter() - started report["ir_optimization_seconds"] = {} for level in LEVELS: diff --git a/scripts/summarize_w2_ci.py b/scripts/summarize_w2_ci.py index 48eeea9..1821926 100644 --- a/scripts/summarize_w2_ci.py +++ b/scripts/summarize_w2_ci.py @@ -154,12 +154,28 @@ def _onnx_operators(report, name): return set(_mapping(inner.get("operators"))) +def parsed_operators(record): + """New reports record successful parsing separately from later execution.""" + evidence = _mapping(record.get("frontend_parse")) + if evidence.get("passed") is True: + return {str(op).lower() for op in _mapping(evidence.get("operators"))} + return set() + + def exercised_operators(reports): """``(frontend_op_types, backend_opcodes)`` actually exercised in this run.""" onnx_seen = set() - for name in (QWEN3_SMALL, QWEN3_PARSE): - if reports.get(name) is not None: - onnx_seen |= _onnx_operators(reports[name], name) + for name in (QWEN3_SMALL, QWEN3_PARSE, RUNTIME_MODEL): + report = _mapping(reports.get(name)) + if "frontend_parse" in report: + onnx_seen |= parsed_operators(report) + # Compatibility with reports written before the explicit parse record. + # An exported graph's operator list alone does not prove parsing passed. + elif name == QWEN3_SMALL and report.get("stage") in ("numeric", "complete"): + onnx_seen |= _onnx_operators(report, name) + elif (name == QWEN3_PARSE and isinstance(report.get("ir"), dict) + and _mapping(report.get("ir_verifier")).get("passed") is True): + onnx_seen |= _onnx_operators(report, name) opcodes = set() # The two-layer model is the only graph the RISC-V QEMU probe compiles, so @@ -171,7 +187,13 @@ def exercised_operators(reports): opcodes |= {opcode_name(op) for op in _onnx_operators(reports[QWEN3_SMALL], QWEN3_SMALL)} # The operator probe records only its families; a passing execution proves them. for case in _cases(reports.get(BACKEND_OPS)): - family = str(case.get("name", "")).split("_", 1)[0] + family = case.get("family") or str(case.get("name", "")).split("_", 1)[0] + if "frontend_parse" in case: + onnx_seen |= parsed_operators(case) + elif isinstance(case.get("ort_vs_numpy"), dict) or case.get("model_sha256"): + # Legacy fixed probe graphs: these fields are written only after + # reference_case returned from parse + IR verification. + onnx_seen |= {op.replace("_", "") for op in FAMILY_OPS.get(family, ())} if any(row.get("passed") is True for row in _rows(case, "executions")): opcodes |= set(FAMILY_OPS.get(family, ())) return {str(op).lower() for op in onnx_seen}, opcodes & set(backend_opcodes()) @@ -188,7 +210,7 @@ def measured(names, exercised): if not names: return "无" hit = [name for name in names if name.lower() in exercised] - return listing(hit) if hit else "无（本次未执行）" + return listing(hit) if hit else "无（没有通过记录）" rows = [ ("IR 层 (`ONNXParser._handle_*`)", listing(supported_frontend), @@ -215,40 +237,85 @@ def precision_legend(): ] +def missing_comparison(record=None): + record = _mapping(record) + stage = record.get("stage") or record.get("error_stage") + reason = {"reference": "参考结果未完成", "ir": "IR 计算未完成", + "compile": "编译失败" if record.get("error") else "编译未完成", + "qemu": "QEMU 运行未完成"}.get(stage, "未执行或未记录比较结果") + return {"missing_reason": reason} + + +def comparison(record, key): + result = record.get(key) + return result if isinstance(result, dict) and result else missing_comparison(record) + + +def fill_missing(rows, planned): + # Reports are external input. Bound the count as well as the file size. + if type(planned) is not int or not 0 <= planned <= 4096: + raise ValueError("Invalid planned comparison count") + return rows + [missing_comparison() for _ in range(max(0, planned - len(rows)))] + + +def planned_cases(report, legacy_count): + return fill_missing(_cases(report), report.get("planned_cases", legacy_count)) + + +def planned_executions(report): + executions = [] + levels = report.get("levels", ("none", "all")) + for case in _cases(report): + rows = _rows(case, "executions") + executions.extend(rows) + present = {row.get("optimization") for row in rows} + executions.extend({"optimization": level, "stage": case.get("stage"), + "error": case.get("error")} + for level in levels if level not in present) + return fill_missing(executions, report.get("planned_executions", len(executions))) + + def precision_comparisons(reports): comparisons = [] def add(label, records): - comparisons.append((label, [row for row in records if isinstance(row, dict)])) + comparisons.append((label, [row if isinstance(row, dict) + else missing_comparison() for row in records])) backend = reports.get(BACKEND_OPS) if backend is not None: - executions = [row for case in _cases(backend) for row in _rows(case, "executions")] + executions = planned_executions(backend) for key, label in (("ir_vs_ort", "IR vs ORT"), ("ir_vs_numpy", "IR vs NumPy"), ("qemu_vs_ort", "QEMU vs ORT"), ("qemu_vs_numpy", "QEMU vs NumPy")): - add(f"w2-backend-ops / {label}", [row.get(key) for row in executions]) + add(f"w2-backend-ops / {label}", [comparison(row, key) for row in executions]) if reports.get(QWEN3_SMALL) is not None: - # The host IR comparison wraps one tensor_diff per checkpoint; report - # the final output, which is what the probe's own gate compares. - records = [] - for case in _cases(reports[QWEN3_SMALL]): - checkpoints = _rows(_mapping(case.get("ir_vs_ort")), "checkpoints") - final = [row for row in checkpoints if row.get("name") == "logits"] - if final: - records.extend(final) - elif isinstance(case.get("ordinary_logits"), dict): - records.append(case["ordinary_logits"].get("ir_vs_ort")) - add("qwen3-small host IR / IR vs ORT（最终 logits）", records) + # Do not substitute the diagnostic graph when ordinary execution fails. + records = [comparison(_mapping(case.get("ordinary_logits")), "ir_vs_ort") + for case in planned_cases(reports[QWEN3_SMALL], 7)] + add("两层模型：IR 解释器与 ORT（最终输出）", records) if reports.get(QWEN3_RISCV) is not None: - add("qwen3-small RV64 QEMU / QEMU vs ORT", - [row for case in _cases(reports[QWEN3_RISCV]) for row in _rows(case, "qemu")]) + report = reports[QWEN3_RISCV] + # Keep both normal and diagnostic results, as requested. Only account + # for missing executions; do not change the scope of this comparison. + rows = [row for case in _cases(report) for row in _rows(case, "qemu")] + planned = _mapping(report.get("timing")).get("planned_count", len(rows)) + add("qwen3-small RV64 QEMU / QEMU vs ORT", fill_missing(rows, planned)) if reports.get(RUNTIME_MODEL) is not None: - executions = [row for case in _cases(reports[RUNTIME_MODEL]) for row in _rows(case, "executions")] - add("w2-runtime-model / vs torch", [row.get("vs_torch") for row in executions]) - add("w2-runtime-model / vs ORT", [row.get("vs_ort") for row in executions]) + report = reports[RUNTIME_MODEL] + engines = ["ort", *(f"ir-{level}" for level in report.get("levels", ("none", "all")))] + executions = [] + for case in planned_cases(report, 2): + rows = _rows(case, "executions") + executions.extend(rows) + present = {row.get("engine") for row in rows} + executions.extend({"engine": engine} for engine in engines if engine not in present) + add("w2-runtime-model / vs torch", [comparison(row, "vs_torch") for row in executions]) + # ORT is the reference here; there is deliberately no ORT-vs-itself test. + add("w2-runtime-model / vs ORT", [comparison(row, "vs_ort") for row in executions + if row.get("engine") != "ort"]) # These two chains publish structural evidence only; say so instead of omitting them. add("w2-runtime（文本层，无数值比较）", []) @@ -318,7 +385,9 @@ def render(directory): "请检查各 W2 job 的日志与产物。\n") lines = ["## W2 测试项\n\n", "### 算子支持\n\n", *operators, - "\n### 精度\n\n", *precision_legend(), precision, "\n\n", METRIC_EXPLANATION, "\n\n", + "\n### 精度\n\n", *precision_legend(), precision, + "\n\nIR 最终输出一行使用只输出最终结果的模型，不用额外输出中间值的版本代替。\n\n", + METRIC_EXPLANATION, "\n\n", "### 耗时\n\n", *timing] rendered = "".join(lines) if len(rendered.encode("utf-8")) > MAX_SUMMARY_BYTES: diff --git a/tests/test_numeric_summary.py b/tests/test_numeric_summary.py index 23f8126..1b4e713 100644 --- a/tests/test_numeric_summary.py +++ b/tests/test_numeric_summary.py @@ -51,6 +51,21 @@ def test_failures_are_not_hidden_by_passing_rows(): assert "未记录 1/3" in result +@pytest.mark.parametrize("render", [compact_metric_table, compact_metric_table_html]) +def test_missing_comparisons_keep_the_count_and_failure_reason(render): + result = render([("partial", [record(), {"missing_reason": "编译失败"}])]) + assert "未完成（通过 1/2，未得到结果 1：编译失败 ×1）" in result + assert "已记录 1/2" in result + assert "未记录 1/2" in result + assert "通过（1/1）" not in result + + +def test_numerical_failures_remain_distinct_from_missing_comparisons(): + failed = dict(record(), passed=False) + result = compact_metric_table([("mixed", [record(), failed, {"missing_reason": "编译失败"}])]) + assert "通过 1/3，数值比较失败 1，未得到结果 1" in result + + def test_undefined_values_show_reason_and_coverage_not_zero(): undefined = dict(passed=True, max_abs=0.0, relative_l2=None, cosine_similarity=None, relative_l2_reason="zero reference norm", cosine_similarity_reason="zero norm") diff --git a/tests/test_qwen3_full_parse.py b/tests/test_qwen3_full_parse.py index 970abec..09575d5 100644 --- a/tests/test_qwen3_full_parse.py +++ b/tests/test_qwen3_full_parse.py @@ -327,6 +327,7 @@ def bad_files(*args): ]) def test_real_decoder_pipeline_and_failure_context(tmp_path, monkeypatch, fault, evidence_locked): """Exercise the stage wiring with real files, parser and audits, cheaply.""" + from collections import Counter import onnx from scratchv.analysis import ir_verifier from probes.w2_qwen3_parse import audit, validation @@ -420,3 +421,12 @@ def write(path, value): assert report["audit"]["return"]["shape"] == [1, 3, 9] for name, evidence in report["evidence"].items(): assert evidence["sha256"] == gate.artifacts.sha256(out / name) + if fault in ("parse", "verifier"): + assert "frontend_parse" not in report + else: + # A later binding/evidence failure does not erase successful parsing. + # Count actual ONNX types, not the deliberately synthetic graph summary. + assert report["frontend_parse"] == { + "passed": True, + "operators": dict(Counter(node.op_type for node in model.graph.node)), + } diff --git a/tests/test_qwen3_small_model.py b/tests/test_qwen3_small_model.py index ca258e9..b4d4daf 100644 --- a/tests/test_qwen3_small_model.py +++ b/tests/test_qwen3_small_model.py @@ -50,6 +50,18 @@ def checkpoints(wrapper, inputs): return dict(zip(wrapper.output_names, result)) +@pytest.fixture(scope="module") +def exported_model(wrapper, inputs, tmp_path_factory): + pytest.importorskip("onnx") + pytest.importorskip("onnxruntime") + pytest.importorskip("onnxscript") + path = tmp_path_factory.mktemp("qwen3-small-export") / "checkpoints.onnx" + original = modeling_qwen3.eager_attention_forward + metadata = export_onnx(wrapper, path, *inputs) + assert_observers_removed(wrapper, original) + return path, metadata + + def raw_logits(model, inputs, positions): return model( input_ids=inputs[0], attention_mask=inputs[1], position_ids=positions, @@ -244,13 +256,12 @@ def test_capture_does_not_intercept_other_official_model_with_more_layers(wrappe assert_observers_removed(wrapper, original) -def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, checkpoints, tmp_path): +def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, checkpoints, exported_model): onnx = pytest.importorskip("onnx") ort = pytest.importorskip("onnxruntime") pytest.importorskip("onnxscript") original = modeling_qwen3.eager_attention_forward - path = tmp_path / "diagnostics.onnx" - metadata = export_onnx(wrapper, path, *inputs) + path, metadata = exported_model assert_observers_removed(wrapper, original) graph = onnx.load(str(path)) onnx.checker.check_model(graph) @@ -267,3 +278,68 @@ def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, che for name, result in zip(wrapper.output_names, actual): np.testing.assert_allclose(result, checkpoints[name].numpy(), atol=1e-5, rtol=1e-5, err_msg=name) + + +@pytest.mark.parametrize("fault", ["normal_parse", "normal_verifier", "diagnostic_parse", + "diagnostic_verifier", "numeric"]) +def test_run_probe_records_only_verified_graphs(tmp_path, monkeypatch, wrapper, exported_model, fault): + """Run the real small-model stages; reuse one export and stop at the fault.""" + from collections import Counter + import shutil + import onnx + from probes.w2_qwen3_small import model, run + from scratchv.analysis import ir_verifier + from scratchv.frontend.onnx_parser import ONNXParser + from scratchv.verification.ir_interpreter import IRInterpreter + + source, metadata = exported_model + monkeypatch.setattr(run, "environment", lambda: {"fixture": "pinned small-model stack"}) + monkeypatch.setattr(model, "build_model", lambda **kwargs: wrapper) + + def reuse_export(wrapper, path, *inputs): + shutil.copyfile(source, path) + return metadata + monkeypatch.setattr(model, "export_onnx", reuse_export) + original_parse, original_verify = ONNXParser.parse, ir_verifier.verify_ir + parsed = [] + + def parse(self, path): + kind = "normal" if str(path).endswith("model.onnx") else "diagnostic" + parsed.append(kind) + if fault == f"{kind}_parse": + raise ValueError(f"injected {fault}") + return original_parse(self, path) + + def verify(program): + if fault == f"{parsed[-1]}_verifier": + return False, [f"injected {fault}"] + return original_verify(program) + + def numeric_failure(*args, **kwargs): + raise ValueError("injected numeric") + + monkeypatch.setattr(ONNXParser, "parse", parse) + monkeypatch.setattr(ir_verifier, "verify_ir", verify) + if fault == "numeric": + monkeypatch.setattr(IRInterpreter, "run", numeric_failure) + report = {"passed": False} + previous = torch.are_deterministic_algorithms_enabled() + previous_warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + try: + with pytest.raises(ValueError, match="injected"): + run.run_probe(tmp_path, report, model_seed=193) + finally: + torch.use_deterministic_algorithms(previous, warn_only=previous_warn_only) + assert not report["passed"] + assert report["planned_cases"] == len(run.input_cases()) == 7 + assert report["stage"] == ("numeric" if fault == "numeric" else "parse") + if fault.startswith("normal"): + assert "frontend_parse" not in report + else: + graphs = ["normal", "diagnostic"] if fault == "numeric" else ["normal"] + operators = Counter() + for kind in graphs: + path = tmp_path / ("model.onnx" if kind == "normal" else "diagnostics.onnx") + operators.update(node.op_type for node in onnx.load(path).graph.node) + assert report["frontend_parse"] == { + "passed": True, "operators": dict(operators), "graphs": graphs} diff --git a/tests/test_w2_acceptance_summary.py b/tests/test_w2_acceptance_summary.py index ac5ca89..199f52c 100644 --- a/tests/test_w2_acceptance_summary.py +++ b/tests/test_w2_acceptance_summary.py @@ -14,7 +14,7 @@ import pytest from scripts.summarize_w2_ci import (FAMILY_OPS, MAX_SUMMARY_BYTES, backend_opcodes, - frontend_operators, main, opcode_name, render) + frontend_operators, main, opcode_name, precision_comparisons, render) ROOT = Path(__file__).resolve().parents[1] DIRECTORIES = ("w2-backend-ops", "w2-runtime", "w2-runtime-model", @@ -27,18 +27,24 @@ def metric(value, name="logits.npy"): def backend_ops_report(): - return {"passed": True, "seconds": 100.0, "cases": [ - {"name": "matmul_matrix", "passed": True, "executions": [ - {"optimization": level, "passed": True, "compile_seconds": 1.0, + return {"passed": True, "seconds": 100.0, "levels": ["none", "all"], + "planned_executions": 4, "cases": [ + {"name": "matmul_matrix", "family": "matmul", "passed": True, "stage": "complete", + "frontend_parse": {"passed": True, "operators": {"MatMul": 1}}, + "ort_vs_numpy": metric(1e-6), "executions": [ + {"optimization": level, "passed": True, "stage": "complete", "compile_seconds": 1.0, "qemu_process_wall_seconds": 2.0, "ir_vs_ort": metric(1e-6), "ir_vs_numpy": metric(2e-6), "qemu_vs_ort": metric(3e-6), "qemu_vs_numpy": metric(4e-6)} for level in ("none", "all")]}, - {"name": "elementwise_broadcast", "passed": True, "executions": [ - {"optimization": "none", "passed": True, "compile_seconds": 1.0, + {"name": "elementwise_broadcast", "family": "elementwise", "passed": True, "stage": "complete", + "frontend_parse": {"passed": True, "operators": {"Add": 1, "Sub": 1, "Mul": 1, "Div": 1}}, + "ort_vs_numpy": metric(1e-6), "executions": [ + {"optimization": level, "passed": True, "stage": "complete", "compile_seconds": 1.0, "qemu_process_wall_seconds": 2.0, "ir_vs_ort": metric(5e-6), "ir_vs_numpy": metric(6e-6), - "qemu_vs_ort": metric(7e-6), "qemu_vs_numpy": metric(8e-6)}]}, + "qemu_vs_ort": metric(7e-6), "qemu_vs_numpy": metric(8e-6)} + for level in ("none", "all")]}, ]} @@ -47,22 +53,33 @@ def reports(): "w2-backend-ops": backend_ops_report(), "w2-runtime": {"gate": "unit:runtime", "passed": True, "seconds": 5.0}, "w2-runtime-model": {"gate": "integration:runtime-model", "passed": True, "seconds": 20.0, - "cases": [{"name": "c0", "executions": [ - {"engine": "ir", "passed": True, "vs_ort": metric(1e-6), - "vs_torch": metric(2e-6)}]}]}, - "qwen3-small": {"passed": True, "seconds": 30.0, + "planned_cases": 2, "levels": ["none", "all"], + "cases": [{"name": name, "executions": [ + {"engine": "ort", "passed": True, "vs_torch": metric(1e-6)}, + {"engine": "ir-none", "passed": True, "vs_ort": metric(1e-6), + "vs_torch": metric(2e-6)}, + {"engine": "ir-all", "passed": True, "vs_ort": metric(1e-6), + "vs_torch": metric(2e-6)}]} + for name in ("chinese", "special_pad_in_prompt")]}, + "qwen3-small": {"passed": True, "stage": "complete", "seconds": 30.0, + "planned_cases": 1, + "frontend_parse": {"passed": True, "operators": {"MatMul": 2, "Add": 3}}, "onnx": {"operators": {"MatMul": 2, "Add": 3}}, - # The host IR comparison wraps one tensor_diff per checkpoint. - "cases": [{"name": "c0", "ir_vs_ort": { - "passed": True, "first_divergence": None, - "checkpoints": [dict(metric(1e-6), name="logits"), - dict(metric(9e-6), name="layer_0.output")]}}]}, + # Ordinary and diagnostic forwards are separate executions. + "cases": [{"name": "c0", "ordinary_logits": {"ir_vs_ort": metric(1e-6)}, + "ir_vs_ort": {"passed": True, "first_divergence": None, + "checkpoints": [dict(metric(2e-6), name="logits"), + dict(metric(9e-6), name="layer_0.output")]}}]}, "qwen3-riscv": {"passed": True, "timing": {"pipeline_seconds": 60.0, "cross_compile_seconds_total": 10.0, "qemu_process_wall_seconds_total": 12.5}, - "cases": [{"name": "c0", "qemu": [metric(1e-6)]}]}, - "qwen3-parse": {"gate": "frontend:parse-qwen3", "passed": True, "seconds": 40.0, - "ir": {"operators": {"matmul": 42}}, + "cases": [{"name": "c0", "qemu": [ + dict(metric(1e-6 if graph == "normal" else 8e-6), graph=graph, optimization=level) + for graph in ("normal", "diagnostic") for level in ("none", "all")]}]}, + "qwen3-parse": {"gate": "frontend:parse-qwen3", "passed": True, "stage": "complete", "seconds": 40.0, + "frontend_parse": {"passed": True, "operators": {"MatMul": 42, "Gather": 1}}, + "ir": {"functions": 1, "operators": {"matmul": 42, "gather": 1}}, + "ir_verifier": {"passed": True}, "graph": {"graph": {"operators": {"MatMul": 42, "Gather": 1}}}}, } @@ -116,7 +133,7 @@ def test_backend_layer_is_not_marked_without_a_passing_qemu_run(tmp_path): execution["passed"] = False payload["w2-backend-ops"]["cases"][1]["executions"] = [] save(tmp_path, payload) - assert row_of(render(tmp_path), "| 汇编层")[2] == "无（本次未执行）" + assert row_of(render(tmp_path), "| 汇编层")[2] == "无（没有通过记录）" def test_precision_legend_explains_every_name_before_the_table(tmp_path): @@ -141,12 +158,12 @@ def test_precision_table_reports_the_worst_recorded_comparison(tmp_path): save(tmp_path, reports()) text = render(tmp_path) qemu = row_of(text, "| w2-backend-ops / QEMU vs ORT") - # Worst of 3e-6, 4e-6, 7e-6 leaves 8e-6 out: that is the NumPy row. + # Worst QEMU/ORT error is 7e-6; 8e-6 belongs only to the NumPy reference row. assert qemu[1] == format(7e-6, ".16e") numpy_row = row_of(text, "| w2-backend-ops / QEMU vs NumPy") assert numpy_row[1] == format(8e-6, ".16e") - # Only the final logits checkpoint counts, not the 9e-6 layer_0.output one. - small = row_of(text, "| qwen3-small host IR / IR vs ORT") + # Use ordinary logits (1e-6), not diagnostic logits (2e-6) or an intermediate (9e-6). + small = row_of(text, "| 两层模型：IR 解释器与 ORT（最终输出）") assert small[1] == format(1e-6, ".16e") @@ -188,15 +205,15 @@ def test_summary_budget_falls_back_without_losing_the_header(tmp_path, monkeypat def test_cli_appends_to_the_github_summary_and_always_succeeds(tmp_path, monkeypatch): destination = tmp_path / "summary.md" - destination.write_text("Prior summary\n") + destination.write_text("Prior summary\n", encoding="utf-8") monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(destination)) assert main(["--report-dir", str(tmp_path / "absent")]) == 0 - assert destination.read_text().startswith("Prior summary\n## W2 测试项\n") + assert destination.read_text(encoding="utf-8").startswith("Prior summary\n## W2 测试项\n") # -S excludes site-packages: the acceptance job installs no dependencies. environment = {key: value for key, value in os.environ.items() if key != "GITHUB_STEP_SUMMARY"} process = subprocess.run([sys.executable, "-S", "-X", "utf8", str(ROOT / "scripts/summarize_w2_ci.py"), "--report-dir", str(tmp_path / "absent")], - cwd=tmp_path, env=environment, text=True, capture_output=True) + cwd=tmp_path, env=environment, text=True, encoding="utf-8", capture_output=True) assert process.returncode == 0, process.stderr assert process.stdout.startswith("## W2 测试项\n") @@ -223,3 +240,209 @@ def test_family_ops_cover_every_operator_the_cases_build(): if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "make_node"} assert built, "no ONNX nodes found in cases.py" assert {opcode_name(op) for op in built} == {op for ops in FAMILY_OPS.values() for op in ops} + + +@pytest.mark.parametrize("missing", [False, True]) +def test_ordinary_logits_failure_or_absence_is_not_hidden_by_diagnostic_pass(tmp_path, missing): + payload = reports() + small = payload["qwen3-small"] + small["passed"] = False + case = small["cases"][0] + case["ir_vs_ort"]["checkpoints"][0] = dict(metric(0), name="logits") + if missing: + case.pop("ordinary_logits") + else: + case["ordinary_logits"]["ir_vs_ort"] = dict(metric(1e-3), passed=False) + save(tmp_path, payload) + row = row_of(render(tmp_path), "| 两层模型：IR 解释器与 ORT（最终输出）") + assert ("未完成" if missing else "未通过") in row[-1] + if missing: + assert "未记录" in row[1] + assert row[1] != format(0, ".16e") + else: + assert row[1] == format(1e-3, ".16e") + assert "失败 1" in row[-1] + + +def test_compile_failure_without_qemu_output_keeps_its_comparison_slot(tmp_path): + payload = {"w2-backend-ops": {"passed": False, "levels": ["none", "all"], + "planned_executions": 2, "cases": [{"name": "matmul_matrix", "executions": [ + {"optimization": "none", "passed": True, "qemu_vs_ort": metric(1e-6)}, + {"optimization": "all", "passed": False, "stage": "compile", "error": "compiler failed"}]}]}} + selected = dict(precision_comparisons(payload))["w2-backend-ops / QEMU vs ORT"] + assert len(selected) == 2 + assert selected[1].get("missing_reason") + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-backend-ops / QEMU vs ORT") + assert "1/2" in row[-1] and "未完成" in row[-1] + assert "未记录 1/2" in row[1] + + +@pytest.mark.parametrize("recorded,planned", [(0, 2), (1, 2), (1, 4)]) +def test_absent_executions_and_remaining_plan_are_not_counted_as_pass(tmp_path, recorded, planned): + executions = ([{"optimization": "none", "passed": True, "qemu_vs_ort": metric(1e-6)}] + if recorded else []) + report = {"passed": False, "levels": ["none", "all"], "planned_executions": planned, + "cases": [{"name": "matmul_matrix", "executions": executions}]} + payload = {"w2-backend-ops": report} + selected = dict(precision_comparisons(payload))["w2-backend-ops / QEMU vs ORT"] + assert len(selected) == planned + assert sum(bool(row.get("missing_reason")) for row in selected) == planned - recorded + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-backend-ops / QEMU vs ORT") + assert "未完成" in row[-1] and f"{recorded}/{planned}" in row[-1] + assert f"未记录 {planned - recorded}/{planned}" in row[1] + + +@pytest.mark.parametrize("chain,stage", [("qwen3-small", "parse"), ("qwen3-parse", "scratchv-parse")]) +def test_failed_frontend_parse_does_not_claim_onnx_inventory_as_measured(tmp_path, chain, stage): + report = reports()[chain] + report.update(passed=False, stage=stage) + report.pop("frontend_parse") + report.pop("ir", None) + report.pop("ir_verifier", None) + report.pop("cases", None) + save(tmp_path, {chain: report}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无（没有通过记录）" + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_backend_only_successful_parse_counts_even_when_later_compile_fails(tmp_path, explicit): + case = {"name": "rmsnorm_hidden_eps_1e-06", "family": "rmsnorm", "passed": False, + "stage": "complete", "ort_vs_numpy": metric(0), + "executions": [{"optimization": "none", "passed": False, + "stage": "compile", "error": "compiler failed"}]} + if explicit: + case["frontend_parse"] = {"passed": True, "operators": { + "Pow": 1, "ReduceMean": 1, "Add": 1, "Sqrt": 1, "Div": 1, "Mul": 1}} + save(tmp_path, {"w2-backend-ops": {"passed": False, "cases": [case]}}) + text = render(tmp_path) + frontend = row_of(text, "| IR 层")[2] + assert "reducemean" in frontend and "sqrt" in frontend and "pow" in frontend + assert row_of(text, "| 汇编层")[2] == "无（没有通过记录）" + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_small_parse_success_is_preserved_after_numeric_runtime_failure(tmp_path, explicit): + report = {"passed": False, "stage": "numeric", "onnx": {"operators": {"MatMul": 2}}, + "cases": [{"name": "first", "passed": False}], "error": "interpreter failed"} + if explicit: + report["frontend_parse"] = {"passed": True, "operators": {"MatMul": 2}} + save(tmp_path, {"qwen3-small": report}) + frontend = row_of(render(tmp_path), "| IR 层")[2] + assert "matmul" in frontend + + +def test_explicit_failed_parse_does_not_fall_back_to_a_later_stage(tmp_path): + report = {"passed": False, "stage": "complete", "onnx": {"operators": {"MatMul": 2}}, + "frontend_parse": {"passed": False, "operators": {"MatMul": 2}}} + save(tmp_path, {"qwen3-small": report}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无（没有通过记录）" + + +def test_runtime_model_expected_engines_keep_missing_results_without_ort_self_comparison(tmp_path): + report = reports()["w2-runtime-model"] + baseline = dict(precision_comparisons({"w2-runtime-model": report})) + assert len(baseline["w2-runtime-model / vs torch"]) == 6 + assert len(baseline["w2-runtime-model / vs ORT"]) == 4 + report["passed"] = False + report["cases"][1]["executions"] = [] + selected = dict(precision_comparisons({"w2-runtime-model": report})) + assert len(selected["w2-runtime-model / vs torch"]) == 6 + assert len(selected["w2-runtime-model / vs ORT"]) == 4 + assert sum(bool(row.get("missing_reason")) for row in selected["w2-runtime-model / vs ORT"]) == 2 + save(tmp_path, {"w2-runtime-model": report}) + row = row_of(render(tmp_path), "| w2-runtime-model / vs ORT") + assert "未记录 2/4" in row[1] and "未完成" in row[-1] + + +def test_qemu_intentionally_retains_normal_and_diagnostic_statistics(tmp_path): + # This PR explicitly keeps the combined scope; changing it is not part of these fixes. + report = reports()["qwen3-riscv"] + for row in report["cases"][0]["qemu"]: + if row["graph"] == "diagnostic": + row.update(relative_l2=8e-5, cosine_similarity=0.75) + payload = {"qwen3-riscv": report} + selected = dict(precision_comparisons(payload))["qwen3-small RV64 QEMU / QEMU vs ORT"] + assert len(selected) == 4 + assert {row["graph"] for row in selected} == {"normal", "diagnostic"} + save(tmp_path, payload) + row = row_of(render(tmp_path), "| qwen3-small RV64 QEMU / QEMU vs ORT") + assert row[1] == format(8e-6, ".16e") + assert row[2] == format(8e-5, ".16e") + assert row[3] == format(0.75, ".17g") + +@pytest.mark.parametrize("legacy", [False, True]) +def test_small_missing_cases_remain_in_expected_count(tmp_path, legacy): + report = reports()["qwen3-small"] + report["passed"] = False + if legacy: + report.pop("planned_cases") + expected = 7 + else: + report["planned_cases"] = 2 + expected = 2 + payload = {"qwen3-small": report} + selected = dict(precision_comparisons(payload))["两层模型：IR 解释器与 ORT（最终输出）"] + assert len(selected) == expected + assert sum(bool(row.get("missing_reason")) for row in selected) == expected - 1 + save(tmp_path, payload) + row = row_of(render(tmp_path), "| 两层模型：IR 解释器与 ORT（最终输出）") + assert "未完成" in row[-1] and f"1/{expected}" in row[-1] + + +def test_runtime_model_missing_case_keeps_both_ir_engines(tmp_path): + report = reports()["w2-runtime-model"] + report["passed"] = False + report["cases"].pop() + payload = {"w2-runtime-model": report} + selected = dict(precision_comparisons(payload)) + assert len(selected["w2-runtime-model / vs torch"]) == 6 + ort = selected["w2-runtime-model / vs ORT"] + assert len(ort) == 4 + assert sum(bool(row.get("missing_reason")) for row in ort) == 2 + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-runtime-model / vs ORT") + assert "未完成" in row[-1] and "2/4" in row[-1] + + +def test_qemu_plan_retains_both_graphs_and_unexecuted_comparisons(tmp_path): + report = reports()["qwen3-riscv"] + report["passed"] = False + report["timing"]["planned_count"] = 6 + payload = {"qwen3-riscv": report} + selected = dict(precision_comparisons(payload))["qwen3-small RV64 QEMU / QEMU vs ORT"] + assert len(selected) == 6 + assert sum(bool(row.get("missing_reason")) for row in selected) == 2 + assert {row.get("graph") for row in selected if not row.get("missing_reason")} == {"normal", "diagnostic"} + save(tmp_path, payload) + row = row_of(render(tmp_path), "| qwen3-small RV64 QEMU / QEMU vs ORT") + assert row[1].startswith(format(8e-6, ".16e")) + assert "未完成" in row[-1] and "4/6" in row[-1] + + +@pytest.mark.parametrize("chain,key", [("w2-backend-ops", "planned_executions"), + ("qwen3-small", "planned_cases"), + ("w2-runtime-model", "planned_cases"), + ("qwen3-riscv", "planned_count")]) +@pytest.mark.parametrize("invalid", [True, -1, "2", 10**9]) +def test_invalid_or_unbounded_planned_count_is_reported(tmp_path, chain, key, invalid): + report = reports()[chain] + target = report["timing"] if chain == "qwen3-riscv" else report + target[key] = invalid + save(tmp_path, {chain: report}) + text = render(tmp_path) + assert "报告结构不符" in text + assert "PASS" not in text + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_backend_onnx_inventory_without_successful_parse_is_not_measured(tmp_path, explicit): + case = {"name": "rmsnorm_hidden_eps_1e-06", "family": "rmsnorm", "passed": False, + "stage": "reference", "error": "parse failed", "executions": []} + if explicit: + # Explicit failed parsing cannot be overridden by stale legacy evidence. + case["frontend_parse"] = {"passed": False, "operators": {"Pow": 1, "ReduceMean": 1}} + case["ort_vs_numpy"] = metric(0) + save(tmp_path, {"w2-backend-ops": {"passed": False, "cases": [case]}}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无（没有通过记录）" diff --git a/tests/test_w2_backend_ops_gate.py b/tests/test_w2_backend_ops_gate.py index cf295fb..c4b439d 100644 --- a/tests/test_w2_backend_ops_gate.py +++ b/tests/test_w2_backend_ops_gate.py @@ -49,6 +49,25 @@ def test_success_requires_both_optimization_levels(tmp_path, monkeypatch, fake_t assert report["passed_executions"] == 2 assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.5} assert report["atol"] == 1e-5 and report["rtol"] == 0 + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} + + +@pytest.mark.parametrize("fault", ["parse", "verifier"]) +def test_failed_frontend_never_records_parsed_operators(tmp_path, monkeypatch, fake_target, fault): + if fault == "parse": + def broken_parse(*args, **kwargs): + raise ValueError("injected parse failure") + monkeypatch.setattr(probe.ONNXParser, "parse", broken_parse) + else: + monkeypatch.setattr(probe, "verify_ir", lambda *args, **kwargs: + (False, ["injected verifier failure"])) + code, report = result(tmp_path) + row = report["cases"][0] + assert code == 1 and not report["passed"] + assert row["stage"] == "reference" and row["executions"] == [] + assert "injected" in row["error"] + assert "frontend_parse" not in row @pytest.mark.parametrize("fault", ["numeric", "nan", "shape", "dtype", "second_level"]) @@ -71,6 +90,8 @@ def execute(*args, **kwargs): assert code == 1 and report["passed"] is False assert report["completed_executions"] == 2 assert report["passed_executions"] == (1 if fault == "second_level" else 0) + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} def test_qemu_timeout_preserves_measured_time_and_failures(tmp_path, monkeypatch, fake_target): @@ -93,6 +114,8 @@ def unexpected(*args, **kwargs): code, report = result(tmp_path) assert code == 1 and report["cases"][0]["executions"] == [] assert "NumPy formula" in report["cases"][0]["error"] + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} def test_compiler_failure_cannot_become_qemu_success(tmp_path, monkeypatch, fake_target): @@ -105,6 +128,8 @@ def unexpected(*args, **kwargs): assert code == 1 and report["completed_executions"] == 0 assert all(item["stage"] == "compile" and "unsupported operator" in item["error"] for item in report["cases"][0]["executions"]) + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} @pytest.mark.parametrize("fault", ["empty", "missing_family", "duplicate_name"]) diff --git a/tests/test_w2_runtime_model.py b/tests/test_w2_runtime_model.py index 333147b..587c9f3 100644 --- a/tests/test_w2_runtime_model.py +++ b/tests/test_w2_runtime_model.py @@ -193,6 +193,23 @@ def unavailable(directory, out, report): assert "probes/w2_runtime_model/run.py" in report["source_sha256"] +def test_run_probe_records_planned_cases_before_tokenizer_failure(tmp_path, monkeypatch): + pytest.importorskip("torch") + pytest.importorskip("transformers") + monkeypatch.setattr(run, "environment", lambda: {}) + monkeypatch.setattr(run, "require_environment", lambda: {}) + + def missing_assets(*args): + raise ValueError("injected tokenizer asset failure") + monkeypatch.setattr(run, "verify_assets", missing_assets) + report = {"passed": False} + with pytest.raises(ValueError, match="injected tokenizer asset failure"): + run.run_probe(tmp_path, tmp_path, report) + assert report["planned_cases"] == len(run.CASES) == 2 + assert report["levels"] == list(run.LEVELS) == ["none", "all"] + assert "cases" not in report and "frontend_parse" not in report + + def test_no_git_source_archive_does_not_block_probe_dispatch(tmp_path, monkeypatch): from probes.w2_runtime import run as runtime # ROOT has no .git entry. Even if this directory lives under another Git