diff --git a/probes/numeric_summary.py b/probes/numeric_summary.py index f4926a4..67a3d98 100644 --- a/probes/numeric_summary.py +++ b/probes/numeric_summary.py @@ -76,8 +76,16 @@ def _table_rows(comparisons): passed = sum(row.get("passed") is True for row in rows) failed = sum(row.get("passed") is False for row in rows) unknown = len(rows) - passed - failed + missing = Counter(str(row["missing_reason"]) for row in rows if row.get("missing_reason")) if not rows: status = "未测(0 条比较)" + elif missing: + reasons = ";".join(f"{reason} ×{count}" for reason, count in missing.items()) + failures = f",数值比较失败 {failed}" if failed else "" + status = (f"未完成(通过 {passed}/{len(rows)}{failures}," + f"未得到结果 {sum(missing.values())}:{reasons})") + if unknown > sum(missing.values()): + status += f";另有 {unknown - sum(missing.values())} 次结果未确认" elif passed == len(rows): status = f"通过({passed}/{len(rows)})" else: diff --git a/probes/w2_backend_ops/run.py b/probes/w2_backend_ops/run.py index 53fc22f..2c01c7a 100644 --- a/probes/w2_backend_ops/run.py +++ b/probes/w2_backend_ops/run.py @@ -6,6 +6,7 @@ from __future__ import annotations import argparse +from collections import Counter import copy import hashlib from html import escape @@ -104,6 +105,10 @@ def run_probe(out, report, *, cc=None, qemu=None, timeout=120): try: path = folder / "model.onnx" expected, parser, program = reference_case(case, path) + row["frontend_parse"] = { + "passed": True, + "operators": dict(sorted(Counter(node.op_type for node in case.model.graph.node).items())), + } np.savez(folder / "inputs.npz", **case.feed) np.save(folder / "numpy.npy", case.expected) np.save(folder / "ort.npy", expected) diff --git a/probes/w2_qwen3_parse/run.py b/probes/w2_qwen3_parse/run.py index 1cf2366..e7e7d22 100644 --- a/probes/w2_qwen3_parse/run.py +++ b/probes/w2_qwen3_parse/run.py @@ -140,6 +140,10 @@ def stage(name, operation): "issues": [str(issue) for issue in issues]} if not verified: raise ValueError("ScratchV IR verifier rejected the complete model; see ir_verifier.issues") + report["frontend_parse"] = { + "passed": True, + "operators": dict(sorted(Counter(node.op_type for node in model.graph.node).items())), + } try: audit = stage("ir-and-bindings", lambda: audit_parsed_model(model, parser, program, model_dir)) except Exception: diff --git a/probes/w2_qwen3_small/run.py b/probes/w2_qwen3_small/run.py index dbcea9f..d4b35bc 100644 --- a/probes/w2_qwen3_small/run.py +++ b/probes/w2_qwen3_small/run.py @@ -199,6 +199,7 @@ def run_probe(out: Path, report: dict, model_seed: int = 0) -> None: name: tensor.detach().cpu().numpy() for name, tensor in wrapper.model.state_dict().items()}) (out / "config.json").write_text(wrapper.model.config.to_json_string(use_diff=False), encoding="utf-8") cases = input_cases() + report["planned_cases"] = len(cases) def reference(feed): args = [torch.from_numpy(feed[name]) for name in ("input_ids", "attention_mask")] @@ -240,12 +241,18 @@ def reference(feed): providers=["CPUExecutionProvider"]) report["stage"] = "parse" parsers, programs = {}, {} + parsed_operators, parsed_graphs = Counter(), [] for kind, filename in [("normal", "model.onnx"), ("diagnostic", "diagnostics.onnx")]: parsers[kind] = ONNXParser() programs[kind] = parsers[kind].parse(str(out / filename)) valid, errors = verify_ir(programs[kind]) if not valid: raise ValueError(f"{kind} IR verification failed: {errors}") + graph = ordinary if kind == "normal" else diagnostic + parsed_operators.update(node.op_type for node in graph.graph.node) + parsed_graphs.append(kind) + report["frontend_parse"] = {"passed": True, "operators": dict(sorted(parsed_operators.items())), + "graphs": list(parsed_graphs)} report["stage"] = "numeric" report["cases"] = [] cached_logits = {} diff --git a/probes/w2_runtime_model/run.py b/probes/w2_runtime_model/run.py index ae8d091..3ebe8f7 100644 --- a/probes/w2_runtime_model/run.py +++ b/probes/w2_runtime_model/run.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse +from collections import Counter import copy from datetime import datetime, timezone import gc @@ -171,7 +172,8 @@ def source_evidence(): def run_probe(directory, out, report): - report.update(stage="environment", environment=environment(), tokenizer_environment=require_environment()) + report.update(stage="environment", planned_cases=len(CASES), levels=list(LEVELS), + environment=environment(), tokenizer_environment=require_environment()) import onnx import onnxruntime as ort import torch @@ -258,6 +260,7 @@ def require_valid(program): check_onnx_contract(exported) report["onnx"] = {"sha256": sha256(out / "model.onnx"), "bytes": (out / "model.onnx").stat().st_size, "nodes": len(exported.graph.node), "opset": 18, + "operators": dict(sorted(Counter(node.op_type for node in exported.graph.node).items())), "exporter": "torch.onnx.export(dynamo=True, optimize=False, fallback=False)", "inputs": {"input_ids": "INT64[1,256]", "attention_mask": "FP32[1,1,256,256]"}, "output": "FP32[1,256,151936]", "output_bytes": SEQ * VOCAB * 4} @@ -292,6 +295,7 @@ def require_valid(program): parser = ONNXParser() program = parser.parse(str(out / "model.onnx")) require_valid(program) + report["frontend_parse"] = {"passed": True, "operators": dict(report["onnx"]["operators"])} report["parse_seconds"] = time.perf_counter() - started report["ir_optimization_seconds"] = {} for level in LEVELS: diff --git a/scripts/summarize_w2_ci.py b/scripts/summarize_w2_ci.py index 48eeea9..1821926 100644 --- a/scripts/summarize_w2_ci.py +++ b/scripts/summarize_w2_ci.py @@ -154,12 +154,28 @@ def _onnx_operators(report, name): return set(_mapping(inner.get("operators"))) +def parsed_operators(record): + """New reports record successful parsing separately from later execution.""" + evidence = _mapping(record.get("frontend_parse")) + if evidence.get("passed") is True: + return {str(op).lower() for op in _mapping(evidence.get("operators"))} + return set() + + def exercised_operators(reports): """``(frontend_op_types, backend_opcodes)`` actually exercised in this run.""" onnx_seen = set() - for name in (QWEN3_SMALL, QWEN3_PARSE): - if reports.get(name) is not None: - onnx_seen |= _onnx_operators(reports[name], name) + for name in (QWEN3_SMALL, QWEN3_PARSE, RUNTIME_MODEL): + report = _mapping(reports.get(name)) + if "frontend_parse" in report: + onnx_seen |= parsed_operators(report) + # Compatibility with reports written before the explicit parse record. + # An exported graph's operator list alone does not prove parsing passed. + elif name == QWEN3_SMALL and report.get("stage") in ("numeric", "complete"): + onnx_seen |= _onnx_operators(report, name) + elif (name == QWEN3_PARSE and isinstance(report.get("ir"), dict) + and _mapping(report.get("ir_verifier")).get("passed") is True): + onnx_seen |= _onnx_operators(report, name) opcodes = set() # The two-layer model is the only graph the RISC-V QEMU probe compiles, so @@ -171,7 +187,13 @@ def exercised_operators(reports): opcodes |= {opcode_name(op) for op in _onnx_operators(reports[QWEN3_SMALL], QWEN3_SMALL)} # The operator probe records only its families; a passing execution proves them. for case in _cases(reports.get(BACKEND_OPS)): - family = str(case.get("name", "")).split("_", 1)[0] + family = case.get("family") or str(case.get("name", "")).split("_", 1)[0] + if "frontend_parse" in case: + onnx_seen |= parsed_operators(case) + elif isinstance(case.get("ort_vs_numpy"), dict) or case.get("model_sha256"): + # Legacy fixed probe graphs: these fields are written only after + # reference_case returned from parse + IR verification. + onnx_seen |= {op.replace("_", "") for op in FAMILY_OPS.get(family, ())} if any(row.get("passed") is True for row in _rows(case, "executions")): opcodes |= set(FAMILY_OPS.get(family, ())) return {str(op).lower() for op in onnx_seen}, opcodes & set(backend_opcodes()) @@ -188,7 +210,7 @@ def measured(names, exercised): if not names: return "无" hit = [name for name in names if name.lower() in exercised] - return listing(hit) if hit else "无(本次未执行)" + return listing(hit) if hit else "无(没有通过记录)" rows = [ ("IR 层 (`ONNXParser._handle_*`)", listing(supported_frontend), @@ -215,40 +237,85 @@ def precision_legend(): ] +def missing_comparison(record=None): + record = _mapping(record) + stage = record.get("stage") or record.get("error_stage") + reason = {"reference": "参考结果未完成", "ir": "IR 计算未完成", + "compile": "编译失败" if record.get("error") else "编译未完成", + "qemu": "QEMU 运行未完成"}.get(stage, "未执行或未记录比较结果") + return {"missing_reason": reason} + + +def comparison(record, key): + result = record.get(key) + return result if isinstance(result, dict) and result else missing_comparison(record) + + +def fill_missing(rows, planned): + # Reports are external input. Bound the count as well as the file size. + if type(planned) is not int or not 0 <= planned <= 4096: + raise ValueError("Invalid planned comparison count") + return rows + [missing_comparison() for _ in range(max(0, planned - len(rows)))] + + +def planned_cases(report, legacy_count): + return fill_missing(_cases(report), report.get("planned_cases", legacy_count)) + + +def planned_executions(report): + executions = [] + levels = report.get("levels", ("none", "all")) + for case in _cases(report): + rows = _rows(case, "executions") + executions.extend(rows) + present = {row.get("optimization") for row in rows} + executions.extend({"optimization": level, "stage": case.get("stage"), + "error": case.get("error")} + for level in levels if level not in present) + return fill_missing(executions, report.get("planned_executions", len(executions))) + + def precision_comparisons(reports): comparisons = [] def add(label, records): - comparisons.append((label, [row for row in records if isinstance(row, dict)])) + comparisons.append((label, [row if isinstance(row, dict) + else missing_comparison() for row in records])) backend = reports.get(BACKEND_OPS) if backend is not None: - executions = [row for case in _cases(backend) for row in _rows(case, "executions")] + executions = planned_executions(backend) for key, label in (("ir_vs_ort", "IR vs ORT"), ("ir_vs_numpy", "IR vs NumPy"), ("qemu_vs_ort", "QEMU vs ORT"), ("qemu_vs_numpy", "QEMU vs NumPy")): - add(f"w2-backend-ops / {label}", [row.get(key) for row in executions]) + add(f"w2-backend-ops / {label}", [comparison(row, key) for row in executions]) if reports.get(QWEN3_SMALL) is not None: - # The host IR comparison wraps one tensor_diff per checkpoint; report - # the final output, which is what the probe's own gate compares. - records = [] - for case in _cases(reports[QWEN3_SMALL]): - checkpoints = _rows(_mapping(case.get("ir_vs_ort")), "checkpoints") - final = [row for row in checkpoints if row.get("name") == "logits"] - if final: - records.extend(final) - elif isinstance(case.get("ordinary_logits"), dict): - records.append(case["ordinary_logits"].get("ir_vs_ort")) - add("qwen3-small host IR / IR vs ORT(最终 logits)", records) + # Do not substitute the diagnostic graph when ordinary execution fails. + records = [comparison(_mapping(case.get("ordinary_logits")), "ir_vs_ort") + for case in planned_cases(reports[QWEN3_SMALL], 7)] + add("两层模型:IR 解释器与 ORT(最终输出)", records) if reports.get(QWEN3_RISCV) is not None: - add("qwen3-small RV64 QEMU / QEMU vs ORT", - [row for case in _cases(reports[QWEN3_RISCV]) for row in _rows(case, "qemu")]) + report = reports[QWEN3_RISCV] + # Keep both normal and diagnostic results, as requested. Only account + # for missing executions; do not change the scope of this comparison. + rows = [row for case in _cases(report) for row in _rows(case, "qemu")] + planned = _mapping(report.get("timing")).get("planned_count", len(rows)) + add("qwen3-small RV64 QEMU / QEMU vs ORT", fill_missing(rows, planned)) if reports.get(RUNTIME_MODEL) is not None: - executions = [row for case in _cases(reports[RUNTIME_MODEL]) for row in _rows(case, "executions")] - add("w2-runtime-model / vs torch", [row.get("vs_torch") for row in executions]) - add("w2-runtime-model / vs ORT", [row.get("vs_ort") for row in executions]) + report = reports[RUNTIME_MODEL] + engines = ["ort", *(f"ir-{level}" for level in report.get("levels", ("none", "all")))] + executions = [] + for case in planned_cases(report, 2): + rows = _rows(case, "executions") + executions.extend(rows) + present = {row.get("engine") for row in rows} + executions.extend({"engine": engine} for engine in engines if engine not in present) + add("w2-runtime-model / vs torch", [comparison(row, "vs_torch") for row in executions]) + # ORT is the reference here; there is deliberately no ORT-vs-itself test. + add("w2-runtime-model / vs ORT", [comparison(row, "vs_ort") for row in executions + if row.get("engine") != "ort"]) # These two chains publish structural evidence only; say so instead of omitting them. add("w2-runtime(文本层,无数值比较)", []) @@ -318,7 +385,9 @@ def render(directory): "请检查各 W2 job 的日志与产物。\n") lines = ["## W2 测试项\n\n", "### 算子支持\n\n", *operators, - "\n### 精度\n\n", *precision_legend(), precision, "\n\n", METRIC_EXPLANATION, "\n\n", + "\n### 精度\n\n", *precision_legend(), precision, + "\n\nIR 最终输出一行使用只输出最终结果的模型,不用额外输出中间值的版本代替。\n\n", + METRIC_EXPLANATION, "\n\n", "### 耗时\n\n", *timing] rendered = "".join(lines) if len(rendered.encode("utf-8")) > MAX_SUMMARY_BYTES: diff --git a/tests/test_numeric_summary.py b/tests/test_numeric_summary.py index 23f8126..1b4e713 100644 --- a/tests/test_numeric_summary.py +++ b/tests/test_numeric_summary.py @@ -51,6 +51,21 @@ def test_failures_are_not_hidden_by_passing_rows(): assert "未记录 1/3" in result +@pytest.mark.parametrize("render", [compact_metric_table, compact_metric_table_html]) +def test_missing_comparisons_keep_the_count_and_failure_reason(render): + result = render([("partial", [record(), {"missing_reason": "编译失败"}])]) + assert "未完成(通过 1/2,未得到结果 1:编译失败 ×1)" in result + assert "已记录 1/2" in result + assert "未记录 1/2" in result + assert "通过(1/1)" not in result + + +def test_numerical_failures_remain_distinct_from_missing_comparisons(): + failed = dict(record(), passed=False) + result = compact_metric_table([("mixed", [record(), failed, {"missing_reason": "编译失败"}])]) + assert "通过 1/3,数值比较失败 1,未得到结果 1" in result + + def test_undefined_values_show_reason_and_coverage_not_zero(): undefined = dict(passed=True, max_abs=0.0, relative_l2=None, cosine_similarity=None, relative_l2_reason="zero reference norm", cosine_similarity_reason="zero norm") diff --git a/tests/test_qwen3_full_parse.py b/tests/test_qwen3_full_parse.py index 970abec..09575d5 100644 --- a/tests/test_qwen3_full_parse.py +++ b/tests/test_qwen3_full_parse.py @@ -327,6 +327,7 @@ def bad_files(*args): ]) def test_real_decoder_pipeline_and_failure_context(tmp_path, monkeypatch, fault, evidence_locked): """Exercise the stage wiring with real files, parser and audits, cheaply.""" + from collections import Counter import onnx from scratchv.analysis import ir_verifier from probes.w2_qwen3_parse import audit, validation @@ -420,3 +421,12 @@ def write(path, value): assert report["audit"]["return"]["shape"] == [1, 3, 9] for name, evidence in report["evidence"].items(): assert evidence["sha256"] == gate.artifacts.sha256(out / name) + if fault in ("parse", "verifier"): + assert "frontend_parse" not in report + else: + # A later binding/evidence failure does not erase successful parsing. + # Count actual ONNX types, not the deliberately synthetic graph summary. + assert report["frontend_parse"] == { + "passed": True, + "operators": dict(Counter(node.op_type for node in model.graph.node)), + } diff --git a/tests/test_qwen3_small_model.py b/tests/test_qwen3_small_model.py index ca258e9..b4d4daf 100644 --- a/tests/test_qwen3_small_model.py +++ b/tests/test_qwen3_small_model.py @@ -50,6 +50,18 @@ def checkpoints(wrapper, inputs): return dict(zip(wrapper.output_names, result)) +@pytest.fixture(scope="module") +def exported_model(wrapper, inputs, tmp_path_factory): + pytest.importorskip("onnx") + pytest.importorskip("onnxruntime") + pytest.importorskip("onnxscript") + path = tmp_path_factory.mktemp("qwen3-small-export") / "checkpoints.onnx" + original = modeling_qwen3.eager_attention_forward + metadata = export_onnx(wrapper, path, *inputs) + assert_observers_removed(wrapper, original) + return path, metadata + + def raw_logits(model, inputs, positions): return model( input_ids=inputs[0], attention_mask=inputs[1], position_ids=positions, @@ -244,13 +256,12 @@ def test_capture_does_not_intercept_other_official_model_with_more_layers(wrappe assert_observers_removed(wrapper, original) -def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, checkpoints, tmp_path): +def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, checkpoints, exported_model): onnx = pytest.importorskip("onnx") ort = pytest.importorskip("onnxruntime") pytest.importorskip("onnxscript") original = modeling_qwen3.eager_attention_forward - path = tmp_path / "diagnostics.onnx" - metadata = export_onnx(wrapper, path, *inputs) + path, metadata = exported_model assert_observers_removed(wrapper, original) graph = onnx.load(str(path)) onnx.checker.check_model(graph) @@ -267,3 +278,68 @@ def test_dynamo_export_names_and_all_checkpoints_round_trip(wrapper, inputs, che for name, result in zip(wrapper.output_names, actual): np.testing.assert_allclose(result, checkpoints[name].numpy(), atol=1e-5, rtol=1e-5, err_msg=name) + + +@pytest.mark.parametrize("fault", ["normal_parse", "normal_verifier", "diagnostic_parse", + "diagnostic_verifier", "numeric"]) +def test_run_probe_records_only_verified_graphs(tmp_path, monkeypatch, wrapper, exported_model, fault): + """Run the real small-model stages; reuse one export and stop at the fault.""" + from collections import Counter + import shutil + import onnx + from probes.w2_qwen3_small import model, run + from scratchv.analysis import ir_verifier + from scratchv.frontend.onnx_parser import ONNXParser + from scratchv.verification.ir_interpreter import IRInterpreter + + source, metadata = exported_model + monkeypatch.setattr(run, "environment", lambda: {"fixture": "pinned small-model stack"}) + monkeypatch.setattr(model, "build_model", lambda **kwargs: wrapper) + + def reuse_export(wrapper, path, *inputs): + shutil.copyfile(source, path) + return metadata + monkeypatch.setattr(model, "export_onnx", reuse_export) + original_parse, original_verify = ONNXParser.parse, ir_verifier.verify_ir + parsed = [] + + def parse(self, path): + kind = "normal" if str(path).endswith("model.onnx") else "diagnostic" + parsed.append(kind) + if fault == f"{kind}_parse": + raise ValueError(f"injected {fault}") + return original_parse(self, path) + + def verify(program): + if fault == f"{parsed[-1]}_verifier": + return False, [f"injected {fault}"] + return original_verify(program) + + def numeric_failure(*args, **kwargs): + raise ValueError("injected numeric") + + monkeypatch.setattr(ONNXParser, "parse", parse) + monkeypatch.setattr(ir_verifier, "verify_ir", verify) + if fault == "numeric": + monkeypatch.setattr(IRInterpreter, "run", numeric_failure) + report = {"passed": False} + previous = torch.are_deterministic_algorithms_enabled() + previous_warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + try: + with pytest.raises(ValueError, match="injected"): + run.run_probe(tmp_path, report, model_seed=193) + finally: + torch.use_deterministic_algorithms(previous, warn_only=previous_warn_only) + assert not report["passed"] + assert report["planned_cases"] == len(run.input_cases()) == 7 + assert report["stage"] == ("numeric" if fault == "numeric" else "parse") + if fault.startswith("normal"): + assert "frontend_parse" not in report + else: + graphs = ["normal", "diagnostic"] if fault == "numeric" else ["normal"] + operators = Counter() + for kind in graphs: + path = tmp_path / ("model.onnx" if kind == "normal" else "diagnostics.onnx") + operators.update(node.op_type for node in onnx.load(path).graph.node) + assert report["frontend_parse"] == { + "passed": True, "operators": dict(operators), "graphs": graphs} diff --git a/tests/test_w2_acceptance_summary.py b/tests/test_w2_acceptance_summary.py index ac5ca89..199f52c 100644 --- a/tests/test_w2_acceptance_summary.py +++ b/tests/test_w2_acceptance_summary.py @@ -14,7 +14,7 @@ import pytest from scripts.summarize_w2_ci import (FAMILY_OPS, MAX_SUMMARY_BYTES, backend_opcodes, - frontend_operators, main, opcode_name, render) + frontend_operators, main, opcode_name, precision_comparisons, render) ROOT = Path(__file__).resolve().parents[1] DIRECTORIES = ("w2-backend-ops", "w2-runtime", "w2-runtime-model", @@ -27,18 +27,24 @@ def metric(value, name="logits.npy"): def backend_ops_report(): - return {"passed": True, "seconds": 100.0, "cases": [ - {"name": "matmul_matrix", "passed": True, "executions": [ - {"optimization": level, "passed": True, "compile_seconds": 1.0, + return {"passed": True, "seconds": 100.0, "levels": ["none", "all"], + "planned_executions": 4, "cases": [ + {"name": "matmul_matrix", "family": "matmul", "passed": True, "stage": "complete", + "frontend_parse": {"passed": True, "operators": {"MatMul": 1}}, + "ort_vs_numpy": metric(1e-6), "executions": [ + {"optimization": level, "passed": True, "stage": "complete", "compile_seconds": 1.0, "qemu_process_wall_seconds": 2.0, "ir_vs_ort": metric(1e-6), "ir_vs_numpy": metric(2e-6), "qemu_vs_ort": metric(3e-6), "qemu_vs_numpy": metric(4e-6)} for level in ("none", "all")]}, - {"name": "elementwise_broadcast", "passed": True, "executions": [ - {"optimization": "none", "passed": True, "compile_seconds": 1.0, + {"name": "elementwise_broadcast", "family": "elementwise", "passed": True, "stage": "complete", + "frontend_parse": {"passed": True, "operators": {"Add": 1, "Sub": 1, "Mul": 1, "Div": 1}}, + "ort_vs_numpy": metric(1e-6), "executions": [ + {"optimization": level, "passed": True, "stage": "complete", "compile_seconds": 1.0, "qemu_process_wall_seconds": 2.0, "ir_vs_ort": metric(5e-6), "ir_vs_numpy": metric(6e-6), - "qemu_vs_ort": metric(7e-6), "qemu_vs_numpy": metric(8e-6)}]}, + "qemu_vs_ort": metric(7e-6), "qemu_vs_numpy": metric(8e-6)} + for level in ("none", "all")]}, ]} @@ -47,22 +53,33 @@ def reports(): "w2-backend-ops": backend_ops_report(), "w2-runtime": {"gate": "unit:runtime", "passed": True, "seconds": 5.0}, "w2-runtime-model": {"gate": "integration:runtime-model", "passed": True, "seconds": 20.0, - "cases": [{"name": "c0", "executions": [ - {"engine": "ir", "passed": True, "vs_ort": metric(1e-6), - "vs_torch": metric(2e-6)}]}]}, - "qwen3-small": {"passed": True, "seconds": 30.0, + "planned_cases": 2, "levels": ["none", "all"], + "cases": [{"name": name, "executions": [ + {"engine": "ort", "passed": True, "vs_torch": metric(1e-6)}, + {"engine": "ir-none", "passed": True, "vs_ort": metric(1e-6), + "vs_torch": metric(2e-6)}, + {"engine": "ir-all", "passed": True, "vs_ort": metric(1e-6), + "vs_torch": metric(2e-6)}]} + for name in ("chinese", "special_pad_in_prompt")]}, + "qwen3-small": {"passed": True, "stage": "complete", "seconds": 30.0, + "planned_cases": 1, + "frontend_parse": {"passed": True, "operators": {"MatMul": 2, "Add": 3}}, "onnx": {"operators": {"MatMul": 2, "Add": 3}}, - # The host IR comparison wraps one tensor_diff per checkpoint. - "cases": [{"name": "c0", "ir_vs_ort": { - "passed": True, "first_divergence": None, - "checkpoints": [dict(metric(1e-6), name="logits"), - dict(metric(9e-6), name="layer_0.output")]}}]}, + # Ordinary and diagnostic forwards are separate executions. + "cases": [{"name": "c0", "ordinary_logits": {"ir_vs_ort": metric(1e-6)}, + "ir_vs_ort": {"passed": True, "first_divergence": None, + "checkpoints": [dict(metric(2e-6), name="logits"), + dict(metric(9e-6), name="layer_0.output")]}}]}, "qwen3-riscv": {"passed": True, "timing": {"pipeline_seconds": 60.0, "cross_compile_seconds_total": 10.0, "qemu_process_wall_seconds_total": 12.5}, - "cases": [{"name": "c0", "qemu": [metric(1e-6)]}]}, - "qwen3-parse": {"gate": "frontend:parse-qwen3", "passed": True, "seconds": 40.0, - "ir": {"operators": {"matmul": 42}}, + "cases": [{"name": "c0", "qemu": [ + dict(metric(1e-6 if graph == "normal" else 8e-6), graph=graph, optimization=level) + for graph in ("normal", "diagnostic") for level in ("none", "all")]}]}, + "qwen3-parse": {"gate": "frontend:parse-qwen3", "passed": True, "stage": "complete", "seconds": 40.0, + "frontend_parse": {"passed": True, "operators": {"MatMul": 42, "Gather": 1}}, + "ir": {"functions": 1, "operators": {"matmul": 42, "gather": 1}}, + "ir_verifier": {"passed": True}, "graph": {"graph": {"operators": {"MatMul": 42, "Gather": 1}}}}, } @@ -116,7 +133,7 @@ def test_backend_layer_is_not_marked_without_a_passing_qemu_run(tmp_path): execution["passed"] = False payload["w2-backend-ops"]["cases"][1]["executions"] = [] save(tmp_path, payload) - assert row_of(render(tmp_path), "| 汇编层")[2] == "无(本次未执行)" + assert row_of(render(tmp_path), "| 汇编层")[2] == "无(没有通过记录)" def test_precision_legend_explains_every_name_before_the_table(tmp_path): @@ -141,12 +158,12 @@ def test_precision_table_reports_the_worst_recorded_comparison(tmp_path): save(tmp_path, reports()) text = render(tmp_path) qemu = row_of(text, "| w2-backend-ops / QEMU vs ORT") - # Worst of 3e-6, 4e-6, 7e-6 leaves 8e-6 out: that is the NumPy row. + # Worst QEMU/ORT error is 7e-6; 8e-6 belongs only to the NumPy reference row. assert qemu[1] == format(7e-6, ".16e") numpy_row = row_of(text, "| w2-backend-ops / QEMU vs NumPy") assert numpy_row[1] == format(8e-6, ".16e") - # Only the final logits checkpoint counts, not the 9e-6 layer_0.output one. - small = row_of(text, "| qwen3-small host IR / IR vs ORT") + # Use ordinary logits (1e-6), not diagnostic logits (2e-6) or an intermediate (9e-6). + small = row_of(text, "| 两层模型:IR 解释器与 ORT(最终输出)") assert small[1] == format(1e-6, ".16e") @@ -188,15 +205,15 @@ def test_summary_budget_falls_back_without_losing_the_header(tmp_path, monkeypat def test_cli_appends_to_the_github_summary_and_always_succeeds(tmp_path, monkeypatch): destination = tmp_path / "summary.md" - destination.write_text("Prior summary\n") + destination.write_text("Prior summary\n", encoding="utf-8") monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(destination)) assert main(["--report-dir", str(tmp_path / "absent")]) == 0 - assert destination.read_text().startswith("Prior summary\n## W2 测试项\n") + assert destination.read_text(encoding="utf-8").startswith("Prior summary\n## W2 测试项\n") # -S excludes site-packages: the acceptance job installs no dependencies. environment = {key: value for key, value in os.environ.items() if key != "GITHUB_STEP_SUMMARY"} process = subprocess.run([sys.executable, "-S", "-X", "utf8", str(ROOT / "scripts/summarize_w2_ci.py"), "--report-dir", str(tmp_path / "absent")], - cwd=tmp_path, env=environment, text=True, capture_output=True) + cwd=tmp_path, env=environment, text=True, encoding="utf-8", capture_output=True) assert process.returncode == 0, process.stderr assert process.stdout.startswith("## W2 测试项\n") @@ -223,3 +240,209 @@ def test_family_ops_cover_every_operator_the_cases_build(): if isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "make_node"} assert built, "no ONNX nodes found in cases.py" assert {opcode_name(op) for op in built} == {op for ops in FAMILY_OPS.values() for op in ops} + + +@pytest.mark.parametrize("missing", [False, True]) +def test_ordinary_logits_failure_or_absence_is_not_hidden_by_diagnostic_pass(tmp_path, missing): + payload = reports() + small = payload["qwen3-small"] + small["passed"] = False + case = small["cases"][0] + case["ir_vs_ort"]["checkpoints"][0] = dict(metric(0), name="logits") + if missing: + case.pop("ordinary_logits") + else: + case["ordinary_logits"]["ir_vs_ort"] = dict(metric(1e-3), passed=False) + save(tmp_path, payload) + row = row_of(render(tmp_path), "| 两层模型:IR 解释器与 ORT(最终输出)") + assert ("未完成" if missing else "未通过") in row[-1] + if missing: + assert "未记录" in row[1] + assert row[1] != format(0, ".16e") + else: + assert row[1] == format(1e-3, ".16e") + assert "失败 1" in row[-1] + + +def test_compile_failure_without_qemu_output_keeps_its_comparison_slot(tmp_path): + payload = {"w2-backend-ops": {"passed": False, "levels": ["none", "all"], + "planned_executions": 2, "cases": [{"name": "matmul_matrix", "executions": [ + {"optimization": "none", "passed": True, "qemu_vs_ort": metric(1e-6)}, + {"optimization": "all", "passed": False, "stage": "compile", "error": "compiler failed"}]}]}} + selected = dict(precision_comparisons(payload))["w2-backend-ops / QEMU vs ORT"] + assert len(selected) == 2 + assert selected[1].get("missing_reason") + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-backend-ops / QEMU vs ORT") + assert "1/2" in row[-1] and "未完成" in row[-1] + assert "未记录 1/2" in row[1] + + +@pytest.mark.parametrize("recorded,planned", [(0, 2), (1, 2), (1, 4)]) +def test_absent_executions_and_remaining_plan_are_not_counted_as_pass(tmp_path, recorded, planned): + executions = ([{"optimization": "none", "passed": True, "qemu_vs_ort": metric(1e-6)}] + if recorded else []) + report = {"passed": False, "levels": ["none", "all"], "planned_executions": planned, + "cases": [{"name": "matmul_matrix", "executions": executions}]} + payload = {"w2-backend-ops": report} + selected = dict(precision_comparisons(payload))["w2-backend-ops / QEMU vs ORT"] + assert len(selected) == planned + assert sum(bool(row.get("missing_reason")) for row in selected) == planned - recorded + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-backend-ops / QEMU vs ORT") + assert "未完成" in row[-1] and f"{recorded}/{planned}" in row[-1] + assert f"未记录 {planned - recorded}/{planned}" in row[1] + + +@pytest.mark.parametrize("chain,stage", [("qwen3-small", "parse"), ("qwen3-parse", "scratchv-parse")]) +def test_failed_frontend_parse_does_not_claim_onnx_inventory_as_measured(tmp_path, chain, stage): + report = reports()[chain] + report.update(passed=False, stage=stage) + report.pop("frontend_parse") + report.pop("ir", None) + report.pop("ir_verifier", None) + report.pop("cases", None) + save(tmp_path, {chain: report}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无(没有通过记录)" + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_backend_only_successful_parse_counts_even_when_later_compile_fails(tmp_path, explicit): + case = {"name": "rmsnorm_hidden_eps_1e-06", "family": "rmsnorm", "passed": False, + "stage": "complete", "ort_vs_numpy": metric(0), + "executions": [{"optimization": "none", "passed": False, + "stage": "compile", "error": "compiler failed"}]} + if explicit: + case["frontend_parse"] = {"passed": True, "operators": { + "Pow": 1, "ReduceMean": 1, "Add": 1, "Sqrt": 1, "Div": 1, "Mul": 1}} + save(tmp_path, {"w2-backend-ops": {"passed": False, "cases": [case]}}) + text = render(tmp_path) + frontend = row_of(text, "| IR 层")[2] + assert "reducemean" in frontend and "sqrt" in frontend and "pow" in frontend + assert row_of(text, "| 汇编层")[2] == "无(没有通过记录)" + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_small_parse_success_is_preserved_after_numeric_runtime_failure(tmp_path, explicit): + report = {"passed": False, "stage": "numeric", "onnx": {"operators": {"MatMul": 2}}, + "cases": [{"name": "first", "passed": False}], "error": "interpreter failed"} + if explicit: + report["frontend_parse"] = {"passed": True, "operators": {"MatMul": 2}} + save(tmp_path, {"qwen3-small": report}) + frontend = row_of(render(tmp_path), "| IR 层")[2] + assert "matmul" in frontend + + +def test_explicit_failed_parse_does_not_fall_back_to_a_later_stage(tmp_path): + report = {"passed": False, "stage": "complete", "onnx": {"operators": {"MatMul": 2}}, + "frontend_parse": {"passed": False, "operators": {"MatMul": 2}}} + save(tmp_path, {"qwen3-small": report}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无(没有通过记录)" + + +def test_runtime_model_expected_engines_keep_missing_results_without_ort_self_comparison(tmp_path): + report = reports()["w2-runtime-model"] + baseline = dict(precision_comparisons({"w2-runtime-model": report})) + assert len(baseline["w2-runtime-model / vs torch"]) == 6 + assert len(baseline["w2-runtime-model / vs ORT"]) == 4 + report["passed"] = False + report["cases"][1]["executions"] = [] + selected = dict(precision_comparisons({"w2-runtime-model": report})) + assert len(selected["w2-runtime-model / vs torch"]) == 6 + assert len(selected["w2-runtime-model / vs ORT"]) == 4 + assert sum(bool(row.get("missing_reason")) for row in selected["w2-runtime-model / vs ORT"]) == 2 + save(tmp_path, {"w2-runtime-model": report}) + row = row_of(render(tmp_path), "| w2-runtime-model / vs ORT") + assert "未记录 2/4" in row[1] and "未完成" in row[-1] + + +def test_qemu_intentionally_retains_normal_and_diagnostic_statistics(tmp_path): + # This PR explicitly keeps the combined scope; changing it is not part of these fixes. + report = reports()["qwen3-riscv"] + for row in report["cases"][0]["qemu"]: + if row["graph"] == "diagnostic": + row.update(relative_l2=8e-5, cosine_similarity=0.75) + payload = {"qwen3-riscv": report} + selected = dict(precision_comparisons(payload))["qwen3-small RV64 QEMU / QEMU vs ORT"] + assert len(selected) == 4 + assert {row["graph"] for row in selected} == {"normal", "diagnostic"} + save(tmp_path, payload) + row = row_of(render(tmp_path), "| qwen3-small RV64 QEMU / QEMU vs ORT") + assert row[1] == format(8e-6, ".16e") + assert row[2] == format(8e-5, ".16e") + assert row[3] == format(0.75, ".17g") + +@pytest.mark.parametrize("legacy", [False, True]) +def test_small_missing_cases_remain_in_expected_count(tmp_path, legacy): + report = reports()["qwen3-small"] + report["passed"] = False + if legacy: + report.pop("planned_cases") + expected = 7 + else: + report["planned_cases"] = 2 + expected = 2 + payload = {"qwen3-small": report} + selected = dict(precision_comparisons(payload))["两层模型:IR 解释器与 ORT(最终输出)"] + assert len(selected) == expected + assert sum(bool(row.get("missing_reason")) for row in selected) == expected - 1 + save(tmp_path, payload) + row = row_of(render(tmp_path), "| 两层模型:IR 解释器与 ORT(最终输出)") + assert "未完成" in row[-1] and f"1/{expected}" in row[-1] + + +def test_runtime_model_missing_case_keeps_both_ir_engines(tmp_path): + report = reports()["w2-runtime-model"] + report["passed"] = False + report["cases"].pop() + payload = {"w2-runtime-model": report} + selected = dict(precision_comparisons(payload)) + assert len(selected["w2-runtime-model / vs torch"]) == 6 + ort = selected["w2-runtime-model / vs ORT"] + assert len(ort) == 4 + assert sum(bool(row.get("missing_reason")) for row in ort) == 2 + save(tmp_path, payload) + row = row_of(render(tmp_path), "| w2-runtime-model / vs ORT") + assert "未完成" in row[-1] and "2/4" in row[-1] + + +def test_qemu_plan_retains_both_graphs_and_unexecuted_comparisons(tmp_path): + report = reports()["qwen3-riscv"] + report["passed"] = False + report["timing"]["planned_count"] = 6 + payload = {"qwen3-riscv": report} + selected = dict(precision_comparisons(payload))["qwen3-small RV64 QEMU / QEMU vs ORT"] + assert len(selected) == 6 + assert sum(bool(row.get("missing_reason")) for row in selected) == 2 + assert {row.get("graph") for row in selected if not row.get("missing_reason")} == {"normal", "diagnostic"} + save(tmp_path, payload) + row = row_of(render(tmp_path), "| qwen3-small RV64 QEMU / QEMU vs ORT") + assert row[1].startswith(format(8e-6, ".16e")) + assert "未完成" in row[-1] and "4/6" in row[-1] + + +@pytest.mark.parametrize("chain,key", [("w2-backend-ops", "planned_executions"), + ("qwen3-small", "planned_cases"), + ("w2-runtime-model", "planned_cases"), + ("qwen3-riscv", "planned_count")]) +@pytest.mark.parametrize("invalid", [True, -1, "2", 10**9]) +def test_invalid_or_unbounded_planned_count_is_reported(tmp_path, chain, key, invalid): + report = reports()[chain] + target = report["timing"] if chain == "qwen3-riscv" else report + target[key] = invalid + save(tmp_path, {chain: report}) + text = render(tmp_path) + assert "报告结构不符" in text + assert "PASS" not in text + + +@pytest.mark.parametrize("explicit", [False, True]) +def test_backend_onnx_inventory_without_successful_parse_is_not_measured(tmp_path, explicit): + case = {"name": "rmsnorm_hidden_eps_1e-06", "family": "rmsnorm", "passed": False, + "stage": "reference", "error": "parse failed", "executions": []} + if explicit: + # Explicit failed parsing cannot be overridden by stale legacy evidence. + case["frontend_parse"] = {"passed": False, "operators": {"Pow": 1, "ReduceMean": 1}} + case["ort_vs_numpy"] = metric(0) + save(tmp_path, {"w2-backend-ops": {"passed": False, "cases": [case]}}) + assert row_of(render(tmp_path), "| IR 层")[2] == "无(没有通过记录)" diff --git a/tests/test_w2_backend_ops_gate.py b/tests/test_w2_backend_ops_gate.py index cf295fb..c4b439d 100644 --- a/tests/test_w2_backend_ops_gate.py +++ b/tests/test_w2_backend_ops_gate.py @@ -49,6 +49,25 @@ def test_success_requires_both_optimization_levels(tmp_path, monkeypatch, fake_t assert report["passed_executions"] == 2 assert report["qemu_process_wall_seconds"] == {"measured_count": 2, "total": 0.5} assert report["atol"] == 1e-5 and report["rtol"] == 0 + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} + + +@pytest.mark.parametrize("fault", ["parse", "verifier"]) +def test_failed_frontend_never_records_parsed_operators(tmp_path, monkeypatch, fake_target, fault): + if fault == "parse": + def broken_parse(*args, **kwargs): + raise ValueError("injected parse failure") + monkeypatch.setattr(probe.ONNXParser, "parse", broken_parse) + else: + monkeypatch.setattr(probe, "verify_ir", lambda *args, **kwargs: + (False, ["injected verifier failure"])) + code, report = result(tmp_path) + row = report["cases"][0] + assert code == 1 and not report["passed"] + assert row["stage"] == "reference" and row["executions"] == [] + assert "injected" in row["error"] + assert "frontend_parse" not in row @pytest.mark.parametrize("fault", ["numeric", "nan", "shape", "dtype", "second_level"]) @@ -71,6 +90,8 @@ def execute(*args, **kwargs): assert code == 1 and report["passed"] is False assert report["completed_executions"] == 2 assert report["passed_executions"] == (1 if fault == "second_level" else 0) + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} def test_qemu_timeout_preserves_measured_time_and_failures(tmp_path, monkeypatch, fake_target): @@ -93,6 +114,8 @@ def unexpected(*args, **kwargs): code, report = result(tmp_path) assert code == 1 and report["cases"][0]["executions"] == [] assert "NumPy formula" in report["cases"][0]["error"] + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} def test_compiler_failure_cannot_become_qemu_success(tmp_path, monkeypatch, fake_target): @@ -105,6 +128,8 @@ def unexpected(*args, **kwargs): assert code == 1 and report["completed_executions"] == 0 assert all(item["stage"] == "compile" and "unsupported operator" in item["error"] for item in report["cases"][0]["executions"]) + assert report["cases"][0]["frontend_parse"] == { + "passed": True, "operators": {"MatMul": 1}} @pytest.mark.parametrize("fault", ["empty", "missing_family", "duplicate_name"]) diff --git a/tests/test_w2_runtime_model.py b/tests/test_w2_runtime_model.py index 333147b..587c9f3 100644 --- a/tests/test_w2_runtime_model.py +++ b/tests/test_w2_runtime_model.py @@ -193,6 +193,23 @@ def unavailable(directory, out, report): assert "probes/w2_runtime_model/run.py" in report["source_sha256"] +def test_run_probe_records_planned_cases_before_tokenizer_failure(tmp_path, monkeypatch): + pytest.importorskip("torch") + pytest.importorskip("transformers") + monkeypatch.setattr(run, "environment", lambda: {}) + monkeypatch.setattr(run, "require_environment", lambda: {}) + + def missing_assets(*args): + raise ValueError("injected tokenizer asset failure") + monkeypatch.setattr(run, "verify_assets", missing_assets) + report = {"passed": False} + with pytest.raises(ValueError, match="injected tokenizer asset failure"): + run.run_probe(tmp_path, tmp_path, report) + assert report["planned_cases"] == len(run.CASES) == 2 + assert report["levels"] == list(run.LEVELS) == ["none", "all"] + assert "cases" not in report and "frontend_parse" not in report + + def test_no_git_source_archive_does_not_block_probe_dispatch(tmp_path, monkeypatch): from probes.w2_runtime import run as runtime # ROOT has no .git entry. Even if this directory lives under another Git