diff --git a/models/experimental/RATIO_CHUNKS.ja.md b/models/experimental/RATIO_CHUNKS.ja.md new file mode 100644 index 0000000..c548534 --- /dev/null +++ b/models/experimental/RATIO_CHUNKS.ja.md @@ -0,0 +1,59 @@ + +# 固定ratioモデルのchunk比較 + +感度実験で選んだ因子0.90を動かさず、同じtuning 8録音のfull cp1252/UTF-8全16入力を +whole / 1 / 7 / 64 / 1024-byte chunkで観測する。各入力4 KiB以下・各process10秒上限。 +標準legacy、identity/filteredの因子1/0.90を比較し、全400観測を保存する。 +因子やtableを調整する試験ではなく、incrementalの採用gateを確認する試験。 +大入力、追加fuzz、P01対象の停止調査は行わない。 + +## 比較の分離 + +- 同じmodelのwholeとの差。 +- 同じchunk scheduleのlegacyとの差。 +- 同じprofile・同じchunk scheduleの因子1との差。 + +各比較で候補数、encoding/language順序、confidence bitsを含む候補一覧、final done、 +first done offsetを別に記録する。feed回数はchunkにより必然的に変わるので差分指標には +含めず、生観測に保持する。first done offsetはfeed後に観測した位置で、内部の厳密な +最初の判定byte位置を表すとは限らない。完全一致を近似float比較へ置き換えない。 + +## 結果 + +cp1252のstrict decode-equivalent件数（各8入力）: + +| chunk | legacy | identity 1 | identity 0.90 | filtered 1 | filtered 0.90 | +| --- | ---: | ---: | ---: | ---: | ---: | +| whole | 8 | 4 | 8 | 5 | 8 | +| 1 | 0 | 0 | 0 | 0 | 0 | +| 7 | 6 | 4 | 4 | 4 | 4 | +| 64 | 8 | 4 | 7 | 5 | 8 | +| 1024 | 8 | 4 | 8 | 5 | 8 | + +UTF-8 exactは全model・全scheduleで8/8。全出力confidenceは有限かつ[0,1]内だった。 +wholeに対する候補完全一致の不一致件数は、1/7/64-byteで各modelとも16/16、 +1024-byteで各modelとも4/16。encoding/language順序の不一致とは分けてreportへ保持する。 +final done / first done offsetのwholeとの差は今回すべて0件。 + +1-byteの悪化はlegacyにも存在し、今回のratio変更だけに起因するとは言えない。 +一方、7-byteでは生成modelがlegacyより悪く、wholeの8/8だけで標準採用する根拠にはならない。 +因子0.90は小規模whole-inputの候補のままとし、incremental gateを達成したとは扱わない。 +この結果を見てtuning gridを増やさない。具体的な原因は別途追跡する。 + +## 再現 + +```sh +uv run --no-project python models/experimental/ratio_chunks.py \ + IDENTITY FILTERED MANIFEST SWEEP BUILDS OUTPUT +``` + +入力は24録音版の親artifact、training/tuning manifest、固定ratio sweep report、 +その検証済みbuild directory。sweep hash、親model、入力hash、source/binaryを照合する。 +wholeの4生成model観測は保存sweepと完全一致を要求する。 +複数buildの標準targetも各入力・chunkで完全一致を要求する。 + +private report: `archives/v3-corpus/paris24-ratio-chunks-v1.json`。 +content hash: `5d28f112f1ed472e1a438df9790f9e2f50c9aa0e216943059e2bb51621d0c8c1`。 +生観測、3種類の差分、accuracy、buildの参照元sweep hashを保存している。 +全条件を2回観測してreport全体のbyte一致を確認した。 +今回の評価対象はtuningであり、独立汎化精度の主張ではない。 diff --git a/models/experimental/ratio_chunks.py b/models/experimental/ratio_chunks.py new file mode 100644 index 0000000..619e606 --- /dev/null +++ b/models/experimental/ratio_chunks.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: MIT +"""Bounded fixed-factor chunk observations; never tune against chunk differences.""" +import argparse +import json +from pathlib import Path + +import engine_comparison +import engine_probe +import paired_controls +import ratio_sweep +import ratio_variant +from model import canonical, digest, write_idempotent +from sequence_contract import content_hash + +SWEEP_HASH = "8ebbe3a8e939488bf89aa1fa2c3d6169a494c99e2cf23faf9dc37419f93cd6d6" +CHUNKS = (0, 1, 7, 64, 1024) + + +def differences(observed, reference): + def names(value): + return [(c["encoding"], c["language"]) for c in value["candidates"]] + return dict(candidate_count=observed["candidate_count"] != reference["candidate_count"], + encoding_language_order=names(observed) != names(reference), + exact_candidates=observed["candidates"] != reference["candidates"], + final_done=observed["final_done"] != reference["final_done"], + first_done_offset=observed["first_done_offset"] != reference["first_done_offset"]) + + +def run(identity, filtered, manifest, root, sweep, directory): + if sweep.get("content_hash") != SWEEP_HASH or content_hash(sweep) != SWEEP_HASH: + raise ValueError("requires the frozen ratio sweep") + if manifest["content_hash"] != identity["contract"]["provenance"]["corpus_content_hash"]: + raise ValueError("requires the fixed training/tuning manifest") + records, _ = paired_controls.select_pairs(identity, filtered, manifest, root, "tuning") + if len(records) != 16 or any(len(data) > engine_probe.LIMIT for _, _, data in records): + raise ValueError("requires all sixteen bounded tuning inputs") + builds, saved = {}, {} + for name, parent in (("identity", identity), ("filtered", filtered)): + for factor in ("1", "0.90"): + key = f"{name}-{factor}" + previous = sweep["results"][name][factor] + variant = previous["variant"] + ratio_variant.validate(variant, parent) + path = directory / f"build-{key}" + binaries, provenance = engine_comparison.verified_build(path, variant) + if provenance != previous["build"]: + raise ValueError("build differs from frozen sweep") + builds[key] = (path, variant, binaries, provenance) + saved[key] = {row["sample_id"]: row for row in previous["documents"]} + documents = [] + for source, sample, data in records: + schedules = {} + for chunk in CHUNKS: + observations = {} + for key, (_, _, binaries, _) in builds.items(): + normal = engine_probe.observe(binaries["uchardet-conformance"], data, chunk) + if "legacy" in observations and normal != observations["legacy"]: + raise ValueError("standard targets differ") + observations["legacy"] = normal + value = engine_probe.observe(binaries["uchardet-conformance-experimental"], data, chunk) + old = saved[key][sample["id"]] + if old["sample_sha256"] != digest(data): + raise ValueError("tuning input differs from sweep") + if chunk == 0 and value != old["observation"]: + raise ValueError("whole-input observation differs from sweep") + observations[key] = value + schedules[str(chunk)] = observations + comparisons = {} + scores = {} + for chunk, observations in schedules.items(): + comparisons[chunk], scores[chunk] = {}, {} + for key, value in observations.items(): + parent_key = key.replace("-0.90", "-1") + comparisons[chunk][key] = dict( + vs_whole=differences(value, schedules["0"][key]), + vs_legacy_same_chunk=differences(value, observations["legacy"]), + vs_unscaled_same_chunk=differences(value, observations[parent_key])) + scores[chunk][key] = engine_comparison.score(value, data, sample["encoding"], source["language"]) + documents.append(dict(sample_id=sample["id"], sample_sha256=sample["sha256"], + encoding=sample["encoding"], byte_length=len(data), + observations=schedules, comparisons=comparisons, scores=scores)) + for path, variant, _, provenance in builds.values(): + if engine_comparison.verified_build(path, variant)[1] != provenance: + raise ValueError("build changed during observation") + summary = {} + for chunk in map(str, CHUNKS): + summary[chunk] = {} + for key in ("legacy", *builds): + rows = [dict(encoding=row["encoding"], score=row["scores"][chunk][key], + observation=row["observations"][chunk][key]) for row in documents] + value = ratio_sweep.summarize(rows) + value["differences"] = { + comparison: {field: sum(row["comparisons"][chunk][key][comparison][field] for row in documents) + for field in documents[0]["comparisons"][chunk][key][comparison]} + for comparison in ("vs_whole", "vs_legacy_same_chunk", "vs_unscaled_same_chunk")} + summary[chunk][key] = value + result = dict(schema="paris24-fixed-ratio-chunks-v1", sweep_hash=SWEEP_HASH, + manifest_hash=manifest["content_hash"], chunks=list(CHUNKS), + documents=documents, summary=summary, + dependencies={p.name: digest(p.read_bytes()) for p in (Path(__file__), + Path(engine_probe.__file__), Path(engine_comparison.__file__), + Path(ratio_sweep.__file__), Path(ratio_variant.__file__), Path(paired_controls.__file__))}) + result["content_hash"] = content_hash(result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("identity", "filtered", "manifest", "sweep", "builds", "output"): + parser.add_argument(name, type=Path) + args = parser.parse_args() + def load(path): + return json.loads(path.read_text(encoding="utf-8")) + result = run(load(args.identity), load(args.filtered), load(args.manifest), + args.manifest.parent, load(args.sweep), args.builds) + write_idempotent(args.output, canonical(result)) + + +if __name__ == "__main__": + main() diff --git a/models/experimental/test_ratio_chunks.py b/models/experimental/test_ratio_chunks.py new file mode 100644 index 0000000..b64ed73 --- /dev/null +++ b/models/experimental/test_ratio_chunks.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: MIT +import copy +import unittest +from unittest.mock import patch + +import ratio_chunks + + +class RatioChunkTests(unittest.TestCase): + def test_confidence_order_and_lifecycle_are_separate(self): + base = dict(candidate_count=2, candidates=[ + dict(encoding="cp1252", language="fr", confidence_bits="3f000000"), + dict(encoding="UTF-8", language=None, confidence_bits="3e000000")], + final_done=True, first_done_offset=128, feed_calls=1) + changed = copy.deepcopy(base) + changed["feed_calls"] = 128 + self.assertFalse(any(ratio_chunks.differences(changed, base).values())) + changed["candidates"][0]["confidence_bits"] = "3f000001" + result = ratio_chunks.differences(changed, base) + self.assertTrue(result["exact_candidates"]) + self.assertFalse(result["encoding_language_order"]) + changed["candidates"].reverse() + self.assertTrue(ratio_chunks.differences(changed, base)["encoding_language_order"]) + changed["first_done_offset"] = 64 + self.assertTrue(ratio_chunks.differences(changed, base)["first_done_offset"]) + + def test_wrong_sweep_rejected_before_process(self): + with patch.object(ratio_chunks.engine_probe, "observe") as observe: + with self.assertRaisesRegex(ValueError, "frozen ratio"): + ratio_chunks.run(None, None, None, None, {}, None) + observe.assert_not_called() + + +if __name__ == "__main__": + unittest.main()