diff --git a/families/llama/runtime/CMakeLists.txt b/families/llama/runtime/CMakeLists.txt index 2dd2425a10..1ff900eeab 100644 --- a/families/llama/runtime/CMakeLists.txt +++ b/families/llama/runtime/CMakeLists.txt @@ -53,7 +53,8 @@ if(TRTMC_BUILD_TESTS) trtmc_core nlohmann_json::nlohmann_json ${TRTMC_CUDART_LIBRARY}) endforeach() add_test(NAME test_llama_speculative_contract COMMAND test_llama_speculative_contract) - foreach(test_name IN ITEMS test_llama_chat_template test_llama_native_kv_cache) + foreach(test_name IN ITEMS test_llama_chat_template test_llama_native_kv_cache + test_llama_pretokenizer_variant) add_executable(${test_name} ${PROJECT_SOURCE_DIR}/families/llama/tests/cpp/${test_name}.cpp ) diff --git a/families/llama/runtime/bpe_tokenizer.cpp b/families/llama/runtime/bpe_tokenizer.cpp index 860defc1df..742a4e7d3c 100644 --- a/families/llama/runtime/bpe_tokenizer.cpp +++ b/families/llama/runtime/bpe_tokenizer.cpp @@ -1272,19 +1272,52 @@ class BpeTokenizer final : public ITokenizer { return pretok::Variant::kLlama; } - // Detect variant from the Split inside a Sequence pre_tokenizer. + // Detect variant from the Split step(s) inside a Sequence pre_tokenizer. + // Some checkpoints isolate digit-grouping (\p{N}{1,3}) as its own Split + // step, ahead of the step whose regex actually identifies the variant + // (e.g. Qwen3's `[^\r\n...`). That earlier step's regex never matches + // any known variant signature and falls through to kLlama, so scanning + // stops before reaching the real classifier. Scan every Split step for + // classification instead of only the first; take the digit-group size + // independently, from whichever step's own regex encodes one, since the + // classifying step and the digit-grouping step are not guaranteed to be + // the same step. static pretok::Variant detect_split_variant(const nlohmann::json& pt, int& digit_group_out) { digit_group_out = 0; if (!pt.contains("pretokenizers")) return pretok::Variant::kLlama; + + pretok::Variant variant = pretok::Variant::kLlama; + bool variant_found = false; for (auto& sub : pt["pretokenizers"]) { if (sub.value("type", "") != "Split") continue; if (!sub.contains("pattern") || !sub["pattern"].contains("Regex")) continue; - return classify_split(sub, digit_group_out); + const auto regex = sub["pattern"]["Regex"].get(); + + if (digit_group_out == 0) { + if (int group = parse_digit_group(regex); group > 0) + digit_group_out = group; + } + + int step_digit_group = 0; // discarded; digit_group_out is tracked separately above + pretok::Variant step_variant = classify_split(sub, step_digit_group); + if (step_variant == pretok::Variant::kLlama) + continue; + if (variant_found && step_variant != variant) { + throw std::runtime_error( + "Ambiguous Sequence pre_tokenizer: Split steps classify to different " + "variants (" + + std::to_string(static_cast(variant)) + " and " + + std::to_string(static_cast(step_variant)) + ")"); + } + if (!variant_found) { + variant = step_variant; + variant_found = true; + } } - return pretok::Variant::kLlama; + return variant; } static bool is_space_split_pre_tokenizer(const nlohmann::json& pt, const std::string& pt_type) { diff --git a/families/llama/tests/MINICPM5_1B.md b/families/llama/tests/MINICPM5_1B.md new file mode 100644 index 0000000000..b52984b66e --- /dev/null +++ b/families/llama/tests/MINICPM5_1B.md @@ -0,0 +1,65 @@ +# MiniCPM5-1B checkpoint coverage + +This adds a pinned E2E selection for `openbmb/MiniCPM5-1B` through the existing +Llama family. It does not add a new architecture or claim GPU qualification. + +The configuration fixture is copied from the Apache-2.0 publisher checkpoint: +https://huggingface.co/openbmb/MiniCPM5-1B/blob/87179e5c1f455ef22e6223592d2d61351b525bfc/config.json + +Unlike the existing 2B checkpoint, the 1B model has hidden width 1536 but +16 attention heads of width 128 (attention width 2048), with two KV heads and +24 layers. CPU tests preserve that explicit width, full-context KV byte +geometry and the two stop IDs. The E2E uses the existing family build/runtime +and reference comparison with thinking disabled, FP16 candidate, FP32 reference, +a 256-token build limit and ten generated tokens. These limits are a small +parity experiment, not a long-context or performance claim. + +Validation remaining: build the pinned checkpoint on authorized GPU hardware +and run `families/llama/tests/test_e2e.py` with `--e2e-model minicpm5-1b`, using +the normal family E2E runtime environment. No large weights were downloaded locally. + +The Llama tokenizer's Sequence/Split classification fix is owned by upstream +PR #1423. Its original commit is included as a dependency for combined testing; +maintainers should review and integrate #1423 first. This checkpoint coverage +must not be treated as complete runtime qualification until target-GPU parity +is verified. + +## Tokenizer-sensitive premerge coverage + +The second premerge case uses raw Chinese punctuation, a ten-digit sequence and +an English continuation prompt. Its `expected_prompt_token_ids` come from the +pinned publisher tokenizer, including its `` post-processor token. It uses the +same FP16/FP32 generation comparison and limits as the chat case; no acceptance +threshold is changed. A separate native prefill receipt assertion requires +16 tokens, including BOS, so the observed 18-token baseline cannot pass merely +by generating similar text. This assertion retains the subsequent Hugging Face +generation comparison; it does not turn the case into a KV-contract-only test. +It checks token count, not equality of every native input ID. Selecting `--e2e-model minicpm5-1b` selects both cases. + +CPU comparison against Hugging Face Tokenizers 0.22.2 found six mismatches out +of 28 text probes on the initial PR source (f0c70eac). Replacing only the Llama +tokenizer source with PR #1423 at f509329e735f89acb05a24da96e1fa72461a35de made +all 28 probes match. That comparison disabled special-token insertion on both +sides and covered digits, Chinese punctuation, code, whitespace and multilingual +text. The new manifest prompt was also checked separately against the publisher +post-processor. These are tokenizer-only CPU results, not model inference proof. + +For example, `1234567890` encodes as `[5645, 12740, 17371, 37]` without special +tokens in the pinned tokenizer. The initial PR runtime instead produced +`[5645, 12740, 1877, 1609]`. The dependency fixes that discrepancy. The branch now includes the original #1423 commit as a history-preserving +dependency merge. The tokenizer implementation retains its original author and +commit; this change does not reimplement it. The combined branch still requires +target-GPU validation before the checkpoint is qualified. + +## Staged checkpoint prerequisite + +In offline mode, the Llama E2E lookup consumes a pre-populated Hugging Face +snapshot cache without refreshing Hub metadata. Community GPU CI stages that cache +before entering its offline test container. For a manual run, populate the cache +with the pinned checkpoint first and set `HF_HUB_OFFLINE=1`; an absent revision or missing `config.json` +still fails rather than selecting another checkpoint. + +This avoids a Hub 1.32 repository-tree lookup that otherwise raises +`OfflineModeIsEnabled` even when the pinned snapshot is already cached. The +regression tests use temporary local snapshots without any tree index or model +weights. No manifest revisions, comparisons or acceptance thresholds are changed. diff --git a/families/llama/tests/cpp/test_llama_pretokenizer_variant.cpp b/families/llama/tests/cpp/test_llama_pretokenizer_variant.cpp new file mode 100644 index 0000000000..f4e89dd0d0 --- /dev/null +++ b/families/llama/tests/cpp/test_llama_pretokenizer_variant.cpp @@ -0,0 +1,191 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// detect_split_variant() previously returned on the first Split step found +// in a Sequence pre_tokenizer, even when that step's regex classifies as +// the generic kLlama default. Checkpoints that isolate digit-grouping +// (\p{N}{1,3}) as its own Split step ahead of the step that actually +// identifies the variant (e.g. Qwen3's `[^\r\n...` signature) were +// misclassified as kLlama, silently disabling variant-specific +// pre-tokenization behavior (trailing-newline attachment, grouped digit +// runs) for the whole checkpoint. +// +// The first two checks below use the pre_tokenizer field exactly as +// published in openbmb/MiniCPM5-2B's tokenizer.json (a real, public +// checkpoint that triggers this misclassification), so a passing result +// here reflects the real checkpoint's own regex shapes, not a synthetic +// stand-in. The third check is a synthetic fixture verifying the +// ambiguous-classification case (two Split steps that each classify to a +// different, non-kLlama variant) throws rather than silently picking one. + +#include "families/llama/runtime/tokenizer.h" + +#include +#include +#include +#include +#include + +namespace { + +int failures = 0; + +void check_ids(const std::vector& actual, const std::vector& expected, + const char* name) { + if (actual == expected) + return; + std::cerr << "FAIL: " << name << " expected"; + for (const int32_t token : expected) + std::cerr << ' ' << token; + std::cerr << " got"; + for (const int32_t token : actual) + std::cerr << ' ' << token; + std::cerr << '\n'; + ++failures; +} + +// MiniCPM5-2B's actual pre_tokenizer field (openbmb/MiniCPM5-2B, +// tokenizer.json): a digit-grouping Split step first, then the real +// classifying (Qwen3-shaped) Split step second, then ByteLevel. +constexpr const char* kMiniCPM5PreTokenizer = R"( + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + { + "type": "Split", + "pattern": {"Regex": "\\p{N}{1,3}"}, + "behavior": "Isolated", + "invert": false + }, + { + "type": "Split", + "pattern": { + "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}+| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + }, + "behavior": "Isolated", + "invert": false + }, + { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": false + } + ] + })"; + +// Trailing newlines after punctuation should stay attached to the same +// pre-token (Qwen3-variant behavior) rather than split off as their own +// whitespace token. Only reachable if the second Split step's regex is +// actually used to classify the variant, not the first. +void check_newline_attachment() { + const std::string tokenizer_json = std::string(R"({ + "model": { + "type": "BPE", + "vocab": { + ".": 0, + "Ċ": 1, + "ĊĊ": 2, + ".ĊĊ": 3 + }, + "merges": ["Ċ Ċ", ". ĊĊ"] + },)") + kMiniCPM5PreTokenizer + R"( + })"; + + auto tokenizer = trtmc::CreateBpeTokenizer(tokenizer_json.data(), tokenizer_json.size(), false); + if (!tokenizer) { + std::cerr << "FAIL: check_newline_attachment: tokenizer was not created\n"; + ++failures; + return; + } + check_ids(tokenizer->encode(".\n\n"), {3}, "check_newline_attachment"); +} + +// Multi-digit runs should be grouped into chunks of up to 3 digits +// (\p{N}{1,3}), each its own pre-token, not left as one unbounded digit +// run (the kLlama-misclassified behavior) and not split into individual +// one-digit pre-tokens (the failure mode of a fix that scans for the +// variant but couples digit-group size to the same winning step, which +// does not carry the {1,3} grouping in this checkpoint's real regex). +void check_digit_grouping() { + const std::string tokenizer_json = std::string(R"({ + "model": { + "type": "BPE", + "vocab": { + "1": 0, + "5": 1, + "0": 2, + "15": 3, + "150": 4, + "1500": 5 + }, + "merges": ["1 5", "15 0", "150 0"] + },)") + kMiniCPM5PreTokenizer + R"( + })"; + + auto tokenizer = trtmc::CreateBpeTokenizer(tokenizer_json.data(), tokenizer_json.size(), false); + if (!tokenizer) { + std::cerr << "FAIL: check_digit_grouping: tokenizer was not created\n"; + ++failures; + return; + } + // Correct (grouped by 3): ["150", "0"] -> ids {4, 2}. + // kLlama misclassification (unpatched bug): one ungrouped word "1500" -> {5}. + // Coupled/naive fix (digit_group taken from the same step as the + // variant, which for this checkpoint's real second step is 0): four + // one-digit words -> {0, 1, 2, 2}. + check_ids(tokenizer->encode("1500"), {4, 2}, "check_digit_grouping"); +} + +// Synthetic fixture: two Split steps that each classify to a different, +// non-kLlama variant. Not a real checkpoint shape -- this exercises the +// ambiguous-classification guard rather than reproducing a reported bug. +void check_ambiguous_variants_throw() { + const std::string tokenizer_json = R"({ + "model": { + "type": "BPE", + "vocab": {"a": 0}, + "merges": [] + }, + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + { + "type": "Split", + "pattern": {"Regex": "[^(\\s)]+"}, + "behavior": "Isolated", + "invert": false + }, + { + "type": "Split", + "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?\\p{L}+"}, + "behavior": "Isolated", + "invert": false + } + ] + } + })"; + + bool threw = false; + try { + trtmc::CreateBpeTokenizer(tokenizer_json.data(), tokenizer_json.size(), false); + } catch (const std::runtime_error&) { + threw = true; + } + if (!threw) { + std::cerr + << "FAIL: check_ambiguous_variants_throw: expected std::runtime_error, none thrown\n"; + ++failures; + } +} + +} // namespace + +int main() { + check_newline_attachment(); + check_digit_grouping(); + check_ambiguous_variants_throw(); + return failures == 0 ? 0 : 1; +} diff --git a/families/llama/tests/fixtures/minicpm5-1b-config.json b/families/llama/tests/fixtures/minicpm5-1b-config.json new file mode 100644 index 0000000000..a2cdeeb3ce --- /dev/null +++ b/families/llama/tests/fixtures/minicpm5-1b-config.json @@ -0,0 +1,30 @@ +{ + "_name_or_path": "openbmb/MiniCPM5-1B", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 0, + "eos_token_id": [ + 1, + 130073 + ], + "pad_token_id": 1, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 4608, + "max_position_embeddings": 131072, + "model_type": "llama", + "num_attention_heads": 16, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "head_dim": 128, + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "rope_scaling": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "5.6.2", + "use_cache": true, + "vocab_size": 130560 +} diff --git a/families/llama/tests/manifests/minicpm5-1b.json b/families/llama/tests/manifests/minicpm5-1b.json new file mode 100644 index 0000000000..066d7aa2f9 --- /dev/null +++ b/families/llama/tests/manifests/minicpm5-1b.json @@ -0,0 +1,50 @@ +{ + "name": "minicpm5-1b", + "hf_id": "openbmb/MiniCPM5-1B", + "hf_revision": "87179e5c1f455ef22e6223592d2d61351b525bfc", + "bundle": "minicpm5-1b.bundle", + "family": "llama", + "task": "text_generation", + "trust_remote_code": false, + "precision": "fp16", + "testcases": [ + { + "name": "minicpm5-1b", + "premerge": true, + "prompt": "What is the capital of France? Answer in one word.", + "max_new_tokens": 10, + "use_chat_template": true, + "reference_precision": "fp32", + "enable_thinking": false + }, + { + "name": "minicpm5-1b-multilingual-digits", + "premerge": true, + "prompt": "你好，世界！\n1234567890\nThe next number after 9 is", + "expected_prompt_token_ids": [ + 0, + 75828, + 68572, + 3708, + 5645, + 12740, + 17371, + 37, + 220, + 608, + 2633, + 1471, + 1302, + 242, + 46, + 357 + ], + "max_new_tokens": 10, + "use_chat_template": false, + "reference_precision": "fp32", + "expected_runtime_prefill_tokens": 16 + } + ], + "max_sequence_length": 256, + "tensor_parallel_size": 1 +} diff --git a/families/llama/tests/runtime_receipt.py b/families/llama/tests/runtime_receipt.py index 7ff8507045..7f66977c34 100644 --- a/families/llama/tests/runtime_receipt.py +++ b/families/llama/tests/runtime_receipt.py @@ -24,6 +24,15 @@ def prefill_observations(stderr: str) -> tuple[tuple[int, int, int], ...]: return tuple(values) +def assert_prefill_token_count(stderr: str, expected_tokens: int) -> None: + """Check one generation's native prompt length, including special tokens.""" + observations = prefill_observations(stderr) + assert len(observations) == 1, f"expected one prefill receipt, got {observations}" + assert observations[0][0] == expected_tokens, ( + f"native prefill token count {observations[0][0]} != {expected_tokens}" + ) + + def assert_native_kv_receipt(payload: dict, case: dict, prompt_tokens: int) -> None: stderr = str(payload["runtime_stderr"]) expected_rows = int(case["expected_kv_cache_rows"]) diff --git a/families/llama/tests/test_checkpoint_cache.py b/families/llama/tests/test_checkpoint_cache.py new file mode 100644 index 0000000000..9f765a0ca5 --- /dev/null +++ b/families/llama/tests/test_checkpoint_cache.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise staged checkpoint lookup without Hub metadata or network access.""" + +import pytest +from huggingface_hub import constants +from huggingface_hub.errors import LocalEntryNotFoundError + +from .test_e2e import _checkpoint + + +@pytest.fixture +def offline_cache(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + return tmp_path + + +def test_pinned_checkpoint_uses_snapshot_without_cached_tree(offline_cache): + revision = "a" * 40 + snapshot = offline_cache / "models--trtmc-test--llama" / "snapshots" / revision + snapshot.mkdir(parents=True) + (snapshot / "config.json").write_text("{}", encoding="utf-8") + + assert _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": revision}) == snapshot + + +def test_missing_checkpoint_is_not_replaced_by_another_revision(offline_cache): + cached = offline_cache / "models--trtmc-test--llama" / "snapshots" / ("a" * 40) + cached.mkdir(parents=True) + (cached / "config.json").write_text("{}", encoding="utf-8") + + with pytest.raises(LocalEntryNotFoundError): + _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": "b" * 40}) + + +def test_staged_snapshot_still_requires_config(offline_cache): + revision = "a" * 40 + snapshot = offline_cache / "models--trtmc-test--llama" / "snapshots" / revision + snapshot.mkdir(parents=True) + + with pytest.raises(AssertionError): + _checkpoint({"hf_id": "trtmc-test/llama", "hf_revision": revision}) diff --git a/families/llama/tests/test_e2e.py b/families/llama/tests/test_e2e.py index b2a724d99f..4d0ff1037c 100644 --- a/families/llama/tests/test_e2e.py +++ b/families/llama/tests/test_e2e.py @@ -565,6 +565,14 @@ def test_e2e(case_name: str, request, tmp_path: Path) -> None: f"(bundle max={manifest['max_sequence_length']})" in payload["runtime_stderr"] ) + if "expected_runtime_prefill_tokens" in case: + from families.llama.tests.runtime_receipt import assert_prefill_token_count + + with evidence_stage("compare"): + assert_prefill_token_count( + payload["runtime_stderr"], int(case["expected_runtime_prefill_tokens"]) + ) + if _NATIVE_KV_FIELDS <= case.keys(): from families.llama.tests.runtime_receipt import assert_native_kv_receipt diff --git a/families/llama/tests/test_minicpm5_config.py b/families/llama/tests/test_minicpm5_config.py new file mode 100644 index 0000000000..33ff8163e6 --- /dev/null +++ b/families/llama/tests/test_minicpm5_config.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned MiniCPM5-1B configuration and checkpoint-selection coverage.""" + +import json +from pathlib import Path + +from ..build_routing import native_kv_cache_geometry +from ..config import ModelConfig +from .test_e2e import _CASES + + +def test_minicpm5_1b_checkpoint_is_selectable(): + manifest, case = _CASES["minicpm5-1b"] + assert manifest["hf_id"] == "openbmb/MiniCPM5-1B" + assert manifest["hf_revision"] == "87179e5c1f455ef22e6223592d2d61351b525bfc" + assert manifest["family"] == "llama" and manifest["tensor_parallel_size"] == 1 + assert case["use_chat_template"] and not case["enable_thinking"] + + +def test_minicpm5_1b_preserves_explicit_attention_width_and_stop_tokens(): + raw = json.loads((Path(__file__).parent / "fixtures/minicpm5-1b-config.json").read_text()) + config = ModelConfig.from_json(json.dumps(raw)) + assert config.hidden_size == 1536 + assert config.head_dim == 128 + assert config.attention_size == 2048 + assert config.attention_size != config.hidden_size + assert native_kv_cache_geometry(config, 131072) == (24576, 3221225472) + assert config.eos_token_id == [1, 130073] + assert config.raw["eos_token_id"] == [1, 130073] diff --git a/families/llama/tests/test_runtime_receipt.py b/families/llama/tests/test_runtime_receipt.py index 81948b9a85..8bb4244cdb 100644 --- a/families/llama/tests/test_runtime_receipt.py +++ b/families/llama/tests/test_runtime_receipt.py @@ -78,3 +78,27 @@ def test_receipt_rejects_missing_decode_step() -> None: payload["decode_ms"] = 0.0 with pytest.raises(AssertionError): assert_native_kv_receipt(payload, _case(), 65) + + +@pytest.mark.parametrize("launches, max_chunk", [(1, 16), (2, 8)]) +def test_prefill_count_includes_bos_without_requiring_a_kv_only_case(launches, max_chunk): + from families.llama.tests.runtime_receipt import assert_prefill_token_count + + stderr = f"[trtmc.prefill] tokens=16 launches={launches} max_chunk={max_chunk}" + assert_prefill_token_count(stderr, 16) + + +@pytest.mark.parametrize( + "stderr", + [ + "", + "[trtmc.prefill] tokens=18 launches=1 max_chunk=18", + "[trtmc.prefill] tokens=8 launches=1 max_chunk=8\n" + "[trtmc.prefill] tokens=8 launches=1 max_chunk=8", + ], +) +def test_prefill_count_rejects_missing_wrong_or_repeated_receipts(stderr): + from families.llama.tests.runtime_receipt import assert_prefill_token_count + + with pytest.raises(AssertionError): + assert_prefill_token_count(stderr, 16) diff --git a/qualification_tests/benchmark_qualification/performance/config/release.yaml b/qualification_tests/benchmark_qualification/performance/config/release.yaml index 11011f7bbd..cb981988e3 100644 --- a/qualification_tests/benchmark_qualification/performance/config/release.yaml +++ b/qualification_tests/benchmark_qualification/performance/config/release.yaml @@ -18,6 +18,10 @@ defaults: asset_loading_included: false excluded_profiles: + - model: minicpm5-1b + reason: >- + Checkpoint E2E coverage is declared, but target-GPU parity and a release + performance workload/receipt remain unverified. - model: qwen3-embedding-0.6b reason: >- The migrated embedding task has no release-performance workload or receipt