From 388db8127a37ef02017a76cbaf2ee2ad39730d13 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 07:35:30 +0000
Subject: [PATCH 01/11] [TRTLLM-14093][test] Add InferenceMAX-style GSM8K
 accuracy eval mode for MiniMax-M3

Add a gsm8k_inferencemax lm-eval task mirroring the public InferenceMAX
(SemiAnalysisAI/InferenceX) GSM8K protocol: chat template + 5-shot
multiturn context, an explicit '#### [number]' answer-format instruction,
a 12288-token generation budget for thinking output, and strict '#### N'
extraction. Scores are directly comparable to
inferencex.semianalysis.com/evaluation, unlike the completion-format
gsm8k task which measures chat/thinking-tuned models several points
lower (M3-NVFP4: ~90.5 vs ~95.5 on the same engine).

- tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencemax/: task yaml
  (InferenceX utils/evals/gsm8k.yaml nearly verbatim; renamed task,
  max_gen_toks pinned so CoT is not truncated at lm-eval's 256 default)
- GSM8KInferenceMax evaluator (AIME2026 local-task pattern), exported
  and registered as a trtllm-eval subcommand
- GSM8KInferenceMax AccuracyTask gated on em_strict via the standard
  hypothesis-test machinery; references measured on 4x B200 and
  annotated with the published vLLM B200 FP4 band (94.84-96.06)
- test_nvfp4 / test_nvfp4_eagle3 gain an eval_mode parameter
  (default | inferencemax); default mode is unchanged, inferencemax
  runs the chat-format task with max_seq_len=16384

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tensorrt_llm/commands/eval.py                 |   7 +-
 tensorrt_llm/evaluate/__init__.py             |   4 +-
 tensorrt_llm/evaluate/lm_eval.py              | 111 ++++++++++++++++++
 .../gsm8k_inferencemax.yaml                   |  54 +++++++++
 .../defs/accuracy/accuracy_core.py            |  30 +++++
 .../references/gsm8k_inferencemax.yaml        |  18 +++
 .../defs/accuracy/test_llm_api_pytorch.py     |  46 +++++---
 7 files changed, 252 insertions(+), 18 deletions(-)
 create mode 100644 tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencemax/gsm8k_inferencemax.yaml
 create mode 100644 tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml

diff --git a/tensorrt_llm/commands/eval.py b/tensorrt_llm/commands/eval.py
index 0e25bb80736f..a26bd7dde606 100644
--- a/tensorrt_llm/commands/eval.py
+++ b/tensorrt_llm/commands/eval.py
@@ -21,9 +21,9 @@
 from .. import LLM as PyTorchLLM
 from ..evaluate import (AALCR, AIME2025, AIME2026, GSM8K, HLE, MMLU, MMMU,
                         ArenaHard, CnnDailymail, CoVoST2, GPQADiamond,
-                        GPQAExtended, GPQAMain, GPQANemoSkills, IFBench,
-                        JsonModeEval, LongBenchV1, LongBenchV2, MMMUPro,
-                        SciCode)
+                        GPQAExtended, GPQAMain, GPQANemoSkills,
+                        GSM8KInferenceMax, IFBench, JsonModeEval, LongBenchV1,
+                        LongBenchV2, MMMUPro, SciCode)
 from ..llmapi import KvCacheConfig
 from ..llmapi.llm_args import TorchLlmArgs
 from ..llmapi.llm_utils import update_llm_args_with_extra_options
@@ -216,6 +216,7 @@ def main(ctx, model: str, tokenizer: Optional[str],
 main.add_command(CnnDailymail.command)
 main.add_command(MMLU.command)
 main.add_command(GSM8K.command)
+main.add_command(GSM8KInferenceMax.command)
 main.add_command(GPQADiamond.command)
 main.add_command(GPQAMain.command)
 main.add_command(GPQAExtended.command)
diff --git a/tensorrt_llm/evaluate/__init__.py b/tensorrt_llm/evaluate/__init__.py
index 7c3cb36895e6..0553ec1a8712 100755
--- a/tensorrt_llm/evaluate/__init__.py
+++ b/tensorrt_llm/evaluate/__init__.py
@@ -18,7 +18,8 @@
 from .covost2 import CoVoST2
 from .json_mode_eval import JsonModeEval
 from .lm_eval import (AIME2025, AIME2026, GSM8K, MMMU, GPQADiamond,
-                      GPQAExtended, GPQAMain, LongBenchV1, MMMUPro)
+                      GPQAExtended, GPQAMain, GSM8KInferenceMax, LongBenchV1,
+                      MMMUPro)
 from .longbench_v2 import LongBenchV2
 from .mmlu import MMLU
 from .nemo_skills_eval import (AALCR, HLE, ArenaHard, GPQANemoSkills, IFBench,
@@ -28,6 +29,7 @@
     "CnnDailymail",
     "MMLU",
     "GSM8K",
+    "GSM8KInferenceMax",
     "GPQADiamond",
     "GPQAMain",
     "GPQAExtended",
diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py
index b077747b6c03..e65363c1a540 100644
--- a/tensorrt_llm/evaluate/lm_eval.py
+++ b/tensorrt_llm/evaluate/lm_eval.py
@@ -809,6 +809,117 @@ def command(ctx, **kwargs) -> None:
         GSM8K.command_harness(ctx, **kwargs)
 
 
+class GSM8KInferenceMax(LmEvalEvaluator):
+    """GSM8K under the InferenceMAX (SemiAnalysisAI/InferenceX) protocol.
+
+    Mirrors the public InferenceMAX accuracy eval (utils/evals/gsm8k.yaml +
+    ``run_lm_eval`` there): chat template applied with 5-shot multiturn
+    context, an explicit "#### [number]" answer-format instruction in
+    ``doc_to_text``, a generation budget large enough for thinking models
+    (``max_gen_toks: 12288``), and strict ``#### N`` extraction. Scores are
+    directly comparable to inferencex.semianalysis.com/evaluation, unlike the
+    completion-format ``gsm8k`` task whose scores run several points lower on
+    chat/thinking-tuned models.
+
+    Task yaml lives under ``tensorrt_llm/evaluate/lm_eval_tasks/
+    gsm8k_inferencemax/`` (upstream lm-eval ships no such variant).
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__("gsm8k_inferencemax", **kwargs)
+
+    @click.command("gsm8k_inferencemax")
+    @click.option("--dataset_path",
+                  type=str,
+                  default=None,
+                  help="The path to GSM8K dataset. "
+                  "If unspecified, the dataset is downloaded from HF hub.")
+    @click.option(
+        "--num_samples",
+        type=int,
+        default=None,
+        help="Number of samples to run the evaluation; None means full dataset."
+    )
+    @click.option("--random_seed",
+                  type=int,
+                  default=0,
+                  help="Random seed for dataset processing.")
+    @click.option("--apply_chat_template",
+                  type=click.BOOL,
+                  default=True,
+                  show_default=True,
+                  help="Whether to apply chat template. Default True — the "
+                  "InferenceMAX protocol always evaluates through the chat "
+                  "template.")
+    @click.option(
+        "--chat_template_kwargs",
+        type=str,
+        default=None,
+        callback=lambda ctx, param, value: json.loads(value) if value else None,
+        help='Chat template kwargs as JSON string. Default None keeps the '
+        'template defaults (e.g. adaptive thinking), matching InferenceMAX.')
+    @click.option("--fewshot_as_multiturn",
+                  type=click.BOOL,
+                  default=True,
+                  show_default=True,
+                  help="Apply fewshot as multiturn. Default True — lm-eval "
+                  "auto-enables multiturn fewshots under a chat template, and "
+                  "InferenceMAX inherits that default.")
+    @click.option("--system_prompt",
+                  type=str,
+                  default=None,
+                  help="System prompt.")
+    @click.option("--max_input_length",
+                  type=int,
+                  default=4096,
+                  help="Maximum prompt length.")
+    @click.option(
+        "--max_output_length",
+        type=int,
+        default=12288,
+        show_default=True,
+        help="Maximum generation length. Mirrors the task yaml "
+        "(``max_gen_toks: 12288``, the InferenceMAX budget of 16384 context "
+        "minus 4096 prompt); thinking models need the room.")
+    @click.option("--temperature",
+                  type=float,
+                  default=None,
+                  help="Sampling temperature. Overrides task yaml gen_kwargs.")
+    @click.option(
+        "--top_p",
+        type=float,
+        default=None,
+        help="Nucleus sampling top_p. Overrides task yaml gen_kwargs.")
+    @click.option("--top_k",
+                  type=int,
+                  default=None,
+                  help="Top-k sampling. Overrides task yaml gen_kwargs.")
+    @click.option("--sampling_seed",
+                  type=int,
+                  default=None,
+                  help="Random seed for generation sampling.")
+    @click.option("--log_samples",
+                  is_flag=True,
+                  default=False,
+                  help="Log sample outputs for debugging.")
+    @click.option("--output_path",
+                  type=str,
+                  default=None,
+                  help="Path to save evaluation results.")
+    @click.option("--output_dir",
+                  type=str,
+                  default=None,
+                  help="Directory to save the task infos.")
+    @click.pass_context
+    @staticmethod
+    def command(ctx, **kwargs) -> None:
+        if kwargs.get("fewshot_as_multiturn", False):
+            assert kwargs.get(
+                "apply_chat_template", False
+            ), "apply_chat_template must be True when fewshot_as_multiturn is True"
+        GSM8KInferenceMax.command_harness(ctx, **kwargs)
+
+
 class GPQADiamond(LmEvalEvaluator):
 
     def __init__(self, **kwargs):
diff --git a/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencemax/gsm8k_inferencemax.yaml b/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencemax/gsm8k_inferencemax.yaml
new file mode 100644
index 000000000000..6c42d508306b
--- /dev/null
+++ b/tensorrt_llm/evaluate/lm_eval_tasks/gsm8k_inferencemax/gsm8k_inferencemax.yaml
@@ -0,0 +1,54 @@
+# GSM8K under the InferenceMAX protocol. Mirrors
+# https://github.com/SemiAnalysisAI/InferenceX utils/evals/gsm8k.yaml (itself
+# derived from upstream lm-evaluation-harness gsm8k.yaml with doc_to_text
+# changed so chat models answer in an extractable format; see
+# lm-evaluation-harness#3411). Differences from that file: the task is renamed
+# to avoid clashing with the upstream registry, and max_gen_toks pins the
+# InferenceMAX generation budget (16384 context minus 4096 prompt) so
+# thinking-model CoT is not truncated at lm-eval's 256-token default.
+tag:
+  - math_word_problems
+task: gsm8k_inferencemax
+dataset_path: openai/gsm8k
+dataset_name: main
+output_type: generate_until
+training_split: train
+fewshot_split: train
+test_split: test
+doc_to_text: "Question: {{question}}\nEnd your response with the answer on the last line, formatted as: #### [number]\nAnswer:"
+doc_to_target: "{{answer}}"
+metric_list:
+  - metric: exact_match
+    aggregation: mean
+    higher_is_better: true
+    ignore_case: true
+    ignore_punctuation: false
+    regexes_to_ignore:
+      - ","
+      - "\\$"
+      - "(?s).*#### "
+      - "\\.$"
+generation_kwargs:
+  until:
+    - "</s>"
+    - "<|im_end|>"
+  do_sample: false
+  temperature: 0.0
+  max_gen_toks: 12288
+repeats: 1
+num_fewshot: 5
+filter_list:
+  - name: "strict-match"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "#### (\\-?[0-9\\.\\,]+)"
+      - function: "take_first"
+  - name: "flexible-extract"
+    filter:
+      - function: "regex"
+        group_select: -1
+        regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)"
+      - function: "take_first"
+metadata:
+  version: 3.0
diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index fedbfddbab90..3d4b2cd66f8f 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -414,6 +414,36 @@ class GSM8K(AccuracyTask):
     EVALUATE_KWARGS = dict(scores_filter=None)
 
 
+class GSM8KInferenceMax(AccuracyTask):
+    # GSM8K under the InferenceMAX (SemiAnalysisAI/InferenceX) protocol: chat
+    # template + 5-shot multiturn + "#### [number]" format instruction + a
+    # 12288-token generation budget for thinking models, gated on strict
+    # "#### N" extraction. Scores are directly comparable to
+    # inferencex.semianalysis.com/evaluation; the completion-format GSM8K
+    # above measures the same model several points lower on chat/thinking
+    # models. Substantially slower than GSM8K: thinking output dominates
+    # (minutes with spec decoding + CUDA graphs, tens of minutes eager).
+    DATASET = "gsm8k_inferencemax"
+    DATASET_DIR = f"{llm_models_root()}/datasets/openai/gsm8k"
+
+    ALPHA = 0.05
+    BETA = 0.2
+    SIGMA = 50
+    NUM_SAMPLES = 1319  # Full sample
+
+    MAX_INPUT_LEN = 4096
+    MAX_OUTPUT_LEN = 12288
+
+    EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8KInferenceMax
+    EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR,
+                            random_seed=0,
+                            apply_chat_template=True,
+                            fewshot_as_multiturn=True)
+
+    # InferenceMAX publishes em_strict as the primary score.
+    EVALUATE_KWARGS = dict(scores_filter="exact_match,strict-match")
+
+
 class GPQADiamond(AccuracyTask):
     DATASET = "gpqa_diamond"
     DATASET_DIR = f"{llm_models_root()}/datasets/gpqa"
diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
new file mode 100644
index 000000000000..ef0b3863bb2a
--- /dev/null
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -0,0 +1,18 @@
+# GSM8K under the InferenceMAX protocol (chat template + thinking + 12288-token
+# generation budget, strict "#### N" extraction) — see GSM8KInferenceMax in
+# accuracy_core.py. References measured on 4x B200 (TP4/EP4, 2026-07-21), full
+# 1319-sample split. For context, the published InferenceMAX vLLM B200 FP4 band
+# for MiniMax-M3 is 94.84-96.06 (spec-off) / 95.15-96.06 (Eagle3); identical
+# configs there scatter up to ~1 point between runs.
+nvidia/MiniMax-M3-NVFP4:
+  # Reference sparse-attention path (triton), BF16 KV cache, no spec decoding.
+  - quant_algo: MIXED_PRECISION
+    accuracy: 95.53
+  # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
+  - quant_algo: MIXED_PRECISION
+    kv_cache_quant_algo: FP8
+    accuracy: 95.91
+  # MSA path + one-model Eagle3 (mean of 3 runs: 95.38 / 95.45 / 95.60).
+  - quant_algo: MIXED_PRECISION
+    spec_dec_algo: Eagle3
+    accuracy: 95.48
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 13a8ad792165..ea69311849e9 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -44,8 +44,9 @@
                         skip_post_hopper, skip_pre_ada, skip_pre_blackwell,
                         skip_pre_hopper, skip_ray)
 from .accuracy_core import (GSM8K, MMLU, CnnDailymail, GPQADiamond,
-                            JsonModeEval, LlmapiAccuracyTestHarness,
-                            LongBenchV1, LongBenchV2)
+                            GSM8KInferenceMax, JsonModeEval,
+                            LlmapiAccuracyTestHarness, LongBenchV1,
+                            LongBenchV2)
 
 
 # Keep helper definitions below imports so new imports do not need E402
@@ -7636,14 +7637,19 @@ def test_mxfp8_piecewise_cuda_graph(self, tp_size, ep_size):
 
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_less_device_memory(140000)
+    @parametrize_with_ids("eval_mode", ["default", "inferencemax"])
     @parametrize_with_ids("use_msa", [False, True])
-    def test_nvfp4(self, use_msa):
+    def test_nvfp4(self, use_msa, eval_mode):
         # NVFP4 checkpoint: MXFP8 base layers with NVFP4 routed experts
         # (MIXED_PRECISION checkpoint). The MSA path runs an FP8 KV cache; the
         # Triton path keeps the KV cache in BF16.
+        # eval_mode selects the accuracy protocol: "default" = completion-format
+        # MMLU + GSM8K; "inferencemax" = chat-format GSM8K matching the public
+        # InferenceMAX benchmark (needs a 16k context for thinking output).
         tp_size = ep_size = 4
         model_name = "nvidia/MiniMax-M3-NVFP4"
         model_path = f"{llm_models_root()}/MiniMax-M3-NVFP4"
+        inferencemax = eval_mode == "inferencemax"
         kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6,
                                         enable_block_reuse=False,
                                         dtype="fp8" if use_msa else "auto")
@@ -7656,23 +7662,28 @@ def test_nvfp4(self, use_msa):
                  kv_cache_config=kv_cache_config,
                  sparse_attention_config=sparse_attention_config,
                  moe_config=moe_config,
-                 max_seq_len=4096,
+                 max_seq_len=16384 if inferencemax else 4096,
                  trust_remote_code=True) as llm:
             assert llm.args.quant_config.quant_algo == QuantAlgo.MIXED_PRECISION
-            task = MMLU(model_name)
-            task.evaluate(llm)
-            task = GSM8K(model_name)
-            task.evaluate(llm)
+            if inferencemax:
+                task = GSM8KInferenceMax(model_name)
+                task.evaluate(llm)
+            else:
+                task = MMLU(model_name)
+                task.evaluate(llm)
+                task = GSM8K(model_name)
+                task.evaluate(llm)
 
     @pytest.mark.skip_less_device(4)
     @pytest.mark.skip_less_device_memory(140000)
+    @parametrize_with_ids("eval_mode", ["default", "inferencemax"])
     @parametrize_with_ids("cuda_graph", [True])
     @parametrize_with_ids("use_msa", [True])
     @parametrize_with_ids("overlap_scheduler", [False, True])
     @parametrize_with_ids("attention_dp", [False, True])
     @parametrize_with_ids("tp_size,ep_size", [(4, 4)])
     def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
-                          overlap_scheduler, use_msa, cuda_graph):
+                          overlap_scheduler, use_msa, cuda_graph, eval_mode):
         if use_msa:
             from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_utils import \
                 msa_package_available
@@ -7680,6 +7691,7 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
                 pytest.skip("MSA kernels (fmha_sm100) not available")
         model_name = "nvidia/MiniMax-M3-NVFP4"
         model_path = f"{llm_models_root()}/MiniMax-M3-NVFP4"
+        inferencemax = eval_mode == "inferencemax"
         max_draft_len = 3
         spec_config = Eagle3DecodingConfig(
             max_draft_len=max_draft_len,
@@ -7697,7 +7709,9 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
                 sparse_attention_config=MiniMaxM3SparseAttentionConfig(
                     implementation="msa" if use_msa else "triton"),
                 moe_config=MoeConfig(backend="CUTLASS"),
-                max_seq_len=4096,
+                # The InferenceMAX protocol needs room for thinking output
+                # (12288 generated tokens on a ~2k chat prompt).
+                max_seq_len=16384 if inferencemax else 4096,
                 # The fmha_sm100 decode planner caps total_q x num_qo_heads at
                 # 65536; with 1 + draft_len = 4 verify tokens per row that
                 # bounds the batch at 1024 / TP-sharded heads (256 unsharded
@@ -7727,10 +7741,14 @@ def drain_spec_stats(llm):
                     steps += sd.get("numRequestsWithDraftTokens", 0)
                 return drafted, accepted, steps
 
-            task = MMLU(model_name)
-            task.evaluate(llm)
-            task = GSM8K(model_name)
-            task.evaluate(llm)
+            if inferencemax:
+                task = GSM8KInferenceMax(model_name)
+                task.evaluate(llm)
+            else:
+                task = MMLU(model_name)
+                task.evaluate(llm)
+                task = GSM8K(model_name)
+                task.evaluate(llm)
 
             # Chat-format acceptance — the drafter's training distribution
             # (Inferact/MiniMax-M3-EAGLE3 card: 0.839 / 3.518). Reuses the

From 544d33c05b304ecd56e94a1bd9e7ab27cd1c63ef Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 07:40:53 +0000
Subject: [PATCH 02/11] [TRTLLM-14093][test] True up gsm8k_inferencemax
 references to in-process measurements

All three inferencemax test variants pass on 4x B200: triton spec-off
95.45 (15m16s), MSA/FP8-KV spec-off 95.30 (7m06s), MSA+Eagle3+graphs
95.22 (10m24s, acceptance 0.842/3.525). The FP8-KV row is now measured
under the config the test actually runs (the provisional value came from
a BF16-KV serve run).

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 .../defs/accuracy/references/gsm8k_inferencemax.yaml     | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index ef0b3863bb2a..dc1b4c815c1b 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -7,12 +7,13 @@
 nvidia/MiniMax-M3-NVFP4:
   # Reference sparse-attention path (triton), BF16 KV cache, no spec decoding.
   - quant_algo: MIXED_PRECISION
-    accuracy: 95.53
+    accuracy: 95.45
   # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
-    accuracy: 95.91
-  # MSA path + one-model Eagle3 (mean of 3 runs: 95.38 / 95.45 / 95.60).
+    accuracy: 95.30
+  # MSA path + one-model Eagle3 + CUDA graphs (serve-based repeats of the
+  # same config: 95.38 / 95.45 / 95.60).
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
-    accuracy: 95.48
+    accuracy: 95.22

From 474dcf10b908840bc2f03d32bac395a58f8a12b5 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 09:22:05 +0000
Subject: [PATCH 03/11] [TRTLLM-14093][test] Set Eagle3 gsm8k_inferencemax
 reference to 10-run mean

10 consecutive fresh-engine runs of test_nvfp4_eagle3[...eval_mode=
inferencemax]: all passed, strict-match 94.62-95.53, mean 95.01, SD 0.26
(tighter than InferenceMAX's own median per-config repeat SD of 0.31).
Acceptance rate 0.835-0.845 / length 3.505-3.534 across all runs.

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 .../defs/accuracy/references/gsm8k_inferencemax.yaml       | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index dc1b4c815c1b..133d53315ad6 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -12,8 +12,9 @@ nvidia/MiniMax-M3-NVFP4:
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
     accuracy: 95.30
-  # MSA path + one-model Eagle3 + CUDA graphs (serve-based repeats of the
-  # same config: 95.38 / 95.45 / 95.60).
+  # MSA path + one-model Eagle3 + CUDA graphs + overlap. Mean of 10 in-process
+  # runs (94.62-95.53, SD 0.26); serve-based repeats of the same config at
+  # concurrency 64 run ~0.4 higher (95.38-95.60).
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
-    accuracy: 95.22
+    accuracy: 95.01

From 4fe6c8088da228ff49edf583faa5023b10901e95 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 19:36:15 +0000
Subject: [PATCH 04/11] [TRTLLM-14093][test] Reproduce InferenceMAX few-shot
 exemplars exactly in gsm8k_inferencemax

A byte-level audit of the in-process evaluator vs the lm-eval CLI path
(which InferenceMAX uses against the serving endpoint) showed identical
chat-template rendering and tokenization but DIFFERENT few-shot
exemplars: LmEvalEvaluator shuffles every dataset split with random_seed
and seeds the few-shot sampler with it, while the CLI samples shots from
the unshuffled train split with lm-eval's default sampler seed (1234).
The exemplar difference produced a systematic ~0.45-point offset at
every engine config, spec-on or off.

Add shuffle_dataset / fewshot_random_seed evaluator params (defaults
preserve existing behavior for all other tasks); GSM8KInferenceMax
defaults to the CLI-faithful combination. The built request messages are
now byte-identical to what the lm-eval CLI sends. Also cap
max_batch_size at 64 in inferencemax test mode to match the InferenceMAX
eval regime (client concurrency 64) and keep decode inside CUDA-graph
coverage.

Revalidated on 4x B200 under the fixed protocol: triton spec-off 95.53
(exactly matches the serve-based measurement), MSA/FP8-KV spec-off
95.68, Eagle3+graphs+overlap 95.53/95.53/95.00 (mean 95.35, vs published
vLLM B200 FP4 Eagle3 mean 95.49). References updated accordingly.

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tensorrt_llm/evaluate/lm_eval.py              | 28 +++++++++++++++----
 .../references/gsm8k_inferencemax.yaml        | 13 +++++----
 .../defs/accuracy/test_llm_api_pytorch.py     | 15 ++++++++--
 3 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py
index e65363c1a540..240e8a8ad7ac 100644
--- a/tensorrt_llm/evaluate/lm_eval.py
+++ b/tensorrt_llm/evaluate/lm_eval.py
@@ -475,7 +475,9 @@ def __init__(self,
                  output_path: Optional[str] = None,
                  output_dir: Optional[str] = None,
                  post_process_fn: Optional[Callable[[str], str]] = None,
-                 preserve_caller_max_tokens: bool = False):
+                 preserve_caller_max_tokens: bool = False,
+                 shuffle_dataset: bool = True,
+                 fewshot_random_seed: Optional[int] = None):
         try:
             import lm_eval
         except ImportError as e:
@@ -509,6 +511,12 @@ def __init__(self,
         # larger than the lm-eval task's max_gen_toks. Used by thinking
         # models (e.g. Kimi K2.5) whose CoT output exceeds the task default.
         self.preserve_caller_max_tokens = preserve_caller_max_tokens
+        # shuffle_dataset=False keeps every split in its published order, and
+        # fewshot_random_seed overrides random_seed for the few-shot sampler.
+        # Together they reproduce the lm-eval CLI's exemplar selection
+        # (unshuffled data, sampler seed 1234) for external comparability.
+        self.shuffle_dataset = shuffle_dataset
+        self.fewshot_random_seed = fewshot_random_seed
 
         task_manager = TaskManager(
             include_path=f"{os.path.dirname(__file__)}/lm_eval_tasks")
@@ -529,13 +537,16 @@ def _adjust_config(task_dict, random_seed):
                     }
                 else:
                     # NOTE: Few-shot random seed
-                    task_obj.set_fewshot_seed(seed=random_seed)
+                    task_obj.set_fewshot_seed(seed=self.fewshot_random_seed
+                                              if self.fewshot_random_seed
+                                              is not None else random_seed)
                     adjusted_task_dict[task_name] = task_obj
 
                     # NOTE: Shuffle dataset
-                    data = adjusted_task_dict[task_name].dataset
-                    for split in data.keys():
-                        data[split] = data[split].shuffle(random_seed)
+                    if self.shuffle_dataset:
+                        data = adjusted_task_dict[task_name].dataset
+                        for split in data.keys():
+                            data[split] = data[split].shuffle(random_seed)
 
             return adjusted_task_dict
 
@@ -823,9 +834,16 @@ class GSM8KInferenceMax(LmEvalEvaluator):
 
     Task yaml lives under ``tensorrt_llm/evaluate/lm_eval_tasks/
     gsm8k_inferencemax/`` (upstream lm-eval ships no such variant).
+
+    Exemplar fidelity: the lm-eval CLI (and therefore InferenceMAX) samples
+    the 5 few-shot exemplars from the unshuffled train split with sampler
+    seed 1234. Defaults below reproduce that; the usual dataset shuffle
+    would silently swap in different exemplars and shift scores.
     """
 
     def __init__(self, **kwargs):
+        kwargs.setdefault("shuffle_dataset", False)
+        kwargs.setdefault("fewshot_random_seed", 1234)
         super().__init__("gsm8k_inferencemax", **kwargs)
 
     @click.command("gsm8k_inferencemax")
diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index 133d53315ad6..206839aaa658 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -6,15 +6,16 @@
 # configs there scatter up to ~1 point between runs.
 nvidia/MiniMax-M3-NVFP4:
   # Reference sparse-attention path (triton), BF16 KV cache, no spec decoding.
+  # Matches the serve-based measurement of the same config exactly.
   - quant_algo: MIXED_PRECISION
-    accuracy: 95.45
+    accuracy: 95.53
   # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
-    accuracy: 95.30
-  # MSA path + one-model Eagle3 + CUDA graphs + overlap. Mean of 10 in-process
-  # runs (94.62-95.53, SD 0.26); serve-based repeats of the same config at
-  # concurrency 64 run ~0.4 higher (95.38-95.60).
+    accuracy: 95.68
+  # MSA path + one-model Eagle3 + CUDA graphs + overlap (mean of 3 runs:
+  # 95.53 / 95.53 / 95.00; serve-based repeats 95.38-95.60; published vLLM
+  # B200 FP4 Eagle3 mean 95.49).
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
-    accuracy: 95.01
+    accuracy: 95.35
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index ea69311849e9..37a406799a6e 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -7663,6 +7663,10 @@ def test_nvfp4(self, use_msa, eval_mode):
                  sparse_attention_config=sparse_attention_config,
                  moe_config=moe_config,
                  max_seq_len=16384 if inferencemax else 4096,
+                 # InferenceMAX evaluates a serving endpoint with client
+                 # concurrency capped at 64 (EVAL_CONCURRENT_REQUESTS); match
+                 # that batching regime.
+                 **({"max_batch_size": 64} if inferencemax else {}),
                  trust_remote_code=True) as llm:
             assert llm.args.quant_config.quant_algo == QuantAlgo.MIXED_PRECISION
             if inferencemax:
@@ -7715,15 +7719,20 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
                 # The fmha_sm100 decode planner caps total_q x num_qo_heads at
                 # 65536; with 1 + draft_len = 4 verify tokens per row that
                 # bounds the batch at 1024 / TP-sharded heads (256 unsharded
-                # under attention DP).
-                max_batch_size=256 if attention_dp else 512,
+                # under attention DP). The inferencemax cap of 64 matches the
+                # InferenceMAX eval regime (client concurrency 64) and keeps
+                # every decode iteration inside the captured graph range,
+                # as in real serving at that concurrency.
+                max_batch_size=64 if inferencemax else
+                (256 if attention_dp else 512),
                 speculative_config=spec_config,
                 # Graphs + spec requires the MSA path: its verify batches
                 # are decode-shaped and capture-safe (the reference path
                 # rejects graphs+spec at creation).
                 cuda_graph_config=CudaGraphConfig(
                     enable_padding=True,
-                    max_batch_size=64 if attention_dp else 128,
+                    max_batch_size=64 if
+                    (inferencemax or attention_dp) else 128,
                 ) if cuda_graph else None,
                 disable_overlap_scheduler=not overlap_scheduler,
                 enable_attention_dp=attention_dp,

From cf5010c7e060ea547ea0637b9a56226a00737a10 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 23:26:25 +0000
Subject: [PATCH 05/11] [TRTLLM-14093][test] Fix test-list IDs for eval_mode
 param; print external reference in accuracy report

- Update QA list, l0_dgx_b200, and waives.txt entries for TestMiniMaxM3
  test_nvfp4 / test_nvfp4_eagle3 to the new parametrize IDs
  (-eval_mode=default), keeping current CI behavior unchanged. The
  waives.txt entries are updated too so the B200/B300/GB300 waivers keep
  matching (the AST validator does not check waives).
- Restructure the spots pre-commit's yapf mangled (pre-computed
  batch_kwargs / max_batch_size / fewshot_seed variables).
- Add an optional reference_note field to accuracy reference rows,
  printed in the hypothesis-testing report; gsm8k_inferencemax rows now
  carry the published InferenceMAX vLLM B200 FP4 scores for direct
  comparison in test logs.

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tensorrt_llm/evaluate/lm_eval.py              |  7 +++--
 .../defs/accuracy/accuracy_core.py            | 10 +++++-
 .../references/gsm8k_inferencemax.yaml        |  9 ++++--
 .../defs/accuracy/test_llm_api_pytorch.py     | 31 ++++++++++---------
 .../test_lists/qa/llm_function_core.txt       |  8 ++---
 .../test_lists/test-db/l0_dgx_b200.yml        |  2 +-
 tests/integration/test_lists/waives.txt       |  6 ++--
 7 files changed, 44 insertions(+), 29 deletions(-)

diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py
index 240e8a8ad7ac..1b3d68725423 100644
--- a/tensorrt_llm/evaluate/lm_eval.py
+++ b/tensorrt_llm/evaluate/lm_eval.py
@@ -537,9 +537,10 @@ def _adjust_config(task_dict, random_seed):
                     }
                 else:
                     # NOTE: Few-shot random seed
-                    task_obj.set_fewshot_seed(seed=self.fewshot_random_seed
-                                              if self.fewshot_random_seed
-                                              is not None else random_seed)
+                    fewshot_seed = (self.fewshot_random_seed
+                                    if self.fewshot_random_seed is not None else
+                                    random_seed)
+                    task_obj.set_fewshot_seed(seed=fewshot_seed)
                     adjusted_task_dict[task_name] = task_obj
 
                     # NOTE: Shuffle dataset
diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index 3d4b2cd66f8f..c4a3fbd846cd 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -78,6 +78,9 @@ class HypothesisTestingParams:
     beta: float = 0.2
     sigma: float = 50.0
     higher_is_better: bool = True
+    # Free-text provenance for the reference row (e.g. an externally
+    # published score to compare against); printed in the report only.
+    reference_note: Optional[str] = None
     theta: float = field(init=False)
     threshold: float = field(init=False)
 
@@ -106,6 +109,10 @@ def report(self, accuracy: Optional[float] = None) -> str:
 Theta (Minimum detectable effect): {self.theta:.3f}
 Reference {self.metric_name}: {self.ref_accuracy:.3f}
 Threshold: {self.threshold:.3f}
+==========================================================="""
+        if self.reference_note is not None:
+            report = f"""{report}
+External reference: {self.reference_note}
 ==========================================================="""
         if accuracy is not None:
             report = f"""{report}
@@ -190,7 +197,8 @@ def get_hypothesis_testing_params(self,
             sigma=entry.get("sigma", self.SIGMA),
             num_samples=entry.get("num_samples", self.NUM_SAMPLES),
             higher_is_better=entry.get("higher_is_better",
-                                       self.HIGHER_IS_BETTER))
+                                       self.HIGHER_IS_BETTER),
+            reference_note=entry.get("reference_note"))
 
     def evaluate(self,
                  llm: Union[PyTorchLLM, AutoDeployLLM],
diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index 206839aaa658..b5691e89dfdc 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -9,13 +9,18 @@ nvidia/MiniMax-M3-NVFP4:
   # Matches the serve-based measurement of the same config exactly.
   - quant_algo: MIXED_PRECISION
     accuracy: 95.53
+    reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
+      n=13 (inferencex.semianalysis.com, 2026-07)"
   # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
     accuracy: 95.68
+    reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
+      n=13 (inferencex.semianalysis.com, 2026-07)"
   # MSA path + one-model Eagle3 + CUDA graphs + overlap (mean of 3 runs:
-  # 95.53 / 95.53 / 95.00; serve-based repeats 95.38-95.60; published vLLM
-  # B200 FP4 Eagle3 mean 95.49).
+  # 95.53 / 95.53 / 95.00; serve-based repeats 95.38-95.60).
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
     accuracy: 95.35
+    reference_note: "published vLLM B200 FP4 Eagle3 mean 95.49 [95.15-96.06]
+      n=8 (inferencex.semianalysis.com, 2026-07)"
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 37a406799a6e..82f8e5660ed0 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -45,8 +45,7 @@
                         skip_pre_hopper, skip_ray)
 from .accuracy_core import (GSM8K, MMLU, CnnDailymail, GPQADiamond,
                             GSM8KInferenceMax, JsonModeEval,
-                            LlmapiAccuracyTestHarness, LongBenchV1,
-                            LongBenchV2)
+                            LlmapiAccuracyTestHarness, LongBenchV1, LongBenchV2)
 
 
 # Keep helper definitions below imports so new imports do not need E402
@@ -7656,6 +7655,9 @@ def test_nvfp4(self, use_msa, eval_mode):
         sparse_attention_config = MiniMaxM3SparseAttentionConfig(
             implementation="msa" if use_msa else "triton")
         moe_config = MoeConfig(backend="CUTLASS")
+        # InferenceMAX evaluates a serving endpoint with client concurrency
+        # capped at 64 (EVAL_CONCURRENT_REQUESTS); match that batching regime.
+        batch_kwargs = {"max_batch_size": 64} if inferencemax else {}
         with LLM(model_path,
                  tensor_parallel_size=tp_size,
                  moe_expert_parallel_size=ep_size,
@@ -7663,10 +7665,7 @@ def test_nvfp4(self, use_msa, eval_mode):
                  sparse_attention_config=sparse_attention_config,
                  moe_config=moe_config,
                  max_seq_len=16384 if inferencemax else 4096,
-                 # InferenceMAX evaluates a serving endpoint with client
-                 # concurrency capped at 64 (EVAL_CONCURRENT_REQUESTS); match
-                 # that batching regime.
-                 **({"max_batch_size": 64} if inferencemax else {}),
+                 **batch_kwargs,
                  trust_remote_code=True) as llm:
             assert llm.args.quant_config.quant_algo == QuantAlgo.MIXED_PRECISION
             if inferencemax:
@@ -7705,6 +7704,16 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
         # reference).
         kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.6,
                                         enable_block_reuse=False)
+        # The fmha_sm100 decode planner caps total_q x num_qo_heads at 65536;
+        # with 1 + draft_len = 4 verify tokens per row that bounds the batch
+        # at 1024 / TP-sharded heads (256 unsharded under attention DP). The
+        # inferencemax cap of 64 matches the InferenceMAX eval regime (client
+        # concurrency 64) and keeps every decode iteration inside the
+        # captured graph range, as in real serving at that concurrency.
+        if inferencemax:
+            max_batch_size = 64
+        else:
+            max_batch_size = 256 if attention_dp else 512
         with LLM(
                 model_path,
                 tensor_parallel_size=tp_size,
@@ -7716,15 +7725,7 @@ def test_nvfp4_eagle3(self, tp_size, ep_size, attention_dp,
                 # The InferenceMAX protocol needs room for thinking output
                 # (12288 generated tokens on a ~2k chat prompt).
                 max_seq_len=16384 if inferencemax else 4096,
-                # The fmha_sm100 decode planner caps total_q x num_qo_heads at
-                # 65536; with 1 + draft_len = 4 verify tokens per row that
-                # bounds the batch at 1024 / TP-sharded heads (256 unsharded
-                # under attention DP). The inferencemax cap of 64 matches the
-                # InferenceMAX eval regime (client concurrency 64) and keeps
-                # every decode iteration inside the captured graph range,
-                # as in real serving at that concurrency.
-                max_batch_size=64 if inferencemax else
-                (256 if attention_dp else 512),
+                max_batch_size=max_batch_size,
                 speculative_config=spec_config,
                 # Graphs + spec requires the MSA path: its verify batches
                 # are decode-shaped and capture-safe (the reference path
diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt
index b8de89d10aa7..0ef1d9c46c65 100644
--- a/tests/integration/test_lists/qa/llm_function_core.txt
+++ b/tests/integration/test_lists/qa/llm_function_core.txt
@@ -687,10 +687,10 @@ accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_si
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=True] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-use_msa=True-cuda_graph=True] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=True-overlap_scheduler=True-use_msa=True-cuda_graph=True] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True-eval_mode=default] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=default] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=True-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=default] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8
 accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm]
diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml
index 1c7cd7b4f40a..cbf548cf51bd 100644
--- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml
+++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml
@@ -55,7 +55,7 @@ l0_dgx_b200:
   - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_nixl[DeepSeek-V3-Lite-fp8]
   - disaggregated/test_disaggregated.py::test_disaggregated_gpt_oss_120b_harmony[gpt_oss/gpt-oss-120b]
   - accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_nvfp4_multi_gpus[latency_adp_lmtp_tp4]
-  - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True] TIMEOUT (60)
+  - accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True-eval_mode=default] TIMEOUT (60)
   - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] TIMEOUT (60)
   - unittest/_torch/modeling/test_modeling_deepseekv4.py
   - accuracy/test_llm_api_pytorch.py::TestDeepSeekV4Flash::test_auto_dtype TIMEOUT (60)
diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt
index 49f7faaa9b0a..faa7c9aaa99f 100644
--- a/tests/integration/test_lists/waives.txt
+++ b/tests/integration/test_lists/waives.txt
@@ -215,7 +215,7 @@ full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt
 full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343)
 full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6384747)
 full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188)
-full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6424188)
+full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6424188)
 full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)
 full:B200/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6344107)
@@ -239,7 +239,7 @@ full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6445375)
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188)
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] SKIP (https://nvbugs/6442594)
-full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6445375)
+full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6445375)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype SKIP (https://nvbugs/6316983)
@@ -274,7 +274,7 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpu
 full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6422502)
-full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6422502)
+full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6422502)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985)
 full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)

From c64ee1a8759b6205ac4c25ce7741aa917acf16b6 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Tue, 21 Jul 2026 23:52:42 +0000
Subject: [PATCH 06/11] [TRTLLM-14093][test] Uniform explicit 94.0 accuracy
 floor; print acceptance references
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add an optional per-row 'threshold' override to the accuracy reference
  yaml (HypothesisTestingParams.threshold_override): when set it replaces
  the sigma-derived hypothesis-test threshold with an explicit
  benchmark-style floor. All three gsm8k_inferencemax rows gate at a
  uniform 94.0 (InferenceMAX gates the same task at 90.0; our measured
  run-to-run SD is 0.26 and lm-eval per-run stderr 0.58, so 94.0 keeps
  >2-sigma headroom below the weakest reference, 95.35).
- Acceptance gates stay at their original values (rate > 0.78,
  length > 3.3 — mutually consistent: at draft_len=3, AL ~= 1 + 3*rate),
  and the test now prints the drafter-card references
  (Inferact/MiniMax-M3-EAGLE3: 0.839 / 3.518) after the checks and in
  the assert messages, so failures are self-explanatory.
- Validated live: fresh-engine runs pass the 94.0 gate (95.15 strict
  on the first post-change run; more repeats in flight).

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 .../defs/accuracy/accuracy_core.py            | 22 +++++++++++++------
 .../references/gsm8k_inferencemax.yaml        | 10 ++++++---
 .../defs/accuracy/test_llm_api_pytorch.py     | 15 +++++++++++--
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index c4a3fbd846cd..e1297c0c92be 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -81,6 +81,10 @@ class HypothesisTestingParams:
     # Free-text provenance for the reference row (e.g. an externally
     # published score to compare against); printed in the report only.
     reference_note: Optional[str] = None
+    # Explicit pass/fail cutoff. When set, it replaces the hypothesis-test
+    # threshold computed from sigma/alpha — use for benchmark-style floors
+    # where a fixed, human-readable gate is preferred.
+    threshold_override: Optional[float] = None
     theta: float = field(init=False)
     threshold: float = field(init=False)
 
@@ -89,12 +93,15 @@ def __post_init__(self) -> None:
                                    sigma=self.sigma,
                                    alpha=self.alpha,
                                    beta=self.beta)
-        self.threshold = compute_threshold(
-            self.num_samples,
-            self.ref_accuracy,
-            sigma=self.sigma,
-            alpha=self.alpha,
-            higher_is_better=self.higher_is_better)
+        if self.threshold_override is not None:
+            self.threshold = self.threshold_override
+        else:
+            self.threshold = compute_threshold(
+                self.num_samples,
+                self.ref_accuracy,
+                sigma=self.sigma,
+                alpha=self.alpha,
+                higher_is_better=self.higher_is_better)
 
     def report(self, accuracy: Optional[float] = None) -> str:
         metric_name = self.metric_name.upper()
@@ -198,7 +205,8 @@ def get_hypothesis_testing_params(self,
             num_samples=entry.get("num_samples", self.NUM_SAMPLES),
             higher_is_better=entry.get("higher_is_better",
                                        self.HIGHER_IS_BETTER),
-            reference_note=entry.get("reference_note"))
+            reference_note=entry.get("reference_note"),
+            threshold_override=entry.get("threshold"))
 
     def evaluate(self,
                  llm: Union[PyTorchLLM, AutoDeployLLM],
diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index b5691e89dfdc..c1229b8fa0f0 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -1,20 +1,23 @@
 # GSM8K under the InferenceMAX protocol (chat template + thinking + 12288-token
 # generation budget, strict "#### N" extraction) — see GSM8KInferenceMax in
 # accuracy_core.py. References measured on 4x B200 (TP4/EP4, 2026-07-21), full
-# 1319-sample split. For context, the published InferenceMAX vLLM B200 FP4 band
-# for MiniMax-M3 is 94.84-96.06 (spec-off) / 95.15-96.06 (Eagle3); identical
-# configs there scatter up to ~1 point between runs.
+# 1319-sample split. All rows gate at a uniform explicit floor of 94.0
+# (InferenceMAX-style): measured run-to-run SD is 0.26 and lm-eval's per-run
+# stderr is 0.58, so the floor keeps >2-sigma headroom below the weakest
+# reference while staying far above the published InferenceMAX gate (90.0).
 nvidia/MiniMax-M3-NVFP4:
   # Reference sparse-attention path (triton), BF16 KV cache, no spec decoding.
   # Matches the serve-based measurement of the same config exactly.
   - quant_algo: MIXED_PRECISION
     accuracy: 95.53
+    threshold: 94.0
     reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
       n=13 (inferencex.semianalysis.com, 2026-07)"
   # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
     accuracy: 95.68
+    threshold: 94.0
     reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
       n=13 (inferencex.semianalysis.com, 2026-07)"
   # MSA path + one-model Eagle3 + CUDA graphs + overlap (mean of 3 runs:
@@ -22,5 +25,6 @@ nvidia/MiniMax-M3-NVFP4:
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
     accuracy: 95.35
+    threshold: 94.0
     reference_note: "published vLLM B200 FP4 Eagle3 mean 95.49 [95.15-96.06]
       n=8 (inferencex.semianalysis.com, 2026-07)"
diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index 82f8e5660ed0..e7cc4820d1bb 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -7783,13 +7783,24 @@ def drain_spec_stats(llm):
             assert steps > 0, "no speculative iterations recorded"
             chat_rate = accepted / drafted
             chat_length = 1 + accepted / steps
+            # References from the drafter card; measured bands on this stack
+            # are 0.835-0.845 / 3.50-3.53 without attention DP and
+            # 0.819-0.846 / 3.46-3.54 with it. The floors are consistent
+            # with each other: at draft_len=3, AL ~= 1 + 3*rate, so 3.3
+            # corresponds to a ~0.78 rate.
+            ref_rate, ref_length = 0.839, 3.518
             print(f"MiniMax-M3 Eagle3 chat-GSM8K acceptance: rate="
                   f"{chat_rate:.3f}, mean acceptance length="
                   f"{chat_length:.3f} ({steps} spec iterations)")
             assert chat_rate > 0.78, \
-                f"Eagle3 chat-GSM8K acceptance rate too low: {chat_rate:.3f}"
+                f"Eagle3 chat-GSM8K acceptance rate too low: {chat_rate:.3f} " \
+                f"(reference {ref_rate})"
             assert chat_length > 3.3, \
-                f"Eagle3 chat-GSM8K acceptance length too low: {chat_length:.3f}"
+                f"Eagle3 chat-GSM8K acceptance length too low: " \
+                f"{chat_length:.3f} (reference {ref_length})"
+            print(f"Eagle3 acceptance references: rate {ref_rate}, length "
+                  f"{ref_length} — from the Inferact/MiniMax-M3-EAGLE3 "
+                  f"drafter model card.")
 
 
 @skip_pre_blackwell

From e5e0cab7412c9c2af0b9c252b89d4a2dfdd7e06f Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Wed, 22 Jul 2026 04:22:29 +0000
Subject: [PATCH 07/11] [TRTLLM-14093][test] Trim acceptance-gate comment to
 reference and thresholds

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tests/integration/defs/accuracy/test_llm_api_pytorch.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index e7cc4820d1bb..e338a894026a 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -7783,11 +7783,9 @@ def drain_spec_stats(llm):
             assert steps > 0, "no speculative iterations recorded"
             chat_rate = accepted / drafted
             chat_length = 1 + accepted / steps
-            # References from the drafter card; measured bands on this stack
-            # are 0.835-0.845 / 3.50-3.53 without attention DP and
-            # 0.819-0.846 / 3.46-3.54 with it. The floors are consistent
-            # with each other: at draft_len=3, AL ~= 1 + 3*rate, so 3.3
-            # corresponds to a ~0.78 rate.
+            # Published reference (Inferact/MiniMax-M3-EAGLE3 drafter card):
+            # rate 0.839, length 3.518. Our testing thresholds: rate > 0.78,
+            # length > 3.3.
             ref_rate, ref_length = 0.839, 3.518
             print(f"MiniMax-M3 Eagle3 chat-GSM8K acceptance: rate="
                   f"{chat_rate:.3f}, mean acceptance length="

From 1e5be7e7a262e268f8a17d89d5ede779332e6586 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Wed, 22 Jul 2026 04:51:09 +0000
Subject: [PATCH 08/11] [TRTLLM-14093][test] Report explicit-floor gates
 without hypothesis-test statistics

With a threshold override active, printing Alpha/Beta/Sigma/Theta
suggests a sigma-derived threshold that differs from the printed one;
show only the fields that apply.

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tests/integration/defs/accuracy/accuracy_core.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index e1297c0c92be..d29238f9e0e4 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -105,7 +105,19 @@ def __post_init__(self) -> None:
 
     def report(self, accuracy: Optional[float] = None) -> str:
         metric_name = self.metric_name.upper()
-        report = f"""===========================================================
+        if self.threshold_override is not None:
+            # The hypothesis-test statistics do not apply to an explicit
+            # floor; printing them would suggest a different threshold.
+            report = f"""===========================================================
+= {metric_name} CHECK
+===========================================================
+#Samples: {self.num_samples}
+Higher is better: {self.higher_is_better}
+Reference {self.metric_name}: {self.ref_accuracy:.3f}
+Threshold (explicit floor): {self.threshold:.3f}
+==========================================================="""
+        else:
+            report = f"""===========================================================
 = {metric_name} HYPOTHESIS TESTING
 ===========================================================
 Alpha (Type I:  False Positive): {self.alpha:.3f}

From 093357a2daee66c6bc253d782e8d67d3cfc16245 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Wed, 22 Jul 2026 04:58:20 +0000
Subject: [PATCH 09/11] [TRTLLM-14093][test] Use published InferenceMAX scores
 as gsm8k_inferencemax references

Reference accuracies are now the published vLLM B200 FP4 means (the
directly comparable numbers under this protocol), with the source
rendered inline on the Reference line instead of a separate External
reference row. Our own TRT-LLM measurements move to per-row comments;
gating is unchanged (explicit 94.0 floor).

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 .../defs/accuracy/accuracy_core.py            | 11 ++++---
 .../references/gsm8k_inferencemax.yaml        | 30 ++++++++++---------
 2 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py
index d29238f9e0e4..aa6b1649a09e 100644
--- a/tests/integration/defs/accuracy/accuracy_core.py
+++ b/tests/integration/defs/accuracy/accuracy_core.py
@@ -105,6 +105,9 @@ def __post_init__(self) -> None:
 
     def report(self, accuracy: Optional[float] = None) -> str:
         metric_name = self.metric_name.upper()
+        reference_line = f"Reference {self.metric_name}: {self.ref_accuracy:.3f}"
+        if self.reference_note is not None:
+            reference_line += f" ({self.reference_note})"
         if self.threshold_override is not None:
             # The hypothesis-test statistics do not apply to an explicit
             # floor; printing them would suggest a different threshold.
@@ -113,7 +116,7 @@ def report(self, accuracy: Optional[float] = None) -> str:
 ===========================================================
 #Samples: {self.num_samples}
 Higher is better: {self.higher_is_better}
-Reference {self.metric_name}: {self.ref_accuracy:.3f}
+{reference_line}
 Threshold (explicit floor): {self.threshold:.3f}
 ==========================================================="""
         else:
@@ -126,12 +129,8 @@ def report(self, accuracy: Optional[float] = None) -> str:
 #Samples: {self.num_samples}
 Higher is better: {self.higher_is_better}
 Theta (Minimum detectable effect): {self.theta:.3f}
-Reference {self.metric_name}: {self.ref_accuracy:.3f}
+{reference_line}
 Threshold: {self.threshold:.3f}
-==========================================================="""
-        if self.reference_note is not None:
-            report = f"""{report}
-External reference: {self.reference_note}
 ==========================================================="""
         if accuracy is not None:
             report = f"""{report}
diff --git a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
index c1229b8fa0f0..3072eb26bbf1 100644
--- a/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
+++ b/tests/integration/defs/accuracy/references/gsm8k_inferencemax.yaml
@@ -1,30 +1,32 @@
 # GSM8K under the InferenceMAX protocol (chat template + thinking + 12288-token
 # generation budget, strict "#### N" extraction) — see GSM8KInferenceMax in
-# accuracy_core.py. References measured on 4x B200 (TP4/EP4, 2026-07-21), full
-# 1319-sample split. All rows gate at a uniform explicit floor of 94.0
-# (InferenceMAX-style): measured run-to-run SD is 0.26 and lm-eval's per-run
-# stderr is 0.58, so the floor keeps >2-sigma headroom below the weakest
-# reference while staying far above the published InferenceMAX gate (90.0).
+# accuracy_core.py. Reference accuracies are the PUBLISHED InferenceMAX vLLM
+# B200 FP4 means (directly comparable under this protocol); our own TRT-LLM
+# measurements on 4x B200 (TP4/EP4, 2026-07-22) are noted per row. All rows
+# gate at a uniform explicit floor of 94.0 (measured run-to-run SD 0.26,
+# lm-eval per-run stderr 0.58; InferenceMAX's own gate for this task is 90.0).
 nvidia/MiniMax-M3-NVFP4:
   # Reference sparse-attention path (triton), BF16 KV cache, no spec decoding.
-  # Matches the serve-based measurement of the same config exactly.
+  # Ours measured: 95.53 (matches the serve-based measurement exactly).
   - quant_algo: MIXED_PRECISION
-    accuracy: 95.53
+    accuracy: 95.46
     threshold: 94.0
     reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
-      n=13 (inferencex.semianalysis.com, 2026-07)"
+      n=13, inferencex.semianalysis.com, 2026-07"
   # MSA path (fmha_sm100), FP8 KV cache, no spec decoding.
+  # Ours measured: 95.68.
   - quant_algo: MIXED_PRECISION
     kv_cache_quant_algo: FP8
-    accuracy: 95.68
+    accuracy: 95.46
     threshold: 94.0
     reference_note: "published vLLM B200 FP4 spec-off mean 95.46 [94.84-95.91]
-      n=13 (inferencex.semianalysis.com, 2026-07)"
-  # MSA path + one-model Eagle3 + CUDA graphs + overlap (mean of 3 runs:
-  # 95.53 / 95.53 / 95.00; serve-based repeats 95.38-95.60).
+      n=13, inferencex.semianalysis.com, 2026-07"
+  # MSA path + one-model Eagle3 + CUDA graphs + overlap.
+  # Ours measured: 95.00-95.68 across 8 fresh-engine runs (mean 95.40);
+  # serve-based repeats 95.38-95.60.
   - quant_algo: MIXED_PRECISION
     spec_dec_algo: Eagle3
-    accuracy: 95.35
+    accuracy: 95.49
     threshold: 94.0
     reference_note: "published vLLM B200 FP4 Eagle3 mean 95.49 [95.15-96.06]
-      n=8 (inferencex.semianalysis.com, 2026-07)"
+      n=8, inferencex.semianalysis.com, 2026-07"

From 7167e1a87ac4c618f7b9a83faa89d6f3ff8b0cf9 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Wed, 22 Jul 2026 05:16:07 +0000
Subject: [PATCH 10/11] [TRTLLM-14093][test] Include threshold and reference
 source in acceptance assert messages

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tests/integration/defs/accuracy/test_llm_api_pytorch.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
index e338a894026a..18982fe9703e 100644
--- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py
+++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py
@@ -7787,18 +7787,19 @@ def drain_spec_stats(llm):
             # rate 0.839, length 3.518. Our testing thresholds: rate > 0.78,
             # length > 3.3.
             ref_rate, ref_length = 0.839, 3.518
+            ref_source = "the Inferact/MiniMax-M3-EAGLE3 drafter model card"
             print(f"MiniMax-M3 Eagle3 chat-GSM8K acceptance: rate="
                   f"{chat_rate:.3f}, mean acceptance length="
                   f"{chat_length:.3f} ({steps} spec iterations)")
             assert chat_rate > 0.78, \
                 f"Eagle3 chat-GSM8K acceptance rate too low: {chat_rate:.3f} " \
-                f"(reference {ref_rate})"
+                f"(threshold 0.78, reference {ref_rate} from {ref_source})"
             assert chat_length > 3.3, \
                 f"Eagle3 chat-GSM8K acceptance length too low: " \
-                f"{chat_length:.3f} (reference {ref_length})"
+                f"{chat_length:.3f} (threshold 3.3, reference {ref_length} " \
+                f"from {ref_source})"
             print(f"Eagle3 acceptance references: rate {ref_rate}, length "
-                  f"{ref_length} — from the Inferact/MiniMax-M3-EAGLE3 "
-                  f"drafter model card.")
+                  f"{ref_length} — from {ref_source}.")
 
 
 @skip_pre_blackwell

From 6da0761e1f9e1558ea4d6e4ec5528b6f19d11a58 Mon Sep 17 00:00:00 2001
From: Zheyu Fu <zheyuf@NVIDIA.com>
Date: Wed, 22 Jul 2026 06:11:37 +0000
Subject: [PATCH 11/11] [TRTLLM-14093][test] Switch MiniMax-M3 QA accuracy
 entries to the InferenceMAX protocol
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All four QA rows now gate on eval_mode=inferencemax, directly comparable
to the published InferenceMAX scores. Measured on 4x B200 (all under the
existing TIMEOUT 180): test_nvfp4 triton 24:56 (95.53), test_nvfp4 MSA
11:27 (95.68), eagle3 adp=False 7:01 (95.35 mean of 3), eagle3 adp=True
7:16 (95.91, acceptance 0.829/3.488 — first-run validated). The
use_msa=False platform waivers (B200/B300/GB300 nvbugs) move to the
inferencemax ID with the entry; l0_dgx_b200 keeps its default-mode
use_msa=True entry, so that ID's coverage is unchanged.

Signed-off-by: Zheyu Fu <zheyuf@NVIDIA.com>
---
 tests/integration/test_lists/qa/llm_function_core.txt | 8 ++++----
 tests/integration/test_lists/waives.txt               | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt
index 0ef1d9c46c65..4aa8b1a1e51e 100644
--- a/tests/integration/test_lists/qa/llm_function_core.txt
+++ b/tests/integration/test_lists/qa/llm_function_core.txt
@@ -687,10 +687,10 @@ accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_si
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=True] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True-eval_mode=default] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=default] TIMEOUT (180)
-accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=True-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=default] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=inferencemax] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=True-eval_mode=inferencemax] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=False-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=inferencemax] TIMEOUT (180)
+accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4_eagle3[tp_size=4-ep_size=4-attention_dp=True-overlap_scheduler=True-use_msa=True-cuda_graph=True-eval_mode=inferencemax] TIMEOUT (180)
 accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_auto_dtype
 accuracy/test_llm_api_pytorch.py::TestMinistral8BInstruct::test_fp8
 accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm]
diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt
index faa7c9aaa99f..4082e9191969 100644
--- a/tests/integration/test_lists/waives.txt
+++ b/tests/integration/test_lists/waives.txt
@@ -215,7 +215,7 @@ full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt
 full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343)
 full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6384747)
 full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188)
-full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6424188)
+full:B200/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=inferencemax] SKIP (https://nvbugs/6424188)
 full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:B200/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)
 full:B200/disaggregated/test_disaggregated.py::test_disaggregated_overlap_gen_first[ctx_pp4-TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6344107)
@@ -239,7 +239,7 @@ full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6445375)
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188)
 full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8_piecewise_cuda_graph[tp_size=8-ep_size=8] SKIP (https://nvbugs/6442594)
-full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6445375)
+full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=inferencemax] SKIP (https://nvbugs/6445375)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)
 full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype SKIP (https://nvbugs/6316983)
@@ -274,7 +274,7 @@ full:GB300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpu
 full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_bfloat16_4gpus[pp4-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[pp4-fp8kv=True-attn_backend=FLASHINFER-torch_compile=False] SKIP (https://nvbugs/6385771)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6422502)
-full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=default] SKIP (https://nvbugs/6422502)
+full:GB300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False-eval_mode=inferencemax] SKIP (https://nvbugs/6422502)
 full:GB300/accuracy/test_llm_api_pytorch.py::TestQwen3_5_35B_A3B::test_fp8_moe_dflash SKIP (https://nvbugs/6316985)
 full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318)
 full:GB300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318)