From 2c13654aff111ceafe9cab7600cee016645f10ce Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:18:00 -0700 Subject: [PATCH 1/7] [None][test] Add the DeepSeek-V4.1-Flash aggregated perf-sanity cases tests/scripts/perf-sanity/aggregated/deepseek_v41_flash_grace_blackwell.yaml came in with DeepSeek-V4.1-Flash support (#19727) but no test list ever named it, so CI never ran it. Add its three server configs (TEP4 con8, DEP4 con128, TEP4 + embedded DSpark con8; all 8k/1k on one 4xGB300 tray) to the GB300 4-GPU post-merge perf-sanity list. - l0_gb300_multi_gpus_perf_sanity.yml: the three cases, TIMEOUT from the validation runs (slowest + 25%, at least +15 min, rounded up to 5, floor 30). - .test_durations: seed the three cases with their measured pytest time so the post-merge shards stay balanced. - deepseek_v41_flash_grace_blackwell.yaml: write kv_cache_config.dtype as auto, its default; the harness records a missing dtype as fp8 in the perf DB. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/.test_durations | 3 +++ .../test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml | 4 ++++ .../aggregated/deepseek_v41_flash_grace_blackwell.yaml | 7 ++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index a73d8ce80930..1834ad01456c 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -904,6 +904,9 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 577.7239285714285, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL]": 408.025, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con9832_ctx1_dep4_gen8_dep8_eplb0_mtp3_ccb-NIXL]": 2200.323, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_dep4_8k1k]": 624.423, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_tep4_8k1k]": 635.625, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_tep4_dspark_8k1k]": 701.992, "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_235b_a22b_fp8_hopper-qwen3_235b_a22b_fp8_tp4_ep4_cutlass_8k1k]": 944.3419230769231, "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 703.392, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_dep4_dspark5_8k1k]": 904.2026666666666, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 00bfa931fd0d..ec05ec485371 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -44,6 +44,10 @@ l0_gb300_multi_gpus_perf_sanity: # deepseek-v4.1-flash-fp4 8k1k dspark (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_tep4_dspark5_8k1k] TIMEOUT (30) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_dep4_dspark5_8k1k] TIMEOUT (35) + # deepseek-v4.1-flash 8k1k (aggregated) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_tep4_8k1k] TIMEOUT (30) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_dep4_8k1k] TIMEOUT (30) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_tep4_dspark_8k1k] TIMEOUT (30) # glm-5.3-flash-fp8 tp4 mtp3 8k1k (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_8k1k] TIMEOUT (30) # nemotron-ultra-v3-fp4 8k64k (ctx_only) diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v41_flash_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v41_flash_grace_blackwell.yaml index 2b82d87ee47d..e998241a8370 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v41_flash_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v41_flash_grace_blackwell.yaml @@ -10,7 +10,9 @@ # derivation changes (see disagg_config_ctxtp4_gentp4_deepseek_v41.yaml): # sparse_attention_config auto-derived from architectures[0]. # kv_cache_config.dtype the CSA2 TRTLLM backend stages BF16 KV and rejects -# anything but auto/bfloat16. +# anything but auto/bfloat16. Written out as `auto` +# (the default) only because the perf harness +# records a missing dtype as fp8. # tokens_per_block model default (128 == sliding_window). # moe_config.backend V4.1's experts are MXFP4; the model selects its # own MoE path. @@ -49,6 +51,7 @@ server_configs: enable_padding: true batch_sizes: [1, 2, 4, 8] kv_cache_config: + dtype: auto enable_block_reuse: false free_gpu_memory_fraction: 0.8 client_configs: @@ -81,6 +84,7 @@ server_configs: enable_padding: true batch_sizes: [1, 2, 4, 8, 16, 32] kv_cache_config: + dtype: auto enable_block_reuse: false free_gpu_memory_fraction: 0.8 client_configs: @@ -110,6 +114,7 @@ server_configs: enable_padding: true batch_sizes: [1, 2, 4, 8] kv_cache_config: + dtype: auto enable_block_reuse: false free_gpu_memory_fraction: 0.8 speculative_config: From 58ef3c97a1d6f60db7b8785f2dd7d51b38f5c849 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:55:54 -0700 Subject: [PATCH 2/7] [None][test] Add GLM-5.3-Flash aggregated and disaggregated perf-sanity cases Four GLM-5.3-Flash cases for the GB300 post-merge perf-sanity lists, on the shapes the GLM-5.3-Flash deployment guide and perf PRs measure: - FP8, attention DP4 / EP4 + MTP3 + FP8 KV, 8k/1k at concurrency 256 (64 per rank): the high-concurrency frontier shape. - FP8, TP4 / EP4 + MTP3, ISL 13312 / OSL 33 at concurrency 8 and 1: the long-prompt, short-answer workload the prefill optimizations target. - NVFP4 checkpoint, TP4 / EP4 + MTP3, FP8 KV, TRTLLM MoE, 8k/1k at concurrency 16 and 1 (new model-path key glm_5.3_flash_nvfp4). - FP8 disaggregated E2E, 1 ctx + 1 gen worker, each TP4 / EP4, MTP3, Python NIXL transceiver, 8k/1k at concurrency 16 (the guide's disaggregated recipe), in the existing GB300 CTX1-GPU4 / GEN1-GPU4 stage, whose split count goes from 1 to 2. In the two multi-client cases the larger client runs first: aggregated clients send no warmup request, and the first requests a fresh server sees are slow, which would otherwise dominate the short concurrency-1 window. TIMEOUT follows the README rule from the validation runs (8-10 min of pytest each, so 30), and .test_durations is seeded with those pytest times. The GB300 4-GPU post-merge stage goes from 4 to 5 shards so that no shard holds more than six tests (longest shard ~96 min). Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 13 +- tests/integration/defs/.test_durations | 4 + tests/integration/defs/perf/_model_paths.py | 1 + .../l0_gb300_multi_gpus_perf_sanity.yml | 5 + ...sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml | 2 + ...gb300_glm53_flash_fp8_grace_blackwell.yaml | 112 ++++++++++++++++- ...300_glm53_flash_nvfp4_grace_blackwell.yaml | 67 ++++++++++ ...x1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml | 114 ++++++++++++++++++ 8 files changed, 306 insertions(+), 12 deletions(-) create mode 100644 tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 15b11713545d..de2b3d0a770c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6677,10 +6677,11 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 1, 4, 1, true, false], // PerfSanity post-merge tests "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 1, 4], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 4, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 4, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 4, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 4, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 5, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 5, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 5, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 5, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 5, 5, 4, 1, true, false], ] SBSASlurmTestConfigs = cbtsResizeSplits(SBSASlurmTestConfigs) fullSet += SBSASlurmTestConfigs.keySet() @@ -6820,12 +6821,12 @@ def launchTestJobs(pipeline, testFilter, globalVars) 32, 8 ) - // Nemotron-Ultra-V3 8k64k con1: ctx1 (1 node, 4 GPUs) + gen1 tep4 (1 node, 4 GPUs) = 8 GPUs + // Nemotron-Ultra-V3 8k64k con1 and GLM-5.3-Flash FP8 8k1k con16: ctx1 (1 node, 4 GPUs) + gen1 tep4 (1 node, 4 GPUs) = 8 GPUs multiNodesSBSAConfigs += buildStageConfigs( "GB300-8_GPUs-2_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE1-GPU4-GEN1-NODE1-GPU4-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4", - 1, + 2, 8, 2 ) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 1834ad01456c..78b1001cc4b6 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -911,7 +911,10 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-dynamo_qwen3_32b_fp8_hopper-qwen3_32b_fp8_tp2_6k1k]": 703.392, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_dep4_dspark5_8k1k]": 904.2026666666666, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_tep4_dspark5_8k1k]": 767.533, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_dep4_mtp3_fp8kv_8k1k]": 606.433, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_8k1k]": 379.83566666666667, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_isl13312_osl33]": 483.808, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_8k1k]": 541.238, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp0_50k2k]": 1863.1629285714287, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp3_50k2k]": 780.4455, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_tep4_mtp6_50k2k]": 1339.9197142857142, @@ -944,6 +947,7 @@ "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL]": 1202.3937142857144, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL]": 1021.8967857142857, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 3149.737866666667, + "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL]": 504.041, "perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1649.4002142857144, "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 519.6357058823529, "perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 342.2766470588235, diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index 48799b59b329..3ff1a7c02f1d 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -92,6 +92,7 @@ # GLM-5.2 NVFP4 (MoE, MLA + DSA on the DeepSeek-V3.2 code path) "glm_5.2_fp4": "GLM-5.2-NVFP4", "glm_5.3_flash_fp8": "GLM-5.3-Flash", + "glm_5.3_flash_nvfp4": "GLM-5.3-Flash-NVFP4", # MiniMax-M3 NVFP4: MXFP8 base layers with NVFP4 routed experts # (the MXFP8 checkpoint is registered as "minimax_m3_mxfp8" above). "minimax_m3_fp4": "MiniMax-M3-NVFP4", diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index ec05ec485371..48b536dbe010 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -50,6 +50,11 @@ l0_gb300_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_v41_flash_grace_blackwell-v41_flash_tep4_dspark_8k1k] TIMEOUT (30) # glm-5.3-flash-fp8 tp4 mtp3 8k1k (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_8k1k] TIMEOUT (30) + # glm-5.3-flash-fp8 attention dp4 mtp3 fp8 kv 8k1k and long prompt (aggregated) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_dep4_mtp3_fp8kv_8k1k] TIMEOUT (30) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_isl13312_osl33] TIMEOUT (30) + # glm-5.3-flash-nvfp4 tp4 mtp3 8k1k (aggregated) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_8k1k] TIMEOUT (30) # nemotron-ultra-v3-fp4 8k64k (ctx_only) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (30) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con9832_ctx1_dep4_gen8_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (60) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml index b89e84509fda..2fb63ac6fab4 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4.yml @@ -16,3 +16,5 @@ l0_gb300_multi_nodes_perf_sanity_ctx1_node1_gpu4_gen1_node1_gpu4: tests: # nemotron-ultra-v3-fp4 8k64k con1 - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (45) + # glm-5.3-flash-fp8 8k1k con16 + - perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (30) diff --git a/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_fp8_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_fp8_grace_blackwell.yaml index 96718643da81..ba2824a8afdf 100644 --- a/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_fp8_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_fp8_grace_blackwell.yaml @@ -2,12 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 # GLM-5.3-Flash FP8 (Glm5NextForConditionalGeneration: KDA + sparse-MLA -# glm_kpool layers) on one 4xGB300 tray, TP4 / EP4, TRTLLM MoE, MTP3. -# The engine settings follow the GLM-5.3-Flash tp4_mtp3 serving config: -# batch 32, 8192 tokens, chunked prefill, overlap scheduler, CUDA graphs -# 1..32 with padding, KV fraction 0.6. The accepted draft length is pinned to -# 2.384 of 3, the value that config serves with, so throughput does not depend -# on acceptance on random prompts. +# glm_kpool layers) on one 4xGB300 tray, TRTLLM MoE, MTP3. The accepted draft +# length is pinned to 2.384 of 3, the value the tp4_mtp3 serving config serves +# with, so throughput does not depend on acceptance on random prompts. +# +# - tp4_mtp3_8k1k: the GLM-5.3-Flash tp4_mtp3 serving config: TP4 / EP4, +# batch 32, 8192 tokens, chunked prefill, overlap scheduler, CUDA graphs +# 1..32 with padding, KV fraction 0.6, BF16 KV (the checkpoint sets no KV +# quantization, so `auto` is BF16). +# - dep4_mtp3_fp8kv_8k1k: the high-concurrency shape of the GLM-5.3-Flash +# deployment guide: attention DP4 / EP4 with 4096 tokens and batch 64 per +# rank (64 x 4 ranks = concurrency 256), FP8 KV cache (sparse MLA runs +# natively on it at attention DP), KV fraction 0.5. +# - tp4_mtp3_isl13312_osl33: long prompt, short answer (ISL 13312 / OSL 33), +# the workload the GLM-5.3-Flash prefill optimizations are measured on: +# TP4 / EP4, 16384 tokens, BF16 KV, concurrency 8 and then 1. metadata: model_name: glm_5.3_flash_fp8 supported_gpus: @@ -55,3 +64,94 @@ server_configs: isl: 8192 osl: 1024 backend: openai + +- name: glm53_flash_fp8_dep4_mtp3_fp8kv_8k1k + model_name: glm_5.3_flash_fp8 + server_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 64 + max_num_tokens: 4096 + max_seq_len: 9344 + trust_remote_code: true + enable_attention_dp: true + enable_chunked_prefill: true + disable_overlap_scheduler: false + disable_mm_encoder: true + print_iter_log: true + enable_iter_perf_stats: true + stream_interval: 20 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.5 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + speculative_config: + decoding_type: MTP + max_draft_len: 3 + client_configs: + - name: con256_iter5_8k1k + concurrency: 256 + iterations: 5 + isl: 8192 + osl: 1024 + backend: openai + +- name: glm53_flash_fp8_tp4_mtp3_isl13312_osl33 + model_name: glm_5.3_flash_fp8 + server_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 16384 + trust_remote_code: true + enable_attention_dp: false + enable_chunked_prefill: true + disable_overlap_scheduler: false + disable_mm_encoder: true + print_iter_log: true + enable_iter_perf_stats: true + stream_interval: 20 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: auto + tokens_per_block: 128 + moe_config: + backend: TRTLLM + speculative_config: + decoding_type: MTP + max_draft_len: 3 + # con8 runs first: aggregated clients send no warmup request, and the first + # requests a fresh server sees are slow (TTFT ~2 s instead of ~0.25 s here), + # which would dominate a short con1 window. + client_configs: + - name: con8_iter20_isl13312_osl33 + concurrency: 8 + iterations: 20 + isl: 13312 + osl: 33 + backend: openai + - name: con1_iter10_isl13312_osl33 + concurrency: 1 + iterations: 10 + isl: 13312 + osl: 33 + backend: openai diff --git a/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml new file mode 100644 index 000000000000..eef0804242c7 --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GLM-5.3-Flash NVFP4 (nvidia/GLM-5.3-Flash-NVFP4: NVFP4 routed experts and +# dense layers; attention and the MTP layer stay unquantized) on one 4xGB300 +# tray, TP4 / EP4, MTP3, at concurrency 16 and then 1. Same engine settings as +# the FP8 tp4_mtp3 case in gb300_glm53_flash_fp8_grace_blackwell.yaml, except: +# kv_cache_config.dtype fp8, which the checkpoint's kv_cache_quant_algo +# selects anyway; written out so the perf DB records it. +# moe_config.backend TRTLLM. `AUTO` resolves this checkpoint to CUTLASS; +# TRTLLM-Gen is the faster NVFP4 MoE on Blackwell. +# The accepted draft length is pinned to 2.384 of 3, as in the FP8 case. +metadata: + model_name: glm_5.3_flash_nvfp4 + supported_gpus: + - GB300 +hardware: + gpus_per_node: 4 +server_configs: +- name: glm53_flash_nvfp4_tp4_mtp3_8k1k + model_name: glm_5.3_flash_nvfp4 + server_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.384 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 32 + max_num_tokens: 8192 + max_seq_len: 9344 + trust_remote_code: true + enable_attention_dp: false + enable_chunked_prefill: true + disable_overlap_scheduler: false + disable_mm_encoder: true + print_iter_log: true + enable_iter_perf_stats: true + stream_interval: 20 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + speculative_config: + decoding_type: MTP + max_draft_len: 3 + # con16 runs first, so the slow first requests of a fresh server (aggregated + # clients send no warmup request) fall in its longer window, not in con1's. + client_configs: + - name: con16_iter5_8k1k + concurrency: 16 + iterations: 5 + isl: 8192 + osl: 1024 + backend: openai + - name: con1_iter10_8k1k + concurrency: 1 + iterations: 10 + isl: 8192 + osl: 1024 + backend: openai diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml new file mode 100644 index 000000000000..23bb50c4e412 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL.yaml @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GLM-5.3-Flash FP8 (Glm5NextForConditionalGeneration), disaggregated E2E, 8K ISL / +# 1K OSL at concurrency 16, on two 4xGB300 trays: 1 ctx worker + 1 gen worker, each +# TP4 / EP4. This is the disaggregated recipe of the GLM-5.3-Flash deployment guide: +# the Python NIXL transceiver on both workers, MTP3 on both, and the ctx worker +# without the overlap scheduler or CUDA graphs. The engine settings otherwise follow +# the aggregated tp4_mtp3 case (gb300_glm53_flash_fp8_grace_blackwell.yaml), which +# runs the same load, so the two can be compared: TRTLLM MoE, BF16 KV (the +# checkpoint sets no KV quantization, so `auto` is BF16), 128-token blocks, and the +# accepted draft length pinned to 2.384 of 3 on the gen worker. +metadata: + model_name: glm_5.3_flash_fp8 + precision: fp8 + model_dir_name: GLM-5.3-Flash + architecture: Glm5NextForConditionalGeneration + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 8k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '16' + input_length: 8192 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + gen_worker_env_var: TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.384 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 32 + max_num_tokens: 512 + max_seq_len: 9344 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + trust_remote_code: true + disable_mm_encoder: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 12, 16, 20, 24, 28, 32] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: auto + tokens_per_block: 128 + moe_config: + backend: TRTLLM + cache_transceiver_config: + kv_transfer_timeout_ms: 600000 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: false + speculative_config: &id001 + decoding_type: MTP + max_draft_len: 3 + num_postprocess_workers: 4 + stream_interval: 20 + ctx: + print_iter_log: true + max_batch_size: 4 + max_num_tokens: 16384 + max_seq_len: 9344 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + trust_remote_code: true + disable_mm_encoder: true + cuda_graph_config: null + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: auto + tokens_per_block: 128 + moe_config: + backend: TRTLLM + cache_transceiver_config: + kv_transfer_timeout_ms: 600000 + backend: NIXL + transceiver_runtime: PYTHON + disable_overlap_scheduler: true + speculative_config: *id001 From 6b7820558e0e07b6fd2559f7b3af434e5bc5472c Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:19:38 -0700 Subject: [PATCH 3/7] [None][test] Add the Qwen3.8-Flash-Next modeling_v2 perf-sanity case Run Qwen3.8-Flash-Next in perf sanity on modeling_v2 by default: one GB300 GPU, `modeling_v2: require` (the qwen3_8_flash_next_nvfp4__sm_103__tp1 target), TP1, MTP3, TRTLLM MoE, bf16 recurrent state, breakable prefill CUDA graphs, 8k/1k at concurrency 1 and 256. It joins the GB300 4-GPU post-merge list (GB300 stages allocate whole 4-GPU trays). The KV cache stays `auto` (bf16) for now: NVFP4 KV works on main but its prefill is not fast yet (#20276), about 6.6 s per 8k prompt, so the concurrency-256 client would not finish in a stage. Switch to `nvfp4` once the NVFP4 KV fast path (#20937) is on main. TIMEOUT 35 from the validation run (1,011 s of pytest), and .test_durations is seeded with it. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/.test_durations | 1 + .../l0_gb300_multi_gpus_perf_sanity.yml | 2 + ...wen3_8_flash_next_fp4_grace_blackwell.yaml | 63 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 tests/scripts/perf-sanity/aggregated/gb300_qwen3_8_flash_next_fp4_grace_blackwell.yaml diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index 78b1001cc4b6..be8d3202f586 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -918,6 +918,7 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp0_50k2k]": 1863.1629285714287, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp3_50k2k]": 780.4455, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_tep4_mtp6_50k2k]": 1339.9197142857142, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_qwen3_8_flash_next_fp4_grace_blackwell-qwen3_8_flash_next_fp4_mv2_tp1_mtp3_8k1k]": 1011.102, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gemma4_26b_a4b_nvfp4_blackwell-gemma4_26b_a4b_nvfp4_tp1_1k1k]": 289.63309090909087, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL]": 284.1654117647059, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb200_qwen3-235b-fp4_8k1k_con1_ctx1_tp1_gen1_tep4_eplb0_mtp0_ccb-NIXL]": 228.4835294117647, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 48b536dbe010..d9edfb55db8e 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -55,6 +55,8 @@ l0_gb300_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_isl13312_osl33] TIMEOUT (30) # glm-5.3-flash-nvfp4 tp4 mtp3 8k1k (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_8k1k] TIMEOUT (30) + # qwen3.8-flash-next-nvfp4 modeling_v2 tp1 mtp3 8k1k (aggregated, 1 GPU) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_qwen3_8_flash_next_fp4_grace_blackwell-qwen3_8_flash_next_fp4_mv2_tp1_mtp3_8k1k] TIMEOUT (35) # nemotron-ultra-v3-fp4 8k64k (ctx_only) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (30) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con9832_ctx1_dep4_gen8_dep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (60) diff --git a/tests/scripts/perf-sanity/aggregated/gb300_qwen3_8_flash_next_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_qwen3_8_flash_next_fp4_grace_blackwell.yaml new file mode 100644 index 000000000000..f2b3465f884c --- /dev/null +++ b/tests/scripts/perf-sanity/aggregated/gb300_qwen3_8_flash_next_fp4_grace_blackwell.yaml @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Qwen3.8-Flash-Next NVFP4 on one GB300 GPU, served by modeling_v2 +# (`modeling_v2: require`, the qwen3_8_flash_next_nvfp4__sm_103__tp1 target), the +# default implementation for Qwen3.8-Flash-Next in perf sanity: TP1, MTP3, +# TRTLLM MoE, bf16 recurrent state, breakable prefill CUDA graphs, 8k/1k at +# concurrency 1 and 256. Acceptance is not pinned, so d_al tracks MTP acceptance. +# +# kv_cache_config.dtype is `auto` (bf16) for now. The recipe's NVFP4 KV cache +# works on main but its prefill is not fast yet (see #20276): every 8k prompt +# takes ~6.6 s to prefill, so con256 would not finish in a stage. Switch to +# `nvfp4` once the NVFP4 KV fast path (#20937) is on main. +# enable_block_reuse is kept from the recipe; main disables reuse for this +# hybrid model until kv_cache_config.mamba_state_config sets a snapshot policy. +metadata: + model_name: qwen3.8_flash_next_fp4 + supported_gpus: + - GB300 +hardware: + gpus_per_node: 1 +server_configs: +- name: qwen3_8_flash_next_fp4_mv2_tp1_mtp3_8k1k + server_env_var: TRTLLM_QWEN4_EXP_PLE_HOST_OFFLOAD=1 + model_name: qwen3.8_flash_next_fp4 + tensor_parallel_size: 1 + moe_expert_parallel_size: 1 + pipeline_parallel_size: 1 + trust_remote_code: true + modeling_v2: require + max_batch_size: 256 + max_seq_len: 10240 + max_num_tokens: 16384 + enable_chunked_prefill: true + kv_cache_config: + free_gpu_memory_fraction: 0.6 + enable_block_reuse: true + mamba_ssm_cache_dtype: bfloat16 + dtype: auto + cuda_graph_config: + max_batch_size: 256 + enable_padding: true + prefill_cuda_graph_backend: breakable + prefill_capture_num_tokens: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 768, 1024, 1280, 1536, 1792, 2048, + 2304, 2560, 2816, 3072, 4096, 6144, 8192, 10240, 12288, 14336, 16384] + speculative_config: + decoding_type: MTP + max_draft_len: 3 + moe_config: + backend: TRTLLM + client_configs: + - name: con1_iter10_8k1k + concurrency: 1 + iterations: 10 + isl: 8192 + osl: 1024 + backend: openai + - name: con256_iter10_8k1k + concurrency: 256 + iterations: 10 + isl: 8192 + osl: 1024 + backend: openai From 34fa28eebf39058b67c9518a3be0a255d47295a3 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:15:35 -0700 Subject: [PATCH 4/7] [None][test] Add Kimi-K3 and MiniMax-M3 fixed-context gen-only perf-sanity cases Track Kimi-K3 and MiniMax-M3 decode at a fixed long context with gen_only_no_context lanes: random tokens, no context servers, gated on the gen worker's device step time. - Kimi-K3 NVFP4 + DSpark: the helix8 bs16 dspark5 gen worker of the existing AgentX con138 recipe, unchanged, decoding 16 requests of 163840 input / 1024 output tokens on 2 GB300 nodes. - MiniMax-M3 NVFP4 + EAGLE3-GQA: the base2 TP4 gen worker of the MiniMax-M3 ctx-only / gen-only study at 153600 input / 1024 output tokens, at concurrency 1 and 64 on 1 GB300 node. TIMEOUTs come from the validation runs (742, 1187 and 1759 s of pytest), and .test_durations is seeded with them. GB300-4 post-merge goes from 5 to 6 shards and the GB300 GEN1-NODE2-GPU8 stage from 4 to 5. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 13 +- tests/integration/defs/.test_durations | 3 + .../l0_gb300_multi_gpus_perf_sanity.yml | 3 + ...ulti_nodes_perf_sanity_gen1_node2_gpu8.yml | 2 + ...p16_gen1_helix8_bs16_dspark5_ccb-NIXL.yaml | 160 +++++++++++++++++ ...x1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 169 ++++++++++++++++++ ...x1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 168 +++++++++++++++++ 7 files changed, 512 insertions(+), 6 deletions(-) create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL.yaml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index de2b3d0a770c..23457def0143 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6677,11 +6677,12 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-4_GPUs-PyTorch-PerfSanity-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 1, 4, 1, true, false], // PerfSanity post-merge tests "GB200-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb200-x4", "l0_gb200_multi_gpus_perf_sanity", 1, 1, 4], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 5, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 5, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 5, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 5, 4, 1, true, false], - "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 5, 5, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-1": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 1, 6, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-2": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 2, 6, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-3": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 3, 6, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-4": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 4, 6, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-5": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 5, 6, 4, 1, true, false], + "GB300-4_GPUs-PyTorch-PerfSanity-Post-Merge-6": ["auto:gb300-x4", "l0_gb300_multi_gpus_perf_sanity", 6, 6, 4, 1, true, false], ] SBSASlurmTestConfigs = cbtsResizeSplits(SBSASlurmTestConfigs) fullSet += SBSASlurmTestConfigs.keySet() @@ -6799,7 +6800,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-8_GPUs-2_Nodes-PyTorch-PerfSanity-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8", - 4, + 5, 8, 2 ) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index be8d3202f586..e94d2d2123dc 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -930,6 +930,9 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 794.9620666666666, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 1609.1394285714287, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 411.37428571428575, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL]": 741.683, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1187.22, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1759.201, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1415.384142857143, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL]": 1434.301857142857, "perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_2_nodes_grace_blackwell-glm5_fp4_dep8_mtp1_8k1k]": 655.066, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index d9edfb55db8e..49084506a752 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -71,3 +71,6 @@ l0_gb300_multi_gpus_perf_sanity: # nemotron-ultra-v3-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (40) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL] TIMEOUT (40) + # minimax-m3-fp4 eagle3 150k1k con1 and con64, TP4 gen worker + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (35) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (45) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml index 8541cf92768c..8bf2f90b23f2 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml @@ -22,3 +22,5 @@ l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8: - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (45) # glm-5-fp4 8k1k con1024 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (30) + # kimi-k3-fp4 dspark 160k1k con16, helix CP8 gen worker + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL] TIMEOUT (30) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL.yaml new file mode 100644 index 000000000000..1fcc0867b98b --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL.yaml @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Kimi-K3 NVFP4 + DSpark decode at a fixed long context, for the gen_only_no_context +# lane: one helix (CP8) gen worker on 2 GB300 nodes decodes 16 requests of exactly +# 163840 input / 1024 output random tokens. This is the gen-only method of the K3 +# GB300 optimization work (balanced ISL 163840, steady-state step time), on the +# helix8 bs16 dspark5 gen worker of +# gb300_kimi-k3-fp4-dspark_agentx_con138_ctx16_dwdp16_gen3_helix8_bs16_dspark5_ccb-NIXL.yaml +# (ported from the bench-trtllm-disagg srt-slurm recipe +# disagg-gb300-16p3d-dep1-helix8-c138-b16-mtp). worker_config and the worker env are +# copied from that file unchanged; the AgentX trace replay, its client env, the +# conversation router and the chat-history proxy options are left out. The ctx +# section (the recipe's 16-GPU DWDP prefill fleet) is not started by +# gen_only_no_context. +metadata: + model_name: kimi_k3_nvfp4 + precision: fp4 + model_dir_name: Kimi-K3-NVFP4 + supported_gpus: [GB300] + script_file: disaggr_torch.slurm + benchmark_type: 160k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '16' + input_length: 163840 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 16 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes + NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 HF_HUB_TRUST_REMOTE_CODE=1 + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True KIMI_K3_MLA_MAX_POSITIONS=1048576 KIMI_K3_ATTN_RES_TOPOLOGY=1 + TRTLLM_KV_TRANSFER_NUM_THREADS=8 TRTLLM_SERVE_ENABLE_MSGSPEC=1 KIMI_K3_FP8_WEIGHT_READ_MOE_MLP=1 + TRTLLM_NVLINK_ONE_SIDED_A2A_FORCE_CFT=0 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.62 + ctx_worker_env_var: CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_V2_CTX_COST_KV_OFFSET=114000 TLLM_V2_CTX_COST_KV_DEPTH_THRESHOLD=143000 + TLLM_V2_CTX_COST_PER_CHUNK_DEPTH_TOKENS=0 TLLM_V2_CTX_DEMOTE_UNCACHED_TOKENS=0 TLLM_V2_CTX_DEMOTE_MAX_ITERS=0 + TRTLLM_CUTEDSL_SM_RESERVE=30 TRTLLM_CUTEDSL_DENSE_GEMM_SCHEDULER=clc_dynamic TLLM_MOE_ASYNC_INPUT_QUANT=0 + TRTLLM_DWDP_PREFETCH_GATHER=1 TRTLLM_DWDP_GATHER_KERNEL=tma TRTLLM_DWDP_GATHER_BLOCKS=30 TRTLLM_DWDP_GATHER_THREADS=128 + TRTLLM_DWDP_GATHER_CLUSTER=2 TLLM_FMHA_LIBS=-flashinfer_trtllm_gen TLLM_TRTLLMGEN_FORCE_SEPARATED_ROUTING=0 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: 0 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + context_parallel_size: 8 + cp_config: + cp_type: HELIX + fifo_version: 2 + tokens_per_block: 64 + use_nccl_for_alltoall: false + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16] + enable_padding: true + enable_attention_dp: false + kv_cache_config: + block_reuse_config: + max_num_turns: 5 + policy: per_conversation + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + mamba_ssm_cache_dtype: float32 + mamba_state_config: + additional_snapshot_offsets_from_end: [0] + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 96 + max_seq_len: 997696 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + block_size: 5 + decoding_type: DSpark + max_draft_len: 5 + speculative_model: Inferact-Kimi-K3-DSpark + stream_interval: 100 + tensor_parallel_size: 1 + trust_remote_code: true + ctx: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + kv_cache_routing_new_conv_placement: least_queued + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: 0 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + dwdp_config: + contention_opt: false + dwdp_size: 16 + num_experts_per_worker: 56 + num_groups: 1 + num_prefetch_experts: 56 + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + avg_seq_len: 149442 + block_reuse_config: + max_num_turns: 5 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + free_gpu_memory_fraction: 0.7 + host_cache_size: 171798691840 + mamba_ssm_cache_dtype: float32 + mamba_state_config: + additional_snapshot_offsets_from_end: [0] + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 996672 + moe_config: + backend: CUTEDSL + max_num_tokens: 8192 + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 10 + tensor_parallel_size: 1 + trust_remote_code: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml new file mode 100644 index 000000000000..2fa1eaef3cbb --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# MiniMax-M3 NVFP4 + EAGLE3-GQA decode at a fixed long context, for the +# gen_only_no_context lane: one TP4 gen worker on one GB300 node decodes 1 request +# of exactly 153600 input / 1024 output random tokens. This is the low-latency end +# (B=1) of the TP4 gen-only sweep of the MiniMax-M3 ctx-only / gen-only +# optimization study; gb300_minimax-m3-fp4_150k1k_con64_... is its B=64 end, on the +# same gen worker. That is the study's base2 gen worker: MSA sparse attention with +# an FP8 indexer, NVFP4 KV cache on KV cache manager v2, TRTLLM MoE with +# low-precision combine, Eagle3 draft 3 with acceptance forced to 1.78 drafted +# tokens per step (the base2 setting), CUDA graphs for batch 1-64. worker_config and +# the env are the base2 worker configs (bench/sweeps/gen-tp4) as written; the +# sweep's command-line shape (gen TP4, batch 64, 512 tokens; ctx TP2, batch 64, +# 16384 tokens) is written into the yaml, the retired max_tokens_in_buffer is +# dropped, three env vars main does not read are left out and TLLM_LOG_LEVEL=INFO is +# added, as in every perf-sanity config. The ctx section (the study's TP2 prefill +# worker) is not started by gen_only_no_context. +metadata: + model_name: minimax_m3_fp4 + precision: fp4 + model_dir_name: MiniMax-M3-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 150k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '1' + input_length: 153600 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 + NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=0 TRTLLM_KV_TRANSFER_NUM_THREADS=4 MIMALLOC_PURGE_DELAY=0 + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1.78 TRTLLM_NIXL_BOUNCE_ENABLE=1 + TRTLLM_NIXL_BOUNCE_MAX_CHUNK_SIZE_BYTES=32MiB TRTLLM_NIXL_BOUNCE_MAX_INFLIGHT_CHUNKS_PER_REQUEST=8 TRTLLM_NIXL_BOUNCE_COPY_STREAM_COUNT=8 + TRTLLM_NIXL_BOUNCE_SCATTER_WORKER_COUNT=4 TRTLLM_NIXL_BOUNCE_MIN_DESCRIPTOR_COUNT=1024 TRTLLM_NIXL_BOUNCE_MAX_AVERAGE_DESCRIPTOR_SIZE_BYTES=16KiB + TRTLLM_NIXL_BOUNCE_ENABLE_EAGER_GATHER=1 TRTLLM_NIXL_BOUNCE_USE_ZERO_COPY_ARGUMENTS=1 TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 + gen_worker_env_var: TRTLLM_ENABLE_TRTLLMGEN_MOE_ROUTING_RENORM_PDL=0 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 512 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + stream_interval: 20 + enable_autotuner: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64] + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + enable_block_reuse: false + event_buffer_max_size: 0 + avg_seq_len: 220000 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 1024 + agent_bounce_buffer_enable: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + return_perf_metrics: false + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + moe_expert_parallel_size: 1 + ctx: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 16384 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + enable_chunked_prefill: true + enable_autotuner: true + stream_interval: 20 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + event_buffer_max_size: 0 + host_cache_size: 193273528320 + enable_block_reuse: true + block_reuse_config: + policy: per_conversation + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 1024 + agent_bounce_buffer_enable: true + moe_expert_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml new file mode 100644 index 000000000000..26143f72d748 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# MiniMax-M3 NVFP4 + EAGLE3-GQA decode at a fixed long context, for the +# gen_only_no_context lane: one TP4 gen worker on one GB300 node decodes 64 requests +# of exactly 153600 input / 1024 output random tokens. This is the "TP4, 150K +# context, B64" decode proxy of the MiniMax-M3 ctx-only / gen-only optimization +# study, on that study's base2 gen worker: MSA sparse attention with an FP8 indexer, +# NVFP4 KV cache on KV cache manager v2, TRTLLM MoE with low-precision combine, +# Eagle3 draft 3 with acceptance forced to 1.78 drafted tokens per step (the base2 +# setting), CUDA graphs for batch 1-64. worker_config and the env are the base2 +# worker configs (bench/sweeps/gen-tp4) as written; the sweep's command-line shape +# (gen TP4, batch 64, 512 tokens; ctx TP2, batch 64, 16384 tokens) is written into +# the yaml, the retired max_tokens_in_buffer is dropped, three env vars main does +# not read are left out and TLLM_LOG_LEVEL=INFO is added, as in every perf-sanity +# config. The ctx section (the study's TP2 prefill worker) is not started by +# gen_only_no_context. +metadata: + model_name: minimax_m3_fp4 + precision: fp4 + model_dir_name: MiniMax-M3-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 150k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '64' + input_length: 153600 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 + NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=0 TRTLLM_KV_TRANSFER_NUM_THREADS=4 MIMALLOC_PURGE_DELAY=0 + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1.78 TRTLLM_NIXL_BOUNCE_ENABLE=1 + TRTLLM_NIXL_BOUNCE_MAX_CHUNK_SIZE_BYTES=32MiB TRTLLM_NIXL_BOUNCE_MAX_INFLIGHT_CHUNKS_PER_REQUEST=8 TRTLLM_NIXL_BOUNCE_COPY_STREAM_COUNT=8 + TRTLLM_NIXL_BOUNCE_SCATTER_WORKER_COUNT=4 TRTLLM_NIXL_BOUNCE_MIN_DESCRIPTOR_COUNT=1024 TRTLLM_NIXL_BOUNCE_MAX_AVERAGE_DESCRIPTOR_SIZE_BYTES=16KiB + TRTLLM_NIXL_BOUNCE_ENABLE_EAGER_GATHER=1 TRTLLM_NIXL_BOUNCE_USE_ZERO_COPY_ARGUMENTS=1 TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 + gen_worker_env_var: TRTLLM_ENABLE_TRTLLMGEN_MOE_ROUTING_RENORM_PDL=0 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 512 + tensor_parallel_size: 4 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + stream_interval: 20 + enable_autotuner: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64] + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + enable_block_reuse: false + event_buffer_max_size: 0 + avg_seq_len: 220000 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 1024 + agent_bounce_buffer_enable: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + return_perf_metrics: false + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + moe_expert_parallel_size: 1 + ctx: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 16384 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + enable_chunked_prefill: true + enable_autotuner: true + stream_interval: 20 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + event_buffer_max_size: 0 + host_cache_size: 193273528320 + enable_block_reuse: true + block_reuse_config: + policy: per_conversation + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 1024 + agent_bounce_buffer_enable: true + moe_expert_parallel_size: 1 From e36410f971d2533331d2f197a5a8c255bf857df6 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:39:56 -0700 Subject: [PATCH 5/7] [None][test] Add the ctx_only and gen_only_no_context lanes of the GLM-5.3-Flash disagg case Run the new GLM-5.3-Flash FP8 disaggregated config's workers on their own too, as other disaggregated configs are: the ctx worker alone (ctx_only, 8k prompts at concurrency 16) and the gen worker alone with no context servers (gen_only_no_context, 16 requests of 8k/1k). Both are one TP4 / EP4 worker on 4 GPUs and join the GB300 4-GPU post-merge list. TIMEOUT 30 for both from the validation runs (419 and 433 s of pytest), and .test_durations is seeded with them. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/.test_durations | 2 ++ .../test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index e94d2d2123dc..c0dfb930951c 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -900,6 +900,7 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 2753.409214285714, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 416.70321428571424, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 1575.1635714285715, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL]": 419.311, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_50k2k_con1197_ctx16_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL]": 1168.722642857143, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 577.7239285714285, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL]": 408.025, @@ -930,6 +931,7 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL]": 794.9620666666666, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 1609.1394285714287, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 411.37428571428575, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL]": 433.262, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL]": 741.683, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1187.22, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1759.201, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index 49084506a752..afbce32d5860 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -41,6 +41,8 @@ l0_gb300_multi_gpus_perf_sanity: # deepseek-v4-pro-fp4 8k1k - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con180_ctx3_dep4_gen1_dep32_eplb384_mtp3_ccb-NIXL] TIMEOUT (45) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (125) + # glm-5.3-flash-fp8 8k1k con16 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (30) # deepseek-v4.1-flash-fp4 8k1k dspark (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_tep4_dspark5_8k1k] TIMEOUT (30) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_dep4_dspark5_8k1k] TIMEOUT (35) @@ -71,6 +73,8 @@ l0_gb300_multi_gpus_perf_sanity: # nemotron-ultra-v3-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL] TIMEOUT (40) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL] TIMEOUT (40) + # glm-5.3-flash-fp8 8k1k con16 + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (30) # minimax-m3-fp4 eagle3 150k1k con1 and con64, TP4 gen worker - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (35) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (45) From accb4db1468a699c967dad84a167a54ffaa602bf Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 08:49:02 -0700 Subject: [PATCH 6/7] [None][test] Set modeling_v2 per MiniMax-M3 gen-only lane Measured on GB300 with #20306's CI wheel (its MiniMax-M3 TP4 modeling_v2 target) against the built-in model in the same wheel, the gen worker's median device step time per decode iteration was 6.07 vs 10.44 ms at concurrency 1 and 47.48 vs 45.56 ms at concurrency 64. So the con1 lane sets `modeling_v2: auto`, and the con64 lane pins `off`. Main has no MiniMax-M3 route, so `auto` runs the built-in model today (same step time as with the option unset) and the lane picks the target up once it is on main. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- ...50k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 7 +++++++ ...0k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 2fa1eaef3cbb..bb9fbd9ce351 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -16,6 +16,12 @@ # dropped, three env vars main does not read are left out and TLLM_LOG_LEVEL=INFO is # added, as in every perf-sanity config. The ctx section (the study's TP2 prefill # worker) is not started by gen_only_no_context. +# +# modeling_v2 is `auto` on the gen worker. Main has no MiniMax-M3 route, so the +# built-in model runs (same device step time as with modeling_v2 unset); once a +# MiniMax-M3 TP4 target is on main, this lane runs it. #20306's TP4 target took +# 6.15 ms per decode step here against 10.54 ms for the built-in model (same +# wheel, GB300). metadata: model_name: minimax_m3_fp4 precision: fp4 @@ -65,6 +71,7 @@ profiling: nsys_on: false worker_config: gen: + modeling_v2: auto print_iter_log: true max_batch_size: 64 max_num_tokens: 512 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 26143f72d748..229cd5b81a5b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -15,6 +15,11 @@ # not read are left out and TLLM_LOG_LEVEL=INFO is added, as in every perf-sanity # config. The ctx section (the study's TP2 prefill worker) is not started by # gen_only_no_context. +# +# modeling_v2 is pinned `off` on the gen worker: at B=64 #20306's MiniMax-M3 TP4 +# target took 47.5 ms per decode step here against 45.6 ms for the built-in model +# (median, same wheel, GB300), so this lane keeps the built-in model. The con1 +# lane, where that target is faster, uses `auto`. metadata: model_name: minimax_m3_fp4 precision: fp4 @@ -64,6 +69,7 @@ profiling: nsys_on: false worker_config: gen: + modeling_v2: 'off' print_iter_log: true max_batch_size: 64 max_num_tokens: 512 From 09d0035a3b5a01313d57e34e925abb1c6a077089 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:09:57 -0700 Subject: [PATCH 7/7] [None][test] Add more GLM-5.3-Flash, Kimi-K3 and MiniMax-M3 perf-sanity lanes - GLM-5.3-Flash NVFP4 at ISL 13312 / OSL 33, concurrency 8 then 1: the workload of the NVFP4 checkpoint's published TensorRT-LLM numbers (#20454), TP4 / EP4, MTP3, FP8 KV, TRTLLM MoE, TRTLLM_LOW_M_GEMM_BACKEND=auto. - Kimi-K3 gen_only_no_context on the helix16 bs32 dspark6 gen worker of main's con88 recipe, unchanged: 32 requests of 163840 input / 1024 output tokens on 4 GB300 nodes. - MiniMax-M3 gen_only_no_context on the study's base2 DEP8 gen worker (attention DP over 2 nodes): 128 requests of 153600 / 1024 tokens; and ctx_only on its TP2 ctx worker with 153600-token prompts. - The MiniMax-M3 ctx workers no longer reuse blocks. The random prompts share no prefix, and with reuse on the first measured ctx_only request reused the warmup request's prompt. TIMEOUTs come from the validation runs (967, 886, 914 and 1059 / 1442 s of pytest), and .test_durations is seeded with them. The GB300 GEN1-NODE2-GPU8 stage goes to 6 shards and GEN1-NODE4-GPU16 to 2. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- jenkins/L0_Test.groovy | 4 +- tests/integration/defs/.test_durations | 4 + .../l0_gb300_multi_gpus_perf_sanity.yml | 3 + ...ulti_nodes_perf_sanity_gen1_node2_gpu8.yml | 2 + ...lti_nodes_perf_sanity_gen1_node4_gpu16.yml | 2 + ...300_glm53_flash_nvfp4_grace_blackwell.yaml | 55 ++++++ ...16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml | 161 ++++++++++++++++ ...1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL.yaml | 174 ++++++++++++++++++ ...x1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 9 +- ...x1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 9 +- 10 files changed, 413 insertions(+), 10 deletions(-) create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml create mode 100644 tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL.yaml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 23457def0143..9360f65b2516 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -6800,7 +6800,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-8_GPUs-2_Nodes-PyTorch-PerfSanity-GEN1-NODE2-GPU8-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8", - 5, + 6, 8, 2 ) @@ -6809,7 +6809,7 @@ def launchTestJobs(pipeline, testFilter, globalVars) "GB300-16_GPUs-4_Nodes-PyTorch-PerfSanity-GEN1-NODE4-GPU16-Post-Merge", "auto:gb300-flex", "l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16", - 1, + 2, 16, 4 ) diff --git a/tests/integration/defs/.test_durations b/tests/integration/defs/.test_durations index c0dfb930951c..e7c3135168c4 100644 --- a/tests/integration/defs/.test_durations +++ b/tests/integration/defs/.test_durations @@ -901,6 +901,7 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con1_ctx1_dep2_gen1_tep8_eplb0_mtp3_ccb-NIXL]": 416.70321428571424, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 1575.1635714285715, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL]": 419.311, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1059.366, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_50k2k_con1197_ctx16_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL]": 1168.722642857143, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 577.7239285714285, "perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_nemotron-ultra-v3-fp4_8k64k_con1_ctx1_dep4_gen1_tep4_eplb0_mtp5_ccb-NIXL]": 408.025, @@ -916,6 +917,7 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_8k1k]": 379.83566666666667, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_isl13312_osl33]": 483.808, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_8k1k]": 541.238, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_isl13312_osl33]": 967.477, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp0_50k2k]": 1863.1629285714287, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_dep4_mtp3_50k2k]": 780.4455, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_nemotron_ultra_v3_fp4_grace_blackwell-nemotron_ultra_v3_fp4_tep4_mtp6_50k2k]": 1339.9197142857142, @@ -933,6 +935,8 @@ "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL]": 411.37428571428575, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL]": 433.262, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL]": 741.683, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL]": 886.427, + "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL]": 913.965, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1187.22, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL]": 1759.201, "perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_nemotron-ultra-v3-fp4_50k2k_con12_ctx1_dep4_gen6_tep4_eplb0_mtp6_ccb-NIXL]": 1415.384142857143, diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml index afbce32d5860..5b8382a52670 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus_perf_sanity.yml @@ -43,6 +43,8 @@ l0_gb300_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] TIMEOUT (125) # glm-5.3-flash-fp8 8k1k con16 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_glm53-flash-fp8_8k1k_con16_ctx1_tep4_gen1_tep4_eplb0_mtp3_ccb-NIXL] TIMEOUT (30) + # minimax-m3-fp4 150k prompts, TP2 ctx worker + - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL] TIMEOUT (40) # deepseek-v4.1-flash-fp4 8k1k dspark (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_tep4_dspark5_8k1k] TIMEOUT (30) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_deepseek_v41_flash_nvfp4_grace_blackwell-v41_flash_nvfp4_dep4_dspark5_8k1k] TIMEOUT (35) @@ -57,6 +59,7 @@ l0_gb300_multi_gpus_perf_sanity: - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_fp8_grace_blackwell-glm53_flash_fp8_tp4_mtp3_isl13312_osl33] TIMEOUT (30) # glm-5.3-flash-nvfp4 tp4 mtp3 8k1k (aggregated) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_8k1k] TIMEOUT (30) + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_glm53_flash_nvfp4_grace_blackwell-glm53_flash_nvfp4_tp4_mtp3_isl13312_osl33] TIMEOUT (35) # qwen3.8-flash-next-nvfp4 modeling_v2 tp1 mtp3 8k1k (aggregated, 1 GPU) - perf/test_perf_sanity.py::test_e2e[aggr_upload-gb300_qwen3_8_flash_next_fp4_grace_blackwell-qwen3_8_flash_next_fp4_mv2_tp1_mtp3_8k1k] TIMEOUT (35) # nemotron-ultra-v3-fp4 8k64k (ctx_only) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml index 8bf2f90b23f2..3ecdceb9ad72 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8.yml @@ -24,3 +24,5 @@ l0_gb300_multi_nodes_perf_sanity_gen1_node2_gpu8: - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] TIMEOUT (30) # kimi-k3-fp4 dspark 160k1k con16, helix CP8 gen worker - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con16_ctx16_dwdp16_gen1_helix8_bs16_dspark5_ccb-NIXL] TIMEOUT (30) + # minimax-m3-fp4 eagle3 150k1k con128, DEP8 gen worker + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL] TIMEOUT (35) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16.yml index 69831ce8cb34..5c4fa8308ab7 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16.yml @@ -16,3 +16,5 @@ l0_gb300_multi_nodes_perf_sanity_gen1_node4_gpu16: tests: # deepseek-v4-pro-fp4 8k1k con666 - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] TIMEOUT (30) + # kimi-k3-fp4 dspark 160k1k con32, helix CP16 gen worker + - perf/test_perf_sanity.py::test_e2e[aggr_upload-gen_only_no_context-gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL] TIMEOUT (30) diff --git a/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml index eef0804242c7..51ea523248dc 100644 --- a/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gb300_glm53_flash_nvfp4_grace_blackwell.yaml @@ -10,6 +10,12 @@ # moe_config.backend TRTLLM. `AUTO` resolves this checkpoint to CUTLASS; # TRTLLM-Gen is the faster NVFP4 MoE on Blackwell. # The accepted draft length is pinned to 2.384 of 3, as in the FP8 case. +# +# glm53_flash_nvfp4_tp4_mtp3_isl13312_osl33 is the workload of the NVFP4 +# checkpoint's published TensorRT-LLM numbers (PR #20454: ISL 13312 / OSL 33, +# MTP3, TP4 / EP4, max_num_tokens 16384, TRTLLM MoE, +# TRTLLM_LOW_M_GEMM_BACKEND=auto, concurrency 8 and 1), with the engine shape +# of the FP8 isl13312_osl33 case. metadata: model_name: glm_5.3_flash_nvfp4 supported_gpus: @@ -65,3 +71,52 @@ server_configs: isl: 8192 osl: 1024 backend: openai +- name: glm53_flash_nvfp4_tp4_mtp3_isl13312_osl33 + model_name: glm_5.3_flash_nvfp4 + server_env_var: PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.384 + TRTLLM_LOW_M_GEMM_BACKEND=auto + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 16384 + trust_remote_code: true + enable_attention_dp: false + enable_chunked_prefill: true + disable_overlap_scheduler: false + disable_mm_encoder: true + print_iter_log: true + enable_iter_perf_stats: true + stream_interval: 20 + num_postprocess_workers: 4 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 4, 8, 16] + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.6 + dtype: fp8 + tokens_per_block: 128 + moe_config: + backend: TRTLLM + speculative_config: + decoding_type: MTP + max_draft_len: 3 + # con8 runs first, so the slow first requests of a fresh server fall in its + # longer window, not in con1's. + client_configs: + - name: con8_iter20_isl13312_osl33 + concurrency: 8 + iterations: 20 + isl: 13312 + osl: 33 + backend: openai + - name: con1_iter10_isl13312_osl33 + concurrency: 1 + iterations: 10 + isl: 13312 + osl: 33 + backend: openai diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml new file mode 100644 index 000000000000..9245b09ffb40 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_kimi-k3-fp4-dspark_160k1k_con32_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Kimi-K3 NVFP4 + DSpark decode at a fixed long context, for the gen_only_no_context +# lane: one helix (CP16) gen worker on 4 GB300 nodes decodes 32 requests of exactly +# 163840 input / 1024 output random tokens (10240 KV tokens per rank). This is the +# gen-only method of the K3 GB300 optimization work (balanced ISL 163840, +# steady-state step time at b32), on the helix16 bs32 dspark6 gen worker of its +# c88 b32 formal point, +# gb300_kimi-k3-fp4-dspark_agentx_con88_ctx16_dwdp16_gen1_helix16_bs32_dspark6_ccb-NIXL.yaml +# (ported from the bench-trtllm-disagg srt-slurm recipe +# disagg-gb300-16p1d-dep1-helix16-c88-b32-mtp). worker_config and the worker env are +# copied from that file unchanged; the AgentX trace replay, its client env, the +# conversation router and the chat-history proxy options are left out. The ctx +# section (the recipe's 16-GPU DWDP prefill fleet) is not started by +# gen_only_no_context. +metadata: + model_name: kimi_k3_nvfp4 + precision: fp4 + model_dir_name: Kimi-K3-NVFP4 + supported_gpus: [GB300] + script_file: disaggr_torch.slurm + benchmark_type: 160k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '32' + input_length: 163840 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 16 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 ENROOT_ALLOW_DEV=yes + NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_ARENA_RESERVE=0 OMP_NUM_THREADS=1 HF_HUB_TRUST_REMOTE_CODE=1 + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True KIMI_K3_MLA_MAX_POSITIONS=1048576 KIMI_K3_ATTN_RES_TOPOLOGY=1 + TRTLLM_KV_TRANSFER_NUM_THREADS=8 TRTLLM_SERVE_ENABLE_MSGSPEC=1 KIMI_K3_FP8_WEIGHT_READ_MOE_MLP=1 + TRTLLM_NVLINK_ONE_SIDED_A2A_FORCE_CFT=0 TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=2.75 + ctx_worker_env_var: CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_V2_CTX_COST_KV_OFFSET=114000 TLLM_V2_CTX_COST_KV_DEPTH_THRESHOLD=143000 + TLLM_V2_CTX_COST_PER_CHUNK_DEPTH_TOKENS=0 TLLM_V2_CTX_DEMOTE_UNCACHED_TOKENS=0 TLLM_V2_CTX_DEMOTE_MAX_ITERS=0 + TRTLLM_CUTEDSL_SM_RESERVE=30 TRTLLM_CUTEDSL_DENSE_GEMM_SCHEDULER=clc_dynamic TLLM_MOE_ASYNC_INPUT_QUANT=0 + TRTLLM_DWDP_PREFETCH_GATHER=1 TRTLLM_DWDP_GATHER_KERNEL=tma TRTLLM_DWDP_GATHER_BLOCKS=30 TRTLLM_DWDP_GATHER_THREADS=128 + TRTLLM_DWDP_GATHER_CLUSTER=2 TLLM_FMHA_LIBS=-flashinfer_trtllm_gen TLLM_TRTLLMGEN_FORCE_SEPARATED_ROUTING=0 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: 0 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + context_parallel_size: 16 + cp_config: + cp_type: HELIX + fifo_version: 2 + tokens_per_block: 64 + use_nccl_for_alltoall: false + cuda_graph_config: + batch_sizes: [1, 2, 4, 8, 16, 24, 32] + enable_padding: true + enable_attention_dp: false + kv_cache_config: + block_reuse_config: + max_num_turns: 5 + policy: per_conversation + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.8 + host_cache_size: 0 + mamba_ssm_cache_dtype: float32 + mamba_state_config: + additional_snapshot_offsets_from_end: [0] + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 32 + max_num_tokens: 224 + max_seq_len: 997696 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 8 + pipeline_parallel_size: 1 + print_iter_log: true + speculative_config: + block_size: 6 + decoding_type: DSpark + max_draft_len: 6 + speculative_model: Inferact-Kimi-K3-DSpark + stream_interval: 100 + tensor_parallel_size: 1 + trust_remote_code: true + ctx: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + kv_cache_routing_new_conv_placement: least_queued + cache_transceiver_config: + backend: NIXL + kv_cache_bounce_size_mb: 0 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: true + dwdp_config: + contention_opt: false + dwdp_size: 16 + num_experts_per_worker: 56 + num_groups: 1 + num_prefetch_experts: 56 + enable_attention_dp: true + enable_chunked_prefill: true + kv_cache_config: + avg_seq_len: 149442 + block_reuse_config: + max_num_turns: 5 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + free_gpu_memory_fraction: 0.7 + host_cache_size: 171798691840 + mamba_ssm_cache_dtype: float32 + mamba_state_config: + additional_snapshot_offsets_from_end: [0] + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 16384 + max_seq_len: 996672 + moe_config: + backend: CUTEDSL + max_num_tokens: 8192 + use_low_precision_moe_combine: true + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 10 + tensor_parallel_size: 1 + trust_remote_code: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL.yaml new file mode 100644 index 000000000000..dee093388b75 --- /dev/null +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con128_ctx1_tp2_gen1_dep8_eplb0_eagle3_ccb-NIXL.yaml @@ -0,0 +1,174 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# MiniMax-M3 NVFP4 + EAGLE3-GQA decode at a fixed long context, for the +# gen_only_no_context lane: one DEP8 gen worker (attention DP over 8 GPUs, 2 GB300 +# nodes, MoE EP8) decodes 128 requests (16 per rank) of exactly 153600 input / 1024 +# output random tokens. This is the DEP8 gen-only sweep of the MiniMax-M3 ctx-only / +# gen-only optimization study (B = 64-192, the decode of its high-throughput +# point), at B=128, on that study's base2 DEP8 gen worker: MSA sparse attention +# with an FP8 indexer, NVFP4 KV cache on KV cache manager v2, TRTLLM MoE with +# low-precision combine, Eagle3 draft 3 with acceptance forced to 1.78 drafted +# tokens per step (the base2 setting), CUDA graphs for batch 1-24 per rank. +# worker_config and the env are the base2 worker configs (bench/sweeps/gen-dep8) as +# written; the sweep's command-line shape (gen TP8 with attention DP, batch 24 per +# rank, 512 tokens; ctx TP2, batch 64, 16384 tokens) is written into the yaml, the +# retired max_tokens_in_buffer is dropped, three env vars main does not read are +# left out and TLLM_LOG_LEVEL=INFO is added, as in every perf-sanity config. The +# ctx section (the study's TP2 prefill worker) is not started by +# gen_only_no_context. modeling_v2 is left at its default: no MiniMax-M3 target +# covers attention DP. +# +# On the ctx worker, block reuse and its 180 GiB host tier (the study's AgentX prefix +# reuse) are off: the fixed random prompts share no prefix, and with reuse on the +# first measured ctx_only request would reuse the warmup request's prompt. +metadata: + model_name: minimax_m3_fp4 + precision: fp4 + model_dir_name: MiniMax-M3-NVFP4 + supported_gpus: + - GB300 + script_file: disaggr_torch.slurm + benchmark_type: 150k1k +slurm: + script_file: disaggr_torch.slurm + partition: + account: + job_time: 02:00:00 + job_name: unified-benchmark + extra_args: --gres=gpu:4 + numa_bind: true +benchmark: + mode: e2e + use_nv_sa_benchmark: false + multi_round: 5 + benchmark_ratio: 0.0 + streaming: true + concurrency_list: '128' + input_length: 153600 + output_length: 1024 + dataset_file: +hardware: + gpus_per_node: 4 + num_ctx_servers: 1 + num_gen_servers: 1 +environment: + container_mount: + container_image: + model_path: + trtllm_repo: '' + build_wheel: false + work_dir: + worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 + NCCL_GRAPH_MIXING_SUPPORT=0 TRTLLM_DISABLE_KV_CACHE_TRANSFER_OVERLAP=0 TRTLLM_KV_TRANSFER_NUM_THREADS=4 MIMALLOC_PURGE_DELAY=0 + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1.78 TRTLLM_NIXL_BOUNCE_ENABLE=1 + TRTLLM_NIXL_BOUNCE_MAX_CHUNK_SIZE_BYTES=32MiB TRTLLM_NIXL_BOUNCE_MAX_INFLIGHT_CHUNKS_PER_REQUEST=8 TRTLLM_NIXL_BOUNCE_COPY_STREAM_COUNT=8 + TRTLLM_NIXL_BOUNCE_SCATTER_WORKER_COUNT=4 TRTLLM_NIXL_BOUNCE_MIN_DESCRIPTOR_COUNT=1024 TRTLLM_NIXL_BOUNCE_MAX_AVERAGE_DESCRIPTOR_SIZE_BYTES=16KiB + TRTLLM_NIXL_BOUNCE_ENABLE_EAGER_GATHER=1 TRTLLM_NIXL_BOUNCE_USE_ZERO_COPY_ARGUMENTS=1 TLLM_ENABLE_CACHE_TRANSFER_PERF_INFO=1 + gen_worker_env_var: TRTLLM_ENABLE_TRTLLMGEN_MOE_ROUTING_RENORM_PDL=0 + server_env_var: TRTLLM_SERVER_DISABLE_GC=1 +profiling: + nsys_on: false +worker_config: + gen: + print_iter_log: true + max_batch_size: 24 + max_num_tokens: 512 + tensor_parallel_size: 8 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + stream_interval: 20 + enable_autotuner: true + cuda_graph_config: + enable_padding: true + batch_sizes: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24] + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + enable_block_reuse: false + event_buffer_max_size: 0 + avg_seq_len: 220000 + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 4096 + agent_bounce_buffer_enable: true + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + return_perf_metrics: false + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + moe_expert_parallel_size: 8 + ctx: + print_iter_log: true + max_batch_size: 64 + max_num_tokens: 16384 + tensor_parallel_size: 2 + pipeline_parallel_size: 1 + context_parallel_size: 1 + enable_attention_dp: false + trust_remote_code: true + max_seq_len: 1048576 + num_postprocess_workers: 8 + reasoning_parser: minimax_m3 + torch_compile_config: + enable_fullgraph: true + enable_inductor: false + enable_piecewise_cuda_graph: true + capture_num_tokens: [1, 512, 1024, 2048] + enable_userbuffers: true + max_num_streams: 3 + enable_chunked_prefill: true + enable_autotuner: true + stream_interval: 20 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + sparse_attention_config: + algorithm: minimax_m3 + implementation: msa + indexer_kv_dtype: fp8 + sparse_disable_index_value: true + fuse_qkv_index_projection: true + kv_cache_config: + dtype: nvfp4 + tokens_per_block: 128 + use_kv_cache_manager_v2: true + free_gpu_memory_fraction: 0.94 + event_buffer_max_size: 0 + enable_block_reuse: false + speculative_config: + decoding_type: Eagle3 + max_draft_len: 3 + speculative_model: MiniMax-M3-EAGLE3-GQA + cache_transceiver_config: + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + kv_cache_bounce_size_mb: 4096 + agent_bounce_buffer_enable: true + moe_expert_parallel_size: 1 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index bb9fbd9ce351..a0acb2883c48 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con1_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -22,6 +22,10 @@ # MiniMax-M3 TP4 target is on main, this lane runs it. #20306's TP4 target took # 6.15 ms per decode step here against 10.54 ms for the built-in model (same # wheel, GB300). +# +# On the ctx worker, block reuse and its 180 GiB host tier (the study's AgentX prefix +# reuse) are off: the fixed random prompts share no prefix, and with reuse on the +# first measured ctx_only request would reuse the warmup request's prompt. metadata: model_name: minimax_m3_fp4 precision: fp4 @@ -159,10 +163,7 @@ worker_config: use_kv_cache_manager_v2: true free_gpu_memory_fraction: 0.94 event_buffer_max_size: 0 - host_cache_size: 193273528320 - enable_block_reuse: true - block_reuse_config: - policy: per_conversation + enable_block_reuse: false speculative_config: decoding_type: Eagle3 max_draft_len: 3 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 229cd5b81a5b..073e12638a35 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_minimax-m3-fp4_150k1k_con64_ctx1_tp2_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -20,6 +20,10 @@ # target took 47.5 ms per decode step here against 45.6 ms for the built-in model # (median, same wheel, GB300), so this lane keeps the built-in model. The con1 # lane, where that target is faster, uses `auto`. +# +# On the ctx worker, block reuse and its 180 GiB host tier (the study's AgentX prefix +# reuse) are off: the fixed random prompts share no prefix, and with reuse on the +# first measured ctx_only request would reuse the warmup request's prompt. metadata: model_name: minimax_m3_fp4 precision: fp4 @@ -157,10 +161,7 @@ worker_config: use_kv_cache_manager_v2: true free_gpu_memory_fraction: 0.94 event_buffer_max_size: 0 - host_cache_size: 193273528320 - enable_block_reuse: true - block_reuse_config: - policy: per_conversation + enable_block_reuse: false speculative_config: decoding_type: Eagle3 max_draft_len: 3