From eb400f571bfee0ca63f964334bebebd8036b6299 Mon Sep 17 00:00:00 2001 From: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:21:33 -0400 Subject: [PATCH 1/3] [None][fix] Correct TPOT metric direction in perf sanity Signed-off-by: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 9 +- .../integration/defs/perf/test_perf_sanity.py | 6 +- .../unittest/others/test_perf_sanity_tpot.py | 113 ++++++++++++++++++ 3 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 tests/unittest/others/test_perf_sanity_tpot.py diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 324d71200619..6a49faa198a3 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -26,10 +26,15 @@ For the underlying regression pipeline architecture (three-layer design, baselin | List | Count | Contents | |------|-------|----------| -| `MAXIMIZE_METRICS` | 8 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + TPOT (`d_mean_tpot`, `d_median_tpot`, `d_p99_tpot`) + spec-decoding `d_al` | -| `MINIMIZE_METRICS` | 14 + 108 | TTFT, ITL, E2EL latencies (mean/median/P99 for each) + the five `d_{mean,median,std,p75,p99}_gen_worker_per_iter_device_step_time` (every mode in `DEVICE_STEP_TIME_MODES`) + the 108 `d_tb_*` lifecycle spans of the `time_breakdown` modifier | +| `MAXIMIZE_METRICS` | 5 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + spec-decoding `d_al` | +| `MINIMIZE_METRICS` | 17 + 108 | TTFT, ITL, TPOT, E2EL latencies (mean/median/P99 for each) + the five `d_{mean,median,std,p75,p99}_gen_worker_per_iter_device_step_time` (every mode in `DEVICE_STEP_TIME_MODES`) + the 108 `d_tb_*` lifecycle spans of the `time_breakdown` modifier | | `REGRESSION_METRICS` | 2 default | `d_token_throughput`, `d_total_token_throughput` — gate pass/fail for all modes **except disagg gen_only**. `d_al` is appended at runtime when any client runs spec decoding. | +TPOT is time per output token, so lower values are better. Its reported delta is +negative for a slowdown, and calculated baselines use P5 of the smoothed history, +as for the other latency metrics. TPOT is not in `REGRESSION_METRICS`, so these +statistics are diagnostic and do not determine the default pass/fail verdict. + **Disagg gen_only override**: For `disagg_upload-gen_only-*` tests, regression is gated on `d_mean_gen_worker_per_iter_device_step_time` **and** `d_median_gen_worker_per_iter_device_step_time`. Token-based throughput numbers are dominated by KV-cache transfer time in gen_only mode and are not a useful regression signal there. The two are gated together because they fail on different shapes of slowdown: the mean catches a cost spread thinly across many iterations, the median catches a shift in the typical iteration while ignoring outliers. A real slowdown moves both; a single anomalous iteration moves only the mean. `d_{std,p75,p99}_...` are uploaded for diagnosis but are **not** gated. A newly added gated metric has no baseline history, and `check_regression` skips any metric whose baseline is absent or non-positive (`continue`), so the median cannot fail a build until enough runs accrue. diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 556021dd2644..03b7703b3148 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1017,9 +1017,6 @@ def add_perf_metric_value( "d_token_throughput", "d_total_token_throughput", "d_user_throughput", - "d_mean_tpot", - "d_median_tpot", - "d_p99_tpot", "d_al", ] @@ -1031,6 +1028,9 @@ def add_perf_metric_value( "d_mean_itl", "d_median_itl", "d_p99_itl", + "d_mean_tpot", + "d_median_tpot", + "d_p99_tpot", "d_mean_e2el", "d_median_e2el", "d_p99_e2el", diff --git a/tests/unittest/others/test_perf_sanity_tpot.py b/tests/unittest/others/test_perf_sanity_tpot.py new file mode 100644 index 000000000000..5d62f0f4861b --- /dev/null +++ b/tests/unittest/others/test_perf_sanity_tpot.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TPOT reporting and regression comparisons use latency semantics.""" + +import ast +import importlib.util +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.cpu_only +_PERF_DIR = Path(__file__).resolve().parents[2] / "integration" / "defs" / "perf" +_TPOT_METRICS = ("d_mean_tpot", "d_median_tpot", "d_p99_tpot") + + +@pytest.fixture +def regression_utils(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + # Load the real comparison code without GPU or OpenSearch dependencies. + alternative = ModuleType("defs.trt_test_alternative") + alternative.print_info = lambda *args, **kwargs: None + alternative.print_warning = lambda *args, **kwargs: None + db_utils = ModuleType("defs.perf.open_search_db_utils") + db_utils.add_id = lambda *args, **kwargs: None + db_utils.get_history_data = lambda *args, **kwargs: None + db_utils.post_new_perf_data = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "defs.trt_test_alternative", alternative) + monkeypatch.setitem(sys.modules, "defs.perf.open_search_db_utils", db_utils) + spec = importlib.util.spec_from_file_location( + "defs.perf.perf_regression_utils", _PERF_DIR / "perf_regression_utils.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def metric_lists() -> dict[str, list[str]]: + # Read the actual harness lists without importing its serving dependencies. + # The unrelated time-breakdown metrics are not exercised by these tests. + namespace = {"TIME_BREAKDOWN_METRICS": ()} + names = {"MAXIMIZE_METRICS", "MINIMIZE_METRICS", "REGRESSION_METRICS"} + tree = ast.parse((_PERF_DIR / "test_perf_sanity.py").read_text()) + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in names + ): + exec( + compile(ast.Module(body=[node], type_ignores=[]), "test_perf_sanity", "exec"), + namespace, + ) + return {name: namespace[name] for name in names} + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +@pytest.mark.parametrize("gate_tpot", [False, True]) +@pytest.mark.parametrize( + "value, expected_diff, is_slowdown", + [(15.24, -150.6578947368421, True), (3.04, 50.0, False)], +) +def test_tpot_report_and_gate( + regression_utils: ModuleType, + metric_lists: dict[str, list[str]], + metric: str, + gate_tpot: bool, + value: float, + expected_diff: float, + is_slowdown: bool, +) -> None: + data = {0: {metric: value}} + regression_utils.prepare_regressive_test_cases( + latest_history_data_dict={}, + latest_baseline_threshold_dict={0: {f"d_baseline_{metric[2:]}": 6.08}}, + history_data_dict={}, + new_data_dict=data, + maximize_metrics=metric_lists["MAXIMIZE_METRICS"], + minimize_metrics=metric_lists["MINIMIZE_METRICS"], + regression_metrics=[metric] if gate_tpot else metric_lists["REGRESSION_METRICS"], + ) + assert f"diff={expected_diff:+.2f}%" in data[0]["s_regression_info"] + assert data[0]["b_is_regression"] is (gate_tpot and is_slowdown) + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +def test_tpot_baseline_uses_low_latency_percentile( + regression_utils: ModuleType, metric_lists: dict[str, list[str]], metric: str +) -> None: + history = [ + {"ts_created": "2026-10-01", metric: 6.0}, + {"ts_created": "2026-10-02", metric: 12.0}, + {"ts_created": "2026-10-03", metric: 18.0}, + ] + baseline = regression_utils.calculate_baseline_metrics( + history, None, metric_lists["MAXIMIZE_METRICS"], metric_lists["MINIMIZE_METRICS"] + ) + # The trailing means are 6, 9, 12; their P5 is 6.3 (P95 would be 11.7). + assert baseline[metric] == pytest.approx(6.3) From 84f2c4a22b6c028b08e78b96e26982c171546d18 Mon Sep 17 00:00:00 2001 From: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:17:17 -0400 Subject: [PATCH 2/3] [None][test] Clarify TPOT baseline override behavior Signed-off-by: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 9 ++-- .../unittest/others/test_perf_sanity_tpot.py | 47 ++++++++++++++++++- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 6a49faa198a3..c7f3358fc003 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -31,9 +31,12 @@ For the underlying regression pipeline architecture (three-layer design, baselin | `REGRESSION_METRICS` | 2 default | `d_token_throughput`, `d_total_token_throughput` — gate pass/fail for all modes **except disagg gen_only**. `d_al` is appended at runtime when any client runs spec decoding. | TPOT is time per output token, so lower values are better. Its reported delta is -negative for a slowdown, and calculated baselines use P5 of the smoothed history, -as for the other latency metrics. TPOT is not in `REGRESSION_METRICS`, so these -statistics are diagnostic and do not determine the default pass/fail verdict. +negative when latency exceeds the selected baseline. A stored `d_baseline_*` +override takes precedence; otherwise, the baseline is recalculated as P5 of the +smoothed history, as for the other latency metrics. Calculated baselines are not +automatically stored as overrides. Historical TPOT measurements remain valid +when the comparison direction changes. TPOT is not in `REGRESSION_METRICS`, so +these statistics are diagnostic and do not determine the default pass/fail verdict. **Disagg gen_only override**: For `disagg_upload-gen_only-*` tests, regression is gated on `d_mean_gen_worker_per_iter_device_step_time` **and** `d_median_gen_worker_per_iter_device_step_time`. Token-based throughput numbers are dominated by KV-cache transfer time in gen_only mode and are not a useful regression signal there. The two are gated together because they fail on different shapes of slowdown: the mean catches a cost spread thinly across many iterations, the median catches a shift in the typical iteration while ignoring outliers. A real slowdown moves both; a single anomalous iteration moves only the mean. `d_{std,p75,p99}_...` are uploaded for diagnosis but are **not** gated. diff --git a/tests/unittest/others/test_perf_sanity_tpot.py b/tests/unittest/others/test_perf_sanity_tpot.py index 5d62f0f4861b..aea72ec6ccc5 100644 --- a/tests/unittest/others/test_perf_sanity_tpot.py +++ b/tests/unittest/others/test_perf_sanity_tpot.py @@ -29,7 +29,7 @@ @pytest.fixture def regression_utils(monkeypatch: pytest.MonkeyPatch) -> ModuleType: - # Load the real comparison code without GPU or OpenSearch dependencies. + """Load the real comparison code without GPU or OpenSearch dependencies.""" alternative = ModuleType("defs.trt_test_alternative") alternative.print_info = lambda *args, **kwargs: None alternative.print_warning = lambda *args, **kwargs: None @@ -49,7 +49,7 @@ def regression_utils(monkeypatch: pytest.MonkeyPatch) -> ModuleType: @pytest.fixture(scope="module") def metric_lists() -> dict[str, list[str]]: - # Read the actual harness lists without importing its serving dependencies. + """Read the harness lists without importing its serving dependencies.""" # The unrelated time-breakdown metrics are not exercised by these tests. namespace = {"TIME_BREAKDOWN_METRICS": ()} names = {"MAXIMIZE_METRICS", "MINIMIZE_METRICS", "REGRESSION_METRICS"} @@ -83,6 +83,7 @@ def test_tpot_report_and_gate( expected_diff: float, is_slowdown: bool, ) -> None: + """Honor a stored baseline while applying latency signs and the selected gate.""" data = {0: {metric: value}} regression_utils.prepare_regressive_test_cases( latest_history_data_dict={}, @@ -101,6 +102,7 @@ def test_tpot_report_and_gate( def test_tpot_baseline_uses_low_latency_percentile( regression_utils: ModuleType, metric_lists: dict[str, list[str]], metric: str ) -> None: + """Compute P5 from historical latency measurements, including older samples.""" history = [ {"ts_created": "2026-10-01", metric: 6.0}, {"ts_created": "2026-10-02", metric: 12.0}, @@ -111,3 +113,44 @@ def test_tpot_baseline_uses_low_latency_percentile( ) # The trailing means are 6, 9, 12; their P5 is 6.3 (P95 would be 11.7). assert baseline[metric] == pytest.approx(6.3) + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +def test_calculated_tpot_baseline_is_not_persisted_as_an_override( + regression_utils: ModuleType, + metric_lists: dict[str, list[str]], + monkeypatch: pytest.MonkeyPatch, + metric: str, +) -> None: + """Reclassifying old measurements recomputes P5 instead of inheriting a calculated P95.""" + history = [ + {"ts_created": "2026-10-01", metric: 6.0}, + {"ts_created": "2026-10-02", metric: 12.0}, + {"ts_created": "2026-10-03", metric: 18.0}, + ] + uploaded = [] + monkeypatch.setattr(regression_utils, "get_job_info", lambda: {"b_is_post_merge": True}) + monkeypatch.setattr(regression_utils, "post_new_perf_data", uploaded.append) + stored_fields = {} + for maximize, expected_baseline in ((True, 11.7), (False, 6.3)): + monkeypatch.setattr( + regression_utils, + "get_history_data", + lambda *args: ({}, {0: stored_fields}, {0: history}), + ) + data = {0: {metric: 9.0}} + regression_utils.process_and_upload_test_results( + new_data_dict=data, + match_keys=[], + maximize_metrics=[metric] if maximize else metric_lists["MAXIMIZE_METRICS"], + minimize_metrics=[] if maximize else metric_lists["MINIMIZE_METRICS"], + regression_metrics=metric_lists["REGRESSION_METRICS"] if not maximize else [], + ) + assert f"baseline={expected_baseline:.4f}" in data[0]["s_regression_info"] + assert data[0]["b_is_regression"] is False + assert uploaded[-1] == data + stored_fields = { + key: value for key, value in data[0].items() if key.startswith("d_baseline_") + } + assert not stored_fields + assert "diff=-42.86%" in data[0]["s_regression_info"] From c639bba3c1e086ad733fe02b1e3b875e95330c87 Mon Sep 17 00:00:00 2001 From: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:23:47 -0400 Subject: [PATCH 3/3] [None][test] Snapshot TPOT upload payloads Signed-off-by: Dan Hansen <218031328+dhansen-nvidia@users.noreply.github.com> --- tests/unittest/others/test_perf_sanity_tpot.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/unittest/others/test_perf_sanity_tpot.py b/tests/unittest/others/test_perf_sanity_tpot.py index aea72ec6ccc5..2c4fcd79b7db 100644 --- a/tests/unittest/others/test_perf_sanity_tpot.py +++ b/tests/unittest/others/test_perf_sanity_tpot.py @@ -17,6 +17,7 @@ import ast import importlib.util import sys +from copy import deepcopy from pathlib import Path from types import ModuleType @@ -130,7 +131,9 @@ def test_calculated_tpot_baseline_is_not_persisted_as_an_override( ] uploaded = [] monkeypatch.setattr(regression_utils, "get_job_info", lambda: {"b_is_post_merge": True}) - monkeypatch.setattr(regression_utils, "post_new_perf_data", uploaded.append) + monkeypatch.setattr( + regression_utils, "post_new_perf_data", lambda payload: uploaded.append(deepcopy(payload)) + ) stored_fields = {} for maximize, expected_baseline in ((True, 11.7), (False, 6.3)): monkeypatch.setattr( @@ -146,11 +149,12 @@ def test_calculated_tpot_baseline_is_not_persisted_as_an_override( minimize_metrics=[] if maximize else metric_lists["MINIMIZE_METRICS"], regression_metrics=metric_lists["REGRESSION_METRICS"] if not maximize else [], ) - assert f"baseline={expected_baseline:.4f}" in data[0]["s_regression_info"] - assert data[0]["b_is_regression"] is False assert uploaded[-1] == data + uploaded_record = uploaded[-1][0] + assert f"baseline={expected_baseline:.4f}" in uploaded_record["s_regression_info"] + assert uploaded_record["b_is_regression"] is False stored_fields = { - key: value for key, value in data[0].items() if key.startswith("d_baseline_") + key: value for key, value in uploaded_record.items() if key.startswith("d_baseline_") } assert not stored_fields - assert "diff=-42.86%" in data[0]["s_regression_info"] + assert "diff=-42.86%" in uploaded[-1][0]["s_regression_info"]