diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 324d71200619..c7f3358fc003 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -26,10 +26,18 @@ For the underlying regression pipeline architecture (three-layer design, baselin | List | Count | Contents | |------|-------|----------| -| `MAXIMIZE_METRICS` | 8 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + TPOT (`d_mean_tpot`, `d_median_tpot`, `d_p99_tpot`) + spec-decoding `d_al` | -| `MINIMIZE_METRICS` | 14 + 108 | TTFT, ITL, E2EL latencies (mean/median/P99 for each) + the five `d_{mean,median,std,p75,p99}_gen_worker_per_iter_device_step_time` (every mode in `DEVICE_STEP_TIME_MODES`) + the 108 `d_tb_*` lifecycle spans of the `time_breakdown` modifier | +| `MAXIMIZE_METRICS` | 5 | Throughputs (`d_seq_throughput`, `d_token_throughput`, `d_total_token_throughput`, `d_user_throughput`) + spec-decoding `d_al` | +| `MINIMIZE_METRICS` | 17 + 108 | TTFT, ITL, TPOT, E2EL latencies (mean/median/P99 for each) + the five `d_{mean,median,std,p75,p99}_gen_worker_per_iter_device_step_time` (every mode in `DEVICE_STEP_TIME_MODES`) + the 108 `d_tb_*` lifecycle spans of the `time_breakdown` modifier | | `REGRESSION_METRICS` | 2 default | `d_token_throughput`, `d_total_token_throughput` — gate pass/fail for all modes **except disagg gen_only**. `d_al` is appended at runtime when any client runs spec decoding. | +TPOT is time per output token, so lower values are better. Its reported delta is +negative when latency exceeds the selected baseline. A stored `d_baseline_*` +override takes precedence; otherwise, the baseline is recalculated as P5 of the +smoothed history, as for the other latency metrics. Calculated baselines are not +automatically stored as overrides. Historical TPOT measurements remain valid +when the comparison direction changes. TPOT is not in `REGRESSION_METRICS`, so +these statistics are diagnostic and do not determine the default pass/fail verdict. + **Disagg gen_only override**: For `disagg_upload-gen_only-*` tests, regression is gated on `d_mean_gen_worker_per_iter_device_step_time` **and** `d_median_gen_worker_per_iter_device_step_time`. Token-based throughput numbers are dominated by KV-cache transfer time in gen_only mode and are not a useful regression signal there. The two are gated together because they fail on different shapes of slowdown: the mean catches a cost spread thinly across many iterations, the median catches a shift in the typical iteration while ignoring outliers. A real slowdown moves both; a single anomalous iteration moves only the mean. `d_{std,p75,p99}_...` are uploaded for diagnosis but are **not** gated. A newly added gated metric has no baseline history, and `check_regression` skips any metric whose baseline is absent or non-positive (`continue`), so the median cannot fail a build until enough runs accrue. diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 556021dd2644..03b7703b3148 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -1017,9 +1017,6 @@ def add_perf_metric_value( "d_token_throughput", "d_total_token_throughput", "d_user_throughput", - "d_mean_tpot", - "d_median_tpot", - "d_p99_tpot", "d_al", ] @@ -1031,6 +1028,9 @@ def add_perf_metric_value( "d_mean_itl", "d_median_itl", "d_p99_itl", + "d_mean_tpot", + "d_median_tpot", + "d_p99_tpot", "d_mean_e2el", "d_median_e2el", "d_p99_e2el", diff --git a/tests/unittest/others/test_perf_sanity_tpot.py b/tests/unittest/others/test_perf_sanity_tpot.py new file mode 100644 index 000000000000..2c4fcd79b7db --- /dev/null +++ b/tests/unittest/others/test_perf_sanity_tpot.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TPOT reporting and regression comparisons use latency semantics.""" + +import ast +import importlib.util +import sys +from copy import deepcopy +from pathlib import Path +from types import ModuleType + +import pytest + +pytestmark = pytest.mark.cpu_only +_PERF_DIR = Path(__file__).resolve().parents[2] / "integration" / "defs" / "perf" +_TPOT_METRICS = ("d_mean_tpot", "d_median_tpot", "d_p99_tpot") + + +@pytest.fixture +def regression_utils(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + """Load the real comparison code without GPU or OpenSearch dependencies.""" + alternative = ModuleType("defs.trt_test_alternative") + alternative.print_info = lambda *args, **kwargs: None + alternative.print_warning = lambda *args, **kwargs: None + db_utils = ModuleType("defs.perf.open_search_db_utils") + db_utils.add_id = lambda *args, **kwargs: None + db_utils.get_history_data = lambda *args, **kwargs: None + db_utils.post_new_perf_data = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "defs.trt_test_alternative", alternative) + monkeypatch.setitem(sys.modules, "defs.perf.open_search_db_utils", db_utils) + spec = importlib.util.spec_from_file_location( + "defs.perf.perf_regression_utils", _PERF_DIR / "perf_regression_utils.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def metric_lists() -> dict[str, list[str]]: + """Read the harness lists without importing its serving dependencies.""" + # The unrelated time-breakdown metrics are not exercised by these tests. + namespace = {"TIME_BREAKDOWN_METRICS": ()} + names = {"MAXIMIZE_METRICS", "MINIMIZE_METRICS", "REGRESSION_METRICS"} + tree = ast.parse((_PERF_DIR / "test_perf_sanity.py").read_text()) + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id in names + ): + exec( + compile(ast.Module(body=[node], type_ignores=[]), "test_perf_sanity", "exec"), + namespace, + ) + return {name: namespace[name] for name in names} + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +@pytest.mark.parametrize("gate_tpot", [False, True]) +@pytest.mark.parametrize( + "value, expected_diff, is_slowdown", + [(15.24, -150.6578947368421, True), (3.04, 50.0, False)], +) +def test_tpot_report_and_gate( + regression_utils: ModuleType, + metric_lists: dict[str, list[str]], + metric: str, + gate_tpot: bool, + value: float, + expected_diff: float, + is_slowdown: bool, +) -> None: + """Honor a stored baseline while applying latency signs and the selected gate.""" + data = {0: {metric: value}} + regression_utils.prepare_regressive_test_cases( + latest_history_data_dict={}, + latest_baseline_threshold_dict={0: {f"d_baseline_{metric[2:]}": 6.08}}, + history_data_dict={}, + new_data_dict=data, + maximize_metrics=metric_lists["MAXIMIZE_METRICS"], + minimize_metrics=metric_lists["MINIMIZE_METRICS"], + regression_metrics=[metric] if gate_tpot else metric_lists["REGRESSION_METRICS"], + ) + assert f"diff={expected_diff:+.2f}%" in data[0]["s_regression_info"] + assert data[0]["b_is_regression"] is (gate_tpot and is_slowdown) + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +def test_tpot_baseline_uses_low_latency_percentile( + regression_utils: ModuleType, metric_lists: dict[str, list[str]], metric: str +) -> None: + """Compute P5 from historical latency measurements, including older samples.""" + history = [ + {"ts_created": "2026-10-01", metric: 6.0}, + {"ts_created": "2026-10-02", metric: 12.0}, + {"ts_created": "2026-10-03", metric: 18.0}, + ] + baseline = regression_utils.calculate_baseline_metrics( + history, None, metric_lists["MAXIMIZE_METRICS"], metric_lists["MINIMIZE_METRICS"] + ) + # The trailing means are 6, 9, 12; their P5 is 6.3 (P95 would be 11.7). + assert baseline[metric] == pytest.approx(6.3) + + +@pytest.mark.parametrize("metric", _TPOT_METRICS) +def test_calculated_tpot_baseline_is_not_persisted_as_an_override( + regression_utils: ModuleType, + metric_lists: dict[str, list[str]], + monkeypatch: pytest.MonkeyPatch, + metric: str, +) -> None: + """Reclassifying old measurements recomputes P5 instead of inheriting a calculated P95.""" + history = [ + {"ts_created": "2026-10-01", metric: 6.0}, + {"ts_created": "2026-10-02", metric: 12.0}, + {"ts_created": "2026-10-03", metric: 18.0}, + ] + uploaded = [] + monkeypatch.setattr(regression_utils, "get_job_info", lambda: {"b_is_post_merge": True}) + monkeypatch.setattr( + regression_utils, "post_new_perf_data", lambda payload: uploaded.append(deepcopy(payload)) + ) + stored_fields = {} + for maximize, expected_baseline in ((True, 11.7), (False, 6.3)): + monkeypatch.setattr( + regression_utils, + "get_history_data", + lambda *args: ({}, {0: stored_fields}, {0: history}), + ) + data = {0: {metric: 9.0}} + regression_utils.process_and_upload_test_results( + new_data_dict=data, + match_keys=[], + maximize_metrics=[metric] if maximize else metric_lists["MAXIMIZE_METRICS"], + minimize_metrics=[] if maximize else metric_lists["MINIMIZE_METRICS"], + regression_metrics=metric_lists["REGRESSION_METRICS"] if not maximize else [], + ) + assert uploaded[-1] == data + uploaded_record = uploaded[-1][0] + assert f"baseline={expected_baseline:.4f}" in uploaded_record["s_regression_info"] + assert uploaded_record["b_is_regression"] is False + stored_fields = { + key: value for key, value in uploaded_record.items() if key.startswith("d_baseline_") + } + assert not stored_fields + assert "diff=-42.86%" in uploaded[-1][0]["s_regression_info"]