From 6ef3dc609f96feae8095711716c62fe35522e2cf Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 06:16:18 -0700 Subject: [PATCH 1/7] [None][test] Add a warmup pass before perf-sanity measured clients Before each measured client of a normal aggregated, ctx_only or disaggregated e2e perf-sanity test, send one un-measured wave of the same traffic: the client's own benchmark_serving command with --num-prompts set to its concurrency, --seed set to the measured seed plus one and --no-test-input. JIT kernels for the dynamic prefill shapes then compile before the measured window. gen_only and gen_only_no_context tests are unchanged. The pass is excluded from every uploaded metric: its output is not parsed, its perf_metrics records are skipped by the time breakdown (shared with the connection warmup), and in e2e it runs before the gen-worker device-step window. Uploaded rows record the request count as l_warmup_pass_requests. TRTLLM_PERF_SANITY_WARMUP_PASS=0 turns it off, e.g. to reproduce a JIT stall in the measured window. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 51 ++- .../integration/defs/perf/test_perf_sanity.py | 303 ++++++++++++++---- tests/integration/defs/perf/warmup_pass.py | 128 ++++++++ .../others/test_perf_sanity_warmup_pass.py | 158 +++++++++ 4 files changed, 579 insertions(+), 61 deletions(-) create mode 100644 tests/integration/defs/perf/warmup_pass.py create mode 100644 tests/unittest/others/test_perf_sanity_warmup_pass.py diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 35e615a6b6c8..eccd9cfc36d0 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -203,6 +203,9 @@ AGG_CONFIG_FOLDER=my/custom/agg DISAGG_CONFIG_FOLDER=my/custom/disagg \ python jenkins/scripts/perf/local/submit.py ... ``` +`TRTLLM_PERF_SANITY_WARMUP_PASS=0`, read by `test_perf_sanity.py` itself, skips the +warmup pass; see [Warmup pass](#warmup-pass). + ## Configuration Files There are two modes for perf sanity tests: aggregated (aggr) and disaggregated (disagg). @@ -338,6 +341,45 @@ queue (nvbugs/6564113). Notes: connection-warmup iterations; start the range later to profile only measured iterations. +### Warmup pass + +Before each measured client of a normal aggregated, `ctx_only` or disaggregated `e2e` +test, the harness (`run_warmup_pass`, `warmup_pass.py`) sends one un-measured wave of +the same traffic. How many prompts arrive together, how they are chunked against +`max_num_tokens` and which decode tokens share a step with them are only known once +traffic arrives, so the server's startup warmup cannot compile every JIT kernel those +prefill shapes need. The pass makes them compile before the measured window. Its +command is the measured client's own `benchmark_serving` command with: +- `--num-prompts` set to the client's concurrency (one wave); +- `--seed` set to the measured seed + 1, so the pass sends other prompts of the same + lengths: other random token ids, or another shuffle of the same dataset file; +- `--no-test-input` added and `--save-request-time-breakdown` dropped. + +Notes: +- Not run for `gen_only` / `gen_only_no_context` (they measure decode only, and the + gen server's benchmark fill gate opens once, so warmup requests would open it + early), for AgentX clients (they run their own warmup) or for accuracy-only + clients. In e2e it runs after the connection warmup. +- `TRTLLM_PERF_SANITY_WARMUP_PASS=0` (also `false`, `no`, `off`) skips it, e.g. to + reproduce a JIT stall inside the measured window. CI leaves it on. With + `jenkins/scripts/perf/local/submit.py`, set + `EXTRA_CONTAINER_EXPORTS='TRTLLM_PERF_SANITY_WARMUP_PASS=0'`; it reaches the + aggregated ranks and every disaggregated role. +- Client log: `trtllm-benchmark-warmup-pass...log`, not parsed + for metrics. The test fails if the pass's benchmark_serving exits non-zero, times + out or has every request fail; if only some fail, it logs a warning and the measured + client runs. +- The pass's perf_metrics records are excluded from the time breakdown (see below), + and in e2e its iterations come before the measured client's gen-log window, so + neither `d_tb_*` nor the gen-worker device step time includes it. +- Uploaded rows carry `l_warmup_pass_requests` (0 when no pass ran). It is not a match + key. +- With a dataset file, a few of the pass's prompts can coincide with the measured + sample (about concurrency / dataset size of them). This only matters for configs + with `enable_block_reuse` on. +- It costs one wave, about 1 / `iterations` of the measured run. Iteration-based nsys + ranges count its iterations too. + ### The optional instrumentation modifier The three shapes that read a disaggregated config (2, 3 and 4 above) take one @@ -409,13 +451,14 @@ mid-record) costs only that line, never the whole file: dropping the disagg serv file would silently reroute the per-request groups to same-role worker fallbacks, which produce plausible values for the wrong phase. -The servers also append records of the e2e connection warmup's requests to these -files. The harness therefore waits until the largest file holds a record for every -successful warmup request and the files hold still, then notes each file's line count; the aggregation skips those leading lines and leaves +The servers also append records of the e2e connection warmup's and the warmup pass's +requests to these files. The harness therefore waits until the largest file holds a +record for every successful warmup request and the files hold still, then notes each +file's line count; the aggregation skips those leading lines and leaves them out of the census. Unlike the measured run's census, this one is a failure: if the count is not reached within `PERF_METRICS_SETTLE_TIMEOUT`, the test fails rather than mixing late warmup records into the breakdown. The log line -`Time breakdown: excluded connection-warmup records from {...}` prints the counts as +`Time breakdown: excluded warmup records from {...}` prints the counts as JSON. To reproduce the uploaded numbers offline, run `time_breakdown_metrics.py --mode e2e --output-dir /perf_metrics --drop-warmup-request --leading-lines-to-skip ''`. diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 08b4c5fb6539..ca1860f588d0 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -54,6 +54,7 @@ format_metric_log_lines, wait_for_perf_metrics_files, ) +from .warmup_pass import build_warmup_pass_cmd, warmup_pass_applies, warmup_pass_enabled SUPPORTED_GPU_MAPPING = { "GB200": "gb200", @@ -70,7 +71,7 @@ # Modes whose measured client sends benchmark_serving's single un-measured test # request (it omits --no-test-input). Unrelated to the connection warmup -# (DisaggTestCmds._run_connection_warmup). +# (DisaggTestCmds._run_connection_warmup) and to the warmup pass (warmup_pass.py). WARMUP_BENCHMARK_MODES = ("e2e", "ctx_only") @@ -1776,6 +1777,144 @@ def _run_benchmark_with_log( return raw_output.decode("utf-8", errors="replace") +def warmup_pass_log_path(test_output_dir: str, server_idx: int, client_idx: int) -> str: + """Log of the warmup pass that precedes client ``client_idx`` of server ``server_idx``.""" + return os.path.join( + test_output_dir, f"trtllm-benchmark-warmup-pass.{server_idx}.{client_idx}.log" + ) + + +def warmup_pass_logs(test_output_dir: str, server_idx: int) -> List[str]: + """Every warmup pass log present for server ``server_idx``.""" + return sorted( + glob.glob(os.path.join(test_output_dir, f"trtllm-benchmark-warmup-pass.{server_idx}.*.log")) + ) + + +def warmup_pass_requests_for( + client_config: Optional["ClientConfig"], benchmark_mode: Optional[str] +) -> int: + """Requests in the warmup pass before this client, or 0 when it gets none. + + Args: + client_config: The measured client's config; None gets no pass. + benchmark_mode: The case's benchmark mode; None or "" for a plain + aggregated case. + """ + if ( + client_config is None + or not warmup_pass_enabled() + or not warmup_pass_applies(benchmark_mode) + ): + return 0 + return client_config.warmup_pass_requests + + +def run_warmup_pass( + client_cmd: List[str], + client_config: "ClientConfig", + num_requests: int, + hostname: str, + port: int, + log_path: str, + timeout: float, +) -> int: + """Send the warmup pass for one measured client; see warmup_pass.py. + + If only some requests fail, the pass still warmed the shapes it reached, so it + logs a warning and the measured client decides whether the test passes. + + Args: + client_cmd: The measured client's command, without host and port. + client_config: The measured client's config; its env is reused. + num_requests: Requests to send, from warmup_pass_requests_for. + hostname: Server host. + port: Server port. + log_path: File that receives the pass's output. + timeout: Seconds before the pass is killed. + + Returns: + The number of warmup requests that succeeded. + + Raises: + RuntimeError: benchmark_serving exited non-zero, timed out, or had every + request fail. + """ + cmd = add_host_port_to_cmd(build_warmup_pass_cmd(client_cmd, num_requests), hostname, port) + env = copy.deepcopy(os.environ) + env.update(client_config.to_env()) + print_info(f"Warmup pass: sending {num_requests} requests. cmd is {cmd}") + start = time.monotonic() + try: + output = _run_benchmark_with_log(cmd, env, log_path, timeout=timeout) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Warmup pass: benchmark_serving exited with code {e.returncode}. See {log_path}" + ) from e + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"Warmup pass: {num_requests} requests did not finish within {timeout:.0f}s. " + f"See {log_path}" + ) from e + elapsed = time.monotonic() - start + failed_match = _FAILED_REQUESTS_RE.search(output) + failed_count = int(failed_match.group(1)) if failed_match else 0 + if failed_count >= num_requests: + raise RuntimeError(f"Warmup pass: all {num_requests} requests failed. See {log_path}") + if failed_count: + print_warning( + f"Warmup pass: {failed_count} of {num_requests} requests failed ({elapsed:.0f}s). " + f"See {log_path}" + ) + else: + print_info(f"Warmup pass: {num_requests} requests done in {elapsed:.0f}s") + return num_requests - failed_count + + +def wait_for_unmeasured_line_counts( + perf_metrics_output_dir: str, num_unmeasured_requests: int +) -> Dict[str, int]: + """Wait for the un-measured requests' perf_metrics records; return per-file line counts. + + The time breakdown skips this many leading lines per file, so a warmup record + written after this snapshot would count as measured. Each server queues a + request's record before it finishes answering that request, so every warmup + record is already queued when the warmup client exits; this waits for them to + be written. First, the largest file must reach num_unmeasured_requests lines. + That file belongs to a server that saw every warmup request: the disagg server, + or a lone ctx, gen or aggregated server. Then the files must hold still for + PERF_METRICS_SETTLE_SECONDS, so the other servers' queued records are written + too. + + Args: + perf_metrics_output_dir: Directory the servers write perf_metrics JSONLs to. + num_unmeasured_requests: Requests that succeeded before the measured client + started (connection warmup and warmup pass). + + Returns: + Line count per perf_metrics JSONL basename. + + Raises: + RuntimeError: That did not happen within PERF_METRICS_SETTLE_TIMEOUT. + """ + _, settle_info = wait_for_perf_metrics_files( + perf_metrics_output_dir, + expected_requests=num_unmeasured_requests, + wait_for_expected_requests=True, + stable_seconds=PERF_METRICS_SETTLE_SECONDS, + timeout_seconds=PERF_METRICS_SETTLE_TIMEOUT, + ) + if not settle_info["stable"]: + raise RuntimeError( + f"Expected {num_unmeasured_requests} warmup records in the largest " + f"perf_metrics file under {perf_metrics_output_dir}, but the files did not " + f"reach that count and stop growing within {PERF_METRICS_SETTLE_TIMEOUT:.0f}s " + f"(line counts: {settle_info['line_counts']}). Failing so that late warmup " + "records are not counted as measured in d_tb_*." + ) + return settle_info["line_counts"] + + class ServerConfig: """Configurations of trtllm-server.""" @@ -2254,6 +2393,20 @@ def num_requests(self) -> int: """ return self.concurrency * self.iterations + @property + def warmup_pass_requests(self) -> int: + """Requests in the warmup pass before this client, or 0 when it gets none. + + One wave at the client's concurrency, so the pass prefills the same batches + the measured run starts with. Only benchmark_serving clients get one: + AgentX runs its own warmup phase, and an accuracy-only client measures no + performance. Whether the case's mode gets a pass at all is decided by + warmup_pass_applies and warmup_pass_enabled. + """ + if self.only_run_accuracy or self.benchmark_client or self.use_nv_sa_benchmark: + return 0 + return self.concurrency + def to_cmd(self) -> List[str]: """Generate benchmark command.""" model_dir = get_model_dir(self.model_name) @@ -2506,8 +2659,8 @@ def append_time_breakdown_metrics( session from inside a post-benchmark hook. ``leading_lines_to_skip`` maps a JSONL basename to the number of lines it held - before the measured clients started (the connection warmup's records); those lines - are left out of both the census and the breakdown. + before the measured clients started (the connection warmup's and the warmup + pass's records); those lines are left out of both the census and the breakdown. """ if not pending_time_breakdown: return @@ -2567,7 +2720,7 @@ def append_time_breakdown_metrics( print_info(f"Time breakdown: excluded the warmup request from {info['warmup_dropped']}") if info["leading_lines_skipped"]: print_info( - "Time breakdown: excluded connection-warmup records from " + "Time breakdown: excluded warmup records from " f"{json.dumps(info['leading_lines_skipped'])}" ) @@ -2604,7 +2757,11 @@ def get_server_logs(self, server_idx) -> List[str]: benchmark_logs = sorted( glob.glob(os.path.join(self.test_output_dir, f"trtllm-benchmark.{server_idx}.*.log")) ) - return [server_file_path, *benchmark_logs] + return [ + server_file_path, + *benchmark_logs, + *warmup_pass_logs(self.test_output_dir, server_idx), + ] def run_cmd(self, server_idx: int) -> List[str]: """Run all clients for a server and return outputs. @@ -2624,6 +2781,13 @@ def run_cmd(self, server_idx: int) -> List[str]: # after the server exits -- i.e. after the finally below, not before it. pending_time_breakdown: List[dict] = [] collect_time_breakdown = bool(self.perf_metrics_output_dir) + # Requests the warmup passes completed before the first measured client. + num_unmeasured_requests = 0 + # Filled only when a warmup pass ran with the time breakdown on. + unmeasured_line_counts: Dict[str, int] = {} + unmeasured_lines_counted = False + for stale_log in warmup_pass_logs(self.test_output_dir, server_idx): + os.remove(stale_log) try: server_hostname = "localhost" @@ -2692,6 +2856,32 @@ def run_cmd(self, server_idx: int) -> List[str]: only_run_accuracy = bool(client_config and client_config.only_run_accuracy) if not only_run_accuracy: + num_warmup_requests = warmup_pass_requests_for( + client_config, self.benchmark_mode + ) + if num_warmup_requests: + num_unmeasured_requests += run_warmup_pass( + client_cmd, + client_config, + num_warmup_requests, + server_hostname, + server_port, + warmup_pass_log_path(self.test_output_dir, server_idx, client_idx), + timeout=min(self.timeout, DEFAULT_TIMEOUT), + ) + # The time breakdown skips only leading lines, so count them + # once, before the first measured client. A time_breakdown + # lane has a single client. + if ( + collect_time_breakdown + and num_unmeasured_requests + and not unmeasured_lines_counted + ): + unmeasured_line_counts = wait_for_unmeasured_line_counts( + self.perf_metrics_output_dir, num_unmeasured_requests + ) + unmeasured_lines_counted = True + client_file_path = os.path.join( self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" ) @@ -2749,7 +2939,12 @@ def run_cmd(self, server_idx: int) -> List[str]: # complete. terminate() is SIGTERM, which trtllm-serve handles as a # graceful shutdown, so PerfMetricsJsonlWriter.close() has run and the # tail of the queue is on disk. - append_time_breakdown_metrics(pending_time_breakdown, outputs, self.perf_metrics_output_dir) + append_time_breakdown_metrics( + pending_time_breakdown, + outputs, + self.perf_metrics_output_dir, + leading_lines_to_skip=unmeasured_line_counts, + ) return outputs @@ -3179,6 +3374,7 @@ def get_server_logs(self, server_idx: int) -> List[str]: connection_warmup_log = self._connection_warmup_log_path(server_idx) if os.path.exists(connection_warmup_log): server_logs.append(connection_warmup_log) + server_logs.extend(warmup_pass_logs(self.test_output_dir, server_idx)) return server_logs @staticmethod @@ -3360,47 +3556,6 @@ def _run_connection_warmup( print_info(f"Connection warmup: {num_requests} requests done in {elapsed:.0f}s") return num_requests - failed_count - def _wait_for_connection_warmup_line_counts( - self, num_succeeded_requests: int - ) -> Dict[str, int]: - """Wait for every warmup perf_metrics record to be written; return per-file line counts. - - The time breakdown skips this many leading lines per file, so a warmup record - written after this snapshot would count as measured. Each server queues a - request's record before it finishes answering that request, so every warmup record - is already queued when the warmup client exits; this waits for them to be written. - First, the largest file must reach num_succeeded_requests lines. That file belongs - to a server that saw every warmup request: the disagg server, or a lone ctx or gen - server. Then the files must hold still for PERF_METRICS_SETTLE_SECONDS, so the - other servers' queued records are written too. - - Args: - num_succeeded_requests: Warmup requests that succeeded. - - Returns: - Line count per perf_metrics JSONL basename. - - Raises: - RuntimeError: That did not happen within PERF_METRICS_SETTLE_TIMEOUT. - """ - _, settle_info = wait_for_perf_metrics_files( - self.perf_metrics_output_dir, - expected_requests=num_succeeded_requests, - wait_for_expected_requests=True, - stable_seconds=PERF_METRICS_SETTLE_SECONDS, - timeout_seconds=PERF_METRICS_SETTLE_TIMEOUT, - ) - if not settle_info["stable"]: - raise RuntimeError( - f"Connection warmup: expected {num_succeeded_requests} warmup records in the " - f"largest perf_metrics file under {self.perf_metrics_output_dir}, but the " - f"files did not reach that count and stop growing within " - f"{PERF_METRICS_SETTLE_TIMEOUT:.0f}s (line counts: " - f"{settle_info['line_counts']}). Failing so that late warmup records are " - "not counted as measured in d_tb_*." - ) - return settle_info["line_counts"] - @staticmethod def _dwdp_ctx_rank(saved_env: Dict[str, str]) -> Tuple[int, int]: """(rank, world size) of this task inside the DWDP ctx MPI step.""" @@ -3665,10 +3820,16 @@ def run_cmd(self, server_idx: int) -> List[str]: # is the single master switch), so there is no second predicate to # keep in sync with it. collect_time_breakdown = bool(self.perf_metrics_output_dir) - # Filled only when the connection warmup ran with the time breakdown on. - connection_warmup_line_counts: Dict[str, int] = {} + # Requests that succeeded before the measured client started: the + # connection warmup's and the warmup pass's. + num_unmeasured_requests = 0 + # Filled only when un-measured requests ran with the time breakdown on. + unmeasured_line_counts: Dict[str, int] = {} + unmeasured_lines_counted = False try: self._remove_stale_connection_warmup_log(server_idx) + for stale_log in warmup_pass_logs(self.test_output_dir, server_idx): + os.remove(stale_log) disagg_server_hostname, disagg_server_port = ( self._get_disagg_server_hostname_and_port(server_idx) ) @@ -3685,15 +3846,9 @@ def run_cmd(self, server_idx: int) -> List[str]: measured_client = self._measured_client_for_connection_warmup(server_idx) if measured_client is not None: - num_succeeded_connection_warmup_requests = self._run_connection_warmup( + num_unmeasured_requests += self._run_connection_warmup( server_idx, disagg_server_hostname, disagg_server_port, measured_client ) - if collect_time_breakdown: - connection_warmup_line_counts = ( - self._wait_for_connection_warmup_line_counts( - num_succeeded_connection_warmup_requests - ) - ) client_configs = self.client_configs.get(server_idx, []) @@ -3705,6 +3860,34 @@ def run_cmd(self, server_idx: int) -> List[str]: only_run_accuracy = bool(client_config and client_config.only_run_accuracy) if not only_run_accuracy: + # Before the gen-log snapshot below, so the device step + # time leaves the pass's iterations out. + num_warmup_requests = warmup_pass_requests_for( + client_config, benchmark_mode_for_idx + ) + if num_warmup_requests: + num_unmeasured_requests += run_warmup_pass( + client_cmd, + client_config, + num_warmup_requests, + disagg_server_hostname, + disagg_server_port, + warmup_pass_log_path(self.test_output_dir, server_idx, client_idx), + timeout=min(self.timeout, DEFAULT_TIMEOUT), + ) + # The time breakdown skips only leading lines, so count + # them once, before the first measured client. A disagg + # lane has a single client. + if ( + collect_time_breakdown + and num_unmeasured_requests + and not unmeasured_lines_counted + ): + unmeasured_line_counts = wait_for_unmeasured_line_counts( + self.perf_metrics_output_dir, num_unmeasured_requests + ) + unmeasured_lines_counted = True + benchmark_file_path = os.path.join( self.test_output_dir, f"trtllm-benchmark.{server_idx}.{client_idx}.log" ) @@ -3839,7 +4022,7 @@ def run_cmd(self, server_idx: int) -> List[str]: self._append_time_breakdown_metrics( pending_time_breakdown, outputs, - leading_lines_to_skip=connection_warmup_line_counts, + leading_lines_to_skip=unmeasured_line_counts, ) return outputs @@ -5222,6 +5405,9 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: } new_data.update(server_config_dict) new_data.update(client_config_dict) + new_data["l_warmup_pass_requests"] = warmup_pass_requests_for( + client_config, self.benchmark_mode + ) # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{server_config.name}-{client_config.name}" @@ -5311,6 +5497,9 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: if num_gen_servers > 0: new_data.update(gen_server_config_dict) new_data.update(client_config_dict) + new_data["l_warmup_pass_requests"] = warmup_pass_requests_for( + client_config, disagg_config.benchmark_mode + ) # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{disagg_config.name}-{client_config.name}" diff --git a/tests/integration/defs/perf/warmup_pass.py b/tests/integration/defs/perf/warmup_pass.py new file mode 100644 index 000000000000..bd87f70c02ba --- /dev/null +++ b/tests/integration/defs/perf/warmup_pass.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The un-measured warmup pass perf-sanity sends before each measured client. + +The server's own startup warmup cannot reach every prefill shape a benchmark +produces: how many prompts arrive together, how they are chunked against +``max_num_tokens`` and which decode tokens share a step with them are only known +once real traffic arrives. JIT kernels keyed on those shapes would otherwise +compile inside the first measured requests. The warmup pass sends that traffic +once beforehand: the measured client's own ``benchmark_serving`` command, cut to +one wave of ``concurrency`` requests with different prompts. Nothing it prints is +parsed or uploaded. + +It runs for plain aggregated cases, ``ctx_only`` and disagg ``e2e``, whose +measured window contains prefill. The ``gen_only`` modes never get one: they +measure decode only, and their generation server's benchmark fill gate opens +once, so warmup requests would open it before the measured burst arrives. + +``TRTLLM_PERF_SANITY_WARMUP_PASS=0`` turns it off, e.g. to reproduce a JIT stall +the pass would otherwise absorb. + +Stdlib-only, so unit tests can load it without torch. +""" + +import os +from typing import List, Mapping, Optional, Sequence + +WARMUP_PASS_ENV = "TRTLLM_PERF_SANITY_WARMUP_PASS" + +# Benchmark modes that get a warmup pass. A plain aggregated case has no +# benchmark mode, which the runner carries as "". +WARMUP_PASS_MODES = ("", "ctx_only", "e2e") + +_DISABLED_VALUES = ("0", "false", "no", "off") + +_NUM_PROMPTS_FLAG = "--num-prompts" +_SEED_FLAG = "--seed" +_NO_TEST_INPUT_FLAG = "--no-test-input" +_SAVE_TIME_BREAKDOWN_FLAG = "--save-request-time-breakdown" +# Flags whose value the warmup pass replaces or drops. +_FLAGS_WITH_REPLACED_VALUE = (_NUM_PROMPTS_FLAG, _SEED_FLAG, _SAVE_TIME_BREAKDOWN_FLAG) +# benchmark_serving's --seed default. +_BENCHMARK_SERVING_DEFAULT_SEED = 0 + + +def warmup_pass_enabled(environ: Optional[Mapping[str, str]] = None) -> bool: + """Return whether the warmup pass is on. + + It is on unless ``TRTLLM_PERF_SANITY_WARMUP_PASS`` is ``0``, ``false``, ``no`` + or ``off`` (any case). + + Args: + environ: Environment to read; ``os.environ`` when None. + """ + env = os.environ if environ is None else environ + return env.get(WARMUP_PASS_ENV, "1").strip().lower() not in _DISABLED_VALUES + + +def warmup_pass_applies(benchmark_mode: Optional[str]) -> bool: + """Return whether a case in this benchmark mode gets a warmup pass. + + Args: + benchmark_mode: The case's benchmark mode; None or "" for a plain + aggregated case. + """ + return (benchmark_mode or "") in WARMUP_PASS_MODES + + +def build_warmup_pass_cmd(measured_cmd: Sequence[str], num_requests: int) -> List[str]: + """Derive the warmup pass's ``benchmark_serving`` command from the measured one. + + Every flag that shapes the traffic is kept: model, tokenizer, dataset and + sequence lengths, maximum concurrency, backend, streaming, chat template. Only + these change: + + * ``--num-prompts`` becomes ``num_requests``. + * ``--seed`` becomes the measured seed plus one, so the pass draws different + prompts with the same length distribution: other random token ids, or + another shuffle of the same dataset file. + * ``--no-test-input`` is added, so no single test request goes first. + * ``--save-request-time-breakdown`` and its value are dropped. + + The measured command must pass each flag and its value as separate items, as + the runner builds it. The caller adds host and port. + + Args: + measured_cmd: The measured client's ``benchmark_serving`` command. + num_requests: Requests the pass sends; at least 1. + + Returns: + The warmup pass command. + + Raises: + ValueError: ``num_requests`` is below 1. + """ + if num_requests < 1: + raise ValueError(f"A warmup pass needs at least one request, got {num_requests}") + warmup_cmd: List[str] = [] + measured_seed = _BENCHMARK_SERVING_DEFAULT_SEED + args = iter(measured_cmd) + for arg in args: + if arg in _FLAGS_WITH_REPLACED_VALUE: + value = next(args, None) + if arg == _SEED_FLAG and value is not None: + measured_seed = int(value) + continue + if arg == _NO_TEST_INPUT_FLAG: + continue + warmup_cmd.append(arg) + return warmup_cmd + [ + _NUM_PROMPTS_FLAG, + str(num_requests), + _SEED_FLAG, + str(measured_seed + 1), + _NO_TEST_INPUT_FLAG, + ] diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass.py b/tests/unittest/others/test_perf_sanity_warmup_pass.py new file mode 100644 index 000000000000..468b052af3a8 --- /dev/null +++ b/tests/unittest/others/test_perf_sanity_warmup_pass.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The perf-sanity warmup pass must send the measured traffic shape and nothing else. + +A pass whose command drifts from the measured one warms the wrong prefill shapes, and +one that reuses the measured prompts lets measured requests hit its KV cache blocks; +either way the measured numbers move without any runtime change. +""" + +import importlib.util +from pathlib import Path +from typing import List, Sequence + +import pytest + +pytestmark = pytest.mark.cpu_only + +# Loaded by path: importing it through the ``defs`` package pulls in torch. +_SPEC = importlib.util.spec_from_file_location( + "warmup_pass", + Path(__file__).resolve().parents[2] / "integration/defs/perf/warmup_pass.py", +) +warmup_pass = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(warmup_pass) + +_BENCHMARK_SERVING = ["python", "-m", "tensorrt_llm.serve.scripts.benchmark_serving"] + +# Shaped like ClientConfig._to_default_benchmark_cmd's output for the three modes. +# fmt: off +_AGGR_RANDOM_CMD = _BENCHMARK_SERVING + [ + "--model", "/models/m", "--tokenizer", "/models/m", + "--num-prompts", "40", "--max-concurrency", "4", + "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", "--no-test-input", + "--dataset-name", "random", "--random-ids", "--tokenize-on-client", + "--random-input-len", "8192", "--random-output-len", "1024", + "--random-range-ratio", "0.0", "--backend", "openai", "--trust-remote-code", +] +_E2E_DATASET_TIME_BREAKDOWN_CMD = _BENCHMARK_SERVING + [ + "--model", "/models/m", "--tokenizer", "/models/m", + "--num-prompts", "43010", "--max-concurrency", "4301", + "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", + "--dataset-name", "trtllm_custom", "--dataset-path", "/datasets/d.json", + "--backend", "openai", "--trust-remote-code", + "--save-request-time-breakdown", "/out/time_breakdown", +] +_CTX_ONLY_CMD = _BENCHMARK_SERVING + [ + "--model", "/models/m", "--tokenizer", "/models/m", + "--num-prompts", "6660", "--max-concurrency", "666", + "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", + "--dataset-name", "random", "--random-ids", "--tokenize-on-client", + "--random-input-len", "7386", "--random-output-len", "1", + "--random-range-ratio", "0.0", "--backend", "openai", "--non-streaming", +] +# fmt: on + +_REPLACED_FLAGS_WITH_VALUE = ("--num-prompts", "--seed", "--save-request-time-breakdown") + + +def _value_of(cmd: Sequence[str], flag: str) -> str: + assert cmd.count(flag) == 1, f"{flag} appears {cmd.count(flag)} times in {cmd}" + return cmd[cmd.index(flag) + 1] + + +def _traffic_shape(cmd: Sequence[str]) -> List[str]: + """The command without the flags the warmup pass may change.""" + shape, args = [], iter(cmd) + for arg in args: + if arg in _REPLACED_FLAGS_WITH_VALUE: + next(args, None) + elif arg != "--no-test-input": + shape.append(arg) + return shape + + +@pytest.mark.parametrize( + "measured_cmd", [_AGGR_RANDOM_CMD, _E2E_DATASET_TIME_BREAKDOWN_CMD, _CTX_ONLY_CMD] +) +def test_warmup_pass_cmd_keeps_the_measured_traffic_shape(measured_cmd: List[str]) -> None: + concurrency = int(_value_of(measured_cmd, "--max-concurrency")) + warmup_cmd = warmup_pass.build_warmup_pass_cmd(measured_cmd, concurrency) + + assert _traffic_shape(warmup_cmd) == _traffic_shape(measured_cmd) + assert _value_of(warmup_cmd, "--num-prompts") == str(concurrency) + assert warmup_cmd.count("--no-test-input") == 1 + assert "--save-request-time-breakdown" not in warmup_cmd + assert "/out/time_breakdown" not in warmup_cmd + + +@pytest.mark.parametrize( + ("seed_args", "expected_warmup_seed"), + [ + # benchmark_serving's own default seed is 0. + ([], "1"), + (["--seed", "7"], "8"), + ], +) +def test_warmup_pass_draws_other_prompts_than_the_measured_client( + seed_args: List[str], expected_warmup_seed: str +) -> None: + measured_cmd = _AGGR_RANDOM_CMD + seed_args + warmup_cmd = warmup_pass.build_warmup_pass_cmd(measured_cmd, 4) + + assert _value_of(warmup_cmd, "--seed") == expected_warmup_seed + + +def test_warmup_pass_cmd_leaves_the_measured_cmd_untouched() -> None: + measured_cmd = list(_E2E_DATASET_TIME_BREAKDOWN_CMD) + warmup_pass.build_warmup_pass_cmd(measured_cmd, 4301) + + assert measured_cmd == _E2E_DATASET_TIME_BREAKDOWN_CMD + + +@pytest.mark.parametrize("num_requests", [0, -1]) +def test_warmup_pass_needs_a_request(num_requests: int) -> None: + with pytest.raises(ValueError): + warmup_pass.build_warmup_pass_cmd(_AGGR_RANDOM_CMD, num_requests) + + +@pytest.mark.parametrize( + ("environ", "enabled"), + [ + ({}, True), + ({warmup_pass.WARMUP_PASS_ENV: "1"}, True), + ({warmup_pass.WARMUP_PASS_ENV: "0"}, False), + ({warmup_pass.WARMUP_PASS_ENV: " False "}, False), + ({warmup_pass.WARMUP_PASS_ENV: "OFF"}, False), + ({warmup_pass.WARMUP_PASS_ENV: "no"}, False), + ], +) +def test_warmup_pass_env_switch(environ: dict, enabled: bool) -> None: + assert warmup_pass.warmup_pass_enabled(environ) is enabled + + +@pytest.mark.parametrize( + ("benchmark_mode", "applies"), + [ + (None, True), + ("", True), + ("ctx_only", True), + ("e2e", True), + ("gen_only", False), + ("gen_only_no_context", False), + ], +) +def test_warmup_pass_modes(benchmark_mode, applies: bool) -> None: + assert warmup_pass.warmup_pass_applies(benchmark_mode) is applies From e62da93adf45978a0f1a27a5963e2f7bc136f8f2 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:06:15 -0700 Subject: [PATCH 2/7] [None][test] Count warmup records per server role; close gen-log windows before a pass Review fixes for the perf-sanity warmup pass: - time_breakdown_metrics: the census is now the complete records of the server role that wrote the most, summed over that role's files, instead of the largest single file. A role writes one file per instance and per frontend process, so with num_serve_frontends > 1 the warmup records were split across files and the warmup wait could never reach its count. Each file keeps its own leading-line count. - DisaggTestCmds: close the previous client's gen-log window before the next client's warmup pass, so neither client's device step time counts the pass's iterations. - warmup_pass.send_warmup_pass now holds the pass's outcome handling (non-zero exit, timeout, some or all requests failed), so CPU tests cover it; the connection warmup shares count_failed_requests. - Tests: warmup records split across frontend files, roles not summed together, and the pass's outcomes. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 11 +- .../integration/defs/perf/test_perf_sanity.py | 104 +++++----- .../defs/perf/time_breakdown_metrics.py | 105 +++++++--- tests/integration/defs/perf/warmup_pass.py | 74 ++++++- .../others/test_perf_sanity_warmup_pass.py | 189 +++++++++++++++++- 5 files changed, 388 insertions(+), 95 deletions(-) diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index eccd9cfc36d0..996f5c203bc0 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -441,8 +441,11 @@ to run after the writers are done. Only the **gen** workers announce that workers and the disaggregated server do not. `wait_for_perf_metrics_files` therefore polls the discovered set until its total size holds still for `PERF_METRICS_SETTLE_SECONDS`, bounded by `PERF_METRICS_SETTLE_TIMEOUT`, before -anything is read, and then compares the largest file's complete-record count against -the client's `--num-prompts`. Both a still-growing set at the timeout and a census +anything is read, and then compares the census against the client's `--num-prompts`. +The census counts the complete records of one server role (the disaggregated server, +the ctx workers, the gen workers, or an aggregated server), summed over that role's +files: a role writes one file per instance and per frontend process +(`num_serve_frontends`). Both a still-growing set at the timeout and a census shortfall are logged as warnings, not failures — a nearly-complete file still yields usable statistics, and the numbers are not regression-gated. This matters because a truncated read is otherwise **invisible**: all 108 fields are still populated and the @@ -452,8 +455,8 @@ file would silently reroute the per-request groups to same-role worker fallbacks which produce plausible values for the wrong phase. The servers also append records of the e2e connection warmup's and the warmup pass's -requests to these files. The harness therefore waits until the largest file holds a -record for every successful warmup request and the files hold still, then notes each +requests to these files. The harness therefore waits until the census holds a record +for every successful warmup request and the files hold still, then notes each file's line count; the aggregation skips those leading lines and leaves them out of the census. Unlike the measured run's census, this one is a failure: if the count is not reached within `PERF_METRICS_SETTLE_TIMEOUT`, the test fails rather diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index ca1860f588d0..4a320d38e1ff 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -54,7 +54,13 @@ format_metric_log_lines, wait_for_perf_metrics_files, ) -from .warmup_pass import build_warmup_pass_cmd, warmup_pass_applies, warmup_pass_enabled +from .warmup_pass import ( + build_warmup_pass_cmd, + count_failed_requests, + send_warmup_pass, + warmup_pass_applies, + warmup_pass_enabled, +) SUPPORTED_GPU_MAPPING = { "GB200": "gb200", @@ -218,9 +224,6 @@ def ensure_bench_serving_repo() -> str: # cold 32-rank gen server finished in 42 s on oci-jhb), so reaching this bound means # the servers are wedged. CONNECTION_WARMUP_TIMEOUT = 600 -# benchmark_serving's "Failed requests:" summary line, its only failure signal: -# benchmark_serving exits 0 even when requests fail. -_FAILED_REQUESTS_RE = re.compile(r"^Failed requests:\s+(\d+)", re.MULTILINE) def server_ready_timeout(default: int, mode: str) -> int: @@ -1821,9 +1824,6 @@ def run_warmup_pass( ) -> int: """Send the warmup pass for one measured client; see warmup_pass.py. - If only some requests fail, the pass still warmed the shapes it reached, so it - logs a warning and the measured client decides whether the test passes. - Args: client_cmd: The measured client's command, without host and port. client_config: The measured client's config; its env is reused. @@ -1840,35 +1840,17 @@ def run_warmup_pass( RuntimeError: benchmark_serving exited non-zero, timed out, or had every request fail. """ - cmd = add_host_port_to_cmd(build_warmup_pass_cmd(client_cmd, num_requests), hostname, port) env = copy.deepcopy(os.environ) env.update(client_config.to_env()) - print_info(f"Warmup pass: sending {num_requests} requests. cmd is {cmd}") - start = time.monotonic() - try: - output = _run_benchmark_with_log(cmd, env, log_path, timeout=timeout) - except subprocess.CalledProcessError as e: - raise RuntimeError( - f"Warmup pass: benchmark_serving exited with code {e.returncode}. See {log_path}" - ) from e - except subprocess.TimeoutExpired as e: - raise RuntimeError( - f"Warmup pass: {num_requests} requests did not finish within {timeout:.0f}s. " - f"See {log_path}" - ) from e - elapsed = time.monotonic() - start - failed_match = _FAILED_REQUESTS_RE.search(output) - failed_count = int(failed_match.group(1)) if failed_match else 0 - if failed_count >= num_requests: - raise RuntimeError(f"Warmup pass: all {num_requests} requests failed. See {log_path}") - if failed_count: - print_warning( - f"Warmup pass: {failed_count} of {num_requests} requests failed ({elapsed:.0f}s). " - f"See {log_path}" - ) - else: - print_info(f"Warmup pass: {num_requests} requests done in {elapsed:.0f}s") - return num_requests - failed_count + return send_warmup_pass( + add_host_port_to_cmd(build_warmup_pass_cmd(client_cmd, num_requests), hostname, port), + num_requests, + run_benchmark=lambda cmd: _run_benchmark_with_log(cmd, env, log_path, timeout=timeout), + log_path=log_path, + timeout=timeout, + log_info=print_info, + log_warning=print_warning, + ) def wait_for_unmeasured_line_counts( @@ -1880,10 +1862,11 @@ def wait_for_unmeasured_line_counts( written after this snapshot would count as measured. Each server queues a request's record before it finishes answering that request, so every warmup record is already queued when the warmup client exits; this waits for them to - be written. First, the largest file must reach num_unmeasured_requests lines. - That file belongs to a server that saw every warmup request: the disagg server, - or a lone ctx, gen or aggregated server. Then the files must hold still for - PERF_METRICS_SETTLE_SECONDS, so the other servers' queued records are written + be written. First, one server role must hold num_unmeasured_requests records, + summed over its files: each role (the disagg server, the ctx workers, the gen + workers, or an aggregated server) sees every request, but writes one file per + instance and per frontend process. Then the files must hold still for + PERF_METRICS_SETTLE_SECONDS, so the other roles' queued records are written too. Args: @@ -1906,9 +1889,9 @@ def wait_for_unmeasured_line_counts( ) if not settle_info["stable"]: raise RuntimeError( - f"Expected {num_unmeasured_requests} warmup records in the largest " - f"perf_metrics file under {perf_metrics_output_dir}, but the files did not " - f"reach that count and stop growing within {PERF_METRICS_SETTLE_TIMEOUT:.0f}s " + f"Expected {num_unmeasured_requests} warmup records from one server role " + f"under {perf_metrics_output_dir}, but the perf_metrics files did not reach " + f"that count and stop growing within {PERF_METRICS_SETTLE_TIMEOUT:.0f}s " f"(line counts: {settle_info['line_counts']}). Failing so that late warmup " "records are not counted as measured in d_tb_*." ) @@ -2870,8 +2853,9 @@ def run_cmd(self, server_idx: int) -> List[str]: timeout=min(self.timeout, DEFAULT_TIMEOUT), ) # The time breakdown skips only leading lines, so count them - # once, before the first measured client. A time_breakdown - # lane has a single client. + # once, before the first measured client. CI's time_breakdown + # lanes have a single client; with more, a later client's pass + # joins the breakdown. if ( collect_time_breakdown and num_unmeasured_requests @@ -3540,8 +3524,7 @@ def _run_connection_warmup( f"{timeout_seconds}s, so the servers look wedged. See {log_path}" ) from e elapsed = time.monotonic() - start - failed_match = _FAILED_REQUESTS_RE.search(output) - failed_count = int(failed_match.group(1)) if failed_match else 0 + failed_count = count_failed_requests(output) if failed_count >= num_requests: raise RuntimeError( f"Connection warmup: all {num_requests} requests failed. See {log_path}" @@ -3860,12 +3843,18 @@ def run_cmd(self, server_idx: int) -> List[str]: only_run_accuracy = bool(client_config and client_config.only_run_accuracy) if not only_run_accuracy: - # Before the gen-log snapshot below, so the device step - # time leaves the pass's iterations out. num_warmup_requests = warmup_pass_requests_for( client_config, benchmark_mode_for_idx ) if num_warmup_requests: + # The pass runs between two clients' gen-log windows, + # so neither one's device step time counts its + # iterations: close the previous client's window + # here, and open this client's after the pass. + if collect_device_step_time and pending_device_step_time: + pending_device_step_time[-1]["end_offsets"] = gen_worker_log_sizes( + self.test_output_dir, self.num_gen_servers + ) num_unmeasured_requests += run_warmup_pass( client_cmd, client_config, @@ -3876,8 +3865,9 @@ def run_cmd(self, server_idx: int) -> List[str]: timeout=min(self.timeout, DEFAULT_TIMEOUT), ) # The time breakdown skips only leading lines, so count - # them once, before the first measured client. A disagg - # lane has a single client. + # them once, before the first measured client. CI's + # time_breakdown lanes have a single client; with more, a + # later client's pass joins the breakdown. if ( collect_time_breakdown and num_unmeasured_requests @@ -3897,11 +3887,12 @@ def run_cmd(self, server_idx: int) -> List[str]: print_info(f"Starting benchmark. cmd is {client_cmd_with_port}") # Snapshot gen_server log sizes so each client's stats - # cover only iterations driven by that client. This is - # also the *end* bound of the previous client's window - # (see the fixup below): taken here, it is necessarily - # after that client returned, so it absorbs whatever - # the gen workers flushed late. Modes outside + # cover only iterations driven by that client. Unless a + # warmup pass closed it above, this is also the *end* + # bound of the previous client's window (see the fixup + # below): taken here, it is necessarily after that + # client returned, so it absorbs whatever the gen + # workers flushed late. Modes outside # DEVICE_STEP_TIME_MODES skip this and must not wait for # the GEN teardown sentinel. gen_log_start_offsets = None @@ -3909,7 +3900,10 @@ def run_cmd(self, server_idx: int) -> List[str]: gen_log_start_offsets = gen_worker_log_sizes( self.test_output_dir, self.num_gen_servers ) - if pending_device_step_time: + if ( + pending_device_step_time + and pending_device_step_time[-1]["end_offsets"] is None + ): # Close the previous client's window here rather # than at its own return: this snapshot is the # first byte of the current client's segment, so diff --git a/tests/integration/defs/perf/time_breakdown_metrics.py b/tests/integration/defs/perf/time_breakdown_metrics.py index 5bbe26532bb1..7da698af72f6 100644 --- a/tests/integration/defs/perf/time_breakdown_metrics.py +++ b/tests/integration/defs/perf/time_breakdown_metrics.py @@ -510,8 +510,8 @@ def compute_time_breakdown_metrics( :func:`_drop_warmup_record`. Reported as ``info["warmup_dropped"]``. leading_lines_to_skip: per file basename, the number of leading lines to ignore because they record traffic sent before the measured clients (perf-sanity's - connection warmup). The servers append every request to the same files, so - those records would otherwise join the population. Reported as + connection warmup and warmup pass). The servers append every request to the + same files, so those records would otherwise join the population. Reported as ``info["leading_lines_skipped"]``. Returns: @@ -668,7 +668,7 @@ def compute_time_breakdown_metrics( # Per file, non-empty only when a line failed to parse (usually a partial write). "skipped_lines": skipped_lines, # Per file, how many leading lines were ignored as pre-measurement traffic (the - # connection warmup). + # connection warmup and the warmup pass). "leading_lines_skipped": leading_lines_skipped, "sample_counts": { name: len( @@ -742,12 +742,53 @@ def _count_lines(path: str) -> int: total += chunk.count(b"\n") -def _census(line_counts: Mapping[str, int], leading_lines_to_skip: Mapping[str, int]) -> int: - """Largest per-file line count after subtracting that file's skipped leading lines.""" - return max( - (count - leading_lines_to_skip.get(name, 0) for name, count in line_counts.items()), - default=0, - ) +# Records _census_role parses to tell a file's role. A file holds one role's records, so a +# few suffice, and a gen worker's records carry an entry per decode step, so they can be large. +_CENSUS_ROLE_SAMPLE_RECORDS = 8 + + +def _census_role(path: str) -> str: + """``_classify`` of the file's first complete records; ``empty`` until it has one.""" + records: List[Dict[str, Any]] = [] + with open(path, "rb") as handle: + for line in handle: + if not line.endswith(b"\n"): + break # A partial write. + try: + record = json.loads(line) + except ValueError: + continue + if isinstance(record, dict): + records.append(record) + if len(records) == _CENSUS_ROLE_SAMPLE_RECORDS: + break + return _classify(path, records) + + +def _census( + line_counts: Mapping[str, int], + leading_lines_to_skip: Mapping[str, int], + roles: Mapping[str, str], +) -> int: + """Complete records of the role that wrote the most, after each file's skipped lines. + + The disagg server, the ctx workers and the gen workers each write one record per + request, and so does an aggregated server, so any one role's records are the run's + census. A role writes several files when it has several instances, or when an instance + runs several frontend processes (``num_serve_frontends``): each process writes its own + file, which holds only its share of the requests. So the census sums each role's files. + + Args: + line_counts: Complete lines per file basename. + leading_lines_to_skip: Lines per file basename to leave out of the count. + roles: ``_census_role`` per file basename; ``empty`` files are left out. + """ + totals: Dict[str, int] = defaultdict(int) + for name, count in line_counts.items(): + role = roles.get(name, "empty") + if role != "empty": + totals[role] += max(0, count - leading_lines_to_skip.get(name, 0)) + return max(totals.values(), default=0) def wait_for_perf_metrics_files( @@ -772,18 +813,19 @@ def wait_for_perf_metrics_files( Poll the discovered set's total byte size until it holds still for ``stable_seconds`` (new files appearing counts as growth), bounded by ``timeout_seconds``. Then, if - ``expected_requests`` is known, compare it against the largest file's complete-line - count -- the disagg server and the aggregated server each write one record per request, - so that file is the run's census -- and report a shortfall. Each file's - ``leading_lines_to_skip`` entry is subtracted from its count first, so earlier traffic - cannot mask a shortfall. ``info["line_counts"]`` stays raw (before the subtraction). + ``expected_requests`` is known, compare it against the census -- the complete records + of the role that wrote the most, summed over that role's files (see ``_census``) -- + and report a shortfall. Each file's ``leading_lines_to_skip`` entry is subtracted from + its count first, so earlier traffic cannot mask a shortfall. ``info["line_counts"]`` + stays raw (before the subtraction). With ``wait_for_expected_requests``, the files also need a census of at least ``expected_requests`` before they count as settled, and an empty directory is polled instead of returned at once. On timeout, ``info["stable"]`` is False and the caller - decides. The connection warmup's snapshot uses this mode because it starts as soon as - the warmup client exits. At that point the servers may not have written their last - records, or even created their files. + decides. The harness snapshots its warmup traffic (the connection warmup and the + warmup pass) in this mode, because the snapshot starts as soon as the last warmup + client exits. At that point the servers may not have written their last records, or + even created their files. ``sleep`` / ``monotonic`` are injected so the unit tests can drive this without wall time. @@ -811,12 +853,25 @@ def snapshot() -> Tuple[List[str], Tuple[Tuple[str, Optional[int]], ...]]: sizes.append((os.path.basename(path), None)) return paths, tuple(sizes) + # A file's role never changes once it has a record, so each file is read once for it. + roles: Dict[str, str] = {} + + def census(paths: List[str], line_counts: Mapping[str, int]) -> int: + for path in paths: + name = os.path.basename(path) + if roles.get(name, "empty") == "empty": + try: + roles[name] = _census_role(path) + except OSError: + roles[name] = "empty" + return _census(line_counts, leading_lines_to_skip, roles) + def holds_expected_requests(paths: List[str]) -> bool: try: line_counts = {os.path.basename(path): _count_lines(path) for path in paths} except OSError: return False - return _census(line_counts, leading_lines_to_skip) >= expected_requests + return census(paths, line_counts) >= expected_requests started = monotonic() paths, fingerprint = snapshot() @@ -863,12 +918,12 @@ def holds_expected_requests(paths: List[str]) -> bool: warnings.append(f"{os.path.basename(path)}: could not count lines ({exc})") if expected_requests and line_counts: - census = _census(line_counts, leading_lines_to_skip) - if census < expected_requests: + num_recorded = census(paths, line_counts) + if num_recorded < expected_requests: warnings.append( - f"the largest perf_metrics file holds {census} complete record(s) but the " - f"client issued {expected_requests} request(s); the breakdown covers a " - "subset of the run" + f"the server role with the most perf_metrics records holds {num_recorded} " + f"complete record(s) but the client issued {expected_requests} request(s); " + "the breakdown covers a subset of the run" ) info = { @@ -931,8 +986,8 @@ def main() -> int: default=None, metavar="JSON", help="per-file leading lines to ignore, as a JSON object of basename -> count; for " - "a disagg e2e run, pass the dict the harness logged after 'Time breakdown: excluded " - "connection-warmup records from' (with --drop-warmup-request) to match what it " + "a run with warmup traffic, pass the dict the harness logged after 'Time breakdown: " + "excluded warmup records from' (with --drop-warmup-request) to match what it " "uploads", ) args = parser.parse_args() diff --git a/tests/integration/defs/perf/warmup_pass.py b/tests/integration/defs/perf/warmup_pass.py index bd87f70c02ba..5b67e6fe9ed2 100644 --- a/tests/integration/defs/perf/warmup_pass.py +++ b/tests/integration/defs/perf/warmup_pass.py @@ -35,7 +35,10 @@ """ import os -from typing import List, Mapping, Optional, Sequence +import re +import subprocess +import time +from typing import Callable, List, Mapping, Optional, Sequence WARMUP_PASS_ENV = "TRTLLM_PERF_SANITY_WARMUP_PASS" @@ -53,6 +56,9 @@ _FLAGS_WITH_REPLACED_VALUE = (_NUM_PROMPTS_FLAG, _SEED_FLAG, _SAVE_TIME_BREAKDOWN_FLAG) # benchmark_serving's --seed default. _BENCHMARK_SERVING_DEFAULT_SEED = 0 +# benchmark_serving's "Failed requests:" summary line, its only failure signal: +# benchmark_serving exits 0 even when requests fail. +_FAILED_REQUESTS_RE = re.compile(r"^Failed requests:\s+(\d+)", re.MULTILINE) def warmup_pass_enabled(environ: Optional[Mapping[str, str]] = None) -> bool: @@ -126,3 +132,69 @@ def build_warmup_pass_cmd(measured_cmd: Sequence[str], num_requests: int) -> Lis str(measured_seed + 1), _NO_TEST_INPUT_FLAG, ] + + +def count_failed_requests(output: str) -> int: + """Return the failed requests a ``benchmark_serving`` output reports, or 0.""" + match = _FAILED_REQUESTS_RE.search(output) + return int(match.group(1)) if match else 0 + + +def send_warmup_pass( + warmup_cmd: List[str], + num_requests: int, + run_benchmark: Callable[[List[str]], str], + log_path: str, + timeout: float, + log_info: Callable[[str], None] = print, + log_warning: Callable[[str], None] = print, +) -> int: + """Send the warmup pass; return how many of its requests succeeded. + + If only some requests fail, the pass still warmed the shapes it reached, so it + logs a warning and the measured client decides whether the test passes. + + Args: + warmup_cmd: The ``build_warmup_pass_cmd`` command, with the server's host + and port. + num_requests: Requests the command sends. + run_benchmark: Runs a command and returns its output. Raises + ``subprocess.CalledProcessError`` if the command exits non-zero, and + ``subprocess.TimeoutExpired`` after ``timeout`` seconds. + log_path: Where ``run_benchmark`` writes the output; named in messages. + timeout: The time limit ``run_benchmark`` applies, in seconds. + log_info: Receives progress messages. + log_warning: Receives the message about a partial failure. + + Returns: + The number of warmup requests that succeeded. + + Raises: + RuntimeError: The command exited non-zero, timed out, or had every + request fail. + """ + log_info(f"Warmup pass: sending {num_requests} requests. cmd is {warmup_cmd}") + start = time.monotonic() + try: + output = run_benchmark(warmup_cmd) + except subprocess.CalledProcessError as e: + raise RuntimeError( + f"Warmup pass: benchmark_serving exited with code {e.returncode}. See {log_path}" + ) from e + except subprocess.TimeoutExpired as e: + raise RuntimeError( + f"Warmup pass: {num_requests} requests did not finish within {timeout:.0f}s. " + f"See {log_path}" + ) from e + elapsed = time.monotonic() - start + failed_count = count_failed_requests(output) + if failed_count >= num_requests: + raise RuntimeError(f"Warmup pass: all {num_requests} requests failed. See {log_path}") + if failed_count: + log_warning( + f"Warmup pass: {failed_count} of {num_requests} requests failed ({elapsed:.0f}s). " + f"See {log_path}" + ) + else: + log_info(f"Warmup pass: {num_requests} requests done in {elapsed:.0f}s") + return num_requests - failed_count diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass.py b/tests/unittest/others/test_perf_sanity_warmup_pass.py index 468b052af3a8..325e40dfb622 100644 --- a/tests/unittest/others/test_perf_sanity_warmup_pass.py +++ b/tests/unittest/others/test_perf_sanity_warmup_pass.py @@ -12,28 +12,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The perf-sanity warmup pass must send the measured traffic shape and nothing else. +"""The perf-sanity warmup pass must send the measured traffic shape and stay unmeasured. A pass whose command drifts from the measured one warms the wrong prefill shapes, and one that reuses the measured prompts lets measured requests hit its KV cache blocks; -either way the measured numbers move without any runtime change. +either way the measured numbers move without any runtime change. The servers also +record the pass's requests in the perf_metrics JSONLs the d_tb_* time breakdown reads, +so those records must be left out, however many files they are split across. """ import importlib.util +import json +import subprocess from pathlib import Path -from typing import List, Sequence +from typing import Callable, List, Sequence, Tuple import pytest pytestmark = pytest.mark.cpu_only -# Loaded by path: importing it through the ``defs`` package pulls in torch. -_SPEC = importlib.util.spec_from_file_location( - "warmup_pass", - Path(__file__).resolve().parents[2] / "integration/defs/perf/warmup_pass.py", -) -warmup_pass = importlib.util.module_from_spec(_SPEC) -_SPEC.loader.exec_module(warmup_pass) + +def _load_perf_module(name: str): + # Loaded by path: importing it through the ``defs`` package pulls in torch. + spec = importlib.util.spec_from_file_location( + name, Path(__file__).resolve().parents[2] / f"integration/defs/perf/{name}.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +warmup_pass = _load_perf_module("warmup_pass") +tbm = _load_perf_module("time_breakdown_metrics") _BENCHMARK_SERVING = ["python", "-m", "tensorrt_llm.serve.scripts.benchmark_serving"] @@ -156,3 +166,162 @@ def test_warmup_pass_env_switch(environ: dict, enabled: bool) -> None: ) def test_warmup_pass_modes(benchmark_mode, applies: bool) -> None: assert warmup_pass.warmup_pass_applies(benchmark_mode) is applies + + +def _send_warmup_pass( + run_benchmark: Callable[[List[str]], str], num_requests: int = 4 +) -> Tuple[int, List[str]]: + """Return send_warmup_pass's successful requests and the warnings it logged.""" + warnings: List[str] = [] + succeeded = warmup_pass.send_warmup_pass( + ["benchmark_serving"], + num_requests, + run_benchmark, + log_path="warmup-pass.log", + timeout=60.0, + log_info=lambda _message: None, + log_warning=warnings.append, + ) + return succeeded, warnings + + +def _benchmark_summary(successful: int, failed: int) -> str: + return f"Successful requests: {successful}\nFailed requests: {failed}\n" + + +@pytest.mark.parametrize( + ("output", "expected_succeeded", "warns"), + [ + (_benchmark_summary(4, 0), 4, False), + # The pass still warmed the shapes it reached; the measured client decides. + (_benchmark_summary(3, 1), 3, True), + ], +) +def test_warmup_pass_returns_its_successful_requests( + output: str, expected_succeeded: int, warns: bool +) -> None: + sent: List[List[str]] = [] + + def run_benchmark(cmd: List[str]) -> str: + sent.append(cmd) + return output + + succeeded, warnings = _send_warmup_pass(run_benchmark) + + assert sent == [["benchmark_serving"]] + assert succeeded == expected_succeeded + assert bool(warnings) is warns + + +@pytest.mark.parametrize( + "error", + [ + subprocess.CalledProcessError(1, ["benchmark_serving"]), + subprocess.TimeoutExpired(["benchmark_serving"], 60.0), + ], +) +def test_warmup_pass_client_error_fails_the_test(error: subprocess.SubprocessError) -> None: + def run_benchmark(_cmd: List[str]) -> str: + raise error + + with pytest.raises(RuntimeError) as excinfo: + _send_warmup_pass(run_benchmark) + assert excinfo.value.__cause__ is error + + +def test_warmup_pass_with_every_request_failed_fails_the_test() -> None: + with pytest.raises(RuntimeError): + _send_warmup_pass(lambda _cmd: _benchmark_summary(0, 4)) + + +class _FakeClock: + """Drives wait_for_perf_metrics_files without wall time; calls on_sleep at each poll.""" + + def __init__(self, on_sleep: Callable[[], None] = lambda: None): + self.now = 0.0 + self._on_sleep = on_sleep + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.now += seconds + self._on_sleep() + + +def _aggregated_server_record(arrival: float, ctx_queue_s: float) -> str: + """One JSONL line of an aggregated server (a ctx_only case), with the fields read here.""" + timing = {"arrival_time": arrival, "first_scheduled_time": arrival + ctx_queue_s} + return json.dumps({"perf_metrics": {"timing_metrics": timing}}) + "\n" + + +def test_warmup_records_split_across_frontend_files_are_skipped(tmp_path: Path) -> None: + # A ctx_only server with two frontend processes (num_serve_frontends: 2). Each + # process writes its own file, so each file holds only its share of the warmup + # records, and the second has not written its last one yet. + frontend_files = [tmp_path / f"perf_metrics-server-node0-{pid}-x.jsonl" for pid in (11, 12)] + warmup_records = [_aggregated_server_record(i * 0.001, ctx_queue_s=5.0) for i in range(8)] + frontend_files[0].write_text("".join(warmup_records[0::2])) + frontend_files[1].write_text("".join(warmup_records[1:-1:2])) + late_records = warmup_records[-1:] + + def flush_late_records() -> None: + with frontend_files[1].open("a") as handle: + handle.writelines(late_records) + late_records.clear() + + clock = _FakeClock(on_sleep=flush_late_records) + _, after_warmup = tbm.wait_for_perf_metrics_files( + str(tmp_path), + expected_requests=len(warmup_records), + wait_for_expected_requests=True, + stable_seconds=0, + sleep=clock.sleep, + monotonic=clock.monotonic, + ) + assert after_warmup["stable"] + # Does nothing if the wait saw the late record. Otherwise the record lands after the + # snapshot, and the assertion below catches the leak. + flush_late_records() + + for frontend_idx, path in enumerate(frontend_files): + with path.open("a") as handle: + handle.writelines( + _aggregated_server_record(100.0 + frontend_idx + i * 0.01, ctx_queue_s=0.1) + for i in range(30) + ) + metrics, _ = tbm.compute_time_breakdown_metrics( + [str(path) for path in frontend_files], + "ctx_only", + leading_lines_to_skip=after_warmup["line_counts"], + ) + + assert metrics["d_tb_ctx_queue_mean"] == pytest.approx(100.0) + + +def test_warmup_census_does_not_add_up_server_roles(tmp_path: Path) -> None: + # A disagg e2e run: the disagg server, the ctx worker and the gen worker each record + # every request. Each has written two of the four warmup records so far, which is + # not all four, however many files hold them. + first_records = { + "perf_metrics-disagg-node0-1-x.jsonl": {"ctx_perf_metrics": {}, "gen_perf_metrics": {}}, + "perf_metrics-server-node1-2-x.jsonl": { + "time_breakdown_metrics": {"ctx_chunk_metrics": []} + }, + "perf_metrics-server-node2-3-x.jsonl": {"time_breakdown_metrics": {"step_metrics": []}}, + } + for name, record in first_records.items(): + (tmp_path / name).write_text((json.dumps(record) + "\n") * 2) + + clock = _FakeClock() + _, info = tbm.wait_for_perf_metrics_files( + str(tmp_path), + expected_requests=4, + wait_for_expected_requests=True, + stable_seconds=0, + timeout_seconds=5.0, + sleep=clock.sleep, + monotonic=clock.monotonic, + ) + + assert not info["stable"] From 8afc2df1726328295108916515d39d48028c585d Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:08:50 -0700 Subject: [PATCH 3/7] [None][test] Add the warmup pass to the single-node perf tests (test_perf.py) test_perf.py (QA's single-node perf tests and l0_perf) now warms each benchmark up with one un-measured wave of its own traffic, as perf-sanity does: - bench: trtllm-bench throughput measures in the process that loads the model, so its own --warmup is raised to one wave (the concurrency, or min(reqs, maxbs) without one), never below what it is today. Those requests come from the measured dataset, so a config that enables KV cache block reuse keeps today's warmup. - serve: before each measured client, the harness sends one wave with the client's benchmark_serving command and other prompts of the same lengths. A real-text client gets its own prompt file with the articles after the measured ones. The pass drops the measured client's result-file flags, so it writes none of its output files. The switch is renamed to TRTLLM_PERF_WARMUP_PASS, since it now controls both harnesses. With it off, every QA and l0_perf id builds the same commands as before. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/perf/README.md | 6 + .../defs/perf/README_test_perf_sanity.md | 12 +- tests/integration/defs/perf/test_perf.py | 103 ++++++++++++++++-- tests/integration/defs/perf/utils.py | 58 ++++++++-- tests/integration/defs/perf/warmup_pass.py | 87 +++++++++++---- .../others/test_perf_sanity_warmup_pass.py | 57 +++++++++- 6 files changed, 272 insertions(+), 51 deletions(-) diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md index 342b6cad5860..9476e8fed79a 100644 --- a/tests/integration/defs/perf/README.md +++ b/tests/integration/defs/perf/README.md @@ -32,6 +32,12 @@ If a CI run fails `sanity_perf_check.py`, it will upload a patch file as an arti This patch will only update the metrics that had a difference which was over the accepted threshold. The patch will also remove/add metrics according to the removed or added tests. +## Warmup pass +Before measuring, `test_perf.py` sends one un-measured wave of each benchmark's own traffic: as many requests as the client keeps in flight (its concurrency, or `min(reqs, maxbs)` without one). How many prompts arrive together and how they are chunked against `max_num_tokens` are only known once traffic arrives, so the server's startup warmup cannot compile every JIT kernel those prefill shapes need; the pass compiles them outside the measured window. +- `bench` cases raise `trtllm-bench throughput --warmup` to one wave, never below what it is without the pass (trtllm-bench's default of 2, or 2 per GPU on multi-GPU cases). trtllm-bench runs those requests in the measured process and reports none of them. They are drawn from the measured dataset, so a case whose config enables KV cache block reuse keeps the smaller warmup. +- `serve` cases send it before each measured client: the client's own `benchmark_serving` command with other prompts of the same lengths (`--seed` + 1; a real-text client gets the articles after the measured ones) and none of its output files. Its log is `benchmark-warmup-pass-*.log` in the working directory. The test fails if it exits non-zero or every request fails. +- `TRTLLM_PERF_WARMUP_PASS=0` (also `false`, `no`, `off`) turns it off, e.g. to reproduce a JIT stall inside the measured window. `test_perf_sanity.py` reads the same switch. + ## Running locally Given a `target_perf_csv_path` you can compare it to another perf csv file. First make sure you install the dependencies: diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 996f5c203bc0..634831b814da 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -203,7 +203,7 @@ AGG_CONFIG_FOLDER=my/custom/agg DISAGG_CONFIG_FOLDER=my/custom/disagg \ python jenkins/scripts/perf/local/submit.py ... ``` -`TRTLLM_PERF_SANITY_WARMUP_PASS=0`, read by `test_perf_sanity.py` itself, skips the +`TRTLLM_PERF_WARMUP_PASS=0`, read by `test_perf_sanity.py` itself, skips the warmup pass; see [Warmup pass](#warmup-pass). ## Configuration Files @@ -353,18 +353,20 @@ command is the measured client's own `benchmark_serving` command with: - `--num-prompts` set to the client's concurrency (one wave); - `--seed` set to the measured seed + 1, so the pass sends other prompts of the same lengths: other random token ids, or another shuffle of the same dataset file; -- `--no-test-input` added and `--save-request-time-breakdown` dropped. +- `--no-test-input` added, and the output-file flags (`--save-request-time-breakdown`, + `--save-result`, `--save-detailed`, `--result-dir`, `--result-filename`) dropped. Notes: - Not run for `gen_only` / `gen_only_no_context` (they measure decode only, and the gen server's benchmark fill gate opens once, so warmup requests would open it early), for AgentX clients (they run their own warmup) or for accuracy-only clients. In e2e it runs after the connection warmup. -- `TRTLLM_PERF_SANITY_WARMUP_PASS=0` (also `false`, `no`, `off`) skips it, e.g. to +- `TRTLLM_PERF_WARMUP_PASS=0` (also `false`, `no`, `off`) skips it, e.g. to reproduce a JIT stall inside the measured window. CI leaves it on. With `jenkins/scripts/perf/local/submit.py`, set - `EXTRA_CONTAINER_EXPORTS='TRTLLM_PERF_SANITY_WARMUP_PASS=0'`; it reaches the - aggregated ranks and every disaggregated role. + `EXTRA_CONTAINER_EXPORTS='TRTLLM_PERF_WARMUP_PASS=0'`; it reaches the + aggregated ranks and every disaggregated role. The same switch controls the + single-node perf tests' warmup pass (`test_perf.py`; see `README.md`). - Client log: `trtllm-benchmark-warmup-pass...log`, not parsed for metrics. The test fails if the pass's benchmark_serving exits non-zero, times out or has every request fail; if only some fail, it logs a warning and the measured diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 02419708bd7f..cba79a73acfc 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -24,7 +24,7 @@ import os import re import shutil -from typing import Dict, List, NamedTuple +from typing import Dict, List, NamedTuple, Optional import pytest import yaml @@ -40,6 +40,8 @@ from .utils import (AbstractPerfScriptTestClass, PerfBenchScriptTestCmds, PerfMetricType, PerfServeScriptTestCmds, generate_test_nodes) +from .warmup_pass import (WarmupPass, build_warmup_pass_cmd, one_wave_requests, + warmup_pass_enabled) if not hasattr(re, "Pattern"): re.Pattern = type(re.compile("")) @@ -49,6 +51,9 @@ TIMING_CACHE_DIR = os.environ.get("TIMING_CACHE_DIR", "") +# trtllm-bench throughput's --warmup default. +TRTLLM_BENCH_DEFAULT_WARMUP = 2 + # Models served with the Nemotron reasoning/tool parsers, a long max model len # and a longer server-start timeout. "nano-v3" and "nemotron-v3" are two # registered names for the same reasoning parser, so Lightning belongs here too. @@ -1115,6 +1120,29 @@ def get_prepare_data_command(self, engine_dir, input_len, return data_cmd + def _get_bench_warmup_requests(self) -> int: + """Return trtllm-bench's --warmup for this case, or 0 for its default. + + trtllm-bench runs these requests in its own process before the measured + ones, at the same concurrency, and reports none of them. With the warmup + pass on, they are one wave of the measured traffic (see warmup_pass.py), + and never fewer than without it. They are drawn from the measured + dataset, though, so a case whose config reuses KV cache blocks keeps the + fewer requests it has without the pass: otherwise its measured requests + would hit the warmup's cached prefixes. + """ + without_pass = 2 * self._config.num_gpus if self._config.num_gpus > 1 else 0 + if not warmup_pass_enabled(): + return without_pass + kv_cache_config = self._get_model_yaml_config().get( + 'kv_cache_config') or {} + if kv_cache_config.get('enable_block_reuse'): + return without_pass + wave = one_wave_requests(self._config.concurrency, + self._config.num_reqs, + self._config.max_batch_size) + return max(without_pass or TRTLLM_BENCH_DEFAULT_WARMUP, wave) + def get_trtllm_bench_command(self, engine_dir): model_dir = self.get_trtllm_bench_model() model_name = self._config.model_name @@ -1149,8 +1177,9 @@ def get_trtllm_bench_command(self, engine_dir): benchmark_cmd += [f"--pp={self._config.pp_size}"] if self._config.streaming == "streaming": benchmark_cmd += [f"--streaming"] - if self._config.num_gpus > 1: - benchmark_cmd += [f"--warmup={2 * self._config.num_gpus}"] + warmup_requests = self._get_bench_warmup_requests() + if warmup_requests: + benchmark_cmd += [f"--warmup={warmup_requests}"] # Add extra-llm-api-config.yml for the PyTorch backend. if self._config.backend == "pytorch": @@ -1227,9 +1256,20 @@ def get_trtllm_serve_server_command(self, engine_dir): return server_cmd - def generate_trtllm_custom_dataset(self, dst_dataset_path: str, - input_len: int, output_len: int, - dataset_source: str): + def generate_trtllm_custom_dataset(self, + dst_dataset_path: str, + input_len: int, + output_len: int, + dataset_source: str, + num_requests: Optional[int] = None, + skip_requests: int = 0): + """Write a real-text dataset of fixed-length prompts. + + Args: + num_requests: Samples to write; the case's request count if None. + skip_requests: Leading usable articles to skip, so that another + file can hold prompts this one does not. + """ # Currently only support cnn_dailymail dataset source. if dataset_source != "cnn_dailymail": raise ValueError( @@ -1260,8 +1300,9 @@ def generate_trtllm_custom_dataset(self, dst_dataset_path: str, if not os.path.exists(os.path.dirname(dst_dataset_path)): os.makedirs(os.path.dirname(dst_dataset_path), exist_ok=True) - num_reqs = self._config.num_reqs + num_reqs = self._config.num_reqs if num_requests is None else num_requests req_count = 0 + num_skipped = 0 with open(dst_dataset_path, "w", encoding="utf-8") as f: for req in dataset: article = req.get("article") @@ -1281,6 +1322,9 @@ def generate_trtllm_custom_dataset(self, dst_dataset_path: str, prompt_ids = (prompt_ids * repeats)[:input_len] elif len(prompt_ids) > input_len: prompt_ids = prompt_ids[:input_len] + if num_skipped < skip_requests: + num_skipped += 1 + continue prompt_text = tokenizer.decode(prompt_ids, skip_special_tokens=False) if self._config.model_name in DEEPSEEK_V41_MODELS: @@ -1416,6 +1460,44 @@ def get_trtllm_serve_client_command(self, client_cmd += ["--trust-remote-code"] return client_cmd + def _get_serve_warmup_pass(self, engine_dir: str, input_len: int, + output_len: int, + client_cmd: List[str]) -> Optional[WarmupPass]: + """Return the warmup pass before a serve client, or None; see warmup_pass.py. + + A real-text client gets a prompt file of its own, with the articles + after the measured ones. The measured file holds exactly the measured + prompts, so another shuffle of it would resend them, and KV cache block + reuse would then serve their prefixes to the measured requests. + """ + if not warmup_pass_enabled(): + return None + num_requests = one_wave_requests(self._config.concurrency, + self._config.num_reqs, + self._config.max_batch_size) + if num_requests < 1: + return None + dataset_source = SPEC_DEC_REAL_DATASET_MODELS.get( + self._config.model_name) + if dataset_source is not None: + warmup_dataset_path = os.path.join( + engine_dir, + f"dataset_custom_{input_len}_{output_len}_warmup_pass.jsonl") + self.generate_trtllm_custom_dataset( + warmup_dataset_path, + input_len, + output_len, + dataset_source=dataset_source, + num_requests=num_requests, + skip_requests=self._config.num_reqs) + client_cmd = self.get_trtllm_serve_client_command( + engine_dir, + input_len, + output_len, + real_dataset_path=warmup_dataset_path) + return WarmupPass(build_warmup_pass_cmd(client_cmd, num_requests), + num_requests) + def get_commands(self): num_gpus = self._config.num_gpus @@ -1428,6 +1510,7 @@ def get_commands(self): if self._config.runtime == "serve": server_cmd = self.get_trtllm_serve_server_command(engine_dir) client_cmds = [] + warmup_passes = [] data_cmds = [] for bs in self._config.batch_sizes: for len_idx, input_len in enumerate(self._config.input_lens): @@ -1453,6 +1536,9 @@ def get_commands(self): output_len, real_dataset_path=real_dataset_path) client_cmds.append(client_cmd) + warmup_passes.append( + self._get_serve_warmup_pass(engine_dir, input_len, + output_len, client_cmd)) server_env = os.environ.copy() if self._config.model_name.startswith("qwen3.8_flash_next_"): server_env["TRTLLM_QWEN4_EXP_PLE_HOST_OFFLOAD"] = "1" @@ -1490,7 +1576,8 @@ def get_commands(self): client_cmds=client_cmds, data_cmds=data_cmds, server_env=server_env, - server_timeout=server_timeout) + server_timeout=server_timeout, + warmup_passes=warmup_passes) # PyTorch loads model checkpoints directly. build_cmd = [] diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py index d5b3cab6e4fc..6f6b37ec9759 100644 --- a/tests/integration/defs/perf/utils.py +++ b/tests/integration/defs/perf/utils.py @@ -32,7 +32,7 @@ import pytest from defs.trt_test_alternative import (Popen, cleanup_process_tree, print_error, - print_info) + print_info, print_warning) from ..common import get_trt_llm_lib_dir from ..local_venv import PythonVenvRunnerImpl @@ -41,6 +41,7 @@ write_gpu_monitoring_no_test_results, write_yaml) from .gpu_clock_lock import GPUClockLock, InvalidGPUMonitoringResultError from .session_data_writer import SessionDataWriter +from .warmup_pass import WarmupPass, send_warmup_pass def clean_myelin_time(log: str): @@ -371,14 +372,20 @@ def _wait_for_server_ready(url: str, class PerfServeScriptTestCmds: """Commands for serve runtime perf tests (server-client model).""" - def __init__(self, server_cmd: List[str], client_cmds: List[List[str]], - data_cmds: List[List[str]], server_env: Dict[str, str], - server_timeout: int): + def __init__(self, + server_cmd: List[str], + client_cmds: List[List[str]], + data_cmds: List[List[str]], + server_env: Dict[str, str], + server_timeout: int, + warmup_passes: Optional[List[Optional[WarmupPass]]] = None): self.server_cmd = server_cmd self.client_cmds = client_cmds self.data_cmds = data_cmds self.server_env = server_env self.server_timeout = server_timeout + # Per client, the warmup pass to send before it, or None. + self.warmup_passes = warmup_passes or [None] * len(client_cmds) self._server_proc = None self._server_log_path = None @@ -462,11 +469,12 @@ def run_cmd(self, cmd_idx: int, venv) -> str: # instead of an empty string. output = self.get_server_log_content() else: - client_cmd = self.client_cmds[cmd_idx - 1 - len(self.data_cmds)] - client_cmd_with_port = client_cmd + [ - "--host", self._host, "--port", - str(self._port) - ] + client_idx = cmd_idx - 1 - len(self.data_cmds) + host_port = ["--host", self._host, "--port", str(self._port)] + warmup_pass = self.warmup_passes[client_idx] + if warmup_pass is not None: + self._send_warmup_pass(warmup_pass, host_port) + client_cmd_with_port = self.client_cmds[client_idx] + host_port print_info( f"Running benchmark client: {' '.join(client_cmd_with_port)}") output = _run_command_with_captured_output(client_cmd_with_port, @@ -474,6 +482,38 @@ def run_cmd(self, cmd_idx: int, venv) -> str: os.environ)) return output + @staticmethod + def _send_warmup_pass(warmup_pass: WarmupPass, + host_port: List[str]) -> None: + """Send a client's warmup pass; its output goes to a log of its own.""" + # Unique per pass: cluster jobs can share a working directory. + with tempfile.NamedTemporaryFile(mode="w", + prefix="benchmark-warmup-pass-", + suffix=".log", + dir=os.getcwd(), + delete=False) as log_file: + log_path = log_file.name + + def run_benchmark(cmd: List[str]) -> str: + try: + output = _run_command_with_captured_output(cmd, + env=copy.deepcopy( + os.environ)) + except subprocess.CalledProcessError as e: + with open(log_path, "wb") as log: + log.write(e.stdout or b"") + raise + with open(log_path, "w", encoding="utf-8") as log: + log.write(output) + return output + + send_warmup_pass(warmup_pass.cmd + host_port, + warmup_pass.num_requests, + run_benchmark, + log_path=log_path, + log_info=print_info, + log_warning=print_warning) + def get_cmd_str(self, cmd_idx) -> str: if cmd_idx <= len(self.data_cmds) - 1: return " ".join(self.data_cmds[cmd_idx]) diff --git a/tests/integration/defs/perf/warmup_pass.py b/tests/integration/defs/perf/warmup_pass.py index 5b67e6fe9ed2..28a990bbd0c5 100644 --- a/tests/integration/defs/perf/warmup_pass.py +++ b/tests/integration/defs/perf/warmup_pass.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The un-measured warmup pass perf-sanity sends before each measured client. +"""The un-measured warmup pass the perf tests send before each measured client. The server's own startup warmup cannot reach every prefill shape a benchmark produces: how many prompts arrive together, how they are chunked against @@ -20,15 +20,21 @@ once real traffic arrives. JIT kernels keyed on those shapes would otherwise compile inside the first measured requests. The warmup pass sends that traffic once beforehand: the measured client's own ``benchmark_serving`` command, cut to -one wave of ``concurrency`` requests with different prompts. Nothing it prints is -parsed or uploaded. +one wave of requests with different prompts. Nothing it prints is parsed or +uploaded. -It runs for plain aggregated cases, ``ctx_only`` and disagg ``e2e``, whose -measured window contains prefill. The ``gen_only`` modes never get one: they -measure decode only, and their generation server's benchmark fill gate opens -once, so warmup requests would open it before the measured burst arrives. +perf-sanity (``test_perf_sanity.py``) runs it for plain aggregated cases, +``ctx_only`` and disagg ``e2e``, whose measured window contains prefill. The +``gen_only`` modes never get one: they measure decode only, and their generation +server's benchmark fill gate opens once, so warmup requests would open it before +the measured burst arrives. -``TRTLLM_PERF_SANITY_WARMUP_PASS=0`` turns it off, e.g. to reproduce a JIT stall +The single-node perf tests (``test_perf.py``) run it for both runtimes. A +``serve`` case sends it before each measured client. A ``bench`` case measures +inside the ``trtllm-bench throughput`` process that loads the model, so its pass +is that command's own ``--warmup`` requests, raised to one wave. + +``TRTLLM_PERF_WARMUP_PASS=0`` turns it off in both, e.g. to reproduce a JIT stall the pass would otherwise absorb. Stdlib-only, so unit tests can load it without torch. @@ -38,9 +44,9 @@ import re import subprocess import time -from typing import Callable, List, Mapping, Optional, Sequence +from typing import Callable, List, Mapping, NamedTuple, Optional, Sequence -WARMUP_PASS_ENV = "TRTLLM_PERF_SANITY_WARMUP_PASS" +WARMUP_PASS_ENV = "TRTLLM_PERF_WARMUP_PASS" # Benchmark modes that get a warmup pass. A plain aggregated case has no # benchmark mode, which the runner carries as "". @@ -51,9 +57,17 @@ _NUM_PROMPTS_FLAG = "--num-prompts" _SEED_FLAG = "--seed" _NO_TEST_INPUT_FLAG = "--no-test-input" -_SAVE_TIME_BREAKDOWN_FLAG = "--save-request-time-breakdown" -# Flags whose value the warmup pass replaces or drops. -_FLAGS_WITH_REPLACED_VALUE = (_NUM_PROMPTS_FLAG, _SEED_FLAG, _SAVE_TIME_BREAKDOWN_FLAG) +# Flags whose value the warmup pass replaces or drops: the request count, the seed, +# and where the measured client writes its time breakdown and result file. +_FLAGS_WITH_REPLACED_VALUE = ( + _NUM_PROMPTS_FLAG, + _SEED_FLAG, + "--save-request-time-breakdown", + "--result-dir", + "--result-filename", +) +# Flags without a value that the warmup pass drops; it adds --no-test-input back. +_DROPPED_FLAGS = (_NO_TEST_INPUT_FLAG, "--save-result", "--save-detailed") # benchmark_serving's --seed default. _BENCHMARK_SERVING_DEFAULT_SEED = 0 # benchmark_serving's "Failed requests:" summary line, its only failure signal: @@ -64,8 +78,8 @@ def warmup_pass_enabled(environ: Optional[Mapping[str, str]] = None) -> bool: """Return whether the warmup pass is on. - It is on unless ``TRTLLM_PERF_SANITY_WARMUP_PASS`` is ``0``, ``false``, ``no`` - or ``off`` (any case). + It is on unless ``TRTLLM_PERF_WARMUP_PASS`` is ``0``, ``false``, ``no`` or + ``off`` (any case). Args: environ: Environment to read; ``os.environ`` when None. @@ -84,6 +98,30 @@ def warmup_pass_applies(benchmark_mode: Optional[str]) -> bool: return (benchmark_mode or "") in WARMUP_PASS_MODES +def one_wave_requests(concurrency: int, num_requests: int, max_batch_size: int) -> int: + """Return the requests in one wave of a benchmark: those it keeps in flight at once. + + That is the client's concurrency limit. A client without one sends every + request at once, so the server's batch size caps the wave. A wave never holds + more requests than the benchmark sends. + + Args: + concurrency: The client's concurrency limit; 0 or less for none. + num_requests: Requests the benchmark sends; 0 or less if unknown. + max_batch_size: The server's maximum batch size. + """ + wave = concurrency if concurrency > 0 else max_batch_size + return min(wave, num_requests) if num_requests > 0 else wave + + +class WarmupPass(NamedTuple): + """The warmup pass to send before one measured client.""" + + # The build_warmup_pass_cmd command, without host and port. + cmd: List[str] + num_requests: int + + def build_warmup_pass_cmd(measured_cmd: Sequence[str], num_requests: int) -> List[str]: """Derive the warmup pass's ``benchmark_serving`` command from the measured one. @@ -96,10 +134,12 @@ def build_warmup_pass_cmd(measured_cmd: Sequence[str], num_requests: int) -> Lis prompts with the same length distribution: other random token ids, or another shuffle of the same dataset file. * ``--no-test-input`` is added, so no single test request goes first. - * ``--save-request-time-breakdown`` and its value are dropped. + * ``--save-request-time-breakdown``, ``--save-result``, ``--save-detailed``, + ``--result-dir`` and ``--result-filename`` are dropped with their values, + so the pass writes none of the measured client's output files. The measured command must pass each flag and its value as separate items, as - the runner builds it. The caller adds host and port. + the runners build it. The caller adds host and port. Args: measured_cmd: The measured client's ``benchmark_serving`` command. @@ -122,7 +162,7 @@ def build_warmup_pass_cmd(measured_cmd: Sequence[str], num_requests: int) -> Lis if arg == _SEED_FLAG and value is not None: measured_seed = int(value) continue - if arg == _NO_TEST_INPUT_FLAG: + if arg in _DROPPED_FLAGS: continue warmup_cmd.append(arg) return warmup_cmd + [ @@ -145,7 +185,7 @@ def send_warmup_pass( num_requests: int, run_benchmark: Callable[[List[str]], str], log_path: str, - timeout: float, + timeout: Optional[float] = None, log_info: Callable[[str], None] = print, log_warning: Callable[[str], None] = print, ) -> int: @@ -160,9 +200,10 @@ def send_warmup_pass( num_requests: Requests the command sends. run_benchmark: Runs a command and returns its output. Raises ``subprocess.CalledProcessError`` if the command exits non-zero, and - ``subprocess.TimeoutExpired`` after ``timeout`` seconds. + ``subprocess.TimeoutExpired`` if it applies a time limit that runs out. log_path: Where ``run_benchmark`` writes the output; named in messages. - timeout: The time limit ``run_benchmark`` applies, in seconds. + timeout: The time limit ``run_benchmark`` applies, in seconds; None if it + applies none. log_info: Receives progress messages. log_warning: Receives the message about a partial failure. @@ -182,9 +223,9 @@ def send_warmup_pass( f"Warmup pass: benchmark_serving exited with code {e.returncode}. See {log_path}" ) from e except subprocess.TimeoutExpired as e: + limit = f" within {timeout:.0f}s" if timeout is not None else "" raise RuntimeError( - f"Warmup pass: {num_requests} requests did not finish within {timeout:.0f}s. " - f"See {log_path}" + f"Warmup pass: {num_requests} requests did not finish{limit}. See {log_path}" ) from e elapsed = time.monotonic() - start failed_count = count_failed_requests(output) diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass.py b/tests/unittest/others/test_perf_sanity_warmup_pass.py index 325e40dfb622..b7f466523999 100644 --- a/tests/unittest/others/test_perf_sanity_warmup_pass.py +++ b/tests/unittest/others/test_perf_sanity_warmup_pass.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""The perf-sanity warmup pass must send the measured traffic shape and stay unmeasured. +"""The perf tests' warmup pass must send the measured traffic shape and stay unmeasured. A pass whose command drifts from the measured one warms the wrong prefill shapes, and one that reuses the measured prompts lets measured requests hit its KV cache blocks; @@ -73,9 +73,29 @@ def _load_perf_module(name: str): "--random-input-len", "7386", "--random-output-len", "1", "--random-range-ratio", "0.0", "--backend", "openai", "--non-streaming", ] +# Shaped like test_perf.py's get_trtllm_serve_client_command for a real-text client +# that saves its detailed results. +_SINGLE_NODE_SERVE_REAL_TEXT_CMD = _BENCHMARK_SERVING + [ + "--model", "/models/m", "--tokenizer", "/models/m", + "--num-prompts", "64", "--tokenize-on-client", "--no-test-input", + "--percentile-metrics", "ttft,tpot,itl,e2el", + "--save-result", "--save-detailed", "--result-dir", "/engine", + "--result-filename", "result_8192_1024.json", + "--dataset-name", "trtllm_custom", "--dataset-path", "/engine/dataset_custom.jsonl", + "--max-concurrency", "16", "--trust-remote-code", +] # fmt: on -_REPLACED_FLAGS_WITH_VALUE = ("--num-prompts", "--seed", "--save-request-time-breakdown") +# The flags the warmup pass may change, and the output files the measured client writes. +_REPLACED_FLAGS_WITH_VALUE = ( + "--num-prompts", + "--seed", + "--save-request-time-breakdown", + "--result-dir", + "--result-filename", +) +_REPLACED_FLAGS = ("--no-test-input", "--save-result", "--save-detailed") +_MEASURED_OUTPUT_FILES = ("/out/time_breakdown", "/engine", "result_8192_1024.json") def _value_of(cmd: Sequence[str], flag: str) -> str: @@ -89,13 +109,19 @@ def _traffic_shape(cmd: Sequence[str]) -> List[str]: for arg in args: if arg in _REPLACED_FLAGS_WITH_VALUE: next(args, None) - elif arg != "--no-test-input": + elif arg not in _REPLACED_FLAGS: shape.append(arg) return shape @pytest.mark.parametrize( - "measured_cmd", [_AGGR_RANDOM_CMD, _E2E_DATASET_TIME_BREAKDOWN_CMD, _CTX_ONLY_CMD] + "measured_cmd", + [ + _AGGR_RANDOM_CMD, + _E2E_DATASET_TIME_BREAKDOWN_CMD, + _CTX_ONLY_CMD, + _SINGLE_NODE_SERVE_REAL_TEXT_CMD, + ], ) def test_warmup_pass_cmd_keeps_the_measured_traffic_shape(measured_cmd: List[str]) -> None: concurrency = int(_value_of(measured_cmd, "--max-concurrency")) @@ -104,8 +130,27 @@ def test_warmup_pass_cmd_keeps_the_measured_traffic_shape(measured_cmd: List[str assert _traffic_shape(warmup_cmd) == _traffic_shape(measured_cmd) assert _value_of(warmup_cmd, "--num-prompts") == str(concurrency) assert warmup_cmd.count("--no-test-input") == 1 - assert "--save-request-time-breakdown" not in warmup_cmd - assert "/out/time_breakdown" not in warmup_cmd + # The pass writes none of the measured client's output files. + assert not set(warmup_cmd) & {"--save-result", "--save-detailed", *_MEASURED_OUTPUT_FILES} + + +@pytest.mark.parametrize( + ("concurrency", "num_requests", "max_batch_size", "expected_wave"), + [ + (4, 40, 512, 4), + # Never more than the benchmark sends. + (16, 8, 512, 8), + # A client without a concurrency limit sends everything at once; the + # server's batch size caps what is in flight. + (-1, 512, 128, 128), + (-1, 32, 512, 32), + (-1, 0, 64, 64), + ], +) +def test_one_wave_is_what_the_client_keeps_in_flight( + concurrency: int, num_requests: int, max_batch_size: int, expected_wave: int +) -> None: + assert warmup_pass.one_wave_requests(concurrency, num_requests, max_batch_size) == expected_wave @pytest.mark.parametrize( From 8ff11015818f0a8b9c846b04a7feae9f2f9e47fc Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:07:07 -0700 Subject: [PATCH 4/7] [None][test] Give only the first measured client a warmup pass while the time breakdown is collected The time breakdown leaves out only the records written before a server's first measured client, and pools every later record into all of that server's rows, so a later client's warmup pass would count as measured. With the breakdown on, warmup_pass_plan now gives the later clients no pass and logs a warning; the uploaded l_warmup_pass_requests reports the plan. Without the breakdown, every client keeps its pass. Adds harness tests that drive AggrTestCmds.run_cmd and DisaggTestCmds.run_cmd with a fake server and benchmark_serving: two clients with the breakdown on (warmup records queue 5 s, measured ones 0.1 s, and the pooled ctx_queue mean must stay at the measured value), two clients without it, and the gen-log windows around a second client's pass. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../defs/perf/README_test_perf_sanity.md | 4 + .../integration/defs/perf/test_perf_sanity.py | 99 +++++-- tests/integration/defs/perf/warmup_pass.py | 24 ++ .../others/test_perf_sanity_warmup_pass.py | 19 ++ .../test_perf_sanity_warmup_pass_harness.py | 279 ++++++++++++++++++ 5 files changed, 400 insertions(+), 25 deletions(-) create mode 100644 tests/unittest/others/test_perf_sanity_warmup_pass_harness.py diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 634831b814da..1ef5f0da58b4 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -374,6 +374,10 @@ Notes: - The pass's perf_metrics records are excluded from the time breakdown (see below), and in e2e its iterations come before the measured client's gen-log window, so neither `d_tb_*` nor the gen-worker device step time includes it. +- With the time breakdown on, only a server's first measured client gets a pass. The + breakdown pools every record after the first measured client's start, so it cannot + leave a later pass out; the later clients log a warning and upload + `l_warmup_pass_requests` 0. CI's time_breakdown lanes have one client per server. - Uploaded rows carry `l_warmup_pass_requests` (0 when no pass ran). It is not a match key. - With a dataset file, a few of the pass's prompts can coincide with the measured diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 4a320d38e1ff..556021dd2644 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -60,6 +60,7 @@ send_warmup_pass, warmup_pass_applies, warmup_pass_enabled, + warmup_pass_plan, ) SUPPORTED_GPU_MAPPING = { @@ -1813,6 +1814,40 @@ def warmup_pass_requests_for( return client_config.warmup_pass_requests +def warmup_pass_plan_for( + client_configs: List[Optional["ClientConfig"]], + benchmark_mode: Optional[str], + time_breakdown: bool, +) -> List[int]: + """Warmup pass size for each client of one server, in run order; 0 for none. + + The run loops send these passes and the uploaded rows report them. While the + time breakdown is collected, only the first measured client gets one; see + warmup_pass_plan. + + Args: + client_configs: The server's client configs in run order; None entries are + measured clients without a config. + benchmark_mode: The case's benchmark mode; None or "" for a plain + aggregated case. + time_breakdown: Whether the time breakdown is collected for the server. + """ + return warmup_pass_plan( + [warmup_pass_requests_for(config, benchmark_mode) for config in client_configs], + [config is None or not config.only_run_accuracy for config in client_configs], + time_breakdown, + ) + + +def warn_skipped_warmup_pass(client_idx: int) -> None: + """Log that a client gets no warmup pass because the time breakdown is on.""" + print_warning( + f"Warmup pass: none before client {client_idx}. With the time breakdown on, only " + "the first measured client gets one: the breakdown cannot leave out the records " + "of a later pass, so they would count as measured." + ) + + def run_warmup_pass( client_cmd: List[str], client_config: "ClientConfig", @@ -2831,17 +2866,21 @@ def run_cmd(self, server_idx: int) -> List[str]: ) write_startup_observations(self.test_output_dir, server_idx, [observation]) + client_cmds = self.client_cmds[server_idx] + loop_configs = [ + client_configs[client_idx] if client_idx < len(client_configs) else None + for client_idx in range(len(client_cmds)) + ] + warmup_plan = warmup_pass_plan_for( + loop_configs, self.benchmark_mode, collect_time_breakdown + ) # Run all clients for this server - for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]): - client_config = ( - client_configs[client_idx] if client_idx < len(client_configs) else None - ) + for client_idx, client_cmd in enumerate(client_cmds): + client_config = loop_configs[client_idx] only_run_accuracy = bool(client_config and client_config.only_run_accuracy) if not only_run_accuracy: - num_warmup_requests = warmup_pass_requests_for( - client_config, self.benchmark_mode - ) + num_warmup_requests = warmup_plan[client_idx] if num_warmup_requests: num_unmeasured_requests += run_warmup_pass( client_cmd, @@ -2852,10 +2891,11 @@ def run_cmd(self, server_idx: int) -> List[str]: warmup_pass_log_path(self.test_output_dir, server_idx, client_idx), timeout=min(self.timeout, DEFAULT_TIMEOUT), ) + elif warmup_pass_requests_for(client_config, self.benchmark_mode): + warn_skipped_warmup_pass(client_idx) # The time breakdown skips only leading lines, so count them - # once, before the first measured client. CI's time_breakdown - # lanes have a single client; with more, a later client's pass - # joins the breakdown. + # once, before the first measured client. warmup_plan gives + # the later clients no pass while it is collected. if ( collect_time_breakdown and num_unmeasured_requests @@ -3834,18 +3874,26 @@ def run_cmd(self, server_idx: int) -> List[str]: ) client_configs = self.client_configs.get(server_idx, []) + client_cmds = self.client_cmds[server_idx] + loop_configs = [ + client_configs[client_idx] if client_idx < len(client_configs) else None + for client_idx in range(len(client_cmds)) + ] + warmup_plan = warmup_pass_plan_for( + loop_configs, benchmark_mode_for_idx, collect_time_breakdown + ) # Run all clients for this server - for client_idx, client_cmd in enumerate(self.client_cmds[server_idx]): - client_config = ( - client_configs[client_idx] if client_idx < len(client_configs) else None - ) + for client_idx, client_cmd in enumerate(client_cmds): + client_config = loop_configs[client_idx] only_run_accuracy = bool(client_config and client_config.only_run_accuracy) if not only_run_accuracy: - num_warmup_requests = warmup_pass_requests_for( + num_warmup_requests = warmup_plan[client_idx] + if not num_warmup_requests and warmup_pass_requests_for( client_config, benchmark_mode_for_idx - ) + ): + warn_skipped_warmup_pass(client_idx) if num_warmup_requests: # The pass runs between two clients' gen-log windows, # so neither one's device step time counts its @@ -3865,9 +3913,8 @@ def run_cmd(self, server_idx: int) -> List[str]: timeout=min(self.timeout, DEFAULT_TIMEOUT), ) # The time breakdown skips only leading lines, so count - # them once, before the first measured client. CI's - # time_breakdown lanes have a single client; with more, a - # later client's pass joins the breakdown. + # them once, before the first measured client. warmup_plan + # gives the later clients no pass while it is collected. if ( collect_time_breakdown and num_unmeasured_requests @@ -5379,6 +5426,9 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: if len(server_perf_results) != len(client_configs): cmd_idx += len(client_configs) continue + warmup_plan = warmup_pass_plan_for( + client_configs, self.benchmark_mode, self.time_breakdown + ) for client_idx, client_config in enumerate(client_configs): client_config_dict = client_config.to_db_data() @@ -5399,9 +5449,7 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: } new_data.update(server_config_dict) new_data.update(client_config_dict) - new_data["l_warmup_pass_requests"] = warmup_pass_requests_for( - client_config, self.benchmark_mode - ) + new_data["l_warmup_pass_requests"] = warmup_plan[client_idx] # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{server_config.name}-{client_config.name}" @@ -5455,6 +5503,9 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: if len(server_perf_results) != len(client_configs): cmd_idx += len(client_configs) continue + warmup_plan = warmup_pass_plan_for( + client_configs, disagg_config.benchmark_mode, self.time_breakdown + ) for client_idx, client_config in enumerate(client_configs): # Skip if metrics missing @@ -5491,9 +5542,7 @@ def add_dict_prefix(config_dict: dict, prefix_name: str) -> dict: if num_gen_servers > 0: new_data.update(gen_server_config_dict) new_data.update(client_config_dict) - new_data["l_warmup_pass_requests"] = warmup_pass_requests_for( - client_config, disagg_config.benchmark_mode - ) + new_data["l_warmup_pass_requests"] = warmup_plan[client_idx] # Add test_case_name for convenient filtering on OpenSearch new_data["s_test_case_name"] = f"{disagg_config.name}-{client_config.name}" diff --git a/tests/integration/defs/perf/warmup_pass.py b/tests/integration/defs/perf/warmup_pass.py index 28a990bbd0c5..a6c68e5fbc60 100644 --- a/tests/integration/defs/perf/warmup_pass.py +++ b/tests/integration/defs/perf/warmup_pass.py @@ -98,6 +98,30 @@ def warmup_pass_applies(benchmark_mode: Optional[str]) -> bool: return (benchmark_mode or "") in WARMUP_PASS_MODES +def warmup_pass_plan( + eligible_requests: Sequence[int], measured: Sequence[bool], time_breakdown: bool +) -> List[int]: + """Return each client's warmup pass size for one server, in run order (0: none). + + The time breakdown leaves out only the records written before the server's + first measured client, and pools every later record. So while it is collected, + only that first measured client gets a pass: a later pass's records would count + as measured for every client of the server. Without it, each client keeps its + own pass. + + Args: + eligible_requests: Each client's pass size on its own; 0 if it gets none. + measured: Whether each client is measured; False for accuracy-only clients. + time_breakdown: Whether the time breakdown is collected for the server. + """ + plan: List[int] = [] + after_first_measured = False + for num_requests, is_measured in zip(eligible_requests, measured): + plan.append(0 if time_breakdown and after_first_measured else num_requests) + after_first_measured = after_first_measured or is_measured + return plan + + def one_wave_requests(concurrency: int, num_requests: int, max_batch_size: int) -> int: """Return the requests in one wave of a benchmark: those it keeps in flight at once. diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass.py b/tests/unittest/others/test_perf_sanity_warmup_pass.py index b7f466523999..fba294e49396 100644 --- a/tests/unittest/others/test_perf_sanity_warmup_pass.py +++ b/tests/unittest/others/test_perf_sanity_warmup_pass.py @@ -213,6 +213,25 @@ def test_warmup_pass_modes(benchmark_mode, applies: bool) -> None: assert warmup_pass.warmup_pass_applies(benchmark_mode) is applies +@pytest.mark.parametrize( + ("eligible", "measured", "time_breakdown", "expected_plan"), + [ + # Without the breakdown, every client keeps its pass. + ([4, 8], [True, True], False, [4, 8]), + # With it, only the first measured client gets one. + ([4, 8], [True, True], True, [4, 0]), + # An accuracy-only client is not measured, so it does not count as first. + ([0, 4, 8], [False, True, True], True, [0, 4, 0]), + # A first client that gets no pass still makes later passes unsafe. + ([0, 8], [True, True], True, [0, 0]), + ], +) +def test_warmup_pass_plan( + eligible: List[int], measured: List[bool], time_breakdown: bool, expected_plan: List[int] +) -> None: + assert warmup_pass.warmup_pass_plan(eligible, measured, time_breakdown) == expected_plan + + def _send_warmup_pass( run_benchmark: Callable[[List[str]], str], num_requests: int = 4 ) -> Tuple[int, List[str]]: diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py b/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py new file mode 100644 index 000000000000..b9bc0bbdb103 --- /dev/null +++ b/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The perf-sanity client loops must keep every warmup pass out of what they measure. + +Drives AggrTestCmds.run_cmd and DisaggTestCmds.run_cmd with fake servers and a fake +benchmark_serving that appends one perf_metrics record per request, so the warmup +pass, the time breakdown and the gen-log windows run as in a real lane. +""" + +import importlib +import json +import os +import re +import sys +import types +from pathlib import Path +from typing import List, Tuple + +import pytest + +pytestmark = pytest.mark.cpu_only + +_TESTS_DIR = Path(__file__).resolve().parents[2] +_DEFS_DIR = _TESTS_DIR / "integration" / "defs" + +# The time breakdown's ctx_queue span, per kind of request the fake client sends. +_WARMUP_PASS_QUEUE_S = 5.0 +_MEASURED_QUEUE_S = 0.1 + + +def _module(name: str, **attrs) -> types.ModuleType: + module = types.ModuleType(name) + module.__dict__.update(attrs) + return module + + +def _package(name: str, path: Path) -> types.ModuleType: + package = _module(name) + package.__path__ = [str(path)] + return package + + +@pytest.fixture +def tps(monkeypatch): + """test_perf_sanity.py, with the imports that need a CI test environment stubbed.""" + noop = lambda *args, **kwargs: None # noqa: E731 + stubs = { + "defs": _package("defs", _DEFS_DIR), + "defs.perf": _package("defs.perf", _DEFS_DIR / "perf"), + "defs.common": _module("defs.common", wait_for_reported_addr=noop), + "defs.trt_test_alternative": _module( + "defs.trt_test_alternative", print_info=print, print_warning=print + ), + "defs.conftest": _module( + "defs.conftest", + get_llm_root=lambda: str(_TESTS_DIR.parent), + llm_models_root=lambda: "/models", + ), + "defs.perf.perf_regression_utils": _module( + "defs.perf.perf_regression_utils", + _percentile=lambda values, percentile: 0.0, + get_job_info=dict, + process_and_upload_test_results=noop, + ), + "test_common": _package("test_common", _TESTS_DIR / "test_common"), + "test_common.error_utils": _module("test_common.error_utils", report_error=noop), + "test_common.http_utils": _module( + "test_common.http_utils", fail_if_proc_died=noop, wait_for_endpoint_ready=noop + ), + "test_common.perf_sanity_matching": _module( + "test_common.perf_sanity_matching", get_test_case_match_keys=list + ), + } + # Set aside any copy imported under the real packages, so the import below binds + # to the stubs. + for name in [n for n in sys.modules if n.startswith("defs.perf.")]: + monkeypatch.delitem(sys.modules, name) + loaded_before = set(sys.modules) + for name, module in stubs.items(): + monkeypatch.setitem(sys.modules, name, module) + module = importlib.import_module("defs.perf.test_perf_sanity") + # A fake server process, ready at once, that the runner can stop. + monkeypatch.setattr(module.subprocess, "Popen", _FakeServer) + monkeypatch.setattr(module, "wait_for_reported_addr", lambda *a, **k: ("localhost", 8000)) + monkeypatch.setattr(module, "wait_for_endpoint_ready", noop) + monkeypatch.setattr(module, "collect_startup_observation", lambda *a, **k: {}) + monkeypatch.setattr(module, "write_startup_observations", noop) + monkeypatch.setattr(module, "PERF_METRICS_SETTLE_SECONDS", 0.0) + monkeypatch.setattr(module, "PERF_METRICS_SETTLE_TIMEOUT", 10.0) + yield module + for name in set(sys.modules) - loaded_before: + if name.split(".")[0] in ("defs", "test_common"): + sys.modules.pop(name, None) + + +class _FakeServer: + def __init__(self, *args, **kwargs): + self.returncode = None + + def poll(self): + return self.returncode + + def terminate(self): + self.returncode = 0 + + def kill(self): + self.returncode = -9 + + def wait(self, timeout=None): + return self.returncode + + +class _FakeBenchmarkServing: + """Records each run and appends one perf_metrics record per request it sends. + + A warmup pass's requests queue for _WARMUP_PASS_QUEUE_S and measured ones for + _MEASURED_QUEUE_S, so any pass record in the breakdown moves its mean. + """ + + def __init__(self, perf_metrics_dir: str): + # No directory: the time breakdown is off and the server writes no records. + self.jsonl = ( + os.path.join(perf_metrics_dir, "perf_metrics-server-node0-1-x.jsonl") + if perf_metrics_dir + else "" + ) + self.runs: List[Tuple[str, int]] = [] + self._arrival = 0.0 + + def __call__(self, cmd, env, log_path, timeout=0): + num_requests = int(cmd[cmd.index("--num-prompts") + 1]) + is_warmup_pass = "warmup-pass" in os.path.basename(log_path) + self.runs.append((os.path.basename(log_path), num_requests)) + if self.jsonl: + queue_s = _WARMUP_PASS_QUEUE_S if is_warmup_pass else _MEASURED_QUEUE_S + # Without --no-test-input, a single test request goes first. + num_records = num_requests + (0 if "--no-test-input" in cmd else 1) + with open(self.jsonl, "a") as handle: + for _ in range(num_records): + self._arrival += 1.0 + timing = { + "arrival_time": self._arrival, + "first_scheduled_time": self._arrival + queue_s, + } + handle.write(json.dumps({"perf_metrics": {"timing_metrics": timing}}) + "\n") + return f"Successful requests: {num_requests}\nFailed requests: 0\n" + + +def _clients(tps, concurrencies, warmup: bool): + return [ + tps.ClientConfig( + {"name": f"con{c}", "concurrency": c, "iterations": 2, "isl": 1024, "osl": 1}, + "m", + warmup=warmup, + ) + for c in concurrencies + ] + + +def _run_aggr(tps, tmp_path, monkeypatch, benchmark_mode: str, time_breakdown: bool): + perf_metrics_dir = tmp_path / "perf_metrics" if time_breakdown else None + if perf_metrics_dir: + perf_metrics_dir.mkdir() + client = _FakeBenchmarkServing(str(perf_metrics_dir) if perf_metrics_dir else "") + monkeypatch.setattr(tps, "_run_benchmark_with_log", client) + configs = _clients(tps, (4, 8), warmup=benchmark_mode == "ctx_only") + cmds = tps.AggrTestCmds( + server_cmds=[["trtllm-serve", "m"]], + client_cmds={0: [config.to_cmd() for config in configs]}, + timeout=tps.DEFAULT_TIMEOUT, + output_dir=str(tmp_path), + test_output_dir=str(tmp_path), + client_configs={0: configs}, + model_name="m", + server_configs=[], + perf_metrics_output_dir=str(perf_metrics_dir) if perf_metrics_dir else "", + benchmark_mode=benchmark_mode, + ) + return cmds.run_cmd(0), client.runs + + +def test_time_breakdown_leaves_out_every_warmup_pass_with_two_clients( + tps, tmp_path, monkeypatch +) -> None: + outputs, runs = _run_aggr(tps, tmp_path, monkeypatch, "ctx_only", time_breakdown=True) + + # Both clients' rows carry the pooled breakdown: no warmup pass record in it. + for output in outputs: + ctx_queue_mean = re.search(r"Time Breakdown ctx_queue mean \(ms\): ([\d.]+)", output) + assert ctx_queue_mean, output + assert float(ctx_queue_mean.group(1)) == pytest.approx(_MEASURED_QUEUE_S * 1000) + # That takes giving only the first client a pass: the breakdown leaves out only + # the records written before it. + assert [name for name, _ in runs] == [ + "trtllm-benchmark-warmup-pass.0.0.log", + "trtllm-benchmark.0.0.log", + "trtllm-benchmark.0.1.log", + ] + + +def test_every_client_gets_a_warmup_pass_without_the_time_breakdown( + tps, tmp_path, monkeypatch +) -> None: + _, runs = _run_aggr(tps, tmp_path, monkeypatch, "", time_breakdown=False) + + assert runs == [ + ("trtllm-benchmark-warmup-pass.0.0.log", 4), + ("trtllm-benchmark.0.0.log", 8), + ("trtllm-benchmark-warmup-pass.0.1.log", 8), + ("trtllm-benchmark.0.1.log", 16), + ] + + +def test_gen_log_windows_leave_out_the_next_clients_warmup_pass(tps, tmp_path, monkeypatch) -> None: + client = _FakeBenchmarkServing("") + monkeypatch.setattr(tps, "_run_benchmark_with_log", client) + # Each snapshot's value is 100 x the benchmark runs before it, so it names its place. + monkeypatch.setattr(tps, "gen_worker_log_sizes", lambda *a: [100 * len(client.runs)]) + monkeypatch.setattr(tps, "expected_gen_only_ngen", lambda *a, **k: None) + windows = [] + cls = tps.DisaggTestCmds + monkeypatch.setattr(cls, "_get_disagg_server_hostname_and_port", lambda self, i: ("h", 1)) + monkeypatch.setattr(cls, "_collect_worker_startup_observations", lambda self, i: []) + monkeypatch.setattr(cls, "_run_connection_warmup", lambda self, *a: 0) + monkeypatch.setattr( + cls, + "_append_gen_worker_device_step_time", + lambda self, pending, outputs: windows.extend(pending), + ) + configs = _clients(tps, (4, 8), warmup=True) + cmds = cls( + server_cmds=[(["ctx"], ["gen"], ["disagg"])], + client_cmds={0: [config.to_cmd() for config in configs]}, + timeout=tps.DEFAULT_TIMEOUT, + hostname="bench-host", + disagg_serving_type="BENCHMARK", + num_ctx_servers=1, + num_gen_servers=1, + output_dir=str(tmp_path), + test_output_dir=str(tmp_path), + client_configs={0: configs}, + server_configs=[ + (None, None, types.SimpleNamespace(benchmark_mode="e2e", server_env_var="")) + ], + ) + cmds.run_cmd(0) + + assert [name for name, _ in client.runs] == [ + "trtllm-benchmark-warmup-pass.0.0.log", + "trtllm-benchmark.0.0.log", + "trtllm-benchmark-warmup-pass.0.1.log", + "trtllm-benchmark.0.1.log", + ] + # Client 0's window opens after its pass and closes before client 1's pass, which + # comes before client 1's window; the last window reads to the end of the log. + assert [(w["start_offsets"], w["end_offsets"]) for w in windows] == [ + ([100], [200]), + ([300], None), + ] + + +def test_warmup_plan_matches_the_run_loops(tps) -> None: + configs = _clients(tps, (4, 8), warmup=True) + + assert tps.warmup_pass_plan_for(configs, "e2e", time_breakdown=True) == [4, 0] + assert tps.warmup_pass_plan_for(configs, "e2e", time_breakdown=False) == [4, 8] + assert tps.warmup_pass_plan_for(configs, "gen_only", time_breakdown=False) == [0, 0] From 954e173aa829f00c9ea64b5a0f8d05dd17ab8f9f Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:15:17 -0700 Subject: [PATCH 5/7] [None][test] Skip a real-text client's warmup pass when the source has no articles to spare A real-text serve client's warmup pass reads the articles after the measured ones. If the source has fewer than one more wave, building that file raised and failed the case before the benchmark started. The shortfall now raises NotEnoughSamplesError (a ValueError), and the pass catches only that: the client then gets no pass and a warning, as before the pass existed. Any other dataset error still fails the case. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/perf/test_perf.py | 31 +++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index cba79a73acfc..e2ee8cbd1f48 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -54,6 +54,11 @@ # trtllm-bench throughput's --warmup default. TRTLLM_BENCH_DEFAULT_WARMUP = 2 + +class NotEnoughSamplesError(ValueError): + """A real-text source holds fewer usable articles than a dataset file asks for.""" + + # Models served with the Nemotron reasoning/tool parsers, a long max model len # and a longer server-start timeout. "nano-v3" and "nemotron-v3" are two # registered names for the same reasoning parser, so Lightning belongs here too. @@ -1366,7 +1371,7 @@ def generate_trtllm_custom_dataset(self, break if req_count < num_reqs: - raise ValueError( + raise NotEnoughSamplesError( f"Cannot sample enough requests from cnn_dailymail: requested={num_reqs}, sampled={req_count}" ) print_info(f"Generated {req_count} samples from {dataset_source} to " @@ -1468,7 +1473,8 @@ def _get_serve_warmup_pass(self, engine_dir: str, input_len: int, A real-text client gets a prompt file of its own, with the articles after the measured ones. The measured file holds exactly the measured prompts, so another shuffle of it would resend them, and KV cache block - reuse would then serve their prefixes to the measured requests. + reuse would then serve their prefixes to the measured requests. If the + source has no articles to spare, the client gets no pass. """ if not warmup_pass_enabled(): return None @@ -1483,13 +1489,20 @@ def _get_serve_warmup_pass(self, engine_dir: str, input_len: int, warmup_dataset_path = os.path.join( engine_dir, f"dataset_custom_{input_len}_{output_len}_warmup_pass.jsonl") - self.generate_trtllm_custom_dataset( - warmup_dataset_path, - input_len, - output_len, - dataset_source=dataset_source, - num_requests=num_requests, - skip_requests=self._config.num_reqs) + try: + self.generate_trtllm_custom_dataset( + warmup_dataset_path, + input_len, + output_len, + dataset_source=dataset_source, + num_requests=num_requests, + skip_requests=self._config.num_reqs) + except NotEnoughSamplesError as error: + print_warning( + f"Warmup pass: none for this client, since {dataset_source} " + f"has no {num_requests} articles to spare after the " + f"{self._config.num_reqs} measured ones: {error}") + return None client_cmd = self.get_trtllm_serve_client_command( engine_dir, input_len, From 72c0e21bb8fe4badf0cebcd47e9aebec9edb740c Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:25:36 -0700 Subject: [PATCH 6/7] [None][test] Test the real-text warmup pass fallback with a finite source Drives MultiMetricPerfTest's dataset generator with a fake cnn_dailymail and tokenizer. A source with exactly the measured request count gives no pass and a warning, and leaves the measured file and count intact; a source with spare articles gives a pass over the articles after the measured ones; any other dataset error still fails the case. The first test fails on the previous head, which aborted the benchmark there. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- .../others/test_perf_real_text_warmup_pass.py | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 tests/unittest/others/test_perf_real_text_warmup_pass.py diff --git a/tests/unittest/others/test_perf_real_text_warmup_pass.py b/tests/unittest/others/test_perf_real_text_warmup_pass.py new file mode 100644 index 000000000000..f160b6287300 --- /dev/null +++ b/tests/unittest/others/test_perf_real_text_warmup_pass.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A real-text serve case's warmup pass must use other prompts, or none, never fail the case. + +test_perf.py builds the pass's prompt file from the articles after the measured ones. +Drives MultiMetricPerfTest's dataset generator with a finite fake source and tokenizer. +""" + +import importlib +import json +import subprocess +import sys +import types +from pathlib import Path +from typing import List + +import pytest + +pytestmark = pytest.mark.cpu_only + +_TESTS_DIR = Path(__file__).resolve().parents[2] +_DEFS_DIR = _TESTS_DIR / "integration" / "defs" +# A real-text spec-dec model: 4 measured requests, waves of 2. +_TEST_ID = ( + "nemotron_3.5_lightning_30b_nvfp4_mtp-serve-pytorch-streaming-float4-maxbs:2-maxnt:8192" + "-input_output_len:128,16-reqs:4-con:2" +) +_INPUT_LEN, _OUTPUT_LEN = 128, 16 +_NUM_MEASURED = 4 + + +def _module(name: str, **attrs) -> types.ModuleType: + module = types.ModuleType(name) + module.__dict__.update(attrs) + return module + + +def _package(name: str, path: Path = None) -> types.ModuleType: + package = _module(name) + package.__path__ = [str(path)] if path else [] + return package + + +class _CharTokenizer: + """One token per character, so decoded prompts show which article they came from.""" + + def encode(self, text: str, add_special_tokens: bool = False) -> List[int]: + return [ord(c) for c in text] + + def decode(self, ids: List[int], skip_special_tokens: bool = False) -> str: + return "".join(chr(i) for i in ids) + + +class _Source: + """A cnn_dailymail stand-in with a fixed list of articles; counts its loads.""" + + def __init__(self, articles: List[str], error_on_load: int = 0, error: Exception = None): + self.articles = articles + self.loads = 0 + self.error_on_load = error_on_load + self.error = error + + def load_dataset(self, *args, **kwargs): + self.loads += 1 + if self.loads == self.error_on_load: + raise self.error + return [{"article": article} for article in self.articles] + + +@pytest.fixture +def perf(monkeypatch): + """test_perf.py with the imports that need a CI test environment stubbed. + + Returns the module and the list its print_warning appends to. + """ + warnings: List[str] = [] + noop = lambda *args, **kwargs: None # noqa: E731 + stubs = { + "defs": _package("defs", _DEFS_DIR), + "defs.perf": _package("defs.perf", _DEFS_DIR / "perf"), + "defs.trt_test_alternative": _module( + "defs.trt_test_alternative", + is_linux=lambda: True, + is_windows=lambda: False, + print_info=noop, + print_warning=warnings.append, + print_error=noop, + Popen=subprocess.Popen, + cleanup_process_tree=noop, + ), + "defs.conftest": _module( + "defs.conftest", + get_device_count=lambda: 8, + llm_models_root=lambda: "/models", + trt_environment=None, + ), + "defs.common": _module("defs.common", get_trt_llm_lib_dir=lambda venv: ""), + "defs.local_venv": _module("defs.local_venv", PythonVenvRunnerImpl=object), + "defs.test_list_parser": _module("defs.test_list_parser", parse_test_list=noop), + "defs.perf.gpu_clock_lock": _module( + "defs.perf.gpu_clock_lock", + GPUClockLock=object, + InvalidGPUMonitoringResultError=RuntimeError, + ), + "defs.perf.data_export": _module( + "defs.perf.data_export", + GPU_MONITORING_FORMAT_KEYS=[], + write_csv=noop, + write_gpu_monitoring_no_test_results=noop, + write_yaml=noop, + ), + "defs.perf.session_data_writer": _module( + "defs.perf.session_data_writer", SessionDataWriter=object + ), + "tensorrt_llm": _package("tensorrt_llm"), + "tensorrt_llm.llmapi": _package("tensorrt_llm.llmapi"), + "tensorrt_llm.llmapi.mpi_session": _module( + "tensorrt_llm.llmapi.mpi_session", get_mpi_world_size=lambda: 1 + ), + "transformers": _module( + "transformers", + AutoTokenizer=types.SimpleNamespace(from_pretrained=lambda *a, **k: _CharTokenizer()), + ), + } + # Set aside any copy imported under the real packages, so the import below binds + # to the stubs. + for name in [n for n in sys.modules if n.startswith("defs.perf.") or n == "allowed_configs"]: + monkeypatch.delitem(sys.modules, name) + loaded_before = set(sys.modules) + for name, module in stubs.items(): + monkeypatch.setitem(sys.modules, name, module) + # test_perf.py imports allowed_configs from its own directory (__extra_import_path__). + monkeypatch.syspath_prepend(str(_DEFS_DIR / "perf")) + module = importlib.import_module("defs.perf.test_perf") + yield module, warnings + for name in set(sys.modules) - loaded_before: + if name.split(".")[0] in ("defs", "allowed_configs"): + sys.modules.pop(name, None) + + +def _make_test(perf_module, tmp_path: Path): + test = perf_module.MultiMetricPerfTest(f"perf/test_perf.py::test_perf[{_TEST_ID}]") + test.set_runtime_configs(str(tmp_path), str(tmp_path), str(tmp_path), str(tmp_path / "cache")) + return test + + +def _use_source(monkeypatch, source: _Source) -> None: + monkeypatch.setitem( + sys.modules, "datasets", _module("datasets", load_dataset=source.load_dataset) + ) + + +def _prompts(path: Path) -> List[str]: + lines = path.read_text().splitlines() + return [json.loads(line)["input"]["messages"][1]["content"] for line in lines] + + +def _measured_dataset(perf_module, test, tmp_path: Path) -> Path: + measured = tmp_path / "dataset_custom.jsonl" + test.generate_trtllm_custom_dataset(str(measured), _INPUT_LEN, _OUTPUT_LEN, "cnn_dailymail") + return measured + + +def test_warmup_pass_is_skipped_when_the_source_has_no_articles_to_spare( + perf, tmp_path, monkeypatch +) -> None: + perf_module, warnings = perf + # Exactly the measured request count of usable articles; the empty one is skipped. + _use_source(monkeypatch, _Source([f"article {i}" for i in range(_NUM_MEASURED)] + [""])) + test = _make_test(perf_module, tmp_path) + measured = _measured_dataset(perf_module, test, tmp_path) + measured_before = measured.read_text() + + warmup_pass = test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) + + assert warmup_pass is None + assert any(warning.startswith("Warmup pass: none") for warning in warnings), warnings + assert measured.read_text() == measured_before + assert len(_prompts(measured)) == _NUM_MEASURED + + +def test_warmup_pass_reads_the_articles_after_the_measured_ones( + perf, tmp_path, monkeypatch +) -> None: + perf_module, warnings = perf + _use_source(monkeypatch, _Source([f"article {i}" for i in range(_NUM_MEASURED + 2)])) + test = _make_test(perf_module, tmp_path) + measured = _measured_dataset(perf_module, test, tmp_path) + + warmup_pass = test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) + + assert warmup_pass is not None and warmup_pass.num_requests == 2 + warmup_file = Path(warmup_pass.cmd[warmup_pass.cmd.index("--dataset-path") + 1]) + warmup_prompts = _prompts(warmup_file) + assert len(warmup_prompts) == 2 + assert not set(warmup_prompts) & set(_prompts(measured)) + assert all(f"article {i}" in p for i, p in zip(range(_NUM_MEASURED, 6), warmup_prompts)) + assert not warnings + + +@pytest.mark.parametrize( + "error", [OSError("cnn_dailymail shard unreadable"), ValueError("unknown split")] +) +def test_other_dataset_errors_still_fail_the_case(perf, tmp_path, monkeypatch, error) -> None: + perf_module, _ = perf + # The measured file loads the source first; loading it again for the pass fails. + source = _Source([f"article {i}" for i in range(8)], error_on_load=2, error=error) + _use_source(monkeypatch, source) + test = _make_test(perf_module, tmp_path) + _measured_dataset(perf_module, test, tmp_path) + + with pytest.raises(type(error)) as excinfo: + test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) + assert excinfo.value is error From 42a1c3ebdb28c4715f4172ecabab7d72c299e5c7 Mon Sep 17 00:00:00 2001 From: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:52:13 -0700 Subject: [PATCH 7/7] [None][test] Drop the warmup pass unit tests and shorten its README notes perf-sanity and test_perf.py are themselves test scripts, so the warmup pass needs no unit tests of its own. The README sections on the pass are shortened to what a reader needs to run or turn off the pass. Signed-off-by: chenfeiz0326 <203214996+chenfeiz0326@users.noreply.github.com> --- tests/integration/defs/perf/README.md | 5 +- .../defs/perf/README_test_perf_sanity.md | 59 +-- tests/integration/defs/perf/warmup_pass.py | 2 - .../others/test_perf_real_text_warmup_pass.py | 226 ---------- .../others/test_perf_sanity_warmup_pass.py | 391 ------------------ .../test_perf_sanity_warmup_pass_harness.py | 279 ------------- 6 files changed, 11 insertions(+), 951 deletions(-) delete mode 100644 tests/unittest/others/test_perf_real_text_warmup_pass.py delete mode 100644 tests/unittest/others/test_perf_sanity_warmup_pass.py delete mode 100644 tests/unittest/others/test_perf_sanity_warmup_pass_harness.py diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md index 9476e8fed79a..d857be62bcbf 100644 --- a/tests/integration/defs/perf/README.md +++ b/tests/integration/defs/perf/README.md @@ -33,10 +33,7 @@ If a CI run fails `sanity_perf_check.py`, it will upload a patch file as an arti This patch will only update the metrics that had a difference which was over the accepted threshold. The patch will also remove/add metrics according to the removed or added tests. ## Warmup pass -Before measuring, `test_perf.py` sends one un-measured wave of each benchmark's own traffic: as many requests as the client keeps in flight (its concurrency, or `min(reqs, maxbs)` without one). How many prompts arrive together and how they are chunked against `max_num_tokens` are only known once traffic arrives, so the server's startup warmup cannot compile every JIT kernel those prefill shapes need; the pass compiles them outside the measured window. -- `bench` cases raise `trtllm-bench throughput --warmup` to one wave, never below what it is without the pass (trtllm-bench's default of 2, or 2 per GPU on multi-GPU cases). trtllm-bench runs those requests in the measured process and reports none of them. They are drawn from the measured dataset, so a case whose config enables KV cache block reuse keeps the smaller warmup. -- `serve` cases send it before each measured client: the client's own `benchmark_serving` command with other prompts of the same lengths (`--seed` + 1; a real-text client gets the articles after the measured ones) and none of its output files. Its log is `benchmark-warmup-pass-*.log` in the working directory. The test fails if it exits non-zero or every request fails. -- `TRTLLM_PERF_WARMUP_PASS=0` (also `false`, `no`, `off`) turns it off, e.g. to reproduce a JIT stall inside the measured window. `test_perf_sanity.py` reads the same switch. +Before measuring, `test_perf.py` warms up with one un-measured wave of the benchmark's own traffic (its concurrency, or `min(reqs, maxbs)` without one), so JIT kernels for the real prefill shapes compile outside the measured window: `bench` cases raise `trtllm-bench --warmup` to that wave, and `serve` cases send it with other prompts before each client. `TRTLLM_PERF_WARMUP_PASS=0` turns it off. ## Running locally Given a `target_perf_csv_path` you can compare it to another perf csv file. diff --git a/tests/integration/defs/perf/README_test_perf_sanity.md b/tests/integration/defs/perf/README_test_perf_sanity.md index 1ef5f0da58b4..324d71200619 100644 --- a/tests/integration/defs/perf/README_test_perf_sanity.md +++ b/tests/integration/defs/perf/README_test_perf_sanity.md @@ -203,9 +203,6 @@ AGG_CONFIG_FOLDER=my/custom/agg DISAGG_CONFIG_FOLDER=my/custom/disagg \ python jenkins/scripts/perf/local/submit.py ... ``` -`TRTLLM_PERF_WARMUP_PASS=0`, read by `test_perf_sanity.py` itself, skips the -warmup pass; see [Warmup pass](#warmup-pass). - ## Configuration Files There are two modes for perf sanity tests: aggregated (aggr) and disaggregated (disagg). @@ -344,47 +341,13 @@ queue (nvbugs/6564113). Notes: ### Warmup pass Before each measured client of a normal aggregated, `ctx_only` or disaggregated `e2e` -test, the harness (`run_warmup_pass`, `warmup_pass.py`) sends one un-measured wave of -the same traffic. How many prompts arrive together, how they are chunked against -`max_num_tokens` and which decode tokens share a step with them are only known once -traffic arrives, so the server's startup warmup cannot compile every JIT kernel those -prefill shapes need. The pass makes them compile before the measured window. Its -command is the measured client's own `benchmark_serving` command with: -- `--num-prompts` set to the client's concurrency (one wave); -- `--seed` set to the measured seed + 1, so the pass sends other prompts of the same - lengths: other random token ids, or another shuffle of the same dataset file; -- `--no-test-input` added, and the output-file flags (`--save-request-time-breakdown`, - `--save-result`, `--save-detailed`, `--result-dir`, `--result-filename`) dropped. - -Notes: -- Not run for `gen_only` / `gen_only_no_context` (they measure decode only, and the - gen server's benchmark fill gate opens once, so warmup requests would open it - early), for AgentX clients (they run their own warmup) or for accuracy-only - clients. In e2e it runs after the connection warmup. -- `TRTLLM_PERF_WARMUP_PASS=0` (also `false`, `no`, `off`) skips it, e.g. to - reproduce a JIT stall inside the measured window. CI leaves it on. With - `jenkins/scripts/perf/local/submit.py`, set - `EXTRA_CONTAINER_EXPORTS='TRTLLM_PERF_WARMUP_PASS=0'`; it reaches the - aggregated ranks and every disaggregated role. The same switch controls the - single-node perf tests' warmup pass (`test_perf.py`; see `README.md`). -- Client log: `trtllm-benchmark-warmup-pass...log`, not parsed - for metrics. The test fails if the pass's benchmark_serving exits non-zero, times - out or has every request fail; if only some fail, it logs a warning and the measured - client runs. -- The pass's perf_metrics records are excluded from the time breakdown (see below), - and in e2e its iterations come before the measured client's gen-log window, so - neither `d_tb_*` nor the gen-worker device step time includes it. -- With the time breakdown on, only a server's first measured client gets a pass. The - breakdown pools every record after the first measured client's start, so it cannot - leave a later pass out; the later clients log a warning and upload - `l_warmup_pass_requests` 0. CI's time_breakdown lanes have one client per server. -- Uploaded rows carry `l_warmup_pass_requests` (0 when no pass ran). It is not a match - key. -- With a dataset file, a few of the pass's prompts can coincide with the measured - sample (about concurrency / dataset size of them). This only matters for configs - with `enable_block_reuse` on. -- It costs one wave, about 1 / `iterations` of the measured run. Iteration-based nsys - ranges count its iterations too. +test, the harness sends one un-measured wave (`concurrency` requests) of the client's +own `benchmark_serving` command with `--seed` + 1, so JIT kernels for the real prefill +shapes compile outside the measured window. `gen_only` modes, AgentX and accuracy-only +clients get none. Its log (`trtllm-benchmark-warmup-pass.*.log`) is not parsed, and its +records stay out of `d_tb_*` and the device step time; with the time breakdown on, only +the first measured client gets one. `TRTLLM_PERF_WARMUP_PASS=0` turns it off (locally, +via `EXTRA_CONTAINER_EXPORTS`). ### The optional instrumentation modifier @@ -447,11 +410,9 @@ to run after the writers are done. Only the **gen** workers announce that workers and the disaggregated server do not. `wait_for_perf_metrics_files` therefore polls the discovered set until its total size holds still for `PERF_METRICS_SETTLE_SECONDS`, bounded by `PERF_METRICS_SETTLE_TIMEOUT`, before -anything is read, and then compares the census against the client's `--num-prompts`. -The census counts the complete records of one server role (the disaggregated server, -the ctx workers, the gen workers, or an aggregated server), summed over that role's -files: a role writes one file per instance and per frontend process -(`num_serve_frontends`). Both a still-growing set at the timeout and a census +anything is read, and then compares the census (the complete records of one server +role, summed over its files) against the client's `--num-prompts`. Both a +still-growing set at the timeout and a census shortfall are logged as warnings, not failures — a nearly-complete file still yields usable statistics, and the numbers are not regression-gated. This matters because a truncated read is otherwise **invisible**: all 108 fields are still populated and the diff --git a/tests/integration/defs/perf/warmup_pass.py b/tests/integration/defs/perf/warmup_pass.py index a6c68e5fbc60..614827b93958 100644 --- a/tests/integration/defs/perf/warmup_pass.py +++ b/tests/integration/defs/perf/warmup_pass.py @@ -36,8 +36,6 @@ ``TRTLLM_PERF_WARMUP_PASS=0`` turns it off in both, e.g. to reproduce a JIT stall the pass would otherwise absorb. - -Stdlib-only, so unit tests can load it without torch. """ import os diff --git a/tests/unittest/others/test_perf_real_text_warmup_pass.py b/tests/unittest/others/test_perf_real_text_warmup_pass.py deleted file mode 100644 index f160b6287300..000000000000 --- a/tests/unittest/others/test_perf_real_text_warmup_pass.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A real-text serve case's warmup pass must use other prompts, or none, never fail the case. - -test_perf.py builds the pass's prompt file from the articles after the measured ones. -Drives MultiMetricPerfTest's dataset generator with a finite fake source and tokenizer. -""" - -import importlib -import json -import subprocess -import sys -import types -from pathlib import Path -from typing import List - -import pytest - -pytestmark = pytest.mark.cpu_only - -_TESTS_DIR = Path(__file__).resolve().parents[2] -_DEFS_DIR = _TESTS_DIR / "integration" / "defs" -# A real-text spec-dec model: 4 measured requests, waves of 2. -_TEST_ID = ( - "nemotron_3.5_lightning_30b_nvfp4_mtp-serve-pytorch-streaming-float4-maxbs:2-maxnt:8192" - "-input_output_len:128,16-reqs:4-con:2" -) -_INPUT_LEN, _OUTPUT_LEN = 128, 16 -_NUM_MEASURED = 4 - - -def _module(name: str, **attrs) -> types.ModuleType: - module = types.ModuleType(name) - module.__dict__.update(attrs) - return module - - -def _package(name: str, path: Path = None) -> types.ModuleType: - package = _module(name) - package.__path__ = [str(path)] if path else [] - return package - - -class _CharTokenizer: - """One token per character, so decoded prompts show which article they came from.""" - - def encode(self, text: str, add_special_tokens: bool = False) -> List[int]: - return [ord(c) for c in text] - - def decode(self, ids: List[int], skip_special_tokens: bool = False) -> str: - return "".join(chr(i) for i in ids) - - -class _Source: - """A cnn_dailymail stand-in with a fixed list of articles; counts its loads.""" - - def __init__(self, articles: List[str], error_on_load: int = 0, error: Exception = None): - self.articles = articles - self.loads = 0 - self.error_on_load = error_on_load - self.error = error - - def load_dataset(self, *args, **kwargs): - self.loads += 1 - if self.loads == self.error_on_load: - raise self.error - return [{"article": article} for article in self.articles] - - -@pytest.fixture -def perf(monkeypatch): - """test_perf.py with the imports that need a CI test environment stubbed. - - Returns the module and the list its print_warning appends to. - """ - warnings: List[str] = [] - noop = lambda *args, **kwargs: None # noqa: E731 - stubs = { - "defs": _package("defs", _DEFS_DIR), - "defs.perf": _package("defs.perf", _DEFS_DIR / "perf"), - "defs.trt_test_alternative": _module( - "defs.trt_test_alternative", - is_linux=lambda: True, - is_windows=lambda: False, - print_info=noop, - print_warning=warnings.append, - print_error=noop, - Popen=subprocess.Popen, - cleanup_process_tree=noop, - ), - "defs.conftest": _module( - "defs.conftest", - get_device_count=lambda: 8, - llm_models_root=lambda: "/models", - trt_environment=None, - ), - "defs.common": _module("defs.common", get_trt_llm_lib_dir=lambda venv: ""), - "defs.local_venv": _module("defs.local_venv", PythonVenvRunnerImpl=object), - "defs.test_list_parser": _module("defs.test_list_parser", parse_test_list=noop), - "defs.perf.gpu_clock_lock": _module( - "defs.perf.gpu_clock_lock", - GPUClockLock=object, - InvalidGPUMonitoringResultError=RuntimeError, - ), - "defs.perf.data_export": _module( - "defs.perf.data_export", - GPU_MONITORING_FORMAT_KEYS=[], - write_csv=noop, - write_gpu_monitoring_no_test_results=noop, - write_yaml=noop, - ), - "defs.perf.session_data_writer": _module( - "defs.perf.session_data_writer", SessionDataWriter=object - ), - "tensorrt_llm": _package("tensorrt_llm"), - "tensorrt_llm.llmapi": _package("tensorrt_llm.llmapi"), - "tensorrt_llm.llmapi.mpi_session": _module( - "tensorrt_llm.llmapi.mpi_session", get_mpi_world_size=lambda: 1 - ), - "transformers": _module( - "transformers", - AutoTokenizer=types.SimpleNamespace(from_pretrained=lambda *a, **k: _CharTokenizer()), - ), - } - # Set aside any copy imported under the real packages, so the import below binds - # to the stubs. - for name in [n for n in sys.modules if n.startswith("defs.perf.") or n == "allowed_configs"]: - monkeypatch.delitem(sys.modules, name) - loaded_before = set(sys.modules) - for name, module in stubs.items(): - monkeypatch.setitem(sys.modules, name, module) - # test_perf.py imports allowed_configs from its own directory (__extra_import_path__). - monkeypatch.syspath_prepend(str(_DEFS_DIR / "perf")) - module = importlib.import_module("defs.perf.test_perf") - yield module, warnings - for name in set(sys.modules) - loaded_before: - if name.split(".")[0] in ("defs", "allowed_configs"): - sys.modules.pop(name, None) - - -def _make_test(perf_module, tmp_path: Path): - test = perf_module.MultiMetricPerfTest(f"perf/test_perf.py::test_perf[{_TEST_ID}]") - test.set_runtime_configs(str(tmp_path), str(tmp_path), str(tmp_path), str(tmp_path / "cache")) - return test - - -def _use_source(monkeypatch, source: _Source) -> None: - monkeypatch.setitem( - sys.modules, "datasets", _module("datasets", load_dataset=source.load_dataset) - ) - - -def _prompts(path: Path) -> List[str]: - lines = path.read_text().splitlines() - return [json.loads(line)["input"]["messages"][1]["content"] for line in lines] - - -def _measured_dataset(perf_module, test, tmp_path: Path) -> Path: - measured = tmp_path / "dataset_custom.jsonl" - test.generate_trtllm_custom_dataset(str(measured), _INPUT_LEN, _OUTPUT_LEN, "cnn_dailymail") - return measured - - -def test_warmup_pass_is_skipped_when_the_source_has_no_articles_to_spare( - perf, tmp_path, monkeypatch -) -> None: - perf_module, warnings = perf - # Exactly the measured request count of usable articles; the empty one is skipped. - _use_source(monkeypatch, _Source([f"article {i}" for i in range(_NUM_MEASURED)] + [""])) - test = _make_test(perf_module, tmp_path) - measured = _measured_dataset(perf_module, test, tmp_path) - measured_before = measured.read_text() - - warmup_pass = test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) - - assert warmup_pass is None - assert any(warning.startswith("Warmup pass: none") for warning in warnings), warnings - assert measured.read_text() == measured_before - assert len(_prompts(measured)) == _NUM_MEASURED - - -def test_warmup_pass_reads_the_articles_after_the_measured_ones( - perf, tmp_path, monkeypatch -) -> None: - perf_module, warnings = perf - _use_source(monkeypatch, _Source([f"article {i}" for i in range(_NUM_MEASURED + 2)])) - test = _make_test(perf_module, tmp_path) - measured = _measured_dataset(perf_module, test, tmp_path) - - warmup_pass = test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) - - assert warmup_pass is not None and warmup_pass.num_requests == 2 - warmup_file = Path(warmup_pass.cmd[warmup_pass.cmd.index("--dataset-path") + 1]) - warmup_prompts = _prompts(warmup_file) - assert len(warmup_prompts) == 2 - assert not set(warmup_prompts) & set(_prompts(measured)) - assert all(f"article {i}" in p for i, p in zip(range(_NUM_MEASURED, 6), warmup_prompts)) - assert not warnings - - -@pytest.mark.parametrize( - "error", [OSError("cnn_dailymail shard unreadable"), ValueError("unknown split")] -) -def test_other_dataset_errors_still_fail_the_case(perf, tmp_path, monkeypatch, error) -> None: - perf_module, _ = perf - # The measured file loads the source first; loading it again for the pass fails. - source = _Source([f"article {i}" for i in range(8)], error_on_load=2, error=error) - _use_source(monkeypatch, source) - test = _make_test(perf_module, tmp_path) - _measured_dataset(perf_module, test, tmp_path) - - with pytest.raises(type(error)) as excinfo: - test._get_serve_warmup_pass(str(tmp_path), _INPUT_LEN, _OUTPUT_LEN, ["client"]) - assert excinfo.value is error diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass.py b/tests/unittest/others/test_perf_sanity_warmup_pass.py deleted file mode 100644 index fba294e49396..000000000000 --- a/tests/unittest/others/test_perf_sanity_warmup_pass.py +++ /dev/null @@ -1,391 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The perf tests' warmup pass must send the measured traffic shape and stay unmeasured. - -A pass whose command drifts from the measured one warms the wrong prefill shapes, and -one that reuses the measured prompts lets measured requests hit its KV cache blocks; -either way the measured numbers move without any runtime change. The servers also -record the pass's requests in the perf_metrics JSONLs the d_tb_* time breakdown reads, -so those records must be left out, however many files they are split across. -""" - -import importlib.util -import json -import subprocess -from pathlib import Path -from typing import Callable, List, Sequence, Tuple - -import pytest - -pytestmark = pytest.mark.cpu_only - - -def _load_perf_module(name: str): - # Loaded by path: importing it through the ``defs`` package pulls in torch. - spec = importlib.util.spec_from_file_location( - name, Path(__file__).resolve().parents[2] / f"integration/defs/perf/{name}.py" - ) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -warmup_pass = _load_perf_module("warmup_pass") -tbm = _load_perf_module("time_breakdown_metrics") - -_BENCHMARK_SERVING = ["python", "-m", "tensorrt_llm.serve.scripts.benchmark_serving"] - -# Shaped like ClientConfig._to_default_benchmark_cmd's output for the three modes. -# fmt: off -_AGGR_RANDOM_CMD = _BENCHMARK_SERVING + [ - "--model", "/models/m", "--tokenizer", "/models/m", - "--num-prompts", "40", "--max-concurrency", "4", - "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", "--no-test-input", - "--dataset-name", "random", "--random-ids", "--tokenize-on-client", - "--random-input-len", "8192", "--random-output-len", "1024", - "--random-range-ratio", "0.0", "--backend", "openai", "--trust-remote-code", -] -_E2E_DATASET_TIME_BREAKDOWN_CMD = _BENCHMARK_SERVING + [ - "--model", "/models/m", "--tokenizer", "/models/m", - "--num-prompts", "43010", "--max-concurrency", "4301", - "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", - "--dataset-name", "trtllm_custom", "--dataset-path", "/datasets/d.json", - "--backend", "openai", "--trust-remote-code", - "--save-request-time-breakdown", "/out/time_breakdown", -] -_CTX_ONLY_CMD = _BENCHMARK_SERVING + [ - "--model", "/models/m", "--tokenizer", "/models/m", - "--num-prompts", "6660", "--max-concurrency", "666", - "--percentile-metrics", "ttft,tpot,itl,e2el", "--ignore-eos", - "--dataset-name", "random", "--random-ids", "--tokenize-on-client", - "--random-input-len", "7386", "--random-output-len", "1", - "--random-range-ratio", "0.0", "--backend", "openai", "--non-streaming", -] -# Shaped like test_perf.py's get_trtllm_serve_client_command for a real-text client -# that saves its detailed results. -_SINGLE_NODE_SERVE_REAL_TEXT_CMD = _BENCHMARK_SERVING + [ - "--model", "/models/m", "--tokenizer", "/models/m", - "--num-prompts", "64", "--tokenize-on-client", "--no-test-input", - "--percentile-metrics", "ttft,tpot,itl,e2el", - "--save-result", "--save-detailed", "--result-dir", "/engine", - "--result-filename", "result_8192_1024.json", - "--dataset-name", "trtllm_custom", "--dataset-path", "/engine/dataset_custom.jsonl", - "--max-concurrency", "16", "--trust-remote-code", -] -# fmt: on - -# The flags the warmup pass may change, and the output files the measured client writes. -_REPLACED_FLAGS_WITH_VALUE = ( - "--num-prompts", - "--seed", - "--save-request-time-breakdown", - "--result-dir", - "--result-filename", -) -_REPLACED_FLAGS = ("--no-test-input", "--save-result", "--save-detailed") -_MEASURED_OUTPUT_FILES = ("/out/time_breakdown", "/engine", "result_8192_1024.json") - - -def _value_of(cmd: Sequence[str], flag: str) -> str: - assert cmd.count(flag) == 1, f"{flag} appears {cmd.count(flag)} times in {cmd}" - return cmd[cmd.index(flag) + 1] - - -def _traffic_shape(cmd: Sequence[str]) -> List[str]: - """The command without the flags the warmup pass may change.""" - shape, args = [], iter(cmd) - for arg in args: - if arg in _REPLACED_FLAGS_WITH_VALUE: - next(args, None) - elif arg not in _REPLACED_FLAGS: - shape.append(arg) - return shape - - -@pytest.mark.parametrize( - "measured_cmd", - [ - _AGGR_RANDOM_CMD, - _E2E_DATASET_TIME_BREAKDOWN_CMD, - _CTX_ONLY_CMD, - _SINGLE_NODE_SERVE_REAL_TEXT_CMD, - ], -) -def test_warmup_pass_cmd_keeps_the_measured_traffic_shape(measured_cmd: List[str]) -> None: - concurrency = int(_value_of(measured_cmd, "--max-concurrency")) - warmup_cmd = warmup_pass.build_warmup_pass_cmd(measured_cmd, concurrency) - - assert _traffic_shape(warmup_cmd) == _traffic_shape(measured_cmd) - assert _value_of(warmup_cmd, "--num-prompts") == str(concurrency) - assert warmup_cmd.count("--no-test-input") == 1 - # The pass writes none of the measured client's output files. - assert not set(warmup_cmd) & {"--save-result", "--save-detailed", *_MEASURED_OUTPUT_FILES} - - -@pytest.mark.parametrize( - ("concurrency", "num_requests", "max_batch_size", "expected_wave"), - [ - (4, 40, 512, 4), - # Never more than the benchmark sends. - (16, 8, 512, 8), - # A client without a concurrency limit sends everything at once; the - # server's batch size caps what is in flight. - (-1, 512, 128, 128), - (-1, 32, 512, 32), - (-1, 0, 64, 64), - ], -) -def test_one_wave_is_what_the_client_keeps_in_flight( - concurrency: int, num_requests: int, max_batch_size: int, expected_wave: int -) -> None: - assert warmup_pass.one_wave_requests(concurrency, num_requests, max_batch_size) == expected_wave - - -@pytest.mark.parametrize( - ("seed_args", "expected_warmup_seed"), - [ - # benchmark_serving's own default seed is 0. - ([], "1"), - (["--seed", "7"], "8"), - ], -) -def test_warmup_pass_draws_other_prompts_than_the_measured_client( - seed_args: List[str], expected_warmup_seed: str -) -> None: - measured_cmd = _AGGR_RANDOM_CMD + seed_args - warmup_cmd = warmup_pass.build_warmup_pass_cmd(measured_cmd, 4) - - assert _value_of(warmup_cmd, "--seed") == expected_warmup_seed - - -def test_warmup_pass_cmd_leaves_the_measured_cmd_untouched() -> None: - measured_cmd = list(_E2E_DATASET_TIME_BREAKDOWN_CMD) - warmup_pass.build_warmup_pass_cmd(measured_cmd, 4301) - - assert measured_cmd == _E2E_DATASET_TIME_BREAKDOWN_CMD - - -@pytest.mark.parametrize("num_requests", [0, -1]) -def test_warmup_pass_needs_a_request(num_requests: int) -> None: - with pytest.raises(ValueError): - warmup_pass.build_warmup_pass_cmd(_AGGR_RANDOM_CMD, num_requests) - - -@pytest.mark.parametrize( - ("environ", "enabled"), - [ - ({}, True), - ({warmup_pass.WARMUP_PASS_ENV: "1"}, True), - ({warmup_pass.WARMUP_PASS_ENV: "0"}, False), - ({warmup_pass.WARMUP_PASS_ENV: " False "}, False), - ({warmup_pass.WARMUP_PASS_ENV: "OFF"}, False), - ({warmup_pass.WARMUP_PASS_ENV: "no"}, False), - ], -) -def test_warmup_pass_env_switch(environ: dict, enabled: bool) -> None: - assert warmup_pass.warmup_pass_enabled(environ) is enabled - - -@pytest.mark.parametrize( - ("benchmark_mode", "applies"), - [ - (None, True), - ("", True), - ("ctx_only", True), - ("e2e", True), - ("gen_only", False), - ("gen_only_no_context", False), - ], -) -def test_warmup_pass_modes(benchmark_mode, applies: bool) -> None: - assert warmup_pass.warmup_pass_applies(benchmark_mode) is applies - - -@pytest.mark.parametrize( - ("eligible", "measured", "time_breakdown", "expected_plan"), - [ - # Without the breakdown, every client keeps its pass. - ([4, 8], [True, True], False, [4, 8]), - # With it, only the first measured client gets one. - ([4, 8], [True, True], True, [4, 0]), - # An accuracy-only client is not measured, so it does not count as first. - ([0, 4, 8], [False, True, True], True, [0, 4, 0]), - # A first client that gets no pass still makes later passes unsafe. - ([0, 8], [True, True], True, [0, 0]), - ], -) -def test_warmup_pass_plan( - eligible: List[int], measured: List[bool], time_breakdown: bool, expected_plan: List[int] -) -> None: - assert warmup_pass.warmup_pass_plan(eligible, measured, time_breakdown) == expected_plan - - -def _send_warmup_pass( - run_benchmark: Callable[[List[str]], str], num_requests: int = 4 -) -> Tuple[int, List[str]]: - """Return send_warmup_pass's successful requests and the warnings it logged.""" - warnings: List[str] = [] - succeeded = warmup_pass.send_warmup_pass( - ["benchmark_serving"], - num_requests, - run_benchmark, - log_path="warmup-pass.log", - timeout=60.0, - log_info=lambda _message: None, - log_warning=warnings.append, - ) - return succeeded, warnings - - -def _benchmark_summary(successful: int, failed: int) -> str: - return f"Successful requests: {successful}\nFailed requests: {failed}\n" - - -@pytest.mark.parametrize( - ("output", "expected_succeeded", "warns"), - [ - (_benchmark_summary(4, 0), 4, False), - # The pass still warmed the shapes it reached; the measured client decides. - (_benchmark_summary(3, 1), 3, True), - ], -) -def test_warmup_pass_returns_its_successful_requests( - output: str, expected_succeeded: int, warns: bool -) -> None: - sent: List[List[str]] = [] - - def run_benchmark(cmd: List[str]) -> str: - sent.append(cmd) - return output - - succeeded, warnings = _send_warmup_pass(run_benchmark) - - assert sent == [["benchmark_serving"]] - assert succeeded == expected_succeeded - assert bool(warnings) is warns - - -@pytest.mark.parametrize( - "error", - [ - subprocess.CalledProcessError(1, ["benchmark_serving"]), - subprocess.TimeoutExpired(["benchmark_serving"], 60.0), - ], -) -def test_warmup_pass_client_error_fails_the_test(error: subprocess.SubprocessError) -> None: - def run_benchmark(_cmd: List[str]) -> str: - raise error - - with pytest.raises(RuntimeError) as excinfo: - _send_warmup_pass(run_benchmark) - assert excinfo.value.__cause__ is error - - -def test_warmup_pass_with_every_request_failed_fails_the_test() -> None: - with pytest.raises(RuntimeError): - _send_warmup_pass(lambda _cmd: _benchmark_summary(0, 4)) - - -class _FakeClock: - """Drives wait_for_perf_metrics_files without wall time; calls on_sleep at each poll.""" - - def __init__(self, on_sleep: Callable[[], None] = lambda: None): - self.now = 0.0 - self._on_sleep = on_sleep - - def monotonic(self) -> float: - return self.now - - def sleep(self, seconds: float) -> None: - self.now += seconds - self._on_sleep() - - -def _aggregated_server_record(arrival: float, ctx_queue_s: float) -> str: - """One JSONL line of an aggregated server (a ctx_only case), with the fields read here.""" - timing = {"arrival_time": arrival, "first_scheduled_time": arrival + ctx_queue_s} - return json.dumps({"perf_metrics": {"timing_metrics": timing}}) + "\n" - - -def test_warmup_records_split_across_frontend_files_are_skipped(tmp_path: Path) -> None: - # A ctx_only server with two frontend processes (num_serve_frontends: 2). Each - # process writes its own file, so each file holds only its share of the warmup - # records, and the second has not written its last one yet. - frontend_files = [tmp_path / f"perf_metrics-server-node0-{pid}-x.jsonl" for pid in (11, 12)] - warmup_records = [_aggregated_server_record(i * 0.001, ctx_queue_s=5.0) for i in range(8)] - frontend_files[0].write_text("".join(warmup_records[0::2])) - frontend_files[1].write_text("".join(warmup_records[1:-1:2])) - late_records = warmup_records[-1:] - - def flush_late_records() -> None: - with frontend_files[1].open("a") as handle: - handle.writelines(late_records) - late_records.clear() - - clock = _FakeClock(on_sleep=flush_late_records) - _, after_warmup = tbm.wait_for_perf_metrics_files( - str(tmp_path), - expected_requests=len(warmup_records), - wait_for_expected_requests=True, - stable_seconds=0, - sleep=clock.sleep, - monotonic=clock.monotonic, - ) - assert after_warmup["stable"] - # Does nothing if the wait saw the late record. Otherwise the record lands after the - # snapshot, and the assertion below catches the leak. - flush_late_records() - - for frontend_idx, path in enumerate(frontend_files): - with path.open("a") as handle: - handle.writelines( - _aggregated_server_record(100.0 + frontend_idx + i * 0.01, ctx_queue_s=0.1) - for i in range(30) - ) - metrics, _ = tbm.compute_time_breakdown_metrics( - [str(path) for path in frontend_files], - "ctx_only", - leading_lines_to_skip=after_warmup["line_counts"], - ) - - assert metrics["d_tb_ctx_queue_mean"] == pytest.approx(100.0) - - -def test_warmup_census_does_not_add_up_server_roles(tmp_path: Path) -> None: - # A disagg e2e run: the disagg server, the ctx worker and the gen worker each record - # every request. Each has written two of the four warmup records so far, which is - # not all four, however many files hold them. - first_records = { - "perf_metrics-disagg-node0-1-x.jsonl": {"ctx_perf_metrics": {}, "gen_perf_metrics": {}}, - "perf_metrics-server-node1-2-x.jsonl": { - "time_breakdown_metrics": {"ctx_chunk_metrics": []} - }, - "perf_metrics-server-node2-3-x.jsonl": {"time_breakdown_metrics": {"step_metrics": []}}, - } - for name, record in first_records.items(): - (tmp_path / name).write_text((json.dumps(record) + "\n") * 2) - - clock = _FakeClock() - _, info = tbm.wait_for_perf_metrics_files( - str(tmp_path), - expected_requests=4, - wait_for_expected_requests=True, - stable_seconds=0, - timeout_seconds=5.0, - sleep=clock.sleep, - monotonic=clock.monotonic, - ) - - assert not info["stable"] diff --git a/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py b/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py deleted file mode 100644 index b9bc0bbdb103..000000000000 --- a/tests/unittest/others/test_perf_sanity_warmup_pass_harness.py +++ /dev/null @@ -1,279 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""The perf-sanity client loops must keep every warmup pass out of what they measure. - -Drives AggrTestCmds.run_cmd and DisaggTestCmds.run_cmd with fake servers and a fake -benchmark_serving that appends one perf_metrics record per request, so the warmup -pass, the time breakdown and the gen-log windows run as in a real lane. -""" - -import importlib -import json -import os -import re -import sys -import types -from pathlib import Path -from typing import List, Tuple - -import pytest - -pytestmark = pytest.mark.cpu_only - -_TESTS_DIR = Path(__file__).resolve().parents[2] -_DEFS_DIR = _TESTS_DIR / "integration" / "defs" - -# The time breakdown's ctx_queue span, per kind of request the fake client sends. -_WARMUP_PASS_QUEUE_S = 5.0 -_MEASURED_QUEUE_S = 0.1 - - -def _module(name: str, **attrs) -> types.ModuleType: - module = types.ModuleType(name) - module.__dict__.update(attrs) - return module - - -def _package(name: str, path: Path) -> types.ModuleType: - package = _module(name) - package.__path__ = [str(path)] - return package - - -@pytest.fixture -def tps(monkeypatch): - """test_perf_sanity.py, with the imports that need a CI test environment stubbed.""" - noop = lambda *args, **kwargs: None # noqa: E731 - stubs = { - "defs": _package("defs", _DEFS_DIR), - "defs.perf": _package("defs.perf", _DEFS_DIR / "perf"), - "defs.common": _module("defs.common", wait_for_reported_addr=noop), - "defs.trt_test_alternative": _module( - "defs.trt_test_alternative", print_info=print, print_warning=print - ), - "defs.conftest": _module( - "defs.conftest", - get_llm_root=lambda: str(_TESTS_DIR.parent), - llm_models_root=lambda: "/models", - ), - "defs.perf.perf_regression_utils": _module( - "defs.perf.perf_regression_utils", - _percentile=lambda values, percentile: 0.0, - get_job_info=dict, - process_and_upload_test_results=noop, - ), - "test_common": _package("test_common", _TESTS_DIR / "test_common"), - "test_common.error_utils": _module("test_common.error_utils", report_error=noop), - "test_common.http_utils": _module( - "test_common.http_utils", fail_if_proc_died=noop, wait_for_endpoint_ready=noop - ), - "test_common.perf_sanity_matching": _module( - "test_common.perf_sanity_matching", get_test_case_match_keys=list - ), - } - # Set aside any copy imported under the real packages, so the import below binds - # to the stubs. - for name in [n for n in sys.modules if n.startswith("defs.perf.")]: - monkeypatch.delitem(sys.modules, name) - loaded_before = set(sys.modules) - for name, module in stubs.items(): - monkeypatch.setitem(sys.modules, name, module) - module = importlib.import_module("defs.perf.test_perf_sanity") - # A fake server process, ready at once, that the runner can stop. - monkeypatch.setattr(module.subprocess, "Popen", _FakeServer) - monkeypatch.setattr(module, "wait_for_reported_addr", lambda *a, **k: ("localhost", 8000)) - monkeypatch.setattr(module, "wait_for_endpoint_ready", noop) - monkeypatch.setattr(module, "collect_startup_observation", lambda *a, **k: {}) - monkeypatch.setattr(module, "write_startup_observations", noop) - monkeypatch.setattr(module, "PERF_METRICS_SETTLE_SECONDS", 0.0) - monkeypatch.setattr(module, "PERF_METRICS_SETTLE_TIMEOUT", 10.0) - yield module - for name in set(sys.modules) - loaded_before: - if name.split(".")[0] in ("defs", "test_common"): - sys.modules.pop(name, None) - - -class _FakeServer: - def __init__(self, *args, **kwargs): - self.returncode = None - - def poll(self): - return self.returncode - - def terminate(self): - self.returncode = 0 - - def kill(self): - self.returncode = -9 - - def wait(self, timeout=None): - return self.returncode - - -class _FakeBenchmarkServing: - """Records each run and appends one perf_metrics record per request it sends. - - A warmup pass's requests queue for _WARMUP_PASS_QUEUE_S and measured ones for - _MEASURED_QUEUE_S, so any pass record in the breakdown moves its mean. - """ - - def __init__(self, perf_metrics_dir: str): - # No directory: the time breakdown is off and the server writes no records. - self.jsonl = ( - os.path.join(perf_metrics_dir, "perf_metrics-server-node0-1-x.jsonl") - if perf_metrics_dir - else "" - ) - self.runs: List[Tuple[str, int]] = [] - self._arrival = 0.0 - - def __call__(self, cmd, env, log_path, timeout=0): - num_requests = int(cmd[cmd.index("--num-prompts") + 1]) - is_warmup_pass = "warmup-pass" in os.path.basename(log_path) - self.runs.append((os.path.basename(log_path), num_requests)) - if self.jsonl: - queue_s = _WARMUP_PASS_QUEUE_S if is_warmup_pass else _MEASURED_QUEUE_S - # Without --no-test-input, a single test request goes first. - num_records = num_requests + (0 if "--no-test-input" in cmd else 1) - with open(self.jsonl, "a") as handle: - for _ in range(num_records): - self._arrival += 1.0 - timing = { - "arrival_time": self._arrival, - "first_scheduled_time": self._arrival + queue_s, - } - handle.write(json.dumps({"perf_metrics": {"timing_metrics": timing}}) + "\n") - return f"Successful requests: {num_requests}\nFailed requests: 0\n" - - -def _clients(tps, concurrencies, warmup: bool): - return [ - tps.ClientConfig( - {"name": f"con{c}", "concurrency": c, "iterations": 2, "isl": 1024, "osl": 1}, - "m", - warmup=warmup, - ) - for c in concurrencies - ] - - -def _run_aggr(tps, tmp_path, monkeypatch, benchmark_mode: str, time_breakdown: bool): - perf_metrics_dir = tmp_path / "perf_metrics" if time_breakdown else None - if perf_metrics_dir: - perf_metrics_dir.mkdir() - client = _FakeBenchmarkServing(str(perf_metrics_dir) if perf_metrics_dir else "") - monkeypatch.setattr(tps, "_run_benchmark_with_log", client) - configs = _clients(tps, (4, 8), warmup=benchmark_mode == "ctx_only") - cmds = tps.AggrTestCmds( - server_cmds=[["trtllm-serve", "m"]], - client_cmds={0: [config.to_cmd() for config in configs]}, - timeout=tps.DEFAULT_TIMEOUT, - output_dir=str(tmp_path), - test_output_dir=str(tmp_path), - client_configs={0: configs}, - model_name="m", - server_configs=[], - perf_metrics_output_dir=str(perf_metrics_dir) if perf_metrics_dir else "", - benchmark_mode=benchmark_mode, - ) - return cmds.run_cmd(0), client.runs - - -def test_time_breakdown_leaves_out_every_warmup_pass_with_two_clients( - tps, tmp_path, monkeypatch -) -> None: - outputs, runs = _run_aggr(tps, tmp_path, monkeypatch, "ctx_only", time_breakdown=True) - - # Both clients' rows carry the pooled breakdown: no warmup pass record in it. - for output in outputs: - ctx_queue_mean = re.search(r"Time Breakdown ctx_queue mean \(ms\): ([\d.]+)", output) - assert ctx_queue_mean, output - assert float(ctx_queue_mean.group(1)) == pytest.approx(_MEASURED_QUEUE_S * 1000) - # That takes giving only the first client a pass: the breakdown leaves out only - # the records written before it. - assert [name for name, _ in runs] == [ - "trtllm-benchmark-warmup-pass.0.0.log", - "trtllm-benchmark.0.0.log", - "trtllm-benchmark.0.1.log", - ] - - -def test_every_client_gets_a_warmup_pass_without_the_time_breakdown( - tps, tmp_path, monkeypatch -) -> None: - _, runs = _run_aggr(tps, tmp_path, monkeypatch, "", time_breakdown=False) - - assert runs == [ - ("trtllm-benchmark-warmup-pass.0.0.log", 4), - ("trtllm-benchmark.0.0.log", 8), - ("trtllm-benchmark-warmup-pass.0.1.log", 8), - ("trtllm-benchmark.0.1.log", 16), - ] - - -def test_gen_log_windows_leave_out_the_next_clients_warmup_pass(tps, tmp_path, monkeypatch) -> None: - client = _FakeBenchmarkServing("") - monkeypatch.setattr(tps, "_run_benchmark_with_log", client) - # Each snapshot's value is 100 x the benchmark runs before it, so it names its place. - monkeypatch.setattr(tps, "gen_worker_log_sizes", lambda *a: [100 * len(client.runs)]) - monkeypatch.setattr(tps, "expected_gen_only_ngen", lambda *a, **k: None) - windows = [] - cls = tps.DisaggTestCmds - monkeypatch.setattr(cls, "_get_disagg_server_hostname_and_port", lambda self, i: ("h", 1)) - monkeypatch.setattr(cls, "_collect_worker_startup_observations", lambda self, i: []) - monkeypatch.setattr(cls, "_run_connection_warmup", lambda self, *a: 0) - monkeypatch.setattr( - cls, - "_append_gen_worker_device_step_time", - lambda self, pending, outputs: windows.extend(pending), - ) - configs = _clients(tps, (4, 8), warmup=True) - cmds = cls( - server_cmds=[(["ctx"], ["gen"], ["disagg"])], - client_cmds={0: [config.to_cmd() for config in configs]}, - timeout=tps.DEFAULT_TIMEOUT, - hostname="bench-host", - disagg_serving_type="BENCHMARK", - num_ctx_servers=1, - num_gen_servers=1, - output_dir=str(tmp_path), - test_output_dir=str(tmp_path), - client_configs={0: configs}, - server_configs=[ - (None, None, types.SimpleNamespace(benchmark_mode="e2e", server_env_var="")) - ], - ) - cmds.run_cmd(0) - - assert [name for name, _ in client.runs] == [ - "trtllm-benchmark-warmup-pass.0.0.log", - "trtllm-benchmark.0.0.log", - "trtllm-benchmark-warmup-pass.0.1.log", - "trtllm-benchmark.0.1.log", - ] - # Client 0's window opens after its pass and closes before client 1's pass, which - # comes before client 1's window; the last window reads to the end of the log. - assert [(w["start_offsets"], w["end_offsets"]) for w in windows] == [ - ([100], [200]), - ([300], None), - ] - - -def test_warmup_plan_matches_the_run_loops(tps) -> None: - configs = _clients(tps, (4, 8), warmup=True) - - assert tps.warmup_pass_plan_for(configs, "e2e", time_breakdown=True) == [4, 0] - assert tps.warmup_pass_plan_for(configs, "e2e", time_breakdown=False) == [4, 8] - assert tps.warmup_pass_plan_for(configs, "gen_only", time_breakdown=False) == [0, 0]