diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst index 2bb93a668087..0b082051c09a 100644 --- a/docs/source/commands/trtllm-serve/trtllm-serve.rst +++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst @@ -79,6 +79,19 @@ The module ``tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py`` implements t shared format. ``--reasoning_parser`` separately controls reasoning extraction; ``nemotron-v3`` and ``nano-v3`` are aliases for the same Nemotron reasoning parser. +Multiple HTTP Frontends +----------------------- + +At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, beta). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL. + +.. note:: + + * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend. + * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more). + * Each request is handled entirely by the frontend that accepted its connection. Server-wide state lives in the launcher: the other frontends forward ``/metrics``, ``/kv_cache_events``, profiling and the Anthropic batches API to it, and ``/prometheus/metrics`` covers every frontend. Per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, ``/health`` describes the frontend that answered, and each frontend writes its own per-request perf metrics file. + * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses. + * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly. + Inference Endpoints ------------------- diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 08d018dcfb44..1b51181d8a8f 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -488,6 +488,34 @@ def _diagnose_port_in_use(port: int) -> str: return "; ".join(details) +def _reserve_multi_frontend_port(host: str, port: int, + family: socket.AddressFamily) -> socket.socket: + """Refuse a port another server uses; return a lock to hold while serving. + + SO_REUSEPORT would let a second server of the same user join the port. A + plain bind catches any listener (SO_REUSEADDR ignores TIME_WAIT), and an + abstract Unix socket named after the port catches a launcher that has not + started listening yet. + """ + lock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + lock.bind(f"\0trtllm-serve-{os.geteuid()}-{port}") + except OSError as e: + lock.close() + raise RuntimeError(f"Port {port} is already reserved by another " + f"multi-frontend trtllm-serve: {e}") from e + try: + with socket.socket(family, socket.SOCK_STREAM) as probe: + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + probe.bind((host, port)) + except OSError as e: + lock.close() + raise RuntimeError( + f"Failed to bind socket to {host}:{port}: {e}. " + f"Port holder(s): {_diagnose_port_in_use(port)}") from e + return lock + + @contextlib.contextmanager def _provision_kv_cache_pool(llm_args: dict, owns_engine: bool = True) -> Iterator[None]: @@ -558,6 +586,38 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None: f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}") +def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool, + port: int, grpc: bool, + report_addr: Optional[str], + enable_sleep_mode: bool) -> None: + """Fall back to one frontend where the default count cannot run. + + An explicit count (CLI, --config YAML or --set) is left alone, so the + guard that owns the conflict still rejects it. + """ + num_frontends = llm_args.get("num_serve_frontends", 1) + if requested or num_frontends <= 1: + return + if grpc: + # OpenEngine supports several frontends, but only when asked to. + reason = "--grpc" + elif port == 0 or report_addr: + reason = "--port 0 / --report_addr" + elif llm_args.get("orchestrator_type") is not None: + reason = f"orchestrator_type={llm_args['orchestrator_type']!r}" + elif llm_args.get("enable_resource_governor"): + reason = "enable_resource_governor" + elif enable_sleep_mode: + reason = "--enable_sleep_mode" + else: + return + logger.info( + f"num_serve_frontends defaults to {num_frontends} for the OpenAI HTTP " + f"server only; running 1 frontend because of {reason}. Pass " + "--num_serve_frontends explicitly to override.") + llm_args["num_serve_frontends"] = 1 + + def launch_server( host: str, port: int, @@ -622,14 +682,17 @@ def launch_server( logger.warning( "num_serve_frontends > 1: stateful Responses API storage " "(store/previous_response_id) is disabled; the per-frontend " - "in-memory store cannot be shared across frontends.") + "in-memory store cannot be shared across frontends. Pass " + "--num_serve_frontends 1 to keep it.") os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1" addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) address_family = socket.AF_INET6 if all( [info[0] == socket.AF_INET6 for info in addr_info]) else socket.AF_INET - with socket.socket(address_family, socket.SOCK_STREAM) as s: + port_lock = (_reserve_multi_frontend_port(host, port, address_family) + if multi_frontend.is_launcher else contextlib.nullcontext()) + with port_lock, socket.socket(address_family, socket.SOCK_STREAM) as s: # port == 0 lets the kernel pick the port; the caller then needs a way # to learn it, either by service discovery or by report_addr. assert port > 0 or disagg_cluster_config is not None or report_addr, ( @@ -1111,11 +1174,14 @@ def launch_visual_gen_server( @stability_option( "--num_serve_frontends", type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS), - default=1, + # LlmArgs keeps 1: only trtllm-serve spawns frontends. + default=8, help="Number of HTTP or OpenEngine frontend processes serving one " "executor; values > 1 share the serving port via " - "SO_REUSEPORT (classic IPC executor path only).", - status="prototype") + "SO_REUSEPORT (classic IPC executor path only). Without an explicit " + "value, --grpc and configurations that support a single frontend only " + "use 1 and log why.", + status="beta") @stability_option("--num_input_processor_workers", type=click.IntRange(min=1), default=8, @@ -1556,6 +1622,15 @@ def _serve_llm(): if enable_sleep_mode and llm_args.get("sleep_config") is None: llm_args["sleep_config"] = SleepConfig() + _resolve_default_num_serve_frontends( + llm_args, + requested=("num_serve_frontends" in explicit_cli_keys + or "num_serve_frontends" in llm_args_extra_dict), + port=port, + grpc=grpc, + report_addr=report_addr, + enable_sleep_mode=enable_sleep_mode) + _apply_effective_telemetry_config(llm_args, telemetry=telemetry) # Preserve requested settings for failures before entering the LLM constructor. try: diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 24e49833df62..5572326a2769 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -595,6 +595,13 @@ def test_openai_prometheus(llm_root, llm_venv): str(test_root / "_test_openai_prometheus.py")]) +def test_openai_multi_frontend(llm_root, llm_venv): + test_root = unittest_path() / "llmapi" / "apps" + llm_venv.run_cmd( + ["-m", "pytest", + str(test_root / "_test_openai_multi_frontend.py")]) + + def test_openai_lora(llm_root, llm_venv): test_root = unittest_path() / "llmapi" / "apps" llm_venv.run_cmd(["-m", "pytest", str(test_root / "_test_openai_lora.py")]) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index deefc7a15ed3..a2241a9376b3 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -107,6 +107,7 @@ l0_b200: - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] - test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16] - test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b] + - test_e2e.py::test_openai_multi_frontend # Covers the GPU cases of tests/unittest/_torch/attention/ on B200, except the paths # ignored below; its cpu_only cases run on CPU (l0_cpu). kernels/ has no entry of its own # on any list, so apart from kernels/parallel_hw_agnostic/ this entry is what runs it diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index edf634c48fcf..5fdfef3a33d5 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -295,6 +295,7 @@ l0_cpu: - unittest/llmapi/test_serialization.py - unittest/llmapi/test_serve_multi_frontend_helpers.py - unittest/llmapi/test_serve_multi_frontend_route_gate.py + - unittest/llmapi/test_serve_num_frontends_default.py - unittest/llmapi/test_serve_report_addr.py - unittest/llmapi/test_serve_sleep_mode.py - unittest/llmapi/test_session_prefetcher.py diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index 8c06a6d0ea57..d7e9092b99b5 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -344,8 +344,8 @@ commands: - "--num_postprocess_workers" num_serve_frontends: type: int - default: 1 - status: prototype + default: 8 + status: beta required: false multiple: false is_flag: false diff --git a/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py b/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py new file mode 100644 index 000000000000..d3e62c139aeb --- /dev/null +++ b/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""trtllm-serve with 8 frontend processes on one engine. + +Each connection may land on any frontend (SO_REUSEPORT), so every request opens +its own. Server-wide state must not depend on which frontend answers. +""" + +import time +from concurrent.futures import ThreadPoolExecutor + +import psutil +import pytest +import requests + +from ..test_llm import get_model_path +from .openai_server import RemoteOpenAIServer + +NUM_FRONTENDS = 8 +NUM_REQUESTS = 64 +NUM_STREAMS = 16 + + +@pytest.fixture(scope="module") +def model_path() -> str: + return get_model_path("Qwen3/Qwen3-0.6B") + + +@pytest.fixture(scope="module") +def server(model_path: str): + args = ["--backend", "pytorch", "--num_serve_frontends", str(NUM_FRONTENDS)] + extra_config = {"return_perf_metrics": True, "enable_iter_perf_stats": True} + with RemoteOpenAIServer(model_path, args, extra_config=extra_config) as server: + # /health answers once the first frontend listens. + deadline = time.monotonic() + 600 + while _listeners(server.port) < NUM_FRONTENDS: + assert time.monotonic() < deadline, _listeners(server.port) + time.sleep(1) + yield server + + +def _listeners(port: int) -> int: + return sum( + 1 + for c in psutil.net_connections("tcp") + if c.status == psutil.CONN_LISTEN and c.laddr and c.laddr.port == port + ) + + +def _finish_reason(server: RemoteOpenAIServer, model: str, i: int, stream: bool): + """Run one completion on a connection of its own.""" + with server.get_client() as client: + output = client.completions.create( + model=model, + prompt=f"Request {i}: the capital of France is", + max_tokens=16, + stream=stream, + ) + if stream: + choices = [chunk.choices[0] for chunk in output if chunk.choices] + else: + choices = output.choices + text = "".join(choice.text for choice in choices) + reasons = [choice.finish_reason for choice in choices if choice.finish_reason] + return reasons[-1] if text and reasons else None + + +def _success_total(server: RemoteOpenAIServer) -> float: + text = requests.get(server.url_for("prometheus", "metrics"), timeout=30).text + return sum( + float(line.rsplit(" ", 1)[1]) + for line in text.splitlines() + if line.startswith("trtllm_request_success_total") + ) + + +def test_frontends_share_one_engine(server: RemoteOpenAIServer, model_path: str): + with ThreadPoolExecutor(16) as pool: + finish_reasons = list( + pool.map( + lambda i: _finish_reason(server, model_path, i, stream=i >= NUM_REQUESTS), + range(NUM_REQUESTS + NUM_STREAMS), + ) + ) + assert all(finish_reasons), finish_reasons + + # Only the launcher drains the engine's stats (every second), so the first + # poll gets all of them, whichever frontend answers. + time.sleep(3) + polls = [len(requests.get(server.url_for("metrics"), timeout=30).json()) for _ in range(16)] + assert polls[0] > 0 and not any(polls[1:]), polls + + # Every scrape counts the requests of all frontends. + totals = [_success_total(server) for _ in range(NUM_FRONTENDS)] + assert totals == [NUM_REQUESTS + NUM_STREAMS] * NUM_FRONTENDS, totals diff --git a/tests/unittest/llmapi/apps/openai_server.py b/tests/unittest/llmapi/apps/openai_server.py index 62b3543d8159..38f74d557229 100644 --- a/tests/unittest/llmapi/apps/openai_server.py +++ b/tests/unittest/llmapi/apps/openai_server.py @@ -15,6 +15,19 @@ from tensorrt_llm.llmapi.disagg_utils import ServerRole +def _picks_num_frontends(cli_args: List[str], + extra_config: Optional[dict]) -> bool: + """Whether a test sets num_serve_frontends (a --set override wins anyway).""" + configs = [extra_config or {}] + for arg, value in zip(cli_args, cli_args[1:] + [""]): + if arg.split("=")[0] == "--num_serve_frontends": + return True + if arg in ("--config", "--extra_llm_api_options") and value: + with open(value) as f: + configs.append(yaml.safe_load(f) or {}) + return any("num_serve_frontends" in config for config in configs) + + class RemoteOpenAIServer: DUMMY_API_KEY = "tensorrt_llm" MAX_SERVER_START_WAIT_S = 7200 # wait for server to start for 7200 seconds (~ 2 hours) for LLM models weight loading @@ -51,6 +64,10 @@ def __init__(self, f.write(yaml.dump(extra_config)) self.extra_config_file = f.name args += ["--extra_llm_api_options", self.extra_config_file] + # trtllm-serve defaults to several frontend processes; test servers + # run one unless the test picks a count. + if not _picks_num_frontends(cli_args or [], extra_config): + args += ["--num_serve_frontends", "1"] launch_cmd = ["trtllm-serve"] + [model] + args if llmapi_launch: # start server with llmapi-launch on multi nodes diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py new file mode 100644 index 000000000000..7bf379263bb3 --- /dev/null +++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU tests for trtllm-serve's multi-frontend default, its fallback to one, and its port guard.""" + +import socket +from unittest import mock + +import pytest + +from tensorrt_llm._utils import get_free_port +from tensorrt_llm.commands import serve as serve_cmd +from tensorrt_llm.commands.serve import ( + _reserve_multi_frontend_port, + _resolve_default_num_serve_frontends, +) + +pytestmark = pytest.mark.cpu_only + + +@pytest.mark.parametrize( + "llm_args,conflict,reason", + [ + pytest.param({}, {"grpc": True}, "--grpc", id="grpc"), + pytest.param({}, {"port": 0}, "--port 0", id="port0"), + pytest.param({}, {"report_addr": "/tmp/bound.addr"}, "--report_addr", id="report_addr"), + pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"), + pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="rg"), + pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"), + ], +) +def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, reason) -> None: + messages = [] + monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg)) + llm_args = {"num_serve_frontends": 8, **llm_args} + no_conflict = dict( + requested=False, + port=8000, + grpc=False, + report_addr=None, + enable_sleep_mode=False, + ) + + _resolve_default_num_serve_frontends(llm_args, **{**no_conflict, **conflict}) + + assert llm_args["num_serve_frontends"] == 1 + assert len(messages) == 1 and reason in messages[0] and "--num_serve_frontends" in messages[0] + + +@pytest.mark.parametrize( + "args,config,expected", + [ + pytest.param([], None, 8, id="default"), + pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"), + pytest.param(["--port", "0"], None, 1, id="port0"), + pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"), + # An explicit count is honored; launch_server's own guard rejects the conflict. + pytest.param(["--port", "0", "--num_serve_frontends", "8"], None, 8, id="explicit"), + ], +) +def test_frontend_count_reaching_launch_server( + monkeypatch, tmp_path, args, config, expected +) -> None: + monkeypatch.setenv("TRTLLM_RUNTIME_CONTROL_API_KEY", "secret") # for --enable_sleep_mode + if config is not None: + (tmp_path / "config.yaml").write_text(config) + args = [*args, "--config", str(tmp_path / "config.yaml")] + with ( + mock.patch.object(serve_cmd, "get_is_diffusion_only_model", return_value=False), + mock.patch.object(serve_cmd, "device_count", return_value=1), + mock.patch.object(serve_cmd, "launch_server") as launch, + ): + serve_cmd.main(args=[str(tmp_path), *args], standalone_mode=False) + + assert launch.call_args.args[2].get("num_serve_frontends", 1) == expected + + +def test_port_used_by_another_server_is_refused() -> None: + # Without the check, SO_REUSEPORT would let this launcher join the port. + with socket.socket() as other: + other.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + other.bind(("127.0.0.1", 0)) + other.listen() + port = other.getsockname()[1] + with pytest.raises(RuntimeError, match="Failed to bind socket"): + _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET) + + +def test_port_reserved_by_another_launcher_is_refused() -> None: + # A launcher binds its port long before it listens: only its lock shows. + port = get_free_port() + with _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET): + with pytest.raises(RuntimeError, match="another multi-frontend trtllm-serve"): + _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET) + # The lock goes with the launcher. + _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET).close()