[None][feat] BREAKING: serve: default num_serve_frontends to 8 with single-frontend fallback

docs/source/commands/trtllm-serve/trtllm-serve.rst

@@ -79,6 +79,19 @@ The module ``tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py`` implements t
 shared format. ``--reasoning_parser`` separately controls reasoning extraction;
 ``nemotron-v3`` and ``nano-v3`` are aliases for the same Nemotron reasoning parser.
 
+Multiple HTTP Frontends
+-----------------------
+
+At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, prototype). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL.
+
+.. note::
+
+   * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
+   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
+   * Each request is handled entirely by the frontend that accepted its connection. Server-wide state lives in the launcher: the other frontends forward ``/metrics``, ``/kv_cache_events``, profiling and the Anthropic batches API to it, and ``/prometheus/metrics`` covers every frontend. Per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, ``/health`` describes the frontend that answered, and each frontend writes its own per-request perf metrics file.
+   * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses.
+   * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
+
 Inference Endpoints
 -------------------
 

tensorrt_llm/commands/serve.py

@@ -488,6 +488,34 @@ def _diagnose_port_in_use(port: int) -> str:
     return "; ".join(details)
 
 
+def _reserve_multi_frontend_port(host: str, port: int,
+                                 family: socket.AddressFamily) -> socket.socket:
+    """Refuse a port another server uses; return a lock to hold while serving.
+
+    SO_REUSEPORT would let a second server of the same user join the port. A
+    plain bind catches any listener (SO_REUSEADDR ignores TIME_WAIT), and an
+    abstract Unix socket named after the port catches a launcher that has not
+    started listening yet.
+    """
+    lock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        lock.bind(f"\0trtllm-serve-{os.geteuid()}-{port}")
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(f"Port {port} is already reserved by another "
+                           f"multi-frontend trtllm-serve: {e}") from e
+    try:
+        with socket.socket(family, socket.SOCK_STREAM) as probe:
+            probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            probe.bind((host, port))
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(
+            f"Failed to bind socket to {host}:{port}: {e}. "
+            f"Port holder(s): {_diagnose_port_in_use(port)}") from e
+    return lock
+
+
 @contextlib.contextmanager
 def _provision_kv_cache_pool(llm_args: dict,
                              owns_engine: bool = True) -> Iterator[None]:
@@ -558,6 +586,43 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
+# One serving process is host-bound at high concurrency. LlmArgs keeps
+# default 1: only trtllm-serve spawns frontends.
+DEFAULT_NUM_SERVE_FRONTENDS = 8
+
+
+def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
+                                         port: int, grpc: bool,
+                                         report_addr: Optional[str],
+                                         enable_sleep_mode: bool) -> None:
+    """Fall back to one frontend where the default count cannot run.
+
+    An explicit count (CLI, --config YAML or --set) is left alone, so the
+    guard that owns the conflict still rejects it.
+    """
+    num_frontends = llm_args.get("num_serve_frontends", 1)
+    if requested or num_frontends <= 1:
+        return
+    if grpc:
+        # OpenEngine supports several frontends, but only when asked to.
+        reason = "--grpc"
+    elif port == 0 or report_addr:
+        reason = "--port 0 / --report_addr"
+    elif llm_args.get("orchestrator_type") is not None:
+        reason = f"orchestrator_type={llm_args['orchestrator_type']!r}"
+    elif llm_args.get("enable_resource_governor"):
+        reason = "enable_resource_governor"
+    elif enable_sleep_mode:
+        reason = "--enable_sleep_mode"
+    else:
+        return
+    logger.info(
+        f"num_serve_frontends defaults to {num_frontends} for the OpenAI HTTP "
+        f"server only; running 1 frontend because of {reason}. Pass "
+        "--num_serve_frontends explicitly to override.")
+    llm_args["num_serve_frontends"] = 1
+
+
 def launch_server(
         host: str,
         port: int,
@@ -622,14 +687,17 @@ def launch_server(
             logger.warning(
                 "num_serve_frontends > 1: stateful Responses API storage "
                 "(store/previous_response_id) is disabled; the per-frontend "
-                "in-memory store cannot be shared across frontends.")
+                "in-memory store cannot be shared across frontends. Pass "
+                "--num_serve_frontends 1 to keep it.")
         os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1"
 
     addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
                                    socket.SOCK_STREAM)
     address_family = socket.AF_INET6 if all(
         [info[0] == socket.AF_INET6 for info in addr_info]) else socket.AF_INET
-    with socket.socket(address_family, socket.SOCK_STREAM) as s:
+    port_lock = (_reserve_multi_frontend_port(host, port, address_family)
+                 if multi_frontend.is_launcher else contextlib.nullcontext())
+    with port_lock, socket.socket(address_family, socket.SOCK_STREAM) as s:
         # port == 0 lets the kernel pick the port; the caller then needs a way
         # to learn it, either by service discovery or by report_addr.
         assert port > 0 or disagg_cluster_config is not None or report_addr, (
@@ -1111,10 +1179,12 @@ def launch_visual_gen_server(
 @stability_option(
     "--num_serve_frontends",
     type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS),
-    default=1,
+    default=DEFAULT_NUM_SERVE_FRONTENDS,
     help="Number of HTTP or OpenEngine frontend processes serving one "
     "executor; values > 1 share the serving port via "
-    "SO_REUSEPORT (classic IPC executor path only).",
+    "SO_REUSEPORT (classic IPC executor path only). Without an explicit "
+    "value, --grpc and configurations that support a single frontend only "
+    "use 1 and log why.",
     status="prototype")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
@@ -1556,6 +1626,15 @@ def _serve_llm():
         if enable_sleep_mode and llm_args.get("sleep_config") is None:
             llm_args["sleep_config"] = SleepConfig()
 
+        _resolve_default_num_serve_frontends(
+            llm_args,
+            requested=("num_serve_frontends" in explicit_cli_keys
+                       or "num_serve_frontends" in llm_args_extra_dict),
+            port=port,
+            grpc=grpc,
+            report_addr=report_addr,
+            enable_sleep_mode=enable_sleep_mode)
+
         _apply_effective_telemetry_config(llm_args, telemetry=telemetry)
         # Preserve requested settings for failures before entering the LLM constructor.
         try:

tests/integration/test_lists/test-db/l0_cpu.yml

@@ -285,6 +285,7 @@ l0_cpu:
   - unittest/llmapi/test_serialization.py
   - unittest/llmapi/test_serve_multi_frontend_helpers.py
   - unittest/llmapi/test_serve_multi_frontend_route_gate.py
+  - unittest/llmapi/test_serve_num_frontends_default.py
   - unittest/llmapi/test_serve_report_addr.py
   - unittest/llmapi/test_serve_sleep_mode.py
   - unittest/llmapi/test_session_prefetcher.py

tests/unittest/api_stability/references/trtllm_serve_cli.yaml

@@ -344,7 +344,7 @@ commands:
         - "--num_postprocess_workers"
       num_serve_frontends:
         type: int
-        default: 1
+        default: 8
         status: prototype
         required: false
         multiple: false

tests/unittest/llmapi/apps/openai_server.py

@@ -15,6 +15,19 @@
 from tensorrt_llm.llmapi.disagg_utils import ServerRole
 
 
+def _picks_num_frontends(cli_args: List[str],
+                         extra_config: Optional[dict]) -> bool:
+    """Whether a test sets num_serve_frontends (a --set override wins anyway)."""
+    configs = [extra_config or {}]
+    for arg, value in zip(cli_args, cli_args[1:] + [""]):
+        if arg.split("=")[0] == "--num_serve_frontends":
+            return True
+        if arg in ("--config", "--extra_llm_api_options") and value:
+            with open(value) as f:
+                configs.append(yaml.safe_load(f) or {})
+    return any("num_serve_frontends" in config for config in configs)
+
+
 class RemoteOpenAIServer:
     DUMMY_API_KEY = "tensorrt_llm"
     MAX_SERVER_START_WAIT_S = 7200  # wait for server to start for 7200 seconds (~ 2 hours) for LLM models weight loading
@@ -51,6 +64,10 @@ def __init__(self,
                 f.write(yaml.dump(extra_config))
                 self.extra_config_file = f.name
             args += ["--extra_llm_api_options", self.extra_config_file]
+        # trtllm-serve defaults to several frontend processes; test servers
+        # run one unless the test picks a count.
+        if not _picks_num_frontends(cli_args or [], extra_config):
+            args += ["--num_serve_frontends", "1"]
         launch_cmd = ["trtllm-serve"] + [model] + args
         if llmapi_launch:
             # start server with llmapi-launch on multi nodes

tests/unittest/llmapi/test_serve_num_frontends_default.py

@@ -0,0 +1,108 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""CPU tests for trtllm-serve's multi-frontend default, its fallback to one, and its port guard."""
+
+import socket
+from unittest import mock
+
+import pytest
+
+from tensorrt_llm._utils import get_free_port
+from tensorrt_llm.commands import serve as serve_cmd
+from tensorrt_llm.commands.serve import (
+    DEFAULT_NUM_SERVE_FRONTENDS,
+    _reserve_multi_frontend_port,
+    _resolve_default_num_serve_frontends,
+)
+
+pytestmark = pytest.mark.cpu_only
+
+
+@pytest.mark.parametrize(
+    "llm_args,conflict,reason",
+    [
+        pytest.param({}, {"grpc": True}, "--grpc", id="grpc"),
+        pytest.param({}, {"port": 0}, "--port 0", id="port0"),
+        pytest.param({}, {"report_addr": "/tmp/bound.addr"}, "--report_addr", id="report_addr"),
+        pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"),
+        pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="rg"),
+        pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"),
+    ],
+)
+def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, reason) -> None:
+    messages = []
+    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **llm_args}
+    no_conflict = dict(
+        requested=False,
+        port=8000,
+        grpc=False,
+        report_addr=None,
+        enable_sleep_mode=False,
+    )
+
+    _resolve_default_num_serve_frontends(llm_args, **{**no_conflict, **conflict})
+
+    assert llm_args["num_serve_frontends"] == 1
+    assert len(messages) == 1 and reason in messages[0] and "--num_serve_frontends" in messages[0]
+
+
+@pytest.mark.parametrize(
+    "args,config,expected",
+    [
+        pytest.param([], None, DEFAULT_NUM_SERVE_FRONTENDS, id="default"),
+        pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"),
+        pytest.param(["--port", "0"], None, 1, id="port0"),
+        pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"),
+        # An explicit count is honored; launch_server's own guard rejects the conflict.
+        pytest.param(["--port", "0", "--num_serve_frontends", "8"], None, 8, id="explicit"),
+    ],
+)
+def test_frontend_count_reaching_launch_server(
+    monkeypatch, tmp_path, args, config, expected
+) -> None:
+    monkeypatch.setenv("TRTLLM_RUNTIME_CONTROL_API_KEY", "secret")  # for --enable_sleep_mode
+    if config is not None:
+        (tmp_path / "config.yaml").write_text(config)
+        args = [*args, "--config", str(tmp_path / "config.yaml")]
+    with (
+        mock.patch.object(serve_cmd, "get_is_diffusion_only_model", return_value=False),
+        mock.patch.object(serve_cmd, "device_count", return_value=1),
+        mock.patch.object(serve_cmd, "launch_server") as launch,
+    ):
+        serve_cmd.main(args=[str(tmp_path), *args], standalone_mode=False)
+
+    assert launch.call_args.args[2].get("num_serve_frontends", 1) == expected
+
+
+def test_port_used_by_another_server_is_refused() -> None:
+    # Without the check, SO_REUSEPORT would let this launcher join the port.
+    with socket.socket() as other:
+        other.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        other.bind(("127.0.0.1", 0))
+        other.listen()
+        port = other.getsockname()[1]
+        with pytest.raises(RuntimeError, match="Failed to bind socket"):
+            _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET)
+
+
+def test_port_reserved_by_another_launcher_is_refused() -> None:
+    # A launcher binds its port long before it listens: only its lock shows.
+    port = get_free_port()
+    with _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET):
+        with pytest.raises(RuntimeError, match="another multi-frontend trtllm-serve"):
+            _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET)
+    # The lock goes with the launcher.
+    _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET).close()