From 45b1a29ceac04082b52063c7bf8d61b36d46d59f Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 15 Oct 2026 19:41:46 -0700
Subject: [PATCH 01/11] [None][feat] serve: default num_serve_frontends to 8
 with single-frontend fallback

Make the plain `trtllm-serve` subcommand run 8 HTTP frontend processes
against one executor by default (`--num_serve_frontends`, added in #16523).
At high concurrency a single serving process is host-bound on its asyncio
loop; several SO_REUSEPORT frontends remove that ceiling.

- The LlmArgs field default stays 1: a bare LLM() has no HTTP frontends to
  fan out to and only trtllm-serve can spawn them, so trtllm-bench / eval /
  Python users do not pay for the multi-frontend IPC topology. Because
  8 != 1 the CLI value survives get_llm_args's default filter; a YAML
  `num_serve_frontends: 1` still overrides the untyped CLI default.
- When the count comes from the default and the configuration can only
  run one frontend (--port 0 / --report_addr, orchestrator_type,
  enable_resource_governor, --enable_sleep_mode) or serves gRPC, log why
  and run 1 frontend instead of failing at startup. An explicit
  --num_serve_frontends > 1 keeps the loud error.
- api_stability reference, new CPU tests (l0_cpu), docs section.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 .../commands/trtllm-serve/trtllm-serve.rst    |  12 ++
 tensorrt_llm/commands/serve.py                |  67 +++++++-
 .../integration/test_lists/test-db/l0_cpu.yml |   1 +
 .../references/trtllm_serve_cli.yaml          |   2 +-
 .../test_serve_num_frontends_default.py       | 152 ++++++++++++++++++
 5 files changed, 230 insertions(+), 4 deletions(-)
 create mode 100644 tests/unittest/llmapi/test_serve_num_frontends_default.py

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 2bb93a668087..5c5a69763793 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -79,6 +79,18 @@ The module ``tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py`` implements t
 shared format. ``--reasoning_parser`` separately controls reasoning extraction;
 ``nemotron-v3`` and ``nano-v3`` are aliases for the same Nemotron reasoning parser.
 
+Multiple HTTP Frontends
+-----------------------
+
+At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, prototype). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL.
+
+.. note::
+
+   * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
+   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
+   * Each request is handled entirely by the frontend that accepted its connection, and per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, and ``/metrics``, ``/perf_metrics``, ``/prometheus/metrics`` and ``/health`` describe the frontend that happened to answer rather than the whole server.
+   * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
+
 Inference Endpoints
 -------------------
 
diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index 08d018dcfb44..74938c377ba2 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -558,6 +558,50 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
+# Default for the plain `trtllm-serve` subcommand. At high concurrency one
+# serving process is host-bound (a single asyncio loop parses every request
+# and writes every streamed chunk), so several frontends per executor pay off
+# by default. The LlmArgs field itself keeps default 1: a bare LLM() has no
+# HTTP frontends to fan out to, and only trtllm-serve can spawn them.
+DEFAULT_NUM_SERVE_FRONTENDS = 8
+
+
+def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
+                                         port: int, grpc: bool,
+                                         report_addr: Optional[str],
+                                         enable_sleep_mode: bool) -> None:
+    """Fall back to one frontend when the *default* count does not apply.
+
+    The default is sized for the OpenAI HTTP server on the classic IPC
+    executor path on a fixed port. When the user did not ask for a specific
+    count (neither on the CLI nor in the --config YAML / --set) and the
+    configuration can only run a single frontend, or serves gRPC, run one
+    instead of failing at startup. An explicit request keeps the loud error
+    from the guard that owns the incompatibility.
+    """
+    num_frontends = llm_args.get("num_serve_frontends", 1)
+    if requested or num_frontends <= 1:
+        return
+    if grpc:
+        # OpenEngine can run several frontends, but only when asked to.
+        reason = "--grpc"
+    elif port == 0 or report_addr:
+        reason = "--port 0 / --report_addr"
+    elif llm_args.get("orchestrator_type") is not None:
+        reason = f"orchestrator_type={llm_args['orchestrator_type']!r}"
+    elif llm_args.get("enable_resource_governor"):
+        reason = "enable_resource_governor"
+    elif enable_sleep_mode:
+        reason = "--enable_sleep_mode"
+    else:
+        return
+    logger.info(
+        f"num_serve_frontends defaults to {num_frontends} for the OpenAI HTTP "
+        f"server only; running 1 frontend because of {reason}. Pass "
+        "--num_serve_frontends explicitly to override.")
+    llm_args["num_serve_frontends"] = 1
+
+
 def launch_server(
         host: str,
         port: int,
@@ -622,7 +666,8 @@ def launch_server(
             logger.warning(
                 "num_serve_frontends > 1: stateful Responses API storage "
                 "(store/previous_response_id) is disabled; the per-frontend "
-                "in-memory store cannot be shared across frontends.")
+                "in-memory store cannot be shared across frontends. Pass "
+                "--num_serve_frontends 1 to keep it.")
         os.environ["TRTLLM_RESPONSES_API_DISABLE_STORE"] = "1"
 
     addr_info = socket.getaddrinfo(host, port, socket.AF_UNSPEC,
@@ -1111,10 +1156,14 @@ def launch_visual_gen_server(
 @stability_option(
     "--num_serve_frontends",
     type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS),
-    default=1,
+    default=DEFAULT_NUM_SERVE_FRONTENDS,
     help="Number of HTTP or OpenEngine frontend processes serving one "
     "executor; values > 1 share the serving port via "
-    "SO_REUSEPORT (classic IPC executor path only).",
+    "SO_REUSEPORT (classic IPC executor path only). The default applies to "
+    "the OpenAI HTTP server and falls back to 1 for configurations that "
+    "support a single frontend only (--port 0/--report_addr, "
+    "orchestrator_type, enable_resource_governor, --enable_sleep_mode) and "
+    "with --grpc; an explicit value is honored or fails instead.",
     status="prototype")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
@@ -1556,6 +1605,18 @@ def _serve_llm():
         if enable_sleep_mode and llm_args.get("sleep_config") is None:
             llm_args["sleep_config"] = SleepConfig()
 
+        # The multi-frontend default only applies where it can run; an
+        # explicit CLI flag, YAML key or --set override is honored as-is (and
+        # may fail loudly).
+        _resolve_default_num_serve_frontends(
+            llm_args,
+            requested=("num_serve_frontends" in explicit_cli_keys
+                       or "num_serve_frontends" in llm_args_extra_dict),
+            port=port,
+            grpc=grpc,
+            report_addr=report_addr,
+            enable_sleep_mode=enable_sleep_mode)
+
         _apply_effective_telemetry_config(llm_args, telemetry=telemetry)
         # Preserve requested settings for failures before entering the LLM constructor.
         try:
diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml
index edf634c48fcf..5fdfef3a33d5 100644
--- a/tests/integration/test_lists/test-db/l0_cpu.yml
+++ b/tests/integration/test_lists/test-db/l0_cpu.yml
@@ -295,6 +295,7 @@ l0_cpu:
   - unittest/llmapi/test_serialization.py
   - unittest/llmapi/test_serve_multi_frontend_helpers.py
   - unittest/llmapi/test_serve_multi_frontend_route_gate.py
+  - unittest/llmapi/test_serve_num_frontends_default.py
   - unittest/llmapi/test_serve_report_addr.py
   - unittest/llmapi/test_serve_sleep_mode.py
   - unittest/llmapi/test_session_prefetcher.py
diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
index 8c06a6d0ea57..e3648746192c 100644
--- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
+++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
@@ -344,7 +344,7 @@ commands:
         - "--num_postprocess_workers"
       num_serve_frontends:
         type: int
-        default: 1
+        default: 8
         status: prototype
         required: false
         multiple: false
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
new file mode 100644
index 000000000000..fdd66db34176
--- /dev/null
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -0,0 +1,152 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit coverage for trtllm-serve's multi-frontend default.
+
+``--num_serve_frontends`` defaults to several HTTP frontend processes while
+the LlmArgs field keeps default 1. The pieces that make the default reach the
+executor -- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML,
+and falling back to a single frontend where more cannot run -- are covered
+here; spawning the frontends needs a GPU and lives in the integration tests.
+"""
+
+from typing import Optional
+
+import pytest
+
+from tensorrt_llm.commands import serve as serve_cmd
+from tensorrt_llm.commands.serve import (
+    DEFAULT_NUM_SERVE_FRONTENDS,
+    _init_multi_frontend_mode,
+    _resolve_default_num_serve_frontends,
+    get_llm_args,
+)
+from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS
+from tensorrt_llm.llmapi.llm_args import BaseLlmArgs
+from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict
+
+pytestmark = pytest.mark.cpu_only
+
+
+def _resolve(
+    llm_args: dict,
+    *,
+    requested: bool = False,
+    port: int = 8000,
+    grpc: bool = False,
+    report_addr: Optional[str] = None,
+) -> int:
+    _resolve_default_num_serve_frontends(
+        llm_args,
+        requested=requested,
+        port=port,
+        grpc=grpc,
+        report_addr=report_addr,
+        enable_sleep_mode=False,
+    )
+    return llm_args.get("num_serve_frontends", 1)
+
+
+def test_cli_default_is_multi_frontend() -> None:
+    option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends")
+    assert option.default == DEFAULT_NUM_SERVE_FRONTENDS
+    assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS
+
+
+def test_llm_args_default_stays_single_frontend() -> None:
+    # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve
+    # spawns them, so the executor-side default must stay at one lane.
+    assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1
+
+
+def test_cli_default_survives_the_llm_args_default_filter() -> None:
+    # get_llm_args drops CLI values equal to the LlmArgs default unless the
+    # flag was typed. The serve default differs from the LlmArgs default, so
+    # it must reach the LLM constructor without being typed on the CLI ...
+    llm_args, _ = get_llm_args(
+        model="m",
+        backend="pytorch",
+        gpus_per_node=1,
+        num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS,
+    )
+    assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS
+    # ... while an untyped 1 collapses onto the LlmArgs default.
+    llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1)
+    assert "num_serve_frontends" not in llm_args
+
+
+def test_yaml_value_overrides_the_cli_default() -> None:
+    llm_args = update_llm_args_with_extra_dict(
+        {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS},
+        {"num_serve_frontends": 1},
+        explicit_cli_keys=set(),
+    )
+    assert llm_args["num_serve_frontends"] == 1
+
+
+def test_explicit_cli_value_wins_over_yaml() -> None:
+    llm_args = update_llm_args_with_extra_dict(
+        {"num_serve_frontends": 4},
+        {"num_serve_frontends": 1},
+        explicit_cli_keys={"num_serve_frontends"},
+    )
+    assert llm_args["num_serve_frontends"] == 4
+
+
+def test_plain_config_keeps_the_default() -> None:
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}
+    assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS
+    assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
+
+
+@pytest.mark.parametrize(
+    "extra_llm_args,kwargs",
+    [
+        ({"orchestrator_type": "rpc"}, {}),
+        ({"orchestrator_type": "ray"}, {}),
+        ({"enable_resource_governor": True}, {}),
+        ({}, {"grpc": True}),
+        ({}, {"port": 0}),
+        ({}, {"report_addr": "/tmp/bound.addr"}),
+    ],
+)
+def test_default_falls_back_to_one_frontend(extra_llm_args: dict, kwargs: dict) -> None:
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
+    assert _resolve(llm_args, **kwargs) == 1
+    # The fallback leaves a consistent single-frontend configuration behind:
+    # the mode resolver no longer sees a launcher and does not raise.
+    if "orchestrator_type" in extra_llm_args:
+        assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
+
+
+def test_explicit_request_is_not_silently_downgraded() -> None:
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"}
+    assert _resolve(llm_args, requested=True) == DEFAULT_NUM_SERVE_FRONTENDS
+    with pytest.raises(ValueError, match="orchestrator_type"):
+        _init_multi_frontend_mode(llm_args, enabled=True)
+
+
+def test_single_frontend_is_left_alone() -> None:
+    llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"}
+    assert _resolve(llm_args) == 1
+    assert _resolve({"orchestrator_type": "rpc"}) == 1
+
+
+def test_fallback_logs_the_reason(monkeypatch: pytest.MonkeyPatch) -> None:
+    messages = []
+    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
+    _resolve({"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"})
+    assert len(messages) == 1
+    assert "orchestrator_type='rpc'" in messages[0]
+    assert "--num_serve_frontends" in messages[0]

From 03f7b474ea669f43143a3da6ed67488364fbf78b Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 17 Sep 2026 18:37:29 -0700
Subject: [PATCH 02/11] [None][feat] serve: drop the dedicated CPU test file
 for the frontend default

The default and its fallback are covered by the api_stability CLI gate and
the e2e runs; keep the PR to the behaviour change.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 .../integration/test_lists/test-db/l0_cpu.yml |   1 -
 .../test_serve_num_frontends_default.py       | 152 ------------------
 2 files changed, 153 deletions(-)
 delete mode 100644 tests/unittest/llmapi/test_serve_num_frontends_default.py

diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml
index 5fdfef3a33d5..edf634c48fcf 100644
--- a/tests/integration/test_lists/test-db/l0_cpu.yml
+++ b/tests/integration/test_lists/test-db/l0_cpu.yml
@@ -295,7 +295,6 @@ l0_cpu:
   - unittest/llmapi/test_serialization.py
   - unittest/llmapi/test_serve_multi_frontend_helpers.py
   - unittest/llmapi/test_serve_multi_frontend_route_gate.py
-  - unittest/llmapi/test_serve_num_frontends_default.py
   - unittest/llmapi/test_serve_report_addr.py
   - unittest/llmapi/test_serve_sleep_mode.py
   - unittest/llmapi/test_session_prefetcher.py
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
deleted file mode 100644
index fdd66db34176..000000000000
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Unit coverage for trtllm-serve's multi-frontend default.
-
-``--num_serve_frontends`` defaults to several HTTP frontend processes while
-the LlmArgs field keeps default 1. The pieces that make the default reach the
-executor -- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML,
-and falling back to a single frontend where more cannot run -- are covered
-here; spawning the frontends needs a GPU and lives in the integration tests.
-"""
-
-from typing import Optional
-
-import pytest
-
-from tensorrt_llm.commands import serve as serve_cmd
-from tensorrt_llm.commands.serve import (
-    DEFAULT_NUM_SERVE_FRONTENDS,
-    _init_multi_frontend_mode,
-    _resolve_default_num_serve_frontends,
-    get_llm_args,
-)
-from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS
-from tensorrt_llm.llmapi.llm_args import BaseLlmArgs
-from tensorrt_llm.llmapi.llm_utils import update_llm_args_with_extra_dict
-
-pytestmark = pytest.mark.cpu_only
-
-
-def _resolve(
-    llm_args: dict,
-    *,
-    requested: bool = False,
-    port: int = 8000,
-    grpc: bool = False,
-    report_addr: Optional[str] = None,
-) -> int:
-    _resolve_default_num_serve_frontends(
-        llm_args,
-        requested=requested,
-        port=port,
-        grpc=grpc,
-        report_addr=report_addr,
-        enable_sleep_mode=False,
-    )
-    return llm_args.get("num_serve_frontends", 1)
-
-
-def test_cli_default_is_multi_frontend() -> None:
-    option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends")
-    assert option.default == DEFAULT_NUM_SERVE_FRONTENDS
-    assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS
-
-
-def test_llm_args_default_stays_single_frontend() -> None:
-    # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve
-    # spawns them, so the executor-side default must stay at one lane.
-    assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1
-
-
-def test_cli_default_survives_the_llm_args_default_filter() -> None:
-    # get_llm_args drops CLI values equal to the LlmArgs default unless the
-    # flag was typed. The serve default differs from the LlmArgs default, so
-    # it must reach the LLM constructor without being typed on the CLI ...
-    llm_args, _ = get_llm_args(
-        model="m",
-        backend="pytorch",
-        gpus_per_node=1,
-        num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS,
-    )
-    assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS
-    # ... while an untyped 1 collapses onto the LlmArgs default.
-    llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1)
-    assert "num_serve_frontends" not in llm_args
-
-
-def test_yaml_value_overrides_the_cli_default() -> None:
-    llm_args = update_llm_args_with_extra_dict(
-        {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS},
-        {"num_serve_frontends": 1},
-        explicit_cli_keys=set(),
-    )
-    assert llm_args["num_serve_frontends"] == 1
-
-
-def test_explicit_cli_value_wins_over_yaml() -> None:
-    llm_args = update_llm_args_with_extra_dict(
-        {"num_serve_frontends": 4},
-        {"num_serve_frontends": 1},
-        explicit_cli_keys={"num_serve_frontends"},
-    )
-    assert llm_args["num_serve_frontends"] == 4
-
-
-def test_plain_config_keeps_the_default() -> None:
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}
-    assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS
-    assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
-
-
-@pytest.mark.parametrize(
-    "extra_llm_args,kwargs",
-    [
-        ({"orchestrator_type": "rpc"}, {}),
-        ({"orchestrator_type": "ray"}, {}),
-        ({"enable_resource_governor": True}, {}),
-        ({}, {"grpc": True}),
-        ({}, {"port": 0}),
-        ({}, {"report_addr": "/tmp/bound.addr"}),
-    ],
-)
-def test_default_falls_back_to_one_frontend(extra_llm_args: dict, kwargs: dict) -> None:
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
-    assert _resolve(llm_args, **kwargs) == 1
-    # The fallback leaves a consistent single-frontend configuration behind:
-    # the mode resolver no longer sees a launcher and does not raise.
-    if "orchestrator_type" in extra_llm_args:
-        assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
-
-
-def test_explicit_request_is_not_silently_downgraded() -> None:
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"}
-    assert _resolve(llm_args, requested=True) == DEFAULT_NUM_SERVE_FRONTENDS
-    with pytest.raises(ValueError, match="orchestrator_type"):
-        _init_multi_frontend_mode(llm_args, enabled=True)
-
-
-def test_single_frontend_is_left_alone() -> None:
-    llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"}
-    assert _resolve(llm_args) == 1
-    assert _resolve({"orchestrator_type": "rpc"}) == 1
-
-
-def test_fallback_logs_the_reason(monkeypatch: pytest.MonkeyPatch) -> None:
-    messages = []
-    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
-    _resolve({"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, "orchestrator_type": "rpc"})
-    assert len(messages) == 1
-    assert "orchestrator_type='rpc'" in messages[0]
-    assert "--num_serve_frontends" in messages[0]

From 9fe82b6a223ccfcd2f9408c0c414503c7b45cce0 Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 15 Oct 2026 20:27:05 -0700
Subject: [PATCH 03/11] [None][feat] serve: address review on the
 multi-frontend default

- Fall back to one frontend by default with --metadata_server_config_file
  or a disagg cluster: every frontend registers the server under its own
  process identity (llm_id, DisaggClusterWorker id), which would list one
  server once per frontend.
- Restore tests/unittest/llmapi/test_serve_num_frontends_default.py
  (l0_cpu): every fallback reason, explicit counts reaching the guard that
  rejects them, CLI / YAML / --set precedence down to the arguments
  launch_server receives, and a YAML count with --grpc being rejected.
- RemoteOpenAIServer keeps test servers at one frontend unless the test
  picks the count (flag, --set, extra_config or YAML).

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 .../commands/trtllm-serve/trtllm-serve.rst    |   2 +-
 tensorrt_llm/commands/serve.py                |  20 +-
 .../integration/test_lists/test-db/l0_cpu.yml |   1 +
 tests/unittest/llmapi/apps/openai_server.py   |  27 ++
 .../test_serve_num_frontends_default.py       | 252 ++++++++++++++++++
 5 files changed, 297 insertions(+), 5 deletions(-)
 create mode 100644 tests/unittest/llmapi/test_serve_num_frontends_default.py

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 5c5a69763793..1543b9e75360 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -87,7 +87,7 @@ At high concurrency a single serving process is host-bound: one asyncio event lo
 .. note::
 
    * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
-   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
+   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor``, ``--enable_sleep_mode``, ``--metadata_server_config_file`` or a disaggregated cluster (``disagg_cluster`` / ``--disagg_cluster_uri``) it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
    * Each request is handled entirely by the frontend that accepted its connection, and per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, and ``/metrics``, ``/perf_metrics``, ``/prometheus/metrics`` and ``/health`` describe the frontend that happened to answer rather than the whole server.
    * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
 
diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index 74938c377ba2..297c41a6e412 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -569,7 +569,9 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
 def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
                                          port: int, grpc: bool,
                                          report_addr: Optional[str],
-                                         enable_sleep_mode: bool) -> None:
+                                         enable_sleep_mode: bool,
+                                         metadata_server: bool,
+                                         disagg_cluster: bool) -> None:
     """Fall back to one frontend when the *default* count does not apply.
 
     The default is sized for the OpenAI HTTP server on the classic IPC
@@ -593,6 +595,12 @@ def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
         reason = "enable_resource_governor"
     elif enable_sleep_mode:
         reason = "--enable_sleep_mode"
+    elif metadata_server or disagg_cluster:
+        # Every frontend registers the server under its own process identity
+        # (llm_id, DisaggClusterWorker id), which would list it once per
+        # frontend.
+        reason = ("--metadata_server_config_file"
+                  if metadata_server else "the disagg_cluster config")
     else:
         return
     logger.info(
@@ -1162,8 +1170,9 @@ def launch_visual_gen_server(
     "SO_REUSEPORT (classic IPC executor path only). The default applies to "
     "the OpenAI HTTP server and falls back to 1 for configurations that "
     "support a single frontend only (--port 0/--report_addr, "
-    "orchestrator_type, enable_resource_governor, --enable_sleep_mode) and "
-    "with --grpc; an explicit value is honored or fails instead.",
+    "orchestrator_type, enable_resource_governor, --enable_sleep_mode, "
+    "--metadata_server_config_file, a disagg cluster) and with --grpc; an "
+    "explicit value is honored or fails instead.",
     status="prototype")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
@@ -1615,7 +1624,10 @@ def _serve_llm():
             port=port,
             grpc=grpc,
             report_addr=report_addr,
-            enable_sleep_mode=enable_sleep_mode)
+            enable_sleep_mode=enable_sleep_mode,
+            metadata_server=metadata_server_config_file is not None,
+            disagg_cluster=bool(
+                llm_args.get("disagg_cluster") or disagg_cluster_uri))
 
         _apply_effective_telemetry_config(llm_args, telemetry=telemetry)
         # Preserve requested settings for failures before entering the LLM constructor.
diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml
index edf634c48fcf..5fdfef3a33d5 100644
--- a/tests/integration/test_lists/test-db/l0_cpu.yml
+++ b/tests/integration/test_lists/test-db/l0_cpu.yml
@@ -295,6 +295,7 @@ l0_cpu:
   - unittest/llmapi/test_serialization.py
   - unittest/llmapi/test_serve_multi_frontend_helpers.py
   - unittest/llmapi/test_serve_multi_frontend_route_gate.py
+  - unittest/llmapi/test_serve_num_frontends_default.py
   - unittest/llmapi/test_serve_report_addr.py
   - unittest/llmapi/test_serve_sleep_mode.py
   - unittest/llmapi/test_session_prefetcher.py
diff --git a/tests/unittest/llmapi/apps/openai_server.py b/tests/unittest/llmapi/apps/openai_server.py
index 62b3543d8159..66450715983d 100644
--- a/tests/unittest/llmapi/apps/openai_server.py
+++ b/tests/unittest/llmapi/apps/openai_server.py
@@ -14,6 +14,28 @@
 from tensorrt_llm._utils import get_free_port
 from tensorrt_llm.llmapi.disagg_utils import ServerRole
 
+_NUM_FRONTENDS_FLAG = "--num_serve_frontends"
+
+
+def _picks_num_frontends(cli_args: List[str],
+                         extra_config: Optional[dict]) -> bool:
+    """Whether a test set num_serve_frontends itself (flag, --set or YAML)."""
+    if extra_config and "num_serve_frontends" in extra_config:
+        return True
+    for i, arg in enumerate(cli_args):
+        value = cli_args[i + 1] if i + 1 < len(cli_args) else ""
+        if arg == _NUM_FRONTENDS_FLAG or arg.startswith(_NUM_FRONTENDS_FLAG +
+                                                        "="):
+            return True
+        if (arg == "--set" and value.startswith("num_serve_frontends=")
+            ) or arg.startswith("--set=num_serve_frontends="):
+            return True
+        if arg in ("--config", "--extra_llm_api_options") and value:
+            with open(value) as f:
+                if "num_serve_frontends" in (yaml.safe_load(f) or {}):
+                    return True
+    return False
+
 
 class RemoteOpenAIServer:
     DUMMY_API_KEY = "tensorrt_llm"
@@ -51,6 +73,11 @@ def __init__(self,
                 f.write(yaml.dump(extra_config))
                 self.extra_config_file = f.name
             args += ["--extra_llm_api_options", self.extra_config_file]
+        # trtllm-serve runs several HTTP frontend processes by default. Keep
+        # test servers at one -- per-process state (Responses store, /health),
+        # startup time and host memory -- unless a test picks the count.
+        if not _picks_num_frontends(cli_args or [], extra_config):
+            args += [_NUM_FRONTENDS_FLAG, "1"]
         launch_cmd = ["trtllm-serve"] + [model] + args
         if llmapi_launch:
             # start server with llmapi-launch on multi nodes
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
new file mode 100644
index 000000000000..070067e2682b
--- /dev/null
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -0,0 +1,252 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit coverage for trtllm-serve's multi-frontend default.
+
+``--num_serve_frontends`` defaults to several HTTP frontend processes while
+the LlmArgs field keeps default 1. What makes the default reach the executor
+-- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML and
+--set, and falling back to a single frontend where more cannot run -- is
+covered here down to the arguments ``launch_server`` receives; spawning the
+frontends needs a GPU engine.
+"""
+
+from typing import Optional
+from unittest import mock
+
+import click
+import pytest
+
+from tensorrt_llm.commands import serve as serve_cmd
+from tensorrt_llm.commands.serve import (
+    DEFAULT_NUM_SERVE_FRONTENDS,
+    _resolve_default_num_serve_frontends,
+    get_llm_args,
+)
+from tensorrt_llm.commands.serve import main as serve_main
+from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS
+from tensorrt_llm.llmapi.llm_args import BaseLlmArgs
+from tensorrt_llm.serve._frontend_processes import _init_multi_frontend_mode
+
+pytestmark = pytest.mark.cpu_only
+
+
+def _resolve(
+    llm_args: dict,
+    *,
+    requested: bool = False,
+    port: int = 8000,
+    grpc: bool = False,
+    report_addr: Optional[str] = None,
+    enable_sleep_mode: bool = False,
+    metadata_server: bool = False,
+    disagg_cluster: bool = False,
+) -> int:
+    _resolve_default_num_serve_frontends(
+        llm_args,
+        requested=requested,
+        port=port,
+        grpc=grpc,
+        report_addr=report_addr,
+        enable_sleep_mode=enable_sleep_mode,
+        metadata_server=metadata_server,
+        disagg_cluster=disagg_cluster,
+    )
+    return llm_args.get("num_serve_frontends", 1)
+
+
+def test_cli_default_is_multi_frontend() -> None:
+    option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends")
+    assert option.default == DEFAULT_NUM_SERVE_FRONTENDS
+    assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS
+
+
+def test_llm_args_default_stays_single_frontend() -> None:
+    # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve
+    # spawns them, so the executor-side default must stay at one lane.
+    assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1
+
+
+def test_cli_default_survives_the_llm_args_default_filter() -> None:
+    # get_llm_args drops CLI values equal to the LlmArgs default unless the
+    # flag was typed. The serve default differs from the LlmArgs default, so
+    # it must reach the LLM constructor without being typed on the CLI ...
+    llm_args, _ = get_llm_args(
+        model="m",
+        backend="pytorch",
+        gpus_per_node=1,
+        num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS,
+    )
+    assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS
+    # ... while an untyped 1 collapses onto the LlmArgs default.
+    llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1)
+    assert "num_serve_frontends" not in llm_args
+
+
+def test_plain_config_keeps_the_default() -> None:
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}
+    assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS
+    assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
+
+
+_FALLBACKS = [
+    pytest.param({}, {"grpc": True}, "--grpc", id="grpc"),
+    pytest.param({}, {"port": 0}, "--port 0", id="port0"),
+    pytest.param({}, {"report_addr": "/tmp/bound.addr"}, "--report_addr", id="report_addr"),
+    pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"),
+    pytest.param({"orchestrator_type": "ray"}, {}, "orchestrator_type='ray'", id="ray"),
+    pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="governor"),
+    pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"),
+    pytest.param(
+        {}, {"metadata_server": True}, "--metadata_server_config_file", id="metadata_server"
+    ),
+    pytest.param({}, {"disagg_cluster": True}, "disagg_cluster", id="disagg_cluster"),
+]
+
+
+@pytest.mark.parametrize("extra_llm_args,kwargs,reason", _FALLBACKS)
+def test_default_falls_back_to_one_frontend(
+    monkeypatch: pytest.MonkeyPatch, extra_llm_args: dict, kwargs: dict, reason: str
+) -> None:
+    messages = []
+    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
+    assert _resolve(llm_args, **kwargs) == 1
+    # The fallback says why, and how to override it.
+    assert len(messages) == 1
+    assert reason in messages[0] and "--num_serve_frontends" in messages[0]
+    # It leaves a consistent single-frontend configuration behind: the mode
+    # resolver no longer sees a launcher and does not raise.
+    assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
+
+
+@pytest.mark.parametrize("extra_llm_args,kwargs,reason", _FALLBACKS)
+def test_explicit_request_is_not_silently_downgraded(
+    extra_llm_args: dict, kwargs: dict, reason: str
+) -> None:
+    # The value must survive so that the guard owning the incompatibility
+    # still raises (or, for OpenEngine gRPC, honors it).
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
+    assert _resolve(llm_args, requested=True, **kwargs) == DEFAULT_NUM_SERVE_FRONTENDS
+
+
+def test_single_frontend_is_left_alone() -> None:
+    llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"}
+    assert _resolve(llm_args) == 1
+    assert _resolve({"orchestrator_type": "rpc"}) == 1
+
+
+def _serve(args: list, monkeypatch: pytest.MonkeyPatch, tmp_path) -> mock.MagicMock:
+    """Run the CLI down to launch_server and return the mocked launcher."""
+    with (
+        mock.patch(
+            "tensorrt_llm.commands.serve.get_is_diffusion_only_model",
+            return_value=False,
+        ),
+        mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1),
+        mock.patch("tensorrt_llm.commands.serve.launch_server") as launch,
+    ):
+        serve_main(args=[str(tmp_path), *args], standalone_mode=False)
+    assert launch.call_count == 1
+    return launch
+
+
+def _launched_frontends(launch: mock.MagicMock) -> int:
+    return launch.call_args.args[2].get("num_serve_frontends", 1)
+
+
+def _yaml(tmp_path, text: str) -> str:
+    path = tmp_path / "config.yaml"
+    path.write_text(text, encoding="utf-8")
+    return str(path)
+
+
+def test_cli_default_reaches_launch_server(monkeypatch, tmp_path) -> None:
+    assert _launched_frontends(_serve([], monkeypatch, tmp_path)) == DEFAULT_NUM_SERVE_FRONTENDS
+
+
+@pytest.mark.parametrize(
+    "args_factory,expected",
+    [
+        pytest.param(lambda p: ["--num_serve_frontends", "1"], 1, id="cli_1"),
+        pytest.param(lambda p: ["--num_serve_frontends", "3"], 3, id="cli_3"),
+        pytest.param(lambda p: ["--config", _yaml(p, "num_serve_frontends: 1\n")], 1, id="yaml_1"),
+        pytest.param(lambda p: ["--set", "num_serve_frontends=2"], 2, id="set_2"),
+        # An explicit count survives a configuration it conflicts with, so
+        # launch_server's own guard rejects it rather than it shrinking to 1.
+        pytest.param(
+            lambda p: ["--num_serve_frontends", "8", "--port", "0"], 8, id="explicit_port0"
+        ),
+    ],
+)
+def test_requested_counts_reach_launch_server(
+    monkeypatch, tmp_path, args_factory, expected: int
+) -> None:
+    launch = _serve(args_factory(tmp_path), monkeypatch, tmp_path)
+    assert _launched_frontends(launch) == expected
+
+
+@pytest.mark.parametrize(
+    "args_factory",
+    [
+        pytest.param(lambda p: ["--port", "0"], id="port0"),
+        pytest.param(lambda p: ["--report_addr", str(p / "bound.addr")], id="report_addr"),
+        pytest.param(lambda p: ["--config", _yaml(p, "orchestrator_type: rpc\n")], id="rpc"),
+        pytest.param(
+            lambda p: [
+                "--server_role",
+                "context",
+                "--metadata_server_config_file",
+                _yaml(p, "server_type: etcd\n"),
+            ],
+            id="metadata_server",
+        ),
+        pytest.param(
+            lambda p: ["--server_role", "context", "--disagg_cluster_uri", "etcd://localhost:2379"],
+            id="disagg_cluster",
+        ),
+    ],
+)
+def test_default_falls_back_before_launch_server(monkeypatch, tmp_path, args_factory) -> None:
+    launch = _serve(args_factory(tmp_path), monkeypatch, tmp_path)
+    assert _launched_frontends(launch) == 1
+
+
+def test_sleep_mode_default_falls_back_before_launch_server(monkeypatch, tmp_path) -> None:
+    monkeypatch.setenv("TRTLLM_RUNTIME_CONTROL_API_KEY", "secret")
+    launch = _serve(["--enable_sleep_mode"], monkeypatch, tmp_path)
+    assert _launched_frontends(launch) == 1
+    assert launch.call_args.kwargs["enable_runtime_control_endpoints"]
+
+
+def test_yaml_count_with_grpc_is_rejected(monkeypatch, tmp_path) -> None:
+    # An explicit YAML count is honored, so the gRPC guard sees it and rejects
+    # several frontends on the SMG protocol instead of silently running one.
+    with (
+        mock.patch(
+            "tensorrt_llm.commands.serve.get_is_diffusion_only_model",
+            return_value=False,
+        ),
+        mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1),
+        pytest.raises(click.UsageError, match="Multiple gRPC frontends"),
+    ):
+        serve_main(
+            args=[
+                str(tmp_path),
+                "--grpc",
+                "--config",
+                _yaml(tmp_path, "num_serve_frontends: 8\n"),
+            ],
+            standalone_mode=False,
+        )

From b908a388b3c97fb80c388e6118f8f97772f8f9ff Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Sat, 24 Oct 2026 08:55:25 -0700
Subject: [PATCH 04/11] [None][test] serve: trim the multi-frontend default
 tests and comments

Drop tests that restate main's behavior or are already covered by
test_llm_args.py and test_grpc_optional.py, merge the CLI cases into
one table, and shorten comments. No behavior change.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 tensorrt_llm/commands/serve.py                |  29 +-
 tests/unittest/llmapi/apps/openai_server.py   |  28 +-
 .../test_serve_num_frontends_default.py       | 260 ++++--------------
 3 files changed, 69 insertions(+), 248 deletions(-)

diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index 297c41a6e412..8d3c44934210 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -558,11 +558,9 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
-# Default for the plain `trtllm-serve` subcommand. At high concurrency one
-# serving process is host-bound (a single asyncio loop parses every request
-# and writes every streamed chunk), so several frontends per executor pay off
-# by default. The LlmArgs field itself keeps default 1: a bare LLM() has no
-# HTTP frontends to fan out to, and only trtllm-serve can spawn them.
+# One serving process is host-bound at high concurrency (a single asyncio loop
+# parses every request and streams every chunk). LlmArgs keeps default 1:
+# only trtllm-serve spawns frontends.
 DEFAULT_NUM_SERVE_FRONTENDS = 8
 
 
@@ -572,20 +570,16 @@ def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
                                          enable_sleep_mode: bool,
                                          metadata_server: bool,
                                          disagg_cluster: bool) -> None:
-    """Fall back to one frontend when the *default* count does not apply.
-
-    The default is sized for the OpenAI HTTP server on the classic IPC
-    executor path on a fixed port. When the user did not ask for a specific
-    count (neither on the CLI nor in the --config YAML / --set) and the
-    configuration can only run a single frontend, or serves gRPC, run one
-    instead of failing at startup. An explicit request keeps the loud error
-    from the guard that owns the incompatibility.
+    """Fall back to one frontend where the default count cannot run.
+
+    An explicit count (CLI, --config YAML or --set) is left alone, so the
+    guard that owns the conflict still rejects it.
     """
     num_frontends = llm_args.get("num_serve_frontends", 1)
     if requested or num_frontends <= 1:
         return
     if grpc:
-        # OpenEngine can run several frontends, but only when asked to.
+        # OpenEngine supports several frontends, but only when asked to.
         reason = "--grpc"
     elif port == 0 or report_addr:
         reason = "--port 0 / --report_addr"
@@ -596,9 +590,7 @@ def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
     elif enable_sleep_mode:
         reason = "--enable_sleep_mode"
     elif metadata_server or disagg_cluster:
-        # Every frontend registers the server under its own process identity
-        # (llm_id, DisaggClusterWorker id), which would list it once per
-        # frontend.
+        # Every frontend would register the server under its own identity.
         reason = ("--metadata_server_config_file"
                   if metadata_server else "the disagg_cluster config")
     else:
@@ -1614,9 +1606,6 @@ def _serve_llm():
         if enable_sleep_mode and llm_args.get("sleep_config") is None:
             llm_args["sleep_config"] = SleepConfig()
 
-        # The multi-frontend default only applies where it can run; an
-        # explicit CLI flag, YAML key or --set override is honored as-is (and
-        # may fail loudly).
         _resolve_default_num_serve_frontends(
             llm_args,
             requested=("num_serve_frontends" in explicit_cli_keys
diff --git a/tests/unittest/llmapi/apps/openai_server.py b/tests/unittest/llmapi/apps/openai_server.py
index 66450715983d..38f74d557229 100644
--- a/tests/unittest/llmapi/apps/openai_server.py
+++ b/tests/unittest/llmapi/apps/openai_server.py
@@ -14,27 +14,18 @@
 from tensorrt_llm._utils import get_free_port
 from tensorrt_llm.llmapi.disagg_utils import ServerRole
 
-_NUM_FRONTENDS_FLAG = "--num_serve_frontends"
-
 
 def _picks_num_frontends(cli_args: List[str],
                          extra_config: Optional[dict]) -> bool:
-    """Whether a test set num_serve_frontends itself (flag, --set or YAML)."""
-    if extra_config and "num_serve_frontends" in extra_config:
-        return True
-    for i, arg in enumerate(cli_args):
-        value = cli_args[i + 1] if i + 1 < len(cli_args) else ""
-        if arg == _NUM_FRONTENDS_FLAG or arg.startswith(_NUM_FRONTENDS_FLAG +
-                                                        "="):
-            return True
-        if (arg == "--set" and value.startswith("num_serve_frontends=")
-            ) or arg.startswith("--set=num_serve_frontends="):
+    """Whether a test sets num_serve_frontends (a --set override wins anyway)."""
+    configs = [extra_config or {}]
+    for arg, value in zip(cli_args, cli_args[1:] + [""]):
+        if arg.split("=")[0] == "--num_serve_frontends":
             return True
         if arg in ("--config", "--extra_llm_api_options") and value:
             with open(value) as f:
-                if "num_serve_frontends" in (yaml.safe_load(f) or {}):
-                    return True
-    return False
+                configs.append(yaml.safe_load(f) or {})
+    return any("num_serve_frontends" in config for config in configs)
 
 
 class RemoteOpenAIServer:
@@ -73,11 +64,10 @@ def __init__(self,
                 f.write(yaml.dump(extra_config))
                 self.extra_config_file = f.name
             args += ["--extra_llm_api_options", self.extra_config_file]
-        # trtllm-serve runs several HTTP frontend processes by default. Keep
-        # test servers at one -- per-process state (Responses store, /health),
-        # startup time and host memory -- unless a test picks the count.
+        # trtllm-serve defaults to several frontend processes; test servers
+        # run one unless the test picks a count.
         if not _picks_num_frontends(cli_args or [], extra_config):
-            args += [_NUM_FRONTENDS_FLAG, "1"]
+            args += ["--num_serve_frontends", "1"]
         launch_cmd = ["trtllm-serve"] + [model] + args
         if llmapi_launch:
             # start server with llmapi-launch on multi nodes
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
index 070067e2682b..a8e965ae5240 100644
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -12,241 +12,83 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Unit coverage for trtllm-serve's multi-frontend default.
+"""CPU tests for trtllm-serve's multi-frontend default and its fallback to one."""
 
-``--num_serve_frontends`` defaults to several HTTP frontend processes while
-the LlmArgs field keeps default 1. What makes the default reach the executor
--- surviving the CLI-vs-LlmArgs default filter, yielding to the YAML and
---set, and falling back to a single frontend where more cannot run -- is
-covered here down to the arguments ``launch_server`` receives; spawning the
-frontends needs a GPU engine.
-"""
-
-from typing import Optional
 from unittest import mock
 
-import click
 import pytest
 
 from tensorrt_llm.commands import serve as serve_cmd
 from tensorrt_llm.commands.serve import (
     DEFAULT_NUM_SERVE_FRONTENDS,
     _resolve_default_num_serve_frontends,
-    get_llm_args,
 )
-from tensorrt_llm.commands.serve import main as serve_main
-from tensorrt_llm.executor.utils import MAX_NUM_FRONTENDS
-from tensorrt_llm.llmapi.llm_args import BaseLlmArgs
-from tensorrt_llm.serve._frontend_processes import _init_multi_frontend_mode
 
 pytestmark = pytest.mark.cpu_only
 
 
-def _resolve(
-    llm_args: dict,
-    *,
-    requested: bool = False,
-    port: int = 8000,
-    grpc: bool = False,
-    report_addr: Optional[str] = None,
-    enable_sleep_mode: bool = False,
-    metadata_server: bool = False,
-    disagg_cluster: bool = False,
-) -> int:
-    _resolve_default_num_serve_frontends(
-        llm_args,
-        requested=requested,
-        port=port,
-        grpc=grpc,
-        report_addr=report_addr,
-        enable_sleep_mode=enable_sleep_mode,
-        metadata_server=metadata_server,
-        disagg_cluster=disagg_cluster,
-    )
-    return llm_args.get("num_serve_frontends", 1)
-
-
-def test_cli_default_is_multi_frontend() -> None:
-    option = next(p for p in serve_cmd.serve.params if p.name == "num_serve_frontends")
-    assert option.default == DEFAULT_NUM_SERVE_FRONTENDS
-    assert 1 < DEFAULT_NUM_SERVE_FRONTENDS <= MAX_NUM_FRONTENDS
-
-
-def test_llm_args_default_stays_single_frontend() -> None:
-    # A bare LLM() has no HTTP frontends to fan out to; only trtllm-serve
-    # spawns them, so the executor-side default must stay at one lane.
-    assert BaseLlmArgs.model_fields["num_serve_frontends"].default == 1
-
-
-def test_cli_default_survives_the_llm_args_default_filter() -> None:
-    # get_llm_args drops CLI values equal to the LlmArgs default unless the
-    # flag was typed. The serve default differs from the LlmArgs default, so
-    # it must reach the LLM constructor without being typed on the CLI ...
-    llm_args, _ = get_llm_args(
-        model="m",
-        backend="pytorch",
-        gpus_per_node=1,
-        num_serve_frontends=DEFAULT_NUM_SERVE_FRONTENDS,
-    )
-    assert llm_args["num_serve_frontends"] == DEFAULT_NUM_SERVE_FRONTENDS
-    # ... while an untyped 1 collapses onto the LlmArgs default.
-    llm_args, _ = get_llm_args(model="m", backend="pytorch", gpus_per_node=1, num_serve_frontends=1)
-    assert "num_serve_frontends" not in llm_args
-
-
-def test_plain_config_keeps_the_default() -> None:
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS}
-    assert _resolve(llm_args) == DEFAULT_NUM_SERVE_FRONTENDS
-    assert _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
-
-
-_FALLBACKS = [
-    pytest.param({}, {"grpc": True}, "--grpc", id="grpc"),
-    pytest.param({}, {"port": 0}, "--port 0", id="port0"),
-    pytest.param({}, {"report_addr": "/tmp/bound.addr"}, "--report_addr", id="report_addr"),
-    pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"),
-    pytest.param({"orchestrator_type": "ray"}, {}, "orchestrator_type='ray'", id="ray"),
-    pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="governor"),
-    pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"),
-    pytest.param(
-        {}, {"metadata_server": True}, "--metadata_server_config_file", id="metadata_server"
-    ),
-    pytest.param({}, {"disagg_cluster": True}, "disagg_cluster", id="disagg_cluster"),
-]
-
-
-@pytest.mark.parametrize("extra_llm_args,kwargs,reason", _FALLBACKS)
-def test_default_falls_back_to_one_frontend(
-    monkeypatch: pytest.MonkeyPatch, extra_llm_args: dict, kwargs: dict, reason: str
-) -> None:
-    messages = []
-    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
-    assert _resolve(llm_args, **kwargs) == 1
-    # The fallback says why, and how to override it.
-    assert len(messages) == 1
-    assert reason in messages[0] and "--num_serve_frontends" in messages[0]
-    # It leaves a consistent single-frontend configuration behind: the mode
-    # resolver no longer sees a launcher and does not raise.
-    assert not _init_multi_frontend_mode(llm_args, enabled=True).is_launcher
-
-
-@pytest.mark.parametrize("extra_llm_args,kwargs,reason", _FALLBACKS)
-def test_explicit_request_is_not_silently_downgraded(
-    extra_llm_args: dict, kwargs: dict, reason: str
-) -> None:
-    # The value must survive so that the guard owning the incompatibility
-    # still raises (or, for OpenEngine gRPC, honors it).
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **extra_llm_args}
-    assert _resolve(llm_args, requested=True, **kwargs) == DEFAULT_NUM_SERVE_FRONTENDS
-
-
-def test_single_frontend_is_left_alone() -> None:
-    llm_args = {"num_serve_frontends": 1, "orchestrator_type": "rpc"}
-    assert _resolve(llm_args) == 1
-    assert _resolve({"orchestrator_type": "rpc"}) == 1
-
-
-def _serve(args: list, monkeypatch: pytest.MonkeyPatch, tmp_path) -> mock.MagicMock:
-    """Run the CLI down to launch_server and return the mocked launcher."""
-    with (
-        mock.patch(
-            "tensorrt_llm.commands.serve.get_is_diffusion_only_model",
-            return_value=False,
-        ),
-        mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1),
-        mock.patch("tensorrt_llm.commands.serve.launch_server") as launch,
-    ):
-        serve_main(args=[str(tmp_path), *args], standalone_mode=False)
-    assert launch.call_count == 1
-    return launch
-
-
-def _launched_frontends(launch: mock.MagicMock) -> int:
-    return launch.call_args.args[2].get("num_serve_frontends", 1)
-
-
-def _yaml(tmp_path, text: str) -> str:
-    path = tmp_path / "config.yaml"
-    path.write_text(text, encoding="utf-8")
-    return str(path)
-
-
-def test_cli_default_reaches_launch_server(monkeypatch, tmp_path) -> None:
-    assert _launched_frontends(_serve([], monkeypatch, tmp_path)) == DEFAULT_NUM_SERVE_FRONTENDS
-
-
 @pytest.mark.parametrize(
-    "args_factory,expected",
+    "llm_args,conflict,reason",
     [
-        pytest.param(lambda p: ["--num_serve_frontends", "1"], 1, id="cli_1"),
-        pytest.param(lambda p: ["--num_serve_frontends", "3"], 3, id="cli_3"),
-        pytest.param(lambda p: ["--config", _yaml(p, "num_serve_frontends: 1\n")], 1, id="yaml_1"),
-        pytest.param(lambda p: ["--set", "num_serve_frontends=2"], 2, id="set_2"),
-        # An explicit count survives a configuration it conflicts with, so
-        # launch_server's own guard rejects it rather than it shrinking to 1.
-        pytest.param(
-            lambda p: ["--num_serve_frontends", "8", "--port", "0"], 8, id="explicit_port0"
-        ),
+        pytest.param({}, {"grpc": True}, "--grpc", id="grpc"),
+        pytest.param({}, {"port": 0}, "--port 0", id="port0"),
+        pytest.param({}, {"report_addr": "/tmp/bound.addr"}, "--report_addr", id="report_addr"),
+        pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"),
+        pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="rg"),
+        pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"),
+        pytest.param({}, {"metadata_server": True}, "--metadata_server_config_file", id="metadata"),
+        pytest.param({}, {"disagg_cluster": True}, "disagg_cluster", id="disagg_cluster"),
     ],
 )
-def test_requested_counts_reach_launch_server(
-    monkeypatch, tmp_path, args_factory, expected: int
-) -> None:
-    launch = _serve(args_factory(tmp_path), monkeypatch, tmp_path)
-    assert _launched_frontends(launch) == expected
+def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, reason) -> None:
+    messages = []
+    monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
+    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **llm_args}
+    no_conflict = dict(
+        requested=False,
+        port=8000,
+        grpc=False,
+        report_addr=None,
+        enable_sleep_mode=False,
+        metadata_server=False,
+        disagg_cluster=False,
+    )
+
+    _resolve_default_num_serve_frontends(llm_args, **{**no_conflict, **conflict})
+
+    assert llm_args["num_serve_frontends"] == 1
+    assert len(messages) == 1 and reason in messages[0] and "--num_serve_frontends" in messages[0]
 
 
 @pytest.mark.parametrize(
-    "args_factory",
+    "args,config,expected",
     [
-        pytest.param(lambda p: ["--port", "0"], id="port0"),
-        pytest.param(lambda p: ["--report_addr", str(p / "bound.addr")], id="report_addr"),
-        pytest.param(lambda p: ["--config", _yaml(p, "orchestrator_type: rpc\n")], id="rpc"),
+        pytest.param([], None, DEFAULT_NUM_SERVE_FRONTENDS, id="default"),
+        pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"),
+        pytest.param(["--port", "0"], None, 1, id="port0"),
+        pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"),
         pytest.param(
-            lambda p: [
-                "--server_role",
-                "context",
-                "--metadata_server_config_file",
-                _yaml(p, "server_type: etcd\n"),
-            ],
-            id="metadata_server",
-        ),
-        pytest.param(
-            lambda p: ["--server_role", "context", "--disagg_cluster_uri", "etcd://localhost:2379"],
+            ["--server_role", "context", "--disagg_cluster_uri", "etcd://localhost:2379"],
+            None,
+            1,
             id="disagg_cluster",
         ),
+        # An explicit count is honored; launch_server's own guard rejects the conflict.
+        pytest.param(["--port", "0", "--num_serve_frontends", "8"], None, 8, id="explicit"),
     ],
 )
-def test_default_falls_back_before_launch_server(monkeypatch, tmp_path, args_factory) -> None:
-    launch = _serve(args_factory(tmp_path), monkeypatch, tmp_path)
-    assert _launched_frontends(launch) == 1
-
-
-def test_sleep_mode_default_falls_back_before_launch_server(monkeypatch, tmp_path) -> None:
-    monkeypatch.setenv("TRTLLM_RUNTIME_CONTROL_API_KEY", "secret")
-    launch = _serve(["--enable_sleep_mode"], monkeypatch, tmp_path)
-    assert _launched_frontends(launch) == 1
-    assert launch.call_args.kwargs["enable_runtime_control_endpoints"]
-
-
-def test_yaml_count_with_grpc_is_rejected(monkeypatch, tmp_path) -> None:
-    # An explicit YAML count is honored, so the gRPC guard sees it and rejects
-    # several frontends on the SMG protocol instead of silently running one.
+def test_frontend_count_reaching_launch_server(
+    monkeypatch, tmp_path, args, config, expected
+) -> None:
+    monkeypatch.setenv("TRTLLM_RUNTIME_CONTROL_API_KEY", "secret")  # for --enable_sleep_mode
+    if config is not None:
+        (tmp_path / "config.yaml").write_text(config)
+        args = [*args, "--config", str(tmp_path / "config.yaml")]
     with (
-        mock.patch(
-            "tensorrt_llm.commands.serve.get_is_diffusion_only_model",
-            return_value=False,
-        ),
-        mock.patch("tensorrt_llm.commands.serve.device_count", return_value=1),
-        pytest.raises(click.UsageError, match="Multiple gRPC frontends"),
+        mock.patch.object(serve_cmd, "get_is_diffusion_only_model", return_value=False),
+        mock.patch.object(serve_cmd, "device_count", return_value=1),
+        mock.patch.object(serve_cmd, "launch_server") as launch,
     ):
-        serve_main(
-            args=[
-                str(tmp_path),
-                "--grpc",
-                "--config",
-                _yaml(tmp_path, "num_serve_frontends: 8\n"),
-            ],
-            standalone_mode=False,
-        )
+        serve_cmd.main(args=[str(tmp_path), *args], standalone_mode=False)
+
+    assert launch.call_args.args[2].get("num_serve_frontends", 1) == expected

From 51cd675a6c50c5230588718c94f01f7df395ecd1 Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Sat, 24 Oct 2026 23:24:21 -0700
Subject: [PATCH 05/11] [None][fix] serve: refuse a port another server uses
 before joining it with SO_REUSEPORT

Every frontend binds the serving port with SO_REUSEPORT, which lets any
socket of the same user that also sets it join the port. With several
frontends by default, a second trtllm-serve started on a busy port would
silently share the first server's traffic instead of failing.

The launcher now probes the port with a plain bind (any listener makes it
fail; TIME_WAIT leftovers do not) and holds an abstract Unix socket named
after the port while it serves, which also covers a second launcher that
starts before the first one listens.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 .../commands/trtllm-serve/trtllm-serve.rst    |  1 +
 tensorrt_llm/commands/serve.py                | 33 ++++++++++++++++++-
 .../test_serve_num_frontends_default.py       | 27 ++++++++++++++-
 3 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 1543b9e75360..84c96b2e13ce 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -89,6 +89,7 @@ At high concurrency a single serving process is host-bound: one asyncio event lo
    * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
    * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor``, ``--enable_sleep_mode``, ``--metadata_server_config_file`` or a disaggregated cluster (``disagg_cluster`` / ``--disagg_cluster_uri``) it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
    * Each request is handled entirely by the frontend that accepted its connection, and per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, and ``/metrics``, ``/perf_metrics``, ``/prometheus/metrics`` and ``/health`` describe the frontend that happened to answer rather than the whole server.
+   * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses.
    * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
 
 Inference Endpoints
diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index 8d3c44934210..fbfdf5a91ef2 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -488,6 +488,35 @@ def _diagnose_port_in_use(port: int) -> str:
     return "; ".join(details)
 
 
+def _reserve_multi_frontend_port(host: str, port: int,
+                                 family: socket.AddressFamily) -> socket.socket:
+    """Refuse a port another server uses; return a lock to hold while serving.
+
+    The frontends bind the port with SO_REUSEPORT, which lets any socket of the
+    same user that also sets it join: a second server on a busy port would share
+    its traffic instead of failing. A plain bind fails on any listener
+    (SO_REUSEADDR keeps TIME_WAIT leftovers out), and an abstract Unix socket
+    named after the port stops a second launcher before this one listens.
+    """
+    lock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        lock.bind(f"\0trtllm-serve-{os.geteuid()}-{port}")
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(f"Port {port} is already reserved by another "
+                           f"multi-frontend trtllm-serve: {e}") from e
+    try:
+        with socket.socket(family, socket.SOCK_STREAM) as probe:
+            probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            probe.bind((host, port))
+    except OSError as e:
+        lock.close()
+        raise RuntimeError(
+            f"Failed to bind socket to {host}:{port}: {e}. "
+            f"Port holder(s): {_diagnose_port_in_use(port)}") from e
+    return lock
+
+
 @contextlib.contextmanager
 def _provision_kv_cache_pool(llm_args: dict,
                              owns_engine: bool = True) -> Iterator[None]:
@@ -674,7 +703,9 @@ def launch_server(
                                    socket.SOCK_STREAM)
     address_family = socket.AF_INET6 if all(
         [info[0] == socket.AF_INET6 for info in addr_info]) else socket.AF_INET
-    with socket.socket(address_family, socket.SOCK_STREAM) as s:
+    port_lock = (_reserve_multi_frontend_port(host, port, address_family)
+                 if multi_frontend.is_launcher else contextlib.nullcontext())
+    with port_lock, socket.socket(address_family, socket.SOCK_STREAM) as s:
         # port == 0 lets the kernel pick the port; the caller then needs a way
         # to learn it, either by service discovery or by report_addr.
         assert port > 0 or disagg_cluster_config is not None or report_addr, (
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
index a8e965ae5240..af8d203097d7 100644
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -12,15 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""CPU tests for trtllm-serve's multi-frontend default and its fallback to one."""
+"""CPU tests for trtllm-serve's multi-frontend default, its fallback to one, and its port guard."""
 
+import socket
 from unittest import mock
 
 import pytest
 
+from tensorrt_llm._utils import get_free_port
 from tensorrt_llm.commands import serve as serve_cmd
 from tensorrt_llm.commands.serve import (
     DEFAULT_NUM_SERVE_FRONTENDS,
+    _reserve_multi_frontend_port,
     _resolve_default_num_serve_frontends,
 )
 
@@ -92,3 +95,25 @@ def test_frontend_count_reaching_launch_server(
         serve_cmd.main(args=[str(tmp_path), *args], standalone_mode=False)
 
     assert launch.call_args.args[2].get("num_serve_frontends", 1) == expected
+
+
+def test_port_used_by_another_server_is_refused() -> None:
+    # Another multi-frontend server listens with SO_REUSEPORT: without the
+    # check, this launcher would join its port and share its traffic.
+    with socket.socket() as other:
+        other.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+        other.bind(("127.0.0.1", 0))
+        other.listen()
+        port = other.getsockname()[1]
+        with pytest.raises(RuntimeError, match="Failed to bind socket"):
+            _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET)
+
+
+def test_port_reserved_by_another_launcher_is_refused() -> None:
+    # A launcher binds its port long before it listens: only its lock shows.
+    port = get_free_port()
+    with _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET):
+        with pytest.raises(RuntimeError, match="another multi-frontend trtllm-serve"):
+            _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET)
+    # The lock goes with the launcher.
+    _reserve_multi_frontend_port("127.0.0.1", port, socket.AF_INET).close()

From bcddbc1ec318a5a15eb25e6472dbda6dd17aa55e Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Wed, 28 Oct 2026 05:14:34 -0700
Subject: [PATCH 06/11] [None][doc] serve: update the multi-frontend notes for
 launcher-owned state

With #19394 merged, /metrics, /kv_cache_events, profiling and the
Anthropic batches API are answered by the launcher for the whole
server, and /prometheus/metrics covers every frontend. /perf_metrics
no longer exists (per-request perf metrics go to per-process files).

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 docs/source/commands/trtllm-serve/trtllm-serve.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 84c96b2e13ce..268471e79c12 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -88,7 +88,7 @@ At high concurrency a single serving process is host-bound: one asyncio event lo
 
    * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
    * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor``, ``--enable_sleep_mode``, ``--metadata_server_config_file`` or a disaggregated cluster (``disagg_cluster`` / ``--disagg_cluster_uri``) it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
-   * Each request is handled entirely by the frontend that accepted its connection, and per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, and ``/metrics``, ``/perf_metrics``, ``/prometheus/metrics`` and ``/health`` describe the frontend that happened to answer rather than the whole server.
+   * Each request is handled entirely by the frontend that accepted its connection. Server-wide state lives in the launcher: the other frontends forward ``/metrics``, ``/kv_cache_events``, profiling and the Anthropic batches API to it, and ``/prometheus/metrics`` covers every frontend. Per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, ``/health`` describes the frontend that answered, and each frontend writes its own per-request perf metrics file.
    * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses.
    * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
 

From 1a9fc590a26d59fcf1c85c99a870597733f193e7 Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Wed, 28 Oct 2026 08:26:02 -0700
Subject: [PATCH 07/11] [None][feat] serve: keep the multi-frontend default
 with a metadata server or disagg cluster

The default fell back to one frontend with --metadata_server_config_file
or a disagg cluster because every frontend registered the server under
its own identity. With #19394 merged, only the launcher registers, so
that fallback is no longer needed.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 .../commands/trtllm-serve/trtllm-serve.rst     |  2 +-
 tensorrt_llm/commands/serve.py                 | 18 ++++--------------
 .../llmapi/test_serve_num_frontends_default.py |  7 ++-----
 3 files changed, 7 insertions(+), 20 deletions(-)

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 268471e79c12..6036d0729935 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -87,7 +87,7 @@ At high concurrency a single serving process is host-bound: one asyncio event lo
 .. note::
 
    * Pass ``--num_serve_frontends 1`` (or set ``num_serve_frontends: 1`` in the ``--config`` YAML) to run a single frontend.
-   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor``, ``--enable_sleep_mode``, ``--metadata_server_config_file`` or a disaggregated cluster (``disagg_cluster`` / ``--disagg_cluster_uri``) it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
+   * The default applies to the OpenAI HTTP server on the default executor path on a fixed port. With ``--port 0`` / ``--report_addr``, ``orchestrator_type`` (``rpc`` / ``ray``), ``enable_resource_governor`` or ``--enable_sleep_mode`` it falls back to one frontend and logs why; an explicit ``--num_serve_frontends`` greater than 1 fails instead. ``--grpc`` also keeps one frontend unless ``--num_serve_frontends`` is given (only ``--grpc-protocol openengine`` supports more).
    * Each request is handled entirely by the frontend that accepted its connection. Server-wide state lives in the launcher: the other frontends forward ``/metrics``, ``/kv_cache_events``, profiling and the Anthropic batches API to it, and ``/prometheus/metrics`` covers every frontend. Per-process state is not shared: the stateful Responses API store (``store`` / ``previous_response_id``) is disabled, ``/health`` describes the frontend that answered, and each frontend writes its own per-request perf metrics file.
    * ``SO_REUSEPORT`` would also let a second server of the same user join the port and take part of its traffic, so the launcher refuses a port that another server already uses.
    * Every extra frontend is a full Python process that imports TensorRT LLM and loads the tokenizer, so budget host memory accordingly.
diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index fbfdf5a91ef2..ee52fe861f6d 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -596,9 +596,7 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
 def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
                                          port: int, grpc: bool,
                                          report_addr: Optional[str],
-                                         enable_sleep_mode: bool,
-                                         metadata_server: bool,
-                                         disagg_cluster: bool) -> None:
+                                         enable_sleep_mode: bool) -> None:
     """Fall back to one frontend where the default count cannot run.
 
     An explicit count (CLI, --config YAML or --set) is left alone, so the
@@ -618,10 +616,6 @@ def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
         reason = "enable_resource_governor"
     elif enable_sleep_mode:
         reason = "--enable_sleep_mode"
-    elif metadata_server or disagg_cluster:
-        # Every frontend would register the server under its own identity.
-        reason = ("--metadata_server_config_file"
-                  if metadata_server else "the disagg_cluster config")
     else:
         return
     logger.info(
@@ -1193,9 +1187,8 @@ def launch_visual_gen_server(
     "SO_REUSEPORT (classic IPC executor path only). The default applies to "
     "the OpenAI HTTP server and falls back to 1 for configurations that "
     "support a single frontend only (--port 0/--report_addr, "
-    "orchestrator_type, enable_resource_governor, --enable_sleep_mode, "
-    "--metadata_server_config_file, a disagg cluster) and with --grpc; an "
-    "explicit value is honored or fails instead.",
+    "orchestrator_type, enable_resource_governor, --enable_sleep_mode) and "
+    "with --grpc; an explicit value is honored or fails instead.",
     status="prototype")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
@@ -1644,10 +1637,7 @@ def _serve_llm():
             port=port,
             grpc=grpc,
             report_addr=report_addr,
-            enable_sleep_mode=enable_sleep_mode,
-            metadata_server=metadata_server_config_file is not None,
-            disagg_cluster=bool(
-                llm_args.get("disagg_cluster") or disagg_cluster_uri))
+            enable_sleep_mode=enable_sleep_mode)
 
         _apply_effective_telemetry_config(llm_args, telemetry=telemetry)
         # Preserve requested settings for failures before entering the LLM constructor.
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
index af8d203097d7..4f7b63c14143 100644
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -39,8 +39,6 @@
         pytest.param({"orchestrator_type": "rpc"}, {}, "orchestrator_type='rpc'", id="rpc"),
         pytest.param({"enable_resource_governor": True}, {}, "enable_resource_governor", id="rg"),
         pytest.param({}, {"enable_sleep_mode": True}, "--enable_sleep_mode", id="sleep_mode"),
-        pytest.param({}, {"metadata_server": True}, "--metadata_server_config_file", id="metadata"),
-        pytest.param({}, {"disagg_cluster": True}, "disagg_cluster", id="disagg_cluster"),
     ],
 )
 def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, reason) -> None:
@@ -53,8 +51,6 @@ def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, rea
         grpc=False,
         report_addr=None,
         enable_sleep_mode=False,
-        metadata_server=False,
-        disagg_cluster=False,
     )
 
     _resolve_default_num_serve_frontends(llm_args, **{**no_conflict, **conflict})
@@ -70,10 +66,11 @@ def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, rea
         pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"),
         pytest.param(["--port", "0"], None, 1, id="port0"),
         pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"),
+        # Only the launcher registers with a disagg cluster, so the default holds.
         pytest.param(
             ["--server_role", "context", "--disagg_cluster_uri", "etcd://localhost:2379"],
             None,
-            1,
+            DEFAULT_NUM_SERVE_FRONTENDS,
             id="disagg_cluster",
         ),
         # An explicit count is honored; launch_server's own guard rejects the conflict.

From b1f03f09df519042afe19c18c5a1dda76a6ebe3f Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Wed, 28 Oct 2026 09:00:59 -0700
Subject: [PATCH 08/11] [None][test] serve: trim the multi-frontend default
 comments and tests

Shorten the port-guard docstring, the default's comment and the
--num_serve_frontends help (the fallback list lives in the docs), and
drop the disagg-cluster CLI case, which no longer differs from the
default. No behavior change.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 tensorrt_llm/commands/serve.py                | 22 ++++++++-----------
 .../test_serve_num_frontends_default.py       | 10 +--------
 2 files changed, 10 insertions(+), 22 deletions(-)

diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index ee52fe861f6d..cc4f9f4e6955 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -492,11 +492,10 @@ def _reserve_multi_frontend_port(host: str, port: int,
                                  family: socket.AddressFamily) -> socket.socket:
     """Refuse a port another server uses; return a lock to hold while serving.
 
-    The frontends bind the port with SO_REUSEPORT, which lets any socket of the
-    same user that also sets it join: a second server on a busy port would share
-    its traffic instead of failing. A plain bind fails on any listener
-    (SO_REUSEADDR keeps TIME_WAIT leftovers out), and an abstract Unix socket
-    named after the port stops a second launcher before this one listens.
+    SO_REUSEPORT would let a second server of the same user join the port. A
+    plain bind catches any listener (SO_REUSEADDR ignores TIME_WAIT), and an
+    abstract Unix socket named after the port catches a launcher that has not
+    started listening yet.
     """
     lock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
     try:
@@ -587,9 +586,8 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
-# One serving process is host-bound at high concurrency (a single asyncio loop
-# parses every request and streams every chunk). LlmArgs keeps default 1:
-# only trtllm-serve spawns frontends.
+# One serving process is host-bound at high concurrency. LlmArgs keeps
+# default 1: only trtllm-serve spawns frontends.
 DEFAULT_NUM_SERVE_FRONTENDS = 8
 
 
@@ -1184,11 +1182,9 @@ def launch_visual_gen_server(
     default=DEFAULT_NUM_SERVE_FRONTENDS,
     help="Number of HTTP or OpenEngine frontend processes serving one "
     "executor; values > 1 share the serving port via "
-    "SO_REUSEPORT (classic IPC executor path only). The default applies to "
-    "the OpenAI HTTP server and falls back to 1 for configurations that "
-    "support a single frontend only (--port 0/--report_addr, "
-    "orchestrator_type, enable_resource_governor, --enable_sleep_mode) and "
-    "with --grpc; an explicit value is honored or fails instead.",
+    "SO_REUSEPORT (classic IPC executor path only). Without an explicit "
+    "value, --grpc and configurations that support a single frontend only "
+    "use 1 and log why.",
     status="prototype")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
index 4f7b63c14143..4ab29d52aab9 100644
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -66,13 +66,6 @@ def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, rea
         pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"),
         pytest.param(["--port", "0"], None, 1, id="port0"),
         pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"),
-        # Only the launcher registers with a disagg cluster, so the default holds.
-        pytest.param(
-            ["--server_role", "context", "--disagg_cluster_uri", "etcd://localhost:2379"],
-            None,
-            DEFAULT_NUM_SERVE_FRONTENDS,
-            id="disagg_cluster",
-        ),
         # An explicit count is honored; launch_server's own guard rejects the conflict.
         pytest.param(["--port", "0", "--num_serve_frontends", "8"], None, 8, id="explicit"),
     ],
@@ -95,8 +88,7 @@ def test_frontend_count_reaching_launch_server(
 
 
 def test_port_used_by_another_server_is_refused() -> None:
-    # Another multi-frontend server listens with SO_REUSEPORT: without the
-    # check, this launcher would join its port and share its traffic.
+    # Without the check, SO_REUSEPORT would let this launcher join the port.
     with socket.socket() as other:
         other.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
         other.bind(("127.0.0.1", 0))

From 441f2c9e910debe6e3dafb35c812d7bb1fe5f73c Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 29 Oct 2026 00:17:58 -0700
Subject: [PATCH 09/11] [None][chore] serve: inline the num_serve_frontends
 default

Write default=8 on the --num_serve_frontends option, as the
neighboring options do, instead of a module constant. The tests use
the literal; the api_stability reference already pins 8.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 tensorrt_llm/commands/serve.py                            | 8 ++------
 tests/unittest/llmapi/test_serve_num_frontends_default.py | 5 ++---
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index cc4f9f4e6955..04c885840817 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -586,11 +586,6 @@ def _share_prometheus_multiproc_dir(llm, ipc_dir: str) -> None:
             f"Shared PROMETHEUS_MULTIPROC_DIR for all frontends: {path}")
 
 
-# One serving process is host-bound at high concurrency. LlmArgs keeps
-# default 1: only trtllm-serve spawns frontends.
-DEFAULT_NUM_SERVE_FRONTENDS = 8
-
-
 def _resolve_default_num_serve_frontends(llm_args: dict, *, requested: bool,
                                          port: int, grpc: bool,
                                          report_addr: Optional[str],
@@ -1179,7 +1174,8 @@ def launch_visual_gen_server(
 @stability_option(
     "--num_serve_frontends",
     type=click.IntRange(min=1, max=MAX_NUM_FRONTENDS),
-    default=DEFAULT_NUM_SERVE_FRONTENDS,
+    # LlmArgs keeps 1: only trtllm-serve spawns frontends.
+    default=8,
     help="Number of HTTP or OpenEngine frontend processes serving one "
     "executor; values > 1 share the serving port via "
     "SO_REUSEPORT (classic IPC executor path only). Without an explicit "
diff --git a/tests/unittest/llmapi/test_serve_num_frontends_default.py b/tests/unittest/llmapi/test_serve_num_frontends_default.py
index 4ab29d52aab9..7bf379263bb3 100644
--- a/tests/unittest/llmapi/test_serve_num_frontends_default.py
+++ b/tests/unittest/llmapi/test_serve_num_frontends_default.py
@@ -22,7 +22,6 @@
 from tensorrt_llm._utils import get_free_port
 from tensorrt_llm.commands import serve as serve_cmd
 from tensorrt_llm.commands.serve import (
-    DEFAULT_NUM_SERVE_FRONTENDS,
     _reserve_multi_frontend_port,
     _resolve_default_num_serve_frontends,
 )
@@ -44,7 +43,7 @@
 def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, reason) -> None:
     messages = []
     monkeypatch.setattr(serve_cmd.logger, "info", lambda msg, *a, **k: messages.append(msg))
-    llm_args = {"num_serve_frontends": DEFAULT_NUM_SERVE_FRONTENDS, **llm_args}
+    llm_args = {"num_serve_frontends": 8, **llm_args}
     no_conflict = dict(
         requested=False,
         port=8000,
@@ -62,7 +61,7 @@ def test_default_falls_back_to_one_frontend(monkeypatch, llm_args, conflict, rea
 @pytest.mark.parametrize(
     "args,config,expected",
     [
-        pytest.param([], None, DEFAULT_NUM_SERVE_FRONTENDS, id="default"),
+        pytest.param([], None, 8, id="default"),
         pytest.param([], "num_serve_frontends: 1", 1, id="yaml_opt_out"),
         pytest.param(["--port", "0"], None, 1, id="port0"),
         pytest.param(["--enable_sleep_mode"], None, 1, id="sleep_mode"),

From bee9afffbbbca5cf83cc6d18a9b43d30b627d7c2 Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 29 Oct 2026 00:32:29 -0700
Subject: [PATCH 10/11] [None][feat] serve: promote --num_serve_frontends to
 beta

The option becomes the default path for every trtllm-serve user, so it
should not stay a prototype. The LlmArgs field, which the LLM API keeps
at 1, stays prototype.

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 docs/source/commands/trtllm-serve/trtllm-serve.rst            | 2 +-
 tensorrt_llm/commands/serve.py                                | 2 +-
 tests/unittest/api_stability/references/trtllm_serve_cli.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/source/commands/trtllm-serve/trtllm-serve.rst b/docs/source/commands/trtllm-serve/trtllm-serve.rst
index 6036d0729935..0b082051c09a 100644
--- a/docs/source/commands/trtllm-serve/trtllm-serve.rst
+++ b/docs/source/commands/trtllm-serve/trtllm-serve.rst
@@ -82,7 +82,7 @@ shared format. ``--reasoning_parser`` separately controls reasoning extraction;
 Multiple HTTP Frontends
 -----------------------
 
-At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, prototype). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL.
+At high concurrency a single serving process is host-bound: one asyncio event loop parses every request and writes every streamed chunk, and queuing on that loop rather than GPU time starts to dominate first-token latency. ``trtllm-serve`` therefore runs **8 HTTP frontend processes** against one executor by default (``--num_serve_frontends``, beta). The launcher process owns the engine; the other frontends attach to it and share the serving port via ``SO_REUSEPORT``, so clients still see a single URL.
 
 .. note::
 
diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py
index 04c885840817..1b51181d8a8f 100644
--- a/tensorrt_llm/commands/serve.py
+++ b/tensorrt_llm/commands/serve.py
@@ -1181,7 +1181,7 @@ def launch_visual_gen_server(
     "SO_REUSEPORT (classic IPC executor path only). Without an explicit "
     "value, --grpc and configurations that support a single frontend only "
     "use 1 and log why.",
-    status="prototype")
+    status="beta")
 @stability_option("--num_input_processor_workers",
                   type=click.IntRange(min=1),
                   default=8,
diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
index e3648746192c..d7e9092b99b5 100644
--- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
+++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml
@@ -345,7 +345,7 @@ commands:
       num_serve_frontends:
         type: int
         default: 8
-        status: prototype
+        status: beta
         required: false
         multiple: false
         is_flag: false

From 536975e03ed5b888939f83600db2500686caa8f9 Mon Sep 17 00:00:00 2001
From: Lance Liao <108499334+lancelly@users.noreply.github.com>
Date: Thu, 29 Oct 2026 00:37:32 -0700
Subject: [PATCH 11/11] [None][test] serve: add a GPU test for 8 frontends on
 one engine

Requests, streams, /metrics and /prometheus/metrics through 8 frontends
on one Qwen3-0.6B engine, each on its own connection: every request
finishes, the launcher drains the stats once, and every scrape counts
the requests of all frontends. Runs in the single-GPU B200 pre-merge
list (review: explicit coverage for the multi-frontend path).

Signed-off-by: Lance Liao <108499334+lancelly@users.noreply.github.com>
---
 tests/integration/defs/test_e2e.py            |   7 ++
 .../test_lists/test-db/l0_b200.yml            |   1 +
 .../apps/_test_openai_multi_frontend.py       | 107 ++++++++++++++++++
 3 files changed, 115 insertions(+)
 create mode 100644 tests/unittest/llmapi/apps/_test_openai_multi_frontend.py

diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py
index 24e49833df62..5572326a2769 100644
--- a/tests/integration/defs/test_e2e.py
+++ b/tests/integration/defs/test_e2e.py
@@ -595,6 +595,13 @@ def test_openai_prometheus(llm_root, llm_venv):
          str(test_root / "_test_openai_prometheus.py")])
 
 
+def test_openai_multi_frontend(llm_root, llm_venv):
+    test_root = unittest_path() / "llmapi" / "apps"
+    llm_venv.run_cmd(
+        ["-m", "pytest",
+         str(test_root / "_test_openai_multi_frontend.py")])
+
+
 def test_openai_lora(llm_root, llm_venv):
     test_root = unittest_path() / "llmapi" / "apps"
     llm_venv.run_cmd(["-m", "pytest", str(test_root / "_test_openai_lora.py")])
diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml
index deefc7a15ed3..a2241a9376b3 100644
--- a/tests/integration/test_lists/test-db/l0_b200.yml
+++ b/tests/integration/test_lists/test-db/l0_b200.yml
@@ -107,6 +107,7 @@ l0_b200:
   - test_e2e.py::test_ptp_quickstart_advanced_mtp[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]
   - test_e2e.py::test_ptp_quickstart_advanced_mtp_eagle[DeepSeek-V3-Lite-BF16-DeepSeek-V3-Lite/bf16]
   - test_e2e.py::test_openai_chat_guided_decoding[openai/gpt-oss-120b]
+  - test_e2e.py::test_openai_multi_frontend
   # Covers the GPU cases of tests/unittest/_torch/attention/ on B200, except the paths
   # ignored below; its cpu_only cases run on CPU (l0_cpu). kernels/ has no entry of its own
   # on any list, so apart from kernels/parallel_hw_agnostic/ this entry is what runs it
diff --git a/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py b/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py
new file mode 100644
index 000000000000..d3e62c139aeb
--- /dev/null
+++ b/tests/unittest/llmapi/apps/_test_openai_multi_frontend.py
@@ -0,0 +1,107 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""trtllm-serve with 8 frontend processes on one engine.
+
+Each connection may land on any frontend (SO_REUSEPORT), so every request opens
+its own. Server-wide state must not depend on which frontend answers.
+"""
+
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+import psutil
+import pytest
+import requests
+
+from ..test_llm import get_model_path
+from .openai_server import RemoteOpenAIServer
+
+NUM_FRONTENDS = 8
+NUM_REQUESTS = 64
+NUM_STREAMS = 16
+
+
+@pytest.fixture(scope="module")
+def model_path() -> str:
+    return get_model_path("Qwen3/Qwen3-0.6B")
+
+
+@pytest.fixture(scope="module")
+def server(model_path: str):
+    args = ["--backend", "pytorch", "--num_serve_frontends", str(NUM_FRONTENDS)]
+    extra_config = {"return_perf_metrics": True, "enable_iter_perf_stats": True}
+    with RemoteOpenAIServer(model_path, args, extra_config=extra_config) as server:
+        # /health answers once the first frontend listens.
+        deadline = time.monotonic() + 600
+        while _listeners(server.port) < NUM_FRONTENDS:
+            assert time.monotonic() < deadline, _listeners(server.port)
+            time.sleep(1)
+        yield server
+
+
+def _listeners(port: int) -> int:
+    return sum(
+        1
+        for c in psutil.net_connections("tcp")
+        if c.status == psutil.CONN_LISTEN and c.laddr and c.laddr.port == port
+    )
+
+
+def _finish_reason(server: RemoteOpenAIServer, model: str, i: int, stream: bool):
+    """Run one completion on a connection of its own."""
+    with server.get_client() as client:
+        output = client.completions.create(
+            model=model,
+            prompt=f"Request {i}: the capital of France is",
+            max_tokens=16,
+            stream=stream,
+        )
+        if stream:
+            choices = [chunk.choices[0] for chunk in output if chunk.choices]
+        else:
+            choices = output.choices
+    text = "".join(choice.text for choice in choices)
+    reasons = [choice.finish_reason for choice in choices if choice.finish_reason]
+    return reasons[-1] if text and reasons else None
+
+
+def _success_total(server: RemoteOpenAIServer) -> float:
+    text = requests.get(server.url_for("prometheus", "metrics"), timeout=30).text
+    return sum(
+        float(line.rsplit(" ", 1)[1])
+        for line in text.splitlines()
+        if line.startswith("trtllm_request_success_total")
+    )
+
+
+def test_frontends_share_one_engine(server: RemoteOpenAIServer, model_path: str):
+    with ThreadPoolExecutor(16) as pool:
+        finish_reasons = list(
+            pool.map(
+                lambda i: _finish_reason(server, model_path, i, stream=i >= NUM_REQUESTS),
+                range(NUM_REQUESTS + NUM_STREAMS),
+            )
+        )
+    assert all(finish_reasons), finish_reasons
+
+    # Only the launcher drains the engine's stats (every second), so the first
+    # poll gets all of them, whichever frontend answers.
+    time.sleep(3)
+    polls = [len(requests.get(server.url_for("metrics"), timeout=30).json()) for _ in range(16)]
+    assert polls[0] > 0 and not any(polls[1:]), polls
+
+    # Every scrape counts the requests of all frontends.
+    totals = [_success_total(server) for _ in range(NUM_FRONTENDS)]
+    assert totals == [NUM_REQUESTS + NUM_STREAMS] * NUM_FRONTENDS, totals