diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 184bd8539e20..a5fe1a2c1592 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -201,8 +201,6 @@ def is_non_default_or_required(param_name, value, explicit_cli_keys): # name (e.g. `--free_gpu_memory_fraction` constructs `kv_cache_config`). cli_derived_fields = { **SERVE_CLI_CONFIG_FIELDS, - "build_config": - ("max_batch_size", "max_num_tokens", "max_beam_width", "max_seq_len"), } if any(s in explicit_cli_keys for s in cli_derived_fields.get(param_name, ())): diff --git a/tensorrt_llm/serve/_perf_metrics_schema.py b/tensorrt_llm/serve/_perf_metrics_schema.py index 5a9d606d85ac..67370c57ba26 100644 --- a/tensorrt_llm/serve/_perf_metrics_schema.py +++ b/tensorrt_llm/serve/_perf_metrics_schema.py @@ -16,6 +16,11 @@ from typing_extensions import NotRequired, TypedDict +# HTTP response header carrying server-side timings; the LLM and VisualGen +# servers both set it. Defined here, in a module with no TensorRT-LLM imports, +# so neither serving path has to import the other's metrics module. +SERVER_TIMING_HEADER = "Server-Timing" + class StepMetrics(TypedDict): forward_start_time: float diff --git a/tensorrt_llm/serve/disagg_coordinator.py b/tensorrt_llm/serve/disagg_coordinator.py index b134f616daf3..c4caa1dbea5a 100644 --- a/tensorrt_llm/serve/disagg_coordinator.py +++ b/tensorrt_llm/serve/disagg_coordinator.py @@ -142,10 +142,10 @@ def __init__( self._config = config self._client_factory = client_factory self._metadata_config = metadata_config - # The coordinator owns routing state, so it builds the owner routers here - # (is_delegating_client=False): one shared namespace-aware core + a single - # ZMQ ingest server. Sole place owner routers are created (fleet workers - # hold only a CoordinatorClient's delegating surfaces). + # The coordinator owns routing state, so it builds the owner routers here: + # one shared namespace-aware core + a single ZMQ ingest server. Sole place + # owner routers are created (fleet workers hold only a CoordinatorClient's + # delegating surfaces). self._metadata_server = create_metadata_server(metadata_config) ctx_servers, gen_servers = get_ctx_gen_server_addrs(config.server_configs) self._ctx_router, self._gen_router = build_disagg_routers( @@ -157,7 +157,6 @@ def __init__( self._metadata_server, server_preparation_func, disagg_node_id=config.node_id, - is_delegating_client=False, ) # The coordinator owns the disagg cluster storage (auto-scaling backend): # it drives the DisaggClusterManager below and, when the storage is an @@ -556,7 +555,6 @@ def __init__( metadata_config, create_metadata_server(metadata_config), disagg_node_id=config.node_id, - is_delegating_client=True, ) self._ctx_router = self._maybe_delegate(ctx_router, "context") self._gen_router = self._maybe_delegate(gen_router, "generation") diff --git a/tensorrt_llm/serve/harmony_adapter.py b/tensorrt_llm/serve/harmony_adapter.py index 1698d67e6695..87c16e5fe980 100644 --- a/tensorrt_llm/serve/harmony_adapter.py +++ b/tensorrt_llm/serve/harmony_adapter.py @@ -956,11 +956,6 @@ def _parse_tool_call_from_harmony_message( if not msg_recipient or msg_recipient == "assistant": return None - # # Clean up function name by removing constraint tokens that may be incorrectly included - # # The harmony library sometimes includes ><|constrain|>json in the recipient - # if ">" in function_name: - # function_name = function_name.split(">")[0] - # Extract arguments from message content function_call_args = "" for content in msg_content: @@ -1434,12 +1429,8 @@ def stateful_stream_harmony_tokens_to_openai_deltas( stream_state = self.create_stream_state(request_id, available_tools, tool_choice) - # decoded_tokens = self.encoding.decode_utf8(tokens) - # logger.info(">> DECODED TOKENS: %r", decoded_tokens) - try: deltas = stream_state.process_token_batch(tokens) - # logger.info(">> GENERATED DELTAS: %s", deltas) return deltas except (HarmonyError, UnicodeDecodeError, ValueError): logger.error( diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 27e1a0b83b3c..aee83b273920 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -1223,12 +1223,6 @@ def postproc_worker_enabled(self) -> bool: return True if self.generator.args.num_postprocess_workers > 0 else False - @property - def _vocab_size(self) -> Optional[int]: - if self.tokenizer is not None and self.tokenizer.tokenizer is not None: - return self.tokenizer.tokenizer.vocab_size - return None - @staticmethod def create_error_response( message: str, diff --git a/tensorrt_llm/serve/perf_metrics.py b/tensorrt_llm/serve/perf_metrics.py index 73846d4ca72c..3978ac455771 100644 --- a/tensorrt_llm/serve/perf_metrics.py +++ b/tensorrt_llm/serve/perf_metrics.py @@ -61,6 +61,7 @@ from tensorrt_llm.llmapi.disagg_utils import ServerRole from tensorrt_llm.logger import logger from tensorrt_llm.serve._perf_metrics_schema import ( + SERVER_TIMING_HEADER, DisaggPerfMetricsRecord, PerfMetrics, PerfMetricsRecord, @@ -194,7 +195,6 @@ def __getattr__(self, key: str): return self._metrics[key] -SERVER_TIMING_HEADER = "Server-Timing" START_END_TIME_HEADER = "X-TRTLLM-Start-End-Time" STEP_METRICS_HEADER = "X-TRTLLM-Step-Metrics" CTX_CHUNK_METRICS_HEADER = "X-TRTLLM-Ctx-Chunk-Metrics" diff --git a/tensorrt_llm/serve/router.py b/tensorrt_llm/serve/router.py index b6117e4ec6f0..970fa212ec85 100644 --- a/tensorrt_llm/serve/router.py +++ b/tensorrt_llm/serve/router.py @@ -94,14 +94,6 @@ async def decrement_load_by(self, num_tokens: int = 0): self._num_active_requests -= 1 self._num_active_tokens -= num_tokens - async def is_healthy(self) -> bool: - try: - async with self._session.get( - f"{self._base_url}/health") as response: - return response.status == 200 - except Exception: - return False - class KvCacheAwareServerState(ServerState): @@ -2047,7 +2039,6 @@ def build_disagg_routers( metadata_server: Optional[JsonDictionary] = None, server_preparation_func: Optional[Callable[[str], Awaitable[None]]] = None, disagg_node_id: int = 0, - is_delegating_client: bool = False, ) -> tuple[Router, Router]: """Build the ctx and gen routers for one disagg process. @@ -2057,10 +2048,7 @@ def build_disagg_routers( caller (:class:`CoordinatorClient`) wraps those in a :class:`CoordinatorDelegatingRouter` so placement is delegated to the coordinator. Round-robin placement stays local. - ``is_delegating_client`` is accepted for call-site symmetry; router - construction itself does not depend on it. """ - del is_delegating_client # no build-time behavior differs by this flag ctx_router = create_router(ctx_router_config, ctx_servers, metadata_server_cfg, metadata_server, server_preparation_func, disagg_node_id) diff --git a/tensorrt_llm/serve/router_utils.py b/tensorrt_llm/serve/router_utils.py index 7f7f78ca4edd..00ddc7d520ec 100644 --- a/tensorrt_llm/serve/router_utils.py +++ b/tensorrt_llm/serve/router_utils.py @@ -20,7 +20,7 @@ import os from collections import OrderedDict from dataclasses import dataclass, field -from typing import Iterable, List, Optional, Union +from typing import Iterable, Optional, Union from tensorrt_llm.bindings.internal.batch_manager import BlockKey as _NativeBlockKey from tensorrt_llm.bindings.internal.batch_manager import BlockKeyHasher as _NativeBlockKeyHasher @@ -83,64 +83,9 @@ "get_request_num_tokens", "block_key_hasher", "BlockHashMixin", - "PrefixBlockSet", ] -class PrefixBlockSet: - """Single-owner block-hash index -- a flat ``set`` of held block hashes. - - A KV-cache block hash folds in its parent chain (the worker computes it with - ``BlockKeyHasher.hash(block_key, parent_hash)``), so every block-hash *value* - is globally unique to one position in one prefix path. A request's ordered - block-hash list is itself the prefix path, so longest-common-prefix against a - set of held blocks is just "walk the list until the first hash the owner - doesn't hold" -- no explicit tree needed. - - This is the exact structure the orchestrator ``KvCacheAwareServerState`` uses - (a ``set[block_hash]`` per server, walked until the first miss). It is the - right index whenever there is only ONE logical owner -- e.g. the centralized - router's per-instance ``combined_trie`` (owner = instance) and each rank's - trie (owner = that rank). Those are only ever queried for a single owner's - prefix depth (:meth:`match_one`), so a ``hash -> {owner}`` reverse map with - per-depth set intersections would be pure overhead here. - - The ``owner_id`` argument on :meth:`add` / :meth:`remove` / :meth:`match_one` - is accepted and ignored so this is a drop-in for the single-owner call sites. - """ - - __slots__ = ("_blocks",) - - def __init__(self) -> None: - self._blocks: set[int] = set() - - def add(self, owner_id: str, block_hashes: Iterable[int]) -> None: - self._blocks.update(block_hashes) - - def remove(self, owner_id: str, block_hashes: Iterable[int]) -> None: - self._blocks.difference_update(block_hashes) - - def remove_worker(self, owner_id: str) -> None: - self._blocks.clear() - - def match_one(self, owner_id: str, block_hashes: List[int]) -> int: - """Consecutive prefix-block count held by the (single) owner. - - Identical to ``KvCacheAwareServerState.matched_tokens``: walk the query - path, counting blocks present in the set, and stop at the first miss. - """ - blocks = self._blocks - depth = 0 - for h in block_hashes: - if h not in blocks: - break - depth += 1 - return depth - - def has_worker(self, owner_id: str) -> bool: - return bool(self._blocks) - - def responses_request_text(request: ResponsesRequest) -> str: """Return the instructions and input as ``role:content`` lines, like a chat request's text. diff --git a/tensorrt_llm/serve/visual_gen_metrics.py b/tensorrt_llm/serve/visual_gen_metrics.py index e72873f280c3..88453c75c671 100644 --- a/tensorrt_llm/serve/visual_gen_metrics.py +++ b/tensorrt_llm/serve/visual_gen_metrics.py @@ -6,10 +6,11 @@ from typing import TYPE_CHECKING, Dict, Optional +from tensorrt_llm.serve._perf_metrics_schema import SERVER_TIMING_HEADER + if TYPE_CHECKING: from tensorrt_llm.visual_gen.output import VisualGenMetrics -SERVER_TIMING_HEADER = "Server-Timing" VISUAL_GEN_PRE_DENOISE_TIMING = "pre_denoise" VISUAL_GEN_DENOISE_TIMING = "denoise" VISUAL_GEN_POST_DENOISE_TIMING = "post_denoise" diff --git a/tests/unittest/llmapi/apps/test_harmony_parsing.py b/tests/unittest/llmapi/apps/test_harmony_parsing.py index 13393bb0fbad..291207fac468 100644 --- a/tests/unittest/llmapi/apps/test_harmony_parsing.py +++ b/tests/unittest/llmapi/apps/test_harmony_parsing.py @@ -39,7 +39,6 @@ handle_streaming_response, ) from tensorrt_llm.serve.openai_protocol import StreamOptions, _logit_bias_to_embedding_bias - from tensorrt_llm.serve.openai_server import OpenAIServer from tensorrt_llm.serve.postprocess_handlers import ( ChatCompletionPostprocArgs, chat_harmony_streaming_post_processor, @@ -1559,9 +1558,6 @@ def test_non_functions_recipient_is_named(self, adapter, recipient): def test_none_tokenizer_num_postprocess_workers(): - server = object.__new__(OpenAIServer) - server.tokenizer = None - assert server._vocab_size is None with pytest.raises(ValueError, match="logit_bias requires a tokenizer"): _logit_bias_to_embedding_bias({"0": 1.0}, vocab_size=None)