From 881d964d0924d44d30f5420b7b8d8a355d7af951 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:20:44 -0700 Subject: [PATCH 01/10] [None][feat] Support image input in the Triton llmapi backend The llmapi Triton backend accepts text only: `_convert_request` decodes `text_input` into a plain str, and a str is all that reaches `generate_async`. There is no way to attach an image, so multimodal models cannot be served through Triton even though the PyTorch backend supports them. Multimodal used to work on Triton via the TensorRT-engine backend (all_models/multimodal, which exposed image_url_input), but that tree was removed in #15907 and was never carried over to the llmapi backend. Add an optional `image_url` input. When present, the text and media are passed to default_multimodal_input_loader, which applies the chat template, inserts the per-architecture image placeholders and loads the images, producing the PromptInputs dict the LLM API expects: {"prompt": ..., "multi_modal_data": {"image": [...]}} Entries may be a URL, a local path or a base64 data URI, since load_image() already dispatches on the URL scheme. Multiple images per request are supported. Requests without `image_url` are unaffected. The loader inputs are resolved on the first request carrying media and cached, so text-only deployments pay nothing. The loader is imported at point of use to preserve the deferred tensorrt_llm import that multi-instance deployments rely on. Testing: the equivalent change on the v1.2.1 backend files was verified on 8xB200 with Qwen/Qwen3-VL-8B-Instruct served through Triton -- single image, multiple images, local paths and text-only requests. This commit is that change ported to main, where deferred-import handling differs; the port has not been re-run on hardware. Co-Authored-By: Claude Opus 5 Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 34 +++++++++++++++++++ .../llmapi/tensorrt_llm/config.pbtxt | 7 ++++ 2 files changed, 41 insertions(+) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index edd77e1a26a7..55e8b1e4ae08 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -251,6 +251,8 @@ def initialize(self, args): self._response_thread = threading.Thread(target=self._response_loop) self._response_thread.start() + self._multimodal_context = None + self.req_id_to_request_data = {} self.triton_user_id_to_req_ids = {} self.lock = threading.Lock() @@ -582,6 +584,16 @@ async def _execute_single_request(self, request): if triton_user_id is not None and triton_user_id != "" and triton_user_id in self.triton_user_id_to_req_ids: del self.triton_user_id_to_req_ids[triton_user_id] + def _get_multimodal_context(self): + """Return (tokenizer, checkpoint dir, HF model_type), resolved once.""" + if self._multimodal_context is None: + hf_model_dir = str(self._llm_engine._hf_model_dir) + with open(os.path.join(hf_model_dir, "config.json")) as f: + model_type = json.load(f)["model_type"] + self._multimodal_context = (self._llm_engine.tokenizer, + hf_model_dir, model_type) + return self._multimodal_context + def _convert_request(self, request): """Helper function to convert the request into a prompt for LLM.generate_async @@ -593,6 +605,7 @@ def _convert_request(self, request): Notes: - The current implementation only supports text_input being a 1D tensor(a single prompt). + - With `image_url`, prompt becomes a multimodal PromptInputs dict. """ text_input = get_input_tensor_by_name(request, 'text_input') if text_input is None: @@ -608,6 +621,27 @@ def _convert_request(self, request): if isinstance(prompt, bytes): prompt = prompt.decode("utf-8") + # The loader applies the chat template and inserts the per-architecture + # image placeholders, so callers send a plain question. + image_url = get_input_tensor_by_name(request, 'image_url') + if image_url is not None and image_url.size > 0: + # Imported here, not at module scope (see note above). + from tensorrt_llm.inputs import default_multimodal_input_loader + + media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + tokenizer, hf_model_dir, model_type = self._get_multimodal_context() + prompt = default_multimodal_input_loader(tokenizer=tokenizer, + model_dir=hf_model_dir, + model_type=model_type, + modality="image", + prompts=[prompt], + media=media, + image_data_format="pt", + device="cpu")[0] + sampling_params = get_sampling_params_from_request(request) output_config = get_output_config_from_request(request) streaming = get_streaming_from_request(request) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt index a3f995aae882..0a77208871b8 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt @@ -51,6 +51,13 @@ input [ data_type: TYPE_STRING dims: [ -1 ] }, + ## Optional multimodal input: URL, local path or data URI, one per image. + { + name: "image_url" + data_type: TYPE_STRING + dims: [ -1 ] + optional: true + }, { name: "streaming" data_type: TYPE_BOOL From 80fd4526e11d803e0b2d00b388be2e677c6219aa Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:29:55 -0700 Subject: [PATCH 02/10] [None][feat] Use the trtllm-serve multimodal path in the Triton llmapi backend Replace default_multimodal_input_loader, which reloads AutoProcessor on every request, with the primitives trtllm-serve uses for v1/chat/completions: MultimodalDataTracker, add_multimodal_placeholders and async_apply_chat_template. The tokenizer, HF processor and pretrained config are resolved once at model load instead of per forward pass. Make _convert_request async so image fetches no longer run on the engine event loop, where a slow remote host would stall every request in flight. Consume the image_url input only when triton_config.multimodal is set, so a deployment that already declares an input with that name keeps its current behavior after an upgrade. Resolve model_type from the config class rather than the instance: composite configs such as Qwen2_5_VLConfig delegate the instance attribute to text_config and report qwen2_5_vl_text instead of the qwen2_5_vl key used by the multimodal placeholder registry. Add unit tests covering the opt-in gate and the prompt construction. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 168 ++++++++++++++---- .../llmapi/tensorrt_llm/1/model.yaml | 4 + .../llmapi/tensorrt_llm/config.pbtxt | 3 +- .../tests/test_llmapi_python_backend.py | 135 +++++++++++++- 4 files changed, 276 insertions(+), 34 deletions(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index 55e8b1e4ae08..822c93e3a106 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -51,6 +51,19 @@ # inherits these FDs, causing child processes to ignore CUDA_VISIBLE_DEVICES. +@dataclass +class MultimodalContext: + """Startup-resolved state needed to turn media URLs into engine inputs. + + Resolved once in `initialize` so that per-request parsing stays cheap, the + same way `trtllm-serve` resolves them in `OpenAIServer.__init__`. + """ + tokenizer: Any + processor: Any + model_config: Any + model_type: str + + @dataclass class RequestData: triton_req_id: int @@ -251,7 +264,13 @@ def initialize(self, args): self._response_thread = threading.Thread(target=self._response_loop) self._response_thread.start() + # Opt-in: a deployment that already defines an `image_url` input + # for another purpose keeps its behavior unless this is enabled. + self.multimodal_enabled = bool( + triton_config.get("multimodal", False)) self._multimodal_context = None + if self.multimodal_enabled: + self._init_multimodal_context() self.req_id_to_request_data = {} self.triton_user_id_to_req_ids = {} @@ -487,7 +506,7 @@ async def _execute_single_request(self, request): # TODO: [JIRA-4496] Implement when request contains batched prompts (prompt, sampling_params, streaming, output_config, - lora_request) = self._convert_request(request) + lora_request) = await self._convert_request(request) if streaming and not self.decoupled: raise pb_utils.TritonModelException( "Streaming is only supported in decoupled mode.") @@ -584,17 +603,114 @@ async def _execute_single_request(self, request): if triton_user_id is not None and triton_user_id != "" and triton_user_id in self.triton_user_id_to_req_ids: del self.triton_user_id_to_req_ids[triton_user_id] - def _get_multimodal_context(self): - """Return (tokenizer, checkpoint dir, HF model_type), resolved once.""" - if self._multimodal_context is None: - hf_model_dir = str(self._llm_engine._hf_model_dir) - with open(os.path.join(hf_model_dir, "config.json")) as f: - model_type = json.load(f)["model_type"] - self._multimodal_context = (self._llm_engine.tokenizer, - hf_model_dir, model_type) - return self._multimodal_context - - def _convert_request(self, request): + def _init_multimodal_context(self): + """Resolve the tokenizer, HF processor and pretrained config once. + + Loading these per request would re-read the checkpoint on every forward + pass, so they are resolved at model load time and reused. + """ + from transformers import AutoProcessor + + from tensorrt_llm._torch.pyexecutor.config_utils import \ + load_pretrained_config + + tokenizer = self._llm_engine.tokenizer + hf_model_dir = self._llm_engine._hf_model_dir or getattr( + getattr(tokenizer, "tokenizer", None), "name_or_path", None) + if hf_model_dir is None: + raise pb_utils.TritonModelException( + "triton_config.multimodal is enabled but the checkpoint directory " + "could not be resolved from the engine.") + hf_model_dir = str(hf_model_dir) + trust_remote_code = self._llm_engine.args.trust_remote_code + try: + processor = AutoProcessor.from_pretrained( + hf_model_dir, trust_remote_code=trust_remote_code) + model_config = load_pretrained_config( + hf_model_dir, + trust_remote_code=trust_remote_code, + checkpoint_format=getattr(self._llm_engine.args, + "checkpoint_format", None)) + except Exception as e: + raise pb_utils.TritonModelException( + f"triton_config.multimodal is enabled but the HF processor/config " + f"for '{hf_model_dir}' could not be loaded: {e}") + + # Read `model_type` from the config class, not the instance: composite + # configs such as Qwen2_5_VLConfig delegate the instance attribute to + # `text_config` and would report "qwen2_5_vl_text" instead of the + # "qwen2_5_vl" key used by the multimodal placeholder registry. This + # matches `tensorrt_llm.serve.chat_utils.resolve_top_level_model_type`, + # inlined to keep the backend off the `tensorrt_llm.serve` import path. + model_type = getattr(type(model_config), "model_type", None) or getattr( + model_config, "model_type", "") + self._multimodal_context = MultimodalContext(tokenizer=tokenizer, + processor=processor, + model_config=model_config, + model_type=model_type) + self.logger.log_info("[trtllm] multimodal input enabled for model_type " + f"'{self._multimodal_context.model_type}'") + + async def _build_multimodal_prompt(self, text, image_url): + """Build a multimodal PromptInputs from a text prompt and image URLs. + + Runs the same sequence `trtllm-serve` uses for `v1/chat/completions`, + but calls the shared `tensorrt_llm.inputs` primitives directly rather + than going through `tensorrt_llm.serve.chat_utils`: importing anything + under `tensorrt_llm.serve` executes `tensorrt_llm/serve/__init__.py`, + which pulls in the whole OpenAI server stack (FastAPI, the `openai` + SDK) that a Triton deployment neither needs nor is guaranteed to have. + """ + from tensorrt_llm.inputs import prompt_inputs + from tensorrt_llm.inputs.utils import (ConversationMessage, + MultimodalDataTracker, + add_multimodal_placeholders, + async_apply_chat_template, + async_load_image) + + ctx = self._multimodal_context + media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + + # `add_data` takes the un-awaited fetch, so several images on one + # request are fetched concurrently when the tracker is drained below. + mm_data_tracker = MultimodalDataTracker(ctx.model_type) + for url in media: + mm_data_tracker.add_data("image", async_load_image(url)) + mm_placeholder_counts = mm_data_tracker.placeholder_counts() + item_order = mm_data_tracker.item_order() + + # The chat template needs the per-architecture image placeholders in + # the message text, so callers send a plain question as `text_input`. + content = add_multimodal_placeholders(ctx.model_type, text, + mm_placeholder_counts, item_order) + conversation = [ + ConversationMessage(role="user", content=content, media=[]) + ] + + prompt_task = async_apply_chat_template( + model_type=ctx.model_type, + tokenizer=ctx.tokenizer, + processor=ctx.processor, + conversation=conversation, + add_generation_prompt=True, + mm_placeholder_counts=[mm_placeholder_counts], + ) + # Render the template while the images are still in flight. + prompt, (mm_data, + _) = await asyncio.gather(prompt_task, + mm_data_tracker.retrieve_all_async()) + + prompt = prompt_inputs(prompt) + if mm_data: + prompt["multi_modal_data"] = mm_data + if item_order: + prompt["mm_item_order"] = item_order + return prompt + + async def _convert_request(self, request): """Helper function to convert the request into a prompt for LLM.generate_async Args: @@ -606,6 +722,7 @@ def _convert_request(self, request): Notes: - The current implementation only supports text_input being a 1D tensor(a single prompt). - With `image_url`, prompt becomes a multimodal PromptInputs dict. + `image_url` is only read when `triton_config.multimodal` is set. """ text_input = get_input_tensor_by_name(request, 'text_input') if text_input is None: @@ -621,26 +738,13 @@ def _convert_request(self, request): if isinstance(prompt, bytes): prompt = prompt.decode("utf-8") - # The loader applies the chat template and inserts the per-architecture - # image placeholders, so callers send a plain question. - image_url = get_input_tensor_by_name(request, 'image_url') - if image_url is not None and image_url.size > 0: - # Imported here, not at module scope (see note above). - from tensorrt_llm.inputs import default_multimodal_input_loader - - media = [ - url.decode("utf-8") if isinstance(url, bytes) else str(url) - for url in image_url.reshape(-1) - ] - tokenizer, hf_model_dir, model_type = self._get_multimodal_context() - prompt = default_multimodal_input_loader(tokenizer=tokenizer, - model_dir=hf_model_dir, - model_type=model_type, - modality="image", - prompts=[prompt], - media=media, - image_data_format="pt", - device="cpu")[0] + # `image_url` is only consulted when the operator opts in, so an existing + # deployment that already declares an input with this name for another + # purpose keeps its current behavior after an upgrade. + if self.multimodal_enabled: + image_url = get_input_tensor_by_name(request, 'image_url') + if image_url is not None and image_url.size > 0: + prompt = await self._build_multimodal_prompt(prompt, image_url) sampling_params = get_sampling_params_from_request(request) output_config = get_output_config_from_request(request) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.yaml b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.yaml index c9ad98f9dfce..6fa9040b026e 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.yaml +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.yaml @@ -17,3 +17,7 @@ pipeline_parallel_size: 1 triton_config: max_batch_size: 0 # The current implementation does not support batching, batch support is tracked in JIRA-4496 decoupled: False + # Accept the optional `image_url` request input and serve multimodal + # models. Off by default so that a deployment already declaring an + # `image_url` input for another purpose is unaffected by an upgrade. + multimodal: False diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt index 0a77208871b8..660c2a222813 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -52,6 +52,7 @@ input [ dims: [ -1 ] }, ## Optional multimodal input: URL, local path or data URI, one per image. + ## Only read when `triton_config.multimodal` is set in model.yaml. { name: "image_url" data_type: TYPE_STRING diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index b6b79e04aeed..8f7616e13bb3 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -1,4 +1,4 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,6 +24,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import asyncio import sys from dataclasses import dataclass from typing import Dict, List, Union @@ -361,3 +362,135 @@ def test_get_parameter(): # Special cases assert get_parameter(model_config, "empty_param") is None assert get_parameter(model_config, "env_var_param") is None + + +def _make_multimodal_model(enabled: bool): + """A bare model with just the multimodal state `_convert_request` reads.""" + model = TritonPythonModel.__new__(TritonPythonModel) + model.multimodal_enabled = enabled + model._multimodal_context = MultimodalContext(tokenizer="tokenizer", + processor="processor", + model_config=MagicMock(), + model_type="qwen2_5_vl") + return model + + +def test_convert_request_ignores_image_url_when_multimodal_disabled(): + # A deployment that already declares an `image_url` input for another + # purpose must be unaffected until it opts in via triton_config.multimodal. + model = _make_multimodal_model(enabled=False) + request = make_mock_triton_request({ + **inputs(), + "image_url": [b"https://example.com/a.jpg"], + }) + + prompt, _, _, _, _ = asyncio.run(model._convert_request(request)) + + assert prompt == "Tell me a story." + + +def test_convert_request_builds_multimodal_prompt_when_enabled(): + model = _make_multimodal_model(enabled=True) + captured = {} + + async def fake_build(text, image_url): + captured["text"] = text + captured["media"] = [ + url.decode("utf-8") for url in image_url.reshape(-1) + ] + return { + "prompt": "rendered", + "multi_modal_data": { + "image": ["decoded"] + } + } + + model._build_multimodal_prompt = fake_build + request = make_mock_triton_request({ + **inputs(), + "image_url": [b"https://example.com/a.jpg"], + }) + + prompt, _, _, _, _ = asyncio.run(model._convert_request(request)) + + assert prompt["multi_modal_data"] == {"image": ["decoded"]} + assert captured["text"] == "Tell me a story." + assert captured["media"] == ["https://example.com/a.jpg"] + + +def test_build_multimodal_prompt_uses_shared_inputs_primitives(): + # Mocks the deferred tensorrt_llm imports so the test does not require a + # built TRT-LLM. Asserts the backend drives the same placeholder/chat + # template primitives `trtllm-serve` uses, without importing + # `tensorrt_llm.serve` (which would pull in the OpenAI server stack). + model = _make_multimodal_model(enabled=True) + captured = {} + + class FakeTracker: + + def __init__(self, model_type): + captured["model_type"] = model_type + self.items = [] + + def add_data(self, modality, data): + self.items.append((modality, data)) + + def placeholder_counts(self): + return {"<|image_pad|>": len(self.items)} + + def item_order(self): + return [{ + "modality": modality, + "index": i, + "placeholder": "<|image_pad|>" + } for i, (modality, _) in enumerate(self.items)] + + async def retrieve_all_async(self): + return ({"image": [await data for _, data in self.items]}, None) + + async def fake_async_load_image(url): + return f"decoded:{url}" + + def fake_add_placeholders(model_type, text, counts, item_order): + captured["placeholder_args"] = (model_type, text, counts, item_order) + return "<|image_pad|>" + text + + async def fake_apply_chat_template(**kwargs): + captured["template_kwargs"] = kwargs + return "rendered:" + kwargs["conversation"][0]["content"] + + inputs_mod = MagicMock() + inputs_mod.prompt_inputs = lambda prompt: {"prompt": prompt} + utils_mod = MagicMock() + utils_mod.ConversationMessage = dict + utils_mod.MultimodalDataTracker = FakeTracker + utils_mod.add_multimodal_placeholders = fake_add_placeholders + utils_mod.async_apply_chat_template = fake_apply_chat_template + utils_mod.async_load_image = fake_async_load_image + + with patch.dict( + sys.modules, { + "tensorrt_llm": MagicMock(), + "tensorrt_llm.inputs": inputs_mod, + "tensorrt_llm.inputs.utils": utils_mod, + }): + prompt = asyncio.run( + model._build_multimodal_prompt( + "Describe this.", np.array([b"https://example.com/a.jpg"]))) + + assert prompt == { + "prompt": + "rendered:<|image_pad|>Describe this.", + "multi_modal_data": { + "image": ["decoded:https://example.com/a.jpg"] + }, + "mm_item_order": [{ + "modality": "image", + "index": 0, + "placeholder": "<|image_pad|>" + }], + } + # The registry key, not the delegated "qwen2_5_vl_text" instance attribute. + assert captured["model_type"] == "qwen2_5_vl" + assert captured["placeholder_args"][1] == "Describe this." + assert captured["template_kwargs"]["add_generation_prompt"] is True From c41fc3ee07e9f759c8a1401edb3700dd508461cf Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:31:39 -0700 Subject: [PATCH 03/10] [None][feat] Keep the Triton llmapi multimodal path working on TRT-LLM 1.2.1 triton_backend/ is not shipped in the tensorrt_llm wheel, so operators copy this template out of a git checkout into their model repository. The TRT-LLM it runs against is therefore not necessarily the one it was written for. async_apply_chat_template, MultimodalDataTracker.item_order() and the fourth argument of add_multimodal_placeholders all post-date v1.2.1, which is what the current tritonserver container ships. Fall back to the older equivalents instead of failing at the first image request, and add a test that exercises the fallback path. Also trim the explanatory comments added in the previous commit. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 68 +++++++++---------- .../tests/test_llmapi_python_backend.py | 66 ++++++++++++++++++ 2 files changed, 99 insertions(+), 35 deletions(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index 822c93e3a106..0cb41225ef2b 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -53,11 +53,7 @@ @dataclass class MultimodalContext: - """Startup-resolved state needed to turn media URLs into engine inputs. - - Resolved once in `initialize` so that per-request parsing stays cheap, the - same way `trtllm-serve` resolves them in `OpenAIServer.__init__`. - """ + """State resolved once at startup and reused for every request.""" tokenizer: Any processor: Any model_config: Any @@ -264,8 +260,6 @@ def initialize(self, args): self._response_thread = threading.Thread(target=self._response_loop) self._response_thread.start() - # Opt-in: a deployment that already defines an `image_url` input - # for another purpose keeps its behavior unless this is enabled. self.multimodal_enabled = bool( triton_config.get("multimodal", False)) self._multimodal_context = None @@ -604,11 +598,7 @@ async def _execute_single_request(self, request): del self.triton_user_id_to_req_ids[triton_user_id] def _init_multimodal_context(self): - """Resolve the tokenizer, HF processor and pretrained config once. - - Loading these per request would re-read the checkpoint on every forward - pass, so they are resolved at model load time and reused. - """ + """Resolve the tokenizer, HF processor and pretrained config once.""" from transformers import AutoProcessor from tensorrt_llm._torch.pyexecutor.config_utils import \ @@ -636,12 +626,8 @@ def _init_multimodal_context(self): f"triton_config.multimodal is enabled but the HF processor/config " f"for '{hf_model_dir}' could not be loaded: {e}") - # Read `model_type` from the config class, not the instance: composite - # configs such as Qwen2_5_VLConfig delegate the instance attribute to - # `text_config` and would report "qwen2_5_vl_text" instead of the - # "qwen2_5_vl" key used by the multimodal placeholder registry. This - # matches `tensorrt_llm.serve.chat_utils.resolve_top_level_model_type`, - # inlined to keep the backend off the `tensorrt_llm.serve` import path. + # Composite configs (e.g. Qwen2_5_VLConfig) delegate the instance + # attribute to `text_config`, so prefer the class attribute. model_type = getattr(type(model_config), "model_type", None) or getattr( model_config, "model_type", "") self._multimodal_context = MultimodalContext(tokenizer=tokenizer, @@ -654,20 +640,27 @@ def _init_multimodal_context(self): async def _build_multimodal_prompt(self, text, image_url): """Build a multimodal PromptInputs from a text prompt and image URLs. - Runs the same sequence `trtllm-serve` uses for `v1/chat/completions`, - but calls the shared `tensorrt_llm.inputs` primitives directly rather - than going through `tensorrt_llm.serve.chat_utils`: importing anything - under `tensorrt_llm.serve` executes `tensorrt_llm/serve/__init__.py`, - which pulls in the whole OpenAI server stack (FastAPI, the `openai` - SDK) that a Triton deployment neither needs nor is guaranteed to have. + Runs the same sequence `trtllm-serve` uses for `v1/chat/completions`. """ from tensorrt_llm.inputs import prompt_inputs from tensorrt_llm.inputs.utils import (ConversationMessage, MultimodalDataTracker, add_multimodal_placeholders, - async_apply_chat_template, + apply_chat_template, async_load_image) + # This directory is not shipped in the tensorrt_llm wheel, so operators + # copy it out of a git checkout into their model repository. The TRT-LLM + # it runs against is therefore not necessarily the one it was written + # for, and the multimodal helpers below gained their current shape after + # v1.2.1. Degrade to the older equivalents rather than fail at runtime. + try: + from tensorrt_llm.inputs.utils import async_apply_chat_template + except ImportError: + + async def async_apply_chat_template(**kwargs): + return await asyncio.to_thread(apply_chat_template, **kwargs) + ctx = self._multimodal_context media = [ url.decode("utf-8") if isinstance(url, bytes) else str(url) @@ -675,17 +668,23 @@ async def _build_multimodal_prompt(self, text, image_url): ] # `add_data` takes the un-awaited fetch, so several images on one - # request are fetched concurrently when the tracker is drained below. + # request are fetched concurrently. mm_data_tracker = MultimodalDataTracker(ctx.model_type) for url in media: mm_data_tracker.add_data("image", async_load_image(url)) mm_placeholder_counts = mm_data_tracker.placeholder_counts() - item_order = mm_data_tracker.item_order() - - # The chat template needs the per-architecture image placeholders in - # the message text, so callers send a plain question as `text_input`. - content = add_multimodal_placeholders(ctx.model_type, text, - mm_placeholder_counts, item_order) + # `item_order`, and the argument it feeds, also post-date v1.2.1. + item_order = getattr(mm_data_tracker, "item_order", lambda: None)() + + # The chat template inserts the per-architecture image placeholders, + # so callers send a plain question as `text_input`. + if item_order: + content = add_multimodal_placeholders(ctx.model_type, text, + mm_placeholder_counts, + item_order) + else: + content = add_multimodal_placeholders(ctx.model_type, text, + mm_placeholder_counts) conversation = [ ConversationMessage(role="user", content=content, media=[]) ] @@ -738,9 +737,8 @@ async def _convert_request(self, request): if isinstance(prompt, bytes): prompt = prompt.decode("utf-8") - # `image_url` is only consulted when the operator opts in, so an existing - # deployment that already declares an input with this name for another - # purpose keeps its current behavior after an upgrade. + # Only read `image_url` when the operator opts in, so a deployment + # already declaring that input keeps its behavior after an upgrade. if self.multimodal_enabled: image_url = get_input_tensor_by_name(request, 'image_url') if image_url is not None and image_url.size > 0: diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index 8f7616e13bb3..ef23091a3db3 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -26,6 +26,7 @@ import asyncio import sys +import types from dataclasses import dataclass from typing import Dict, List, Union from unittest.mock import MagicMock, patch @@ -494,3 +495,68 @@ async def fake_apply_chat_template(**kwargs): assert captured["model_type"] == "qwen2_5_vl" assert captured["placeholder_args"][1] == "Describe this." assert captured["template_kwargs"]["add_generation_prompt"] is True + + +def test_build_multimodal_prompt_falls_back_on_older_trtllm(): + # The backend template is copied out of a git checkout, so it can run + # against a TRT-LLM older than the one it was written for. On <= v1.2.1 + # there is no async_apply_chat_template, the tracker has no item_order(), + # and add_multimodal_placeholders takes three arguments. + model = _make_multimodal_model(enabled=True) + captured = {} + + class OldTracker: + + def __init__(self, model_type): + self.items = [] + + def add_data(self, modality, data): + self.items.append(data) + + def placeholder_counts(self): + return {"<|image_pad|>": len(self.items)} + + async def retrieve_all_async(self): + return ({"image": [await data for data in self.items]}, None) + + async def fake_async_load_image(url): + return f"decoded:{url}" + + def fake_add_placeholders(model_type, text, counts): + captured["placeholder_argcount"] = 3 + return "<|image_pad|>" + text + + def fake_apply_chat_template(**kwargs): + captured["used_sync_template"] = True + return "rendered:" + kwargs["conversation"][0]["content"] + + # SimpleNamespace, not MagicMock: the missing attribute must raise + # ImportError so the compatibility path is the one under test. + utils_mod = types.SimpleNamespace( + ConversationMessage=dict, + MultimodalDataTracker=OldTracker, + add_multimodal_placeholders=fake_add_placeholders, + apply_chat_template=fake_apply_chat_template, + async_load_image=fake_async_load_image, + ) + inputs_mod = MagicMock() + inputs_mod.prompt_inputs = lambda prompt: {"prompt": prompt} + + with patch.dict( + sys.modules, { + "tensorrt_llm": MagicMock(), + "tensorrt_llm.inputs": inputs_mod, + "tensorrt_llm.inputs.utils": utils_mod, + }): + prompt = asyncio.run( + model._build_multimodal_prompt( + "Describe this.", np.array([b"https://example.com/a.jpg"]))) + + assert prompt == { + "prompt": "rendered:<|image_pad|>Describe this.", + "multi_modal_data": { + "image": ["decoded:https://example.com/a.jpg"] + }, + } + assert "mm_item_order" not in prompt + assert captured == {"placeholder_argcount": 3, "used_sync_template": True} From 54c65ef1972b76ee55662790c5d7c8347c4e7dac Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:18:09 -0700 Subject: [PATCH 04/10] [None][fix] Import test symbols explicitly to satisfy ruff F405 The test module pulls the backend in with `from model import *`, so every symbol it uses from there is reported as possibly-undefined. Import the two names the new tests need, and `json`, explicitly. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- triton_backend/all_models/tests/test_llmapi_python_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index ef23091a3db3..3a3197290eea 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio +import json import sys import types from dataclasses import dataclass @@ -45,6 +46,7 @@ get_streaming_from_request) # Use PYTHONPATH=../llmapi/tensorrt_llm/1/ from model import * +from model import MultimodalContext, TritonPythonModel # explicit: avoids F405 @dataclass From 41134c366bdc064ace6c183513190fac959339f0 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:21:24 -0700 Subject: [PATCH 05/10] [None][fix] Drop the redundant star import from the llmapi backend tests The previous commit imported the symbols the tests use from `model` explicitly, which left `from model import *` unused; autoflake removes it and pre-commit then fails on the modified file. Remove it here instead. Nothing else in the module resolves through it -- ruff reports no undefined names -- and the `patch("model....")` targets keep working because the module is still imported. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- triton_backend/all_models/tests/test_llmapi_python_backend.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index 3a3197290eea..3dddaeee85ff 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -45,8 +45,7 @@ get_sampling_params_from_request, get_streaming_from_request) # Use PYTHONPATH=../llmapi/tensorrt_llm/1/ -from model import * -from model import MultimodalContext, TritonPythonModel # explicit: avoids F405 +from model import MultimodalContext, TritonPythonModel @dataclass From f1166de1bc74d5bc06eb7523a9bf1e6da2bd0e40 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:37:57 -0700 Subject: [PATCH 06/10] [None][fix] Report preprocessing failures instead of hanging the client `_execute_single_request` inferred cancellation from the request's absence in `req_id_to_request_data`, but the entry is only added after preprocessing and `generate_async` succeed. A failure before that point -- an unreachable image URL, a decode error, a template error -- therefore looked like a cancellation, so no response was sent and the client waited forever. Track registration explicitly and only treat absence as cancellation once the request has been registered. Add tests for both halves: an error before registration is reported with COMPLETE_FINAL, and a failure after the cancellation loop removed the entry stays silent. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 7 +- .../tests/test_llmapi_python_backend.py | 89 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index 0cb41225ef2b..d496339c57a9 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -494,6 +494,10 @@ async def _execute_single_request(self, request): # Unique request id used to identify each triton request triton_req_id = str(randint(0, sys.maxsize)) + # Absence from `req_id_to_request_data` only means "cancelled" once the + # request has been added to it. Before that it just means preprocessing + # has not finished, and an error still has to be reported to the client. + request_registered = False try: from tensorrt_llm import SamplingParams @@ -523,6 +527,7 @@ async def _execute_single_request(self, request): # TODO: [JIRA-4496] Add all batched request ids to the set self.triton_user_id_to_req_ids[triton_user_id].add( triton_req_id) + request_registered = True async for request_output in response_iterator: # Send each response if streaming. @@ -575,7 +580,7 @@ async def _execute_single_request(self, request): # already sent a COMPLETE_FINAL on this response_sender; they # remove the entry from req_id_to_request_data as the signal. with self.lock: - was_cancelled = (triton_req_id + was_cancelled = (request_registered and triton_req_id not in self.req_id_to_request_data) if not was_cancelled: error = pb_utils.TritonError(f"Error generating request: {e}") diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index 3dddaeee85ff..cc6f1bc90299 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -27,6 +27,7 @@ import asyncio import json import sys +import threading import types from dataclasses import dataclass from typing import Dict, List, Union @@ -561,3 +562,91 @@ def fake_apply_chat_template(**kwargs): } assert "mm_item_order" not in prompt assert captured == {"placeholder_argcount": 3, "used_sync_template": True} + + +def _bare_model_for_execute(): + """A model with only the state `_execute_single_request` touches.""" + model = TritonPythonModel.__new__(TritonPythonModel) + model.logger = MagicMock() + model.lock = threading.Lock() + model.req_id_to_request_data = {} + model.triton_user_id_to_req_ids = {} + model._ongoing_request_count = 0 + model.decoupled = False + model.output_dtype = np.object_ + return model + + +class _RecordingSender: + + def __init__(self): + self.sent = [] + + def send(self, response, flags=None): + self.sent.append((response, flags)) + + +def test_execute_single_request_reports_preprocessing_failure(): + # A bad image URL fails inside _convert_request, before the request is + # registered in req_id_to_request_data. The error must still reach the + # client with COMPLETE_FINAL, otherwise it waits forever. + model = _bare_model_for_execute() + sender = _RecordingSender() + request = make_mock_triton_request({"text_input": ["describe this"]}) + request.get_response_sender = lambda: sender + request.request_id = lambda: "triton-user-1" + + async def failing_convert(_request): + raise RuntimeError( + "Cannot connect to host example.invalid:443 [Name or service not known]" + ) + + model._convert_request = failing_convert + + with patch.dict(sys.modules, {"tensorrt_llm": MagicMock()}): + with pytest.raises(RuntimeError): + asyncio.run(model._execute_single_request(request)) + + pb_utils = sys.modules["triton_python_backend_utils"] + assert len(sender.sent) == 1, "client must receive exactly one response" + response, flags = sender.sent[0] + assert flags == pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL + assert response.has_error() + assert "example.invalid" in response.error.message + + +def test_execute_single_request_skips_response_when_cancelled(): + # Once the request IS registered, an empty map means the cancellation loop + # already sent COMPLETE_FINAL and removed the entry, so the error handler + # must stay silent rather than send a second final response. + model = _bare_model_for_execute() + sender = _RecordingSender() + request = make_mock_triton_request({"text_input": ["describe this"]}) + request.get_response_sender = lambda: sender + request.request_id = lambda: "triton-user-2" + + async def convert(_request): + return ("a prompt", {}, False, {}, None) + + class CancellingIterator: + + def __aiter__(self): + return self + + async def __anext__(self): + # Stand in for cancellation_loop: it sends COMPLETE_FINAL and drops + # the entry while generation is in flight. + with model.lock: + model.req_id_to_request_data.clear() + raise RuntimeError("request aborted") + + engine = MagicMock() + engine.generate_async.return_value = CancellingIterator() + model._convert_request = convert + model._llm_engine = engine + + with patch.dict(sys.modules, {"tensorrt_llm": MagicMock()}): + with pytest.raises(RuntimeError): + asyncio.run(model._execute_single_request(request)) + + assert sender.sent == [], "must not double-send after cancellation" From bc250d5c91c388c2c99a1eb37bf0eaddc11a55d3 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:52:29 -0700 Subject: [PATCH 07/10] [None][feat] Share the multimodal request-building path with trtllm-serve The Triton backend rebuilt the multimodal prompt itself and had already diverged: it inserted string placeholders unconditionally, while the serve path only does so for ContentFormat.STRING. For models whose registry entry is ContentFormat.OPENAI -- llava_next, gemma4, gemma4_unified, step3p7vl -- apply_chat_template() rebuilds the content from content_parts and adds its own image part, so one image produced two markers. Extract that per-message logic into `apply_mm_placeholders` in tensorrt_llm/inputs/utils.py and call it from parse_chat_messages_coroutines, so there is one implementation. Add `async_build_multimodal_prompt` alongside it for callers whose request format is a prompt plus a list of media references rather than OpenAI chat messages, and use it from the backend. The backend now holds the tokenizer, processor and model type as plain attributes and delegates the whole path in one call, dropping its local placeholder handling, its MultimodalContext dataclass and the version shim that guarded APIs this helper now owns. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- tensorrt_llm/inputs/__init__.py | 3 + tensorrt_llm/inputs/utils.py | 130 ++++++++++++- tensorrt_llm/serve/chat_utils.py | 28 +-- .../all_models/llmapi/tensorrt_llm/1/model.py | 121 +++--------- .../tests/test_llmapi_python_backend.py | 183 +++--------------- 5 files changed, 188 insertions(+), 277 deletions(-) diff --git a/tensorrt_llm/inputs/__init__.py b/tensorrt_llm/inputs/__init__.py index 0330b9d509c6..4d49dd8038d6 100644 --- a/tensorrt_llm/inputs/__init__.py +++ b/tensorrt_llm/inputs/__init__.py @@ -22,6 +22,7 @@ ALL_SUPPORTED_MULTIMODAL_MODELS, ALL_SUPPORTED_VIDEO_MODELS, ConversationMessage, MultimodalData, MultimodalDataTracker, add_multimodal_placeholders, apply_chat_template, + apply_mm_placeholders, async_build_multimodal_prompt, async_load_audio, async_load_image, async_load_video, convert_image_mode, default_multimodal_input_loader, encode_base64_content_from_url, encode_base64_image, @@ -63,6 +64,8 @@ "async_load_image", "async_load_video", "add_multimodal_placeholders", + "apply_mm_placeholders", + "async_build_multimodal_prompt", "apply_chat_template", "convert_image_mode", "default_multimodal_input_loader", diff --git a/tensorrt_llm/inputs/utils.py b/tensorrt_llm/inputs/utils.py index 8a6d406604be..d7395d0ca86f 100644 --- a/tensorrt_llm/inputs/utils.py +++ b/tensorrt_llm/inputs/utils.py @@ -20,7 +20,9 @@ from tensorrt_llm.inputs.content_format import (ContentFormat, detect_content_format) -from tensorrt_llm.inputs.media_io import (_get_aiohttp_session, +from tensorrt_llm.inputs.data import prompt_inputs +from tensorrt_llm.inputs.media_io import (MEDIA_IO_REGISTRY, + _get_aiohttp_session, _load_and_convert_image, _load_video_by_cv2, _normalize_file_uri, @@ -820,6 +822,132 @@ async def async_apply_chat_template( ) +def apply_mm_placeholders( + model_type: str, + message: ConversationMessage, + placeholder_counts: Dict[str, int], + mm_data_tracker: "MultimodalDataTracker", + *, + item_order_start: int = 0, + content_format: Optional[ContentFormat] = None, +) -> None: + """Insert this message's multimodal placeholders into its text, in place. + + Only `ContentFormat.STRING` templates need them. For `OPENAI`, + `apply_chat_template` rebuilds `content` from `content_parts` via + `_build_openai_content`, so pre-inserting here would render every media + item twice: once as text and once as the template's own content part. + + When the model opts into interleaving and `content_parts` is available the + placeholders keep their original positions; otherwise they are placed in + bulk according to the registered placement. + """ + if not placeholder_counts: + return + + if content_format is None: + registry_format = MULTIMODAL_PLACEHOLDER_REGISTRY.get_content_format( + model_type) + content_format = (registry_format if registry_format is not None else + ContentFormat.STRING) + if content_format != ContentFormat.STRING: + return + + content_parts = message.get("content_parts") + interleave = MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders( + model_type) + if content_parts and interleave: + message["content"] = interleave_mm_placeholders( + model_type, content_parts, placeholder_counts, + mm_data_tracker.placeholder_modalities()) + else: + message["content"] = add_multimodal_placeholders( + model_type, + message["content"], + placeholder_counts, + item_order=mm_data_tracker.item_order()[item_order_start:], + ) + + +async def async_build_multimodal_prompt( + *, + model_type: str, + tokenizer: Union[TransformersTokenizer, TokenizerBase], + processor: ProcessorMixin, + prompt: str, + media: List[str], + modality: str = "image", + add_generation_prompt: bool = True, + chat_template: Optional[str] = None, + chat_template_kwargs: Optional[Dict[str, Any]] = None, + media_io_kwargs: Optional[Dict[str, Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Package one text prompt plus media URLs into engine inputs. + + For callers whose request format is a plain prompt and a list of media + references rather than OpenAI chat messages -- the Triton `llmapi` backend, + for example. Runs the same steps `trtllm-serve` runs for + `v1/chat/completions`: resolve placeholders for the model's content format, + render the chat template, and attach the fetched media. + + The media fetches and the template rendering are awaited together, so + several items on one request are loaded concurrently. + + Args: + model_type: Top-level HF `model_type`, the multimodal registry key. + prompt: The user's text, without any placeholder tokens. + media: URLs, `data:` URIs or local paths, one per item. + modality: Registered media modality, e.g. `"image"`. + + Returns: + A `TextPrompt` carrying `multi_modal_data`, ready for + `LLM.generate_async`. + """ + media_io_cls = MEDIA_IO_REGISTRY.get(modality) + if media_io_cls is None: + raise ValueError(f"Unsupported modality {modality!r}. " + f"Registered modalities: {list(MEDIA_IO_REGISTRY)}") + media_io = media_io_cls.create((media_io_kwargs or {}).get(modality), None) + + mm_data_tracker = MultimodalDataTracker(model_type) + content_parts: List[Union[str, Dict[str, Any]]] = [] + for index, item in enumerate(media): + # `add_data` takes the un-awaited fetch; they run concurrently below. + mm_data_tracker.add_data(modality, media_io.async_load(item)) + content_parts.append({"type": modality, "media_index": index}) + content_parts.append(prompt) + + message = ConversationMessage(role="user", + content=prompt, + media=[], + content_parts=content_parts) + placeholder_counts = mm_data_tracker.placeholder_counts() + apply_mm_placeholders(model_type, message, placeholder_counts, + mm_data_tracker) + + prompt_task = async_apply_chat_template( + model_type=model_type, + tokenizer=tokenizer, + processor=processor, + conversation=[message], + add_generation_prompt=add_generation_prompt, + mm_placeholder_counts=[placeholder_counts], + chat_template=chat_template, + chat_template_kwargs=chat_template_kwargs, + ) + rendered, (mm_data, + _) = await asyncio.gather(prompt_task, + mm_data_tracker.retrieve_all_async()) + + inputs = prompt_inputs(rendered) + if mm_data: + inputs["multi_modal_data"] = mm_data + item_order = mm_data_tracker.item_order() + if item_order: + inputs["mm_item_order"] = item_order + return inputs + + def default_multimodal_input_loader( *, tokenizer: Optional[Union[TransformersTokenizer, TokenizerBase]], diff --git a/tensorrt_llm/serve/chat_utils.py b/tensorrt_llm/serve/chat_utils.py index d60d93f2ce45..c1f56b98af01 100644 --- a/tensorrt_llm/serve/chat_utils.py +++ b/tensorrt_llm/serve/chat_utils.py @@ -17,12 +17,11 @@ from tensorrt_llm.inputs import (ContentFormat, ConversationMessage, MultimodalData, MultimodalDataTracker, - add_multimodal_placeholders, load_base64_image_embeds) from tensorrt_llm.inputs.media_io import MEDIA_IO_REGISTRY, BaseMediaIO from tensorrt_llm.inputs.multimodal import MultimodalServerConfig from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY -from tensorrt_llm.inputs.utils import interleave_mm_placeholders +from tensorrt_llm.inputs.utils import apply_mm_placeholders from tensorrt_llm.logger import logger @@ -497,25 +496,12 @@ def parse_chat_messages_coroutines( placeholder] = msg_placeholder_counts.get( placeholder, 0) + 1 - if msg_placeholder_counts and content_format == ContentFormat.STRING: - # For STRING format, use interleaving when the model opts in - # and content_parts is available, otherwise fall back to bulk - # prepend/append according to placeholder_placement. - content_parts = parsed_msg.get("content_parts") - interleave = MULTIMODAL_PLACEHOLDER_REGISTRY.get_interleave_placeholders( - model_type) - if content_parts and interleave: - parsed_msg["content"] = interleave_mm_placeholders( - model_type, content_parts, msg_placeholder_counts, - mm_data_tracker.placeholder_modalities()) - else: - msg_item_order = mm_data_tracker.item_order()[item_order_start:] - parsed_msg["content"] = add_multimodal_placeholders( - type(model_config).model_type, - parsed_msg["content"], - msg_placeholder_counts, - item_order=msg_item_order, - ) + apply_mm_placeholders(model_type, + parsed_msg, + msg_placeholder_counts, + mm_data_tracker, + item_order_start=item_order_start, + content_format=content_format) mm_placeholder_counts.append(msg_placeholder_counts) # ``item_order`` is populated synchronously by ``add_data``, so it can diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index d496339c57a9..cc3d762a5d44 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -51,15 +51,6 @@ # inherits these FDs, causing child processes to ignore CUDA_VISIBLE_DEVICES. -@dataclass -class MultimodalContext: - """State resolved once at startup and reused for every request.""" - tokenizer: Any - processor: Any - model_config: Any - model_type: str - - @dataclass class RequestData: triton_req_id: int @@ -262,9 +253,8 @@ def initialize(self, args): self.multimodal_enabled = bool( triton_config.get("multimodal", False)) - self._multimodal_context = None if self.multimodal_enabled: - self._init_multimodal_context() + self._init_multimodal() self.req_id_to_request_data = {} self.triton_user_id_to_req_ids = {} @@ -602,16 +592,17 @@ async def _execute_single_request(self, request): if triton_user_id is not None and triton_user_id != "" and triton_user_id in self.triton_user_id_to_req_ids: del self.triton_user_id_to_req_ids[triton_user_id] - def _init_multimodal_context(self): - """Resolve the tokenizer, HF processor and pretrained config once.""" + def _init_multimodal(self): + """Resolve the tokenizer, HF processor and model type once.""" from transformers import AutoProcessor from tensorrt_llm._torch.pyexecutor.config_utils import \ load_pretrained_config - tokenizer = self._llm_engine.tokenizer + self._mm_tokenizer = self._llm_engine.tokenizer hf_model_dir = self._llm_engine._hf_model_dir or getattr( - getattr(tokenizer, "tokenizer", None), "name_or_path", None) + getattr(self._mm_tokenizer, "tokenizer", None), "name_or_path", + None) if hf_model_dir is None: raise pb_utils.TritonModelException( "triton_config.multimodal is enabled but the checkpoint directory " @@ -619,7 +610,7 @@ def _init_multimodal_context(self): hf_model_dir = str(hf_model_dir) trust_remote_code = self._llm_engine.args.trust_remote_code try: - processor = AutoProcessor.from_pretrained( + self._mm_processor = AutoProcessor.from_pretrained( hf_model_dir, trust_remote_code=trust_remote_code) model_config = load_pretrained_config( hf_model_dir, @@ -633,86 +624,11 @@ def _init_multimodal_context(self): # Composite configs (e.g. Qwen2_5_VLConfig) delegate the instance # attribute to `text_config`, so prefer the class attribute. - model_type = getattr(type(model_config), "model_type", None) or getattr( - model_config, "model_type", "") - self._multimodal_context = MultimodalContext(tokenizer=tokenizer, - processor=processor, - model_config=model_config, - model_type=model_type) + self._mm_model_type = getattr(type(model_config), + "model_type", None) or getattr( + model_config, "model_type", "") self.logger.log_info("[trtllm] multimodal input enabled for model_type " - f"'{self._multimodal_context.model_type}'") - - async def _build_multimodal_prompt(self, text, image_url): - """Build a multimodal PromptInputs from a text prompt and image URLs. - - Runs the same sequence `trtllm-serve` uses for `v1/chat/completions`. - """ - from tensorrt_llm.inputs import prompt_inputs - from tensorrt_llm.inputs.utils import (ConversationMessage, - MultimodalDataTracker, - add_multimodal_placeholders, - apply_chat_template, - async_load_image) - - # This directory is not shipped in the tensorrt_llm wheel, so operators - # copy it out of a git checkout into their model repository. The TRT-LLM - # it runs against is therefore not necessarily the one it was written - # for, and the multimodal helpers below gained their current shape after - # v1.2.1. Degrade to the older equivalents rather than fail at runtime. - try: - from tensorrt_llm.inputs.utils import async_apply_chat_template - except ImportError: - - async def async_apply_chat_template(**kwargs): - return await asyncio.to_thread(apply_chat_template, **kwargs) - - ctx = self._multimodal_context - media = [ - url.decode("utf-8") if isinstance(url, bytes) else str(url) - for url in image_url.reshape(-1) - ] - - # `add_data` takes the un-awaited fetch, so several images on one - # request are fetched concurrently. - mm_data_tracker = MultimodalDataTracker(ctx.model_type) - for url in media: - mm_data_tracker.add_data("image", async_load_image(url)) - mm_placeholder_counts = mm_data_tracker.placeholder_counts() - # `item_order`, and the argument it feeds, also post-date v1.2.1. - item_order = getattr(mm_data_tracker, "item_order", lambda: None)() - - # The chat template inserts the per-architecture image placeholders, - # so callers send a plain question as `text_input`. - if item_order: - content = add_multimodal_placeholders(ctx.model_type, text, - mm_placeholder_counts, - item_order) - else: - content = add_multimodal_placeholders(ctx.model_type, text, - mm_placeholder_counts) - conversation = [ - ConversationMessage(role="user", content=content, media=[]) - ] - - prompt_task = async_apply_chat_template( - model_type=ctx.model_type, - tokenizer=ctx.tokenizer, - processor=ctx.processor, - conversation=conversation, - add_generation_prompt=True, - mm_placeholder_counts=[mm_placeholder_counts], - ) - # Render the template while the images are still in flight. - prompt, (mm_data, - _) = await asyncio.gather(prompt_task, - mm_data_tracker.retrieve_all_async()) - - prompt = prompt_inputs(prompt) - if mm_data: - prompt["multi_modal_data"] = mm_data - if item_order: - prompt["mm_item_order"] = item_order - return prompt + f"'{self._mm_model_type}'") async def _convert_request(self, request): """Helper function to convert the request into a prompt for LLM.generate_async @@ -747,7 +663,20 @@ async def _convert_request(self, request): if self.multimodal_enabled: image_url = get_input_tensor_by_name(request, 'image_url') if image_url is not None and image_url.size > 0: - prompt = await self._build_multimodal_prompt(prompt, image_url) + from tensorrt_llm.inputs import async_build_multimodal_prompt + + prompt = await async_build_multimodal_prompt( + model_type=self._mm_model_type, + tokenizer=self._mm_tokenizer, + processor=self._mm_processor, + prompt=prompt, + media=[ + url.decode("utf-8") + if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ], + modality="image", + ) sampling_params = get_sampling_params_from_request(request) output_config = get_output_config_from_request(request) diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index cc6f1bc90299..c2b008e95ca7 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -28,7 +28,6 @@ import json import sys import threading -import types from dataclasses import dataclass from typing import Dict, List, Union from unittest.mock import MagicMock, patch @@ -46,7 +45,7 @@ get_sampling_params_from_request, get_streaming_from_request) # Use PYTHONPATH=../llmapi/tensorrt_llm/1/ -from model import MultimodalContext, TritonPythonModel +from model import TritonPythonModel @dataclass @@ -371,10 +370,9 @@ def _make_multimodal_model(enabled: bool): """A bare model with just the multimodal state `_convert_request` reads.""" model = TritonPythonModel.__new__(TritonPythonModel) model.multimodal_enabled = enabled - model._multimodal_context = MultimodalContext(tokenizer="tokenizer", - processor="processor", - model_config=MagicMock(), - model_type="qwen2_5_vl") + model._mm_tokenizer = "tokenizer" + model._mm_processor = "processor" + model._mm_model_type = "qwen2_5_vl" return model @@ -392,15 +390,15 @@ def test_convert_request_ignores_image_url_when_multimodal_disabled(): assert prompt == "Tell me a story." -def test_convert_request_builds_multimodal_prompt_when_enabled(): +def test_convert_request_delegates_to_shared_inputs_helper(): + # The backend must not build the multimodal prompt itself: placeholder + # handling depends on the model's ContentFormat, which the shared helper in + # tensorrt_llm.inputs owns. Assert we hand it the right arguments. model = _make_multimodal_model(enabled=True) captured = {} - async def fake_build(text, image_url): - captured["text"] = text - captured["media"] = [ - url.decode("utf-8") for url in image_url.reshape(-1) - ] + async def fake_helper(**kwargs): + captured.update(kwargs) return { "prompt": "rendered", "multi_modal_data": { @@ -408,160 +406,27 @@ async def fake_build(text, image_url): } } - model._build_multimodal_prompt = fake_build + inputs_mod = MagicMock() + inputs_mod.async_build_multimodal_prompt = fake_helper request = make_mock_triton_request({ **inputs(), - "image_url": [b"https://example.com/a.jpg"], + "image_url": [b"https://example.com/a.jpg", b"/tmp/b.png"], }) - prompt, _, _, _, _ = asyncio.run(model._convert_request(request)) + with patch.dict(sys.modules, { + "tensorrt_llm": MagicMock(), + "tensorrt_llm.inputs": inputs_mod, + }): + prompt, _, _, _, _ = asyncio.run(model._convert_request(request)) assert prompt["multi_modal_data"] == {"image": ["decoded"]} - assert captured["text"] == "Tell me a story." - assert captured["media"] == ["https://example.com/a.jpg"] - - -def test_build_multimodal_prompt_uses_shared_inputs_primitives(): - # Mocks the deferred tensorrt_llm imports so the test does not require a - # built TRT-LLM. Asserts the backend drives the same placeholder/chat - # template primitives `trtllm-serve` uses, without importing - # `tensorrt_llm.serve` (which would pull in the OpenAI server stack). - model = _make_multimodal_model(enabled=True) - captured = {} - - class FakeTracker: - - def __init__(self, model_type): - captured["model_type"] = model_type - self.items = [] - - def add_data(self, modality, data): - self.items.append((modality, data)) - - def placeholder_counts(self): - return {"<|image_pad|>": len(self.items)} - - def item_order(self): - return [{ - "modality": modality, - "index": i, - "placeholder": "<|image_pad|>" - } for i, (modality, _) in enumerate(self.items)] - - async def retrieve_all_async(self): - return ({"image": [await data for _, data in self.items]}, None) - - async def fake_async_load_image(url): - return f"decoded:{url}" - - def fake_add_placeholders(model_type, text, counts, item_order): - captured["placeholder_args"] = (model_type, text, counts, item_order) - return "<|image_pad|>" + text - - async def fake_apply_chat_template(**kwargs): - captured["template_kwargs"] = kwargs - return "rendered:" + kwargs["conversation"][0]["content"] - - inputs_mod = MagicMock() - inputs_mod.prompt_inputs = lambda prompt: {"prompt": prompt} - utils_mod = MagicMock() - utils_mod.ConversationMessage = dict - utils_mod.MultimodalDataTracker = FakeTracker - utils_mod.add_multimodal_placeholders = fake_add_placeholders - utils_mod.async_apply_chat_template = fake_apply_chat_template - utils_mod.async_load_image = fake_async_load_image - - with patch.dict( - sys.modules, { - "tensorrt_llm": MagicMock(), - "tensorrt_llm.inputs": inputs_mod, - "tensorrt_llm.inputs.utils": utils_mod, - }): - prompt = asyncio.run( - model._build_multimodal_prompt( - "Describe this.", np.array([b"https://example.com/a.jpg"]))) - - assert prompt == { - "prompt": - "rendered:<|image_pad|>Describe this.", - "multi_modal_data": { - "image": ["decoded:https://example.com/a.jpg"] - }, - "mm_item_order": [{ - "modality": "image", - "index": 0, - "placeholder": "<|image_pad|>" - }], - } - # The registry key, not the delegated "qwen2_5_vl_text" instance attribute. assert captured["model_type"] == "qwen2_5_vl" - assert captured["placeholder_args"][1] == "Describe this." - assert captured["template_kwargs"]["add_generation_prompt"] is True - - -def test_build_multimodal_prompt_falls_back_on_older_trtllm(): - # The backend template is copied out of a git checkout, so it can run - # against a TRT-LLM older than the one it was written for. On <= v1.2.1 - # there is no async_apply_chat_template, the tracker has no item_order(), - # and add_multimodal_placeholders takes three arguments. - model = _make_multimodal_model(enabled=True) - captured = {} - - class OldTracker: - - def __init__(self, model_type): - self.items = [] - - def add_data(self, modality, data): - self.items.append(data) - - def placeholder_counts(self): - return {"<|image_pad|>": len(self.items)} - - async def retrieve_all_async(self): - return ({"image": [await data for data in self.items]}, None) - - async def fake_async_load_image(url): - return f"decoded:{url}" - - def fake_add_placeholders(model_type, text, counts): - captured["placeholder_argcount"] = 3 - return "<|image_pad|>" + text - - def fake_apply_chat_template(**kwargs): - captured["used_sync_template"] = True - return "rendered:" + kwargs["conversation"][0]["content"] - - # SimpleNamespace, not MagicMock: the missing attribute must raise - # ImportError so the compatibility path is the one under test. - utils_mod = types.SimpleNamespace( - ConversationMessage=dict, - MultimodalDataTracker=OldTracker, - add_multimodal_placeholders=fake_add_placeholders, - apply_chat_template=fake_apply_chat_template, - async_load_image=fake_async_load_image, - ) - inputs_mod = MagicMock() - inputs_mod.prompt_inputs = lambda prompt: {"prompt": prompt} - - with patch.dict( - sys.modules, { - "tensorrt_llm": MagicMock(), - "tensorrt_llm.inputs": inputs_mod, - "tensorrt_llm.inputs.utils": utils_mod, - }): - prompt = asyncio.run( - model._build_multimodal_prompt( - "Describe this.", np.array([b"https://example.com/a.jpg"]))) - - assert prompt == { - "prompt": "rendered:<|image_pad|>Describe this.", - "multi_modal_data": { - "image": ["decoded:https://example.com/a.jpg"] - }, - } - assert "mm_item_order" not in prompt - assert captured == {"placeholder_argcount": 3, "used_sync_template": True} + assert captured["tokenizer"] == "tokenizer" + assert captured["processor"] == "processor" + assert captured["modality"] == "image" + assert captured["prompt"] == "Tell me a story." + # Bytes tensors are decoded, order preserved. + assert captured["media"] == ["https://example.com/a.jpg", "/tmp/b.png"] def _bare_model_for_execute(): From 71497609e2cfdd58a079aca93c38ac91f5f60078 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:08:35 -0700 Subject: [PATCH 08/10] [None][fix] Accept only remote and inline image_url values `image_url` is client-controlled and the media loaders open local paths and `file://` with the server process's permissions, so a request could make the server read any image file it can reach. Restrict the input to http, https and data URLs. Inline `data:` payloads are kept because they carry their own bytes and never touch the filesystem, which is the exposure being closed. This is deliberately narrower than `trtllm-serve`, whose media loading is unrestricted, and is meant to hold until the allowed scope is agreed with the deployment owner; a configured allowlist root would be the natural way to widen it later. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 30 +++++++++-- .../tests/test_llmapi_python_backend.py | 53 ++++++++++++++++++- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index cc3d762a5d44..a13be6cfee74 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -35,6 +35,7 @@ from dataclasses import dataclass from random import randint from typing import Any +from urllib.parse import urlparse import numpy as np import pandas as pd @@ -85,6 +86,24 @@ def get_model_config(filename, include_keys=None, exclude_keys=None): return engine_config +# `image_url` is client-controlled, and the media loaders happily open local +# paths and `file://` with the server process's permissions. Until the allowed +# scope is agreed with the deployment owner, accept only remote fetches and +# inline data, so a request cannot make the server read its filesystem. +ALLOWED_MEDIA_SCHEMES = ("http", "https", "data") + + +def validate_media_urls(urls): + """Reject media references that would read the server's filesystem.""" + for url in urls: + if urlparse(url).scheme not in ALLOWED_MEDIA_SCHEMES: + raise pb_utils.TritonModelException( + f"Unsupported image_url {url!r}: only " + f"{', '.join(ALLOWED_MEDIA_SCHEMES)} are accepted. Local paths " + "and file:// URLs are rejected because the input is " + "client-controlled.") + + def get_input_scalar_by_name(request, name, expected_batch_size=1, @@ -665,16 +684,17 @@ async def _convert_request(self, request): if image_url is not None and image_url.size > 0: from tensorrt_llm.inputs import async_build_multimodal_prompt + media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + validate_media_urls(media) prompt = await async_build_multimodal_prompt( model_type=self._mm_model_type, tokenizer=self._mm_tokenizer, processor=self._mm_processor, prompt=prompt, - media=[ - url.decode("utf-8") - if isinstance(url, bytes) else str(url) - for url in image_url.reshape(-1) - ], + media=media, modality="image", ) diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index c2b008e95ca7..7dad0c14fdc7 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -45,7 +45,7 @@ get_sampling_params_from_request, get_streaming_from_request) # Use PYTHONPATH=../llmapi/tensorrt_llm/1/ -from model import TritonPythonModel +from model import TritonPythonModel, validate_media_urls @dataclass @@ -515,3 +515,54 @@ async def __anext__(self): asyncio.run(model._execute_single_request(request)) assert sender.sent == [], "must not double-send after cancellation" + + +@pytest.mark.parametrize("url", [ + "/etc/passwd", + "/lustre/private/secret.png", + "file:///etc/passwd", + "relative/path.jpg", + "ftp://example.com/a.jpg", +]) +def test_validate_media_urls_rejects_local_and_unknown_schemes(url): + # image_url is client-controlled; a local path would make the server read + # its own filesystem with the Triton process's permissions. + with pytest.raises(MockTritonModelException) as excinfo: + validate_media_urls([url]) + assert "image_url" in str(excinfo.value) + + +@pytest.mark.parametrize("url", [ + "http://images.example.com/a.jpg", + "https://images.example.com/a.jpg", + "data:image/png;base64,iVBORw0KGgo=", +]) +def test_validate_media_urls_accepts_remote_and_inline(url): + validate_media_urls([url]) + + +def test_convert_request_rejects_local_path_before_loading(): + # The rejection must happen before the media loader is reached. + model = _make_multimodal_model(enabled=True) + called = False + + async def fake_helper(**kwargs): + nonlocal called + called = True + return {"prompt": "rendered"} + + inputs_mod = MagicMock() + inputs_mod.async_build_multimodal_prompt = fake_helper + request = make_mock_triton_request({ + **inputs(), + "image_url": [b"/etc/passwd"], + }) + + with patch.dict(sys.modules, { + "tensorrt_llm": MagicMock(), + "tensorrt_llm.inputs": inputs_mod, + }): + with pytest.raises(MockTritonModelException): + asyncio.run(model._convert_request(request)) + + assert not called, "loader must not be reached for a rejected URL" From 0a71057313ec0626527e164a8fc0205c5822e2f5 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:15:56 -0700 Subject: [PATCH 09/10] [None][doc] Document the allowed scope of image_url access State in config.pbtxt which media references are accepted and why local paths are not, so an integrator sees the policy next to the input they are wiring up rather than having to read model.py. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/config.pbtxt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt index 660c2a222813..dfb4232a5070 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt @@ -51,8 +51,14 @@ input [ data_type: TYPE_STRING dims: [ -1 ] }, - ## Optional multimodal input: URL, local path or data URI, one per image. - ## Only read when `triton_config.multimodal` is set in model.yaml. + ## Optional multimodal input, one entry per image. Only read when + ## `triton_config.multimodal` is set in model.yaml. + ## + ## Allowed scope of access: `http(s)://` URLs the server can reach, and + ## inline `data:` URIs. Local filesystem paths and `file://` URLs are + ## rejected: this input is client-controlled, so accepting them would let a + ## caller make the server read any image file its process can open. Widen + ## this only against an agreed allowlist for the deployment. { name: "image_url" data_type: TYPE_STRING From 56d32b03e55d39be429ba5095f7410e129f362b5 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:21:25 -0700 Subject: [PATCH 10/10] [None][fix] Restrict image_url to http(s) URLs Drop `data:` from the allowlist so the accepted scope is strictly web URLs, and trim the surrounding comments. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../all_models/llmapi/tensorrt_llm/1/model.py | 16 ++++------------ .../all_models/llmapi/tensorrt_llm/config.pbtxt | 8 +------- .../tests/test_llmapi_python_backend.py | 9 ++++----- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py index a13be6cfee74..c2dd22455053 100755 --- a/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py +++ b/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py @@ -86,22 +86,17 @@ def get_model_config(filename, include_keys=None, exclude_keys=None): return engine_config -# `image_url` is client-controlled, and the media loaders happily open local -# paths and `file://` with the server process's permissions. Until the allowed -# scope is agreed with the deployment owner, accept only remote fetches and -# inline data, so a request cannot make the server read its filesystem. -ALLOWED_MEDIA_SCHEMES = ("http", "https", "data") +# `image_url` is client-controlled, so restrict it to web URLs for security. +ALLOWED_MEDIA_SCHEMES = ("http", "https") def validate_media_urls(urls): - """Reject media references that would read the server's filesystem.""" + """Reject anything that is not a web URL.""" for url in urls: if urlparse(url).scheme not in ALLOWED_MEDIA_SCHEMES: raise pb_utils.TritonModelException( f"Unsupported image_url {url!r}: only " - f"{', '.join(ALLOWED_MEDIA_SCHEMES)} are accepted. Local paths " - "and file:// URLs are rejected because the input is " - "client-controlled.") + f"{', '.join(ALLOWED_MEDIA_SCHEMES)} URLs are accepted.") def get_input_scalar_by_name(request, @@ -503,9 +498,6 @@ async def _execute_single_request(self, request): # Unique request id used to identify each triton request triton_req_id = str(randint(0, sys.maxsize)) - # Absence from `req_id_to_request_data` only means "cancelled" once the - # request has been added to it. Before that it just means preprocessing - # has not finished, and an error still has to be reported to the client. request_registered = False try: diff --git a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt index dfb4232a5070..ea69fc227321 100644 --- a/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt +++ b/triton_backend/all_models/llmapi/tensorrt_llm/config.pbtxt @@ -51,14 +51,8 @@ input [ data_type: TYPE_STRING dims: [ -1 ] }, - ## Optional multimodal input, one entry per image. Only read when + ## Optional http(s) image URLs, one per image. Only read when ## `triton_config.multimodal` is set in model.yaml. - ## - ## Allowed scope of access: `http(s)://` URLs the server can reach, and - ## inline `data:` URIs. Local filesystem paths and `file://` URLs are - ## rejected: this input is client-controlled, so accepting them would let a - ## caller make the server read any image file its process can open. Widen - ## this only against an agreed allowlist for the deployment. { name: "image_url" data_type: TYPE_STRING diff --git a/triton_backend/all_models/tests/test_llmapi_python_backend.py b/triton_backend/all_models/tests/test_llmapi_python_backend.py index 7dad0c14fdc7..2427104fc1f5 100644 --- a/triton_backend/all_models/tests/test_llmapi_python_backend.py +++ b/triton_backend/all_models/tests/test_llmapi_python_backend.py @@ -523,10 +523,9 @@ async def __anext__(self): "file:///etc/passwd", "relative/path.jpg", "ftp://example.com/a.jpg", + "data:image/png;base64,iVBORw0KGgo=", ]) -def test_validate_media_urls_rejects_local_and_unknown_schemes(url): - # image_url is client-controlled; a local path would make the server read - # its own filesystem with the Triton process's permissions. +def test_validate_media_urls_rejects_non_web_urls(url): with pytest.raises(MockTritonModelException) as excinfo: validate_media_urls([url]) assert "image_url" in str(excinfo.value) @@ -535,9 +534,9 @@ def test_validate_media_urls_rejects_local_and_unknown_schemes(url): @pytest.mark.parametrize("url", [ "http://images.example.com/a.jpg", "https://images.example.com/a.jpg", - "data:image/png;base64,iVBORw0KGgo=", + "HTTPS://images.example.com/a.jpg", ]) -def test_validate_media_urls_accepts_remote_and_inline(url): +def test_validate_media_urls_accepts_web_urls(url): validate_media_urls([url])