From 1485ea61309f70796ada8d1abc09b33e03ca142f Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:55:43 -0700 Subject: [PATCH 1/6] docs: Add Qwen2.5-VL guide for the TRT-LLM PyTorch backend Add Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md, documenting how to serve a multimodal (vision) model on Triton via the TensorRT-LLM PyTorch backend through the llmapi backend. No engine build is required. Also add a deprecation banner to the Llava1.5 TensorRT-LLM guide, whose prebuilt-engine multimodal path is end-of-life as of TensorRT-LLM v1.2. The multimodal image_url input and triton_config.multimodal opt-in used by the new guide are added by NVIDIA/TensorRT-LLM#18381, which is not yet merged; the guide states this prominently. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../Llava1.5/llava_trtllm_guide.md | 7 + .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 308 ++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md diff --git a/Popular_Models_Guide/Llava1.5/llava_trtllm_guide.md b/Popular_Models_Guide/Llava1.5/llava_trtllm_guide.md index a8b8e802..80591ec6 100644 --- a/Popular_Models_Guide/Llava1.5/llava_trtllm_guide.md +++ b/Popular_Models_Guide/Llava1.5/llava_trtllm_guide.md @@ -28,6 +28,13 @@ # Deploying Hugging Face Llava1.5-7b Model in Triton +> [!WARNING] +> **Deprecated.** This guide describes the prebuilt-TensorRT-engine multimodal +> path (`tensorrtllm_backend`'s `all_models/multimodal`), which TensorRT-LLM has +> declared end-of-life as of TensorRT-LLM v1.2. It is no longer maintained. +> Use the TensorRT-LLM PyTorch backend instead; see +> [Deploying Hugging Face Qwen2.5-VL Model in Triton](../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md). + TensorRT-LLM is Nvidia's recommended solution of running Large Language Models(LLMs) on Nvidia GPUs. Read more about TensoRT-LLM [here](https://github.com/NVIDIA/TensorRT-LLM) and Triton's TensorRT-LLM Backend [here](https://github.com/triton-inference-server/tensorrtllm_backend). diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md new file mode 100644 index 00000000..630e9c5c --- /dev/null +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -0,0 +1,308 @@ + + +# Deploying Hugging Face Qwen2.5-VL Model in Triton + +This guide shows how to serve a multimodal (vision-language) model on Triton +Inference Server using the +[TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) PyTorch backend through +the [LLM API](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/llm-api/README.md), +exposed by Triton's `llmapi` backend. + +> [!IMPORTANT] +> **This workflow depends on an unmerged TensorRT-LLM change.** +> Image support in the Triton `llmapi` backend (the optional `image_url` input +> and the `triton_config.multimodal` opt-in used below) is added by +> [NVIDIA/TensorRT-LLM#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381), +> which has not been merged and is not present in any released TensorRT-LLM +> version or container image. Until that PR lands you must build the +> `llmapi` backend files from that branch; a stock container will not accept an +> `image_url` input. + +> [!NOTE] +> This guide replaces +> [the Llava1.5 TensorRT-LLM guide](../Llava1.5/llava_trtllm_guide.md), which +> uses the prebuilt-TensorRT-engine multimodal path that TensorRT-LLM has +> declared end-of-life as of TensorRT-LLM v1.2. See +> [triton-inference-server/server#8945](https://github.com/triton-inference-server/server/issues/8945). + +## Why the PyTorch backend + +The deprecated multimodal path (`tensorrtllm_backend`'s `all_models/multimodal`) +required two ahead-of-time compilation steps before you could serve anything: a +`trtllm-build` invocation to produce the LLM engine, and a separate visual +engine build for the vision encoder. Both artifacts had to be rebuilt whenever +the model, precision, or maximum sequence length changed. + +The PyTorch backend needs **no compilation and no engine build at all**. The +model repository is four plain Python/text files, TensorRT-LLM is a +pip-installed wheel inside the container, and the weights are loaded directly +from a Hugging Face snapshot at startup. This is the single biggest practical +difference between the two workflows. + +LLaVA-1.5 itself is not a drop-in replacement target here. TensorRT-LLM's +[supported models matrix](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/models/supported-models.md) +lists `LlavaNextForConditionalGeneration` and `LlavaLlamaModel` (VILA) among the +supported multimodal architectures, but not `LlavaForConditionalGeneration`, +which is the architecture of `llava-hf/llava-1.5-7b-hf`. This guide therefore +uses [`Qwen/Qwen2.5-VL-3B-Instruct`](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct). + +## What was validated + +| Item | Value | +| ---- | ----- | +| Container | `nvcr.io/nvidia/tritonserver:26.07-trtllm-python-py3` | +| Triton | 2.71.0 | +| TensorRT-LLM | 1.2.1 | +| CUDA | 13.1 | +| Model | `Qwen/Qwen2.5-VL-3B-Instruct` | +| Hardware | 1x NVIDIA B200 | + +## Prerequisites + +### Container + +```bash +docker run --rm -it --gpus all --network host \ + -v ${PWD}:/workspace -w /workspace \ + nvcr.io/nvidia/tritonserver:26.07-trtllm-python-py3 +``` + +### Known issue: the container's `openai` package is too old + +The 26.07 image ships `openai 1.107.3`, which is older than what +`tensorrt_llm/serve/responses_utils.py` requires. Loading a model fails with: + +``` +ImportError: cannot import name 'PartReasoningText' +``` + +Because `tensorrt_llm/_torch/pyexecutor/py_executor.py` imports +`tensorrt_llm.serve`, this breaks loading of **any** model on the `llmapi` +backend, not just multimodal ones. Work around it by installing a newer `openai` +into an overlay directory and putting that directory on `PYTHONPATH`, which +avoids modifying the container's site-packages: + +```bash +pip install --target=/workspace/pylibs -U openai +export PYTHONPATH=/workspace/pylibs +``` + +### Model weights + +Provide either a local Hugging Face snapshot directory or the Hugging Face model +id `Qwen/Qwen2.5-VL-3B-Instruct`. If you use the model id, the container needs +network access to huggingface.co at startup. + +## Preparing the model repository + +Copy the four `llmapi` backend files from TensorRT-LLM's +`triton_backend/all_models/llmapi/tensorrt_llm/` into a model repository: + +``` +model_repo/ +└── tensorrt_llm/ + ├── config.pbtxt + └── 1/ + ├── model.py + ├── helpers.py + └── model.yaml +``` + +Note that the TensorRT-LLM Triton backend sources now live in the +[NVIDIA/TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) repository under +`triton_backend/`; the standalone `tensorrtllm_backend` repository has been +superseded. + +Only `1/model.yaml` needs editing: + +```yaml +model: /path/to/Qwen2.5-VL-3B-Instruct # HF snapshot dir or HF model id +backend: "pytorch" +tensor_parallel_size: 1 +kv_cache_config: + free_gpu_memory_fraction: 0.5 + +triton_config: + max_batch_size: 0 + decoupled: False + multimodal: True # opt-in; default False +``` + +`triton_config.multimodal` defaults to `False`. This is deliberate: existing +deployments that already declare their own `image_url` input keep their current +behavior when they upgrade. The flip side is that if you forget to set it, any +`image_url` values you send are **silently ignored** and you get a text-only +answer, so set it explicitly for multimodal models. + +## Starting the server + +In Slurm/MPI environments, launch through `trtllm-llmapi-launch`: + +```bash +trtllm-llmapi-launch tritonserver --model-repository=/path/to/model_repo \ + --http-port=8000 --grpc-port=8001 --metrics-port=8002 +``` + +Running plain `tritonserver` fails at engine start with: + +``` +mpi4py.MPI.Exception: MPI_ERR_SPAWN: could not spawn processes +``` + +The LLM API uses `MpiPoolSession` to spawn its workers, and +`trtllm-llmapi-launch` (which sets `TLLM_SPAWN_PROXY_PROCESS=1`) is the +supported wrapper for that. + +Startup takes roughly 70 seconds. Wait for `Started HTTPService` in the log. A +successful multimodal start also logs: + +``` +[trtllm] multimodal input enabled for model_type 'qwen2_5_vl' +``` + +You can poll readiness with: + +```bash +curl -s -o /dev/null -w '%{http_code}' http://localhost:8000/v2/health/ready +``` + +which returns `200` once the server is up. + +## Sending an inference request + +Requests go to the standard Triton HTTP inference endpoint, +`POST /v2/models/tensorrt_llm/infer`. Inputs are Triton tensors, not OpenAI-style +chat JSON: + +| Input | Datatype | Shape | Description | +| ----- | -------- | ----- | ----------- | +| `text_input` | `BYTES` | `[1]` | The plain question. The backend applies the chat template and inserts the per-architecture image placeholders, so do **not** add `<\|vision_start\|>` or similar tokens yourself. | +| `image_url` | `BYTES` | `[N]` | One entry per image. Accepts an `http(s)` URL, a local filesystem path readable by the server, or a `data:image/...;base64,...` URI. | +| `sampling_param_max_tokens` | `INT32` | `[1]` | Maximum number of tokens to generate. | +| `sampling_param_exclude_input_from_output` | `BOOL` | `[1]` | Set to `true`; otherwise the rendered prompt is echoed back in `text_output`. | + +The only output is `text_output`. + +### Python client + +This client uses only the standard library: + +```python +import json +import urllib.request + +URL = "http://localhost:8000/v2/models/tensorrt_llm/infer" + + +def ask(prompt, images, max_tokens=64): + """Send a prompt plus one or more images and return the generated text.""" + body = { + "inputs": [ + {"name": "text_input", "shape": [1], "datatype": "BYTES", + "data": [prompt]}, + {"name": "image_url", "shape": [len(images)], "datatype": "BYTES", + "data": images}, + {"name": "sampling_param_max_tokens", "shape": [1], + "datatype": "INT32", "data": [max_tokens]}, + {"name": "sampling_param_exclude_input_from_output", "shape": [1], + "datatype": "BOOL", "data": [True]}, + ], + "outputs": [{"name": "text_output"}], + } + request = urllib.request.Request( + URL, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=300) as response: + result = json.load(response) + return result["outputs"][0]["data"][0].strip() + + +if __name__ == "__main__": + print(ask( + "What color is the bus and what does the sign say?", + ["http://images.cocodataset.org/test2017/000000155781.jpg"], + )) +``` + +Expected output: + +``` +The bus is yellow and white, and the sign on the bus says "Out of Service." +``` + +### Multiple images + +Pass more than one entry in `image_url`; the shape must match the number of +entries: + +```python +ask( + "Describe each image.", + [ + "http://images.cocodataset.org/test2017/000000155781.jpg", + "/workspace/images/second.jpg", + ], +) +``` + +The model enumerates both images and describes each one in the order they were +sent. + +### Image source equivalence + +A `data:image/...;base64,...` URI and a local file path produce the same answer +as the `http` URL for the same image, so you can pick whichever form fits your +deployment. Local paths must be readable by the server process, not the client. + +### Error behavior + +An unreachable image URL surfaces as a Triton error rather than silently +degrading to a text-only answer, for example: + +``` +[trtllm] Error generating request: Cannot connect to host example.invalid:443 +``` + +## Troubleshooting + +| Symptom | Cause and fix | +| ------- | ------------- | +| `mpi4py.MPI.Exception: MPI_ERR_SPAWN: could not spawn processes` | `tritonserver` was started directly. The LLM API spawns workers via `MpiPoolSession`; start it with `trtllm-llmapi-launch` instead. | +| `ImportError: cannot import name 'PartReasoningText'` | The container's `openai` package is too old for `tensorrt_llm.serve`, which is imported unconditionally by the PyTorch executor. Install a newer `openai` into an overlay directory and export it on `PYTHONPATH` (see [Prerequisites](#known-issue-the-containers-openai-package-is-too-old)). | +| `ConnectionRefusedError` from the client | The server is not up yet. Startup takes roughly 70 seconds; wait for `Started HTTPService` in the log, or poll until `curl -s -o /dev/null -w '%{http_code}' http://localhost:8000/v2/health/ready` returns `200`. | +| Images appear to be ignored and answers are text-only | `triton_config.multimodal` is not set to `True` in `1/model.yaml`. It defaults to `False` and image inputs are silently dropped. | + +## References + +- [TensorRT-LLM LLM API](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/llm-api/README.md) +- [TensorRT-LLM supported models](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/models/supported-models.md) +- [NVIDIA/TensorRT-LLM#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381) - adds multimodal input to the Triton `llmapi` backend +- [Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct) From d1482ffa32fba44c8d32da6d345e98d30f73d2ec Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:40:11 -0700 Subject: [PATCH 2/6] docs: simplify the Qwen2.5-VL setup steps all_models/llmapi/ holds exactly one model directory, so it can be used as a Triton model repository directly instead of copying files into a new one. Give the concrete clone and model.yaml commands, and add a curl example with the response it returns. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 63 +++++++++++++++---- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md index 630e9c5c..5dd71d20 100644 --- a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -121,11 +121,18 @@ network access to huggingface.co at startup. ## Preparing the model repository -Copy the four `llmapi` backend files from TensorRT-LLM's -`triton_backend/all_models/llmapi/tensorrt_llm/` into a model repository: +The Triton backend sources live in the +[NVIDIA/TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) repository under +`triton_backend/`; the standalone `tensorrtllm_backend` repository has been +superseded. They are plain Python files and are not shipped in the +`tensorrt_llm` wheel, so fetch them from a checkout. + +`triton_backend/all_models/llmapi/` contains exactly one model directory +(`tensorrt_llm/`), so it doubles as a Triton model repository and needs no +copying: ``` -model_repo/ +all_models/llmapi/ <- point --model-repository here └── tensorrt_llm/ ├── config.pbtxt └── 1/ @@ -134,15 +141,24 @@ model_repo/ └── model.yaml ``` -Note that the TensorRT-LLM Triton backend sources now live in the -[NVIDIA/TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) repository under -`triton_backend/`; the standalone `tensorrtllm_backend` repository has been -superseded. +> [!NOTE] +> Until [NVIDIA/TensorRT-LLM#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381) +> merges, the `image_url` input and the `triton_config.multimodal` option below +> exist only on that pull request's branch. Clone the fork shown here for now; +> once it lands, clone `https://github.com/NVIDIA/TensorRT-LLM.git` instead. + +```bash +git clone --depth 1 --branch feat/triton-llmapi-multimodal-image \ + https://github.com/faradawn/TensorRT-LLM.git /workspace/trtllm-pr +``` -Only `1/model.yaml` needs editing: +Then point `1/model.yaml` at the model and turn on the multimodal opt-in. This +edits the file in place inside the checkout, which leaves that clone's +`git status` dirty — fine for a throwaway container: -```yaml -model: /path/to/Qwen2.5-VL-3B-Instruct # HF snapshot dir or HF model id +```bash +cat > /workspace/trtllm-pr/triton_backend/all_models/llmapi/tensorrt_llm/1/model.yaml <<'EOF' +model: Qwen/Qwen2.5-VL-3B-Instruct backend: "pytorch" tensor_parallel_size: 1 kv_cache_config: @@ -151,9 +167,13 @@ kv_cache_config: triton_config: max_batch_size: 0 decoupled: False - multimodal: True # opt-in; default False + multimodal: True +EOF ``` +`model` accepts a Hugging Face model id (downloaded to `HF_HOME`) or a local +snapshot directory. + `triton_config.multimodal` defaults to `False`. This is deliberate: existing deployments that already declare their own `image_url` input keep their current behavior when they upgrade. The flip side is that if you forget to set it, any @@ -165,7 +185,8 @@ answer, so set it explicitly for multimodal models. In Slurm/MPI environments, launch through `trtllm-llmapi-launch`: ```bash -trtllm-llmapi-launch tritonserver --model-repository=/path/to/model_repo \ +trtllm-llmapi-launch tritonserver \ + --model-repository=/workspace/trtllm-pr/triton_backend/all_models/llmapi \ --http-port=8000 --grpc-port=8001 --metrics-port=8002 ``` @@ -209,6 +230,24 @@ chat JSON: The only output is `text_output`. +### Quick check with `curl` + +```bash +curl -s http://localhost:8000/v2/models/tensorrt_llm/infer -H 'Content-Type: application/json' -d '{ + "inputs": [ + {"name":"text_input","shape":[1],"datatype":"BYTES","data":["What color is the bus and what does the sign say?"]}, + {"name":"image_url","shape":[1],"datatype":"BYTES","data":["http://images.cocodataset.org/test2017/000000155781.jpg"]}, + {"name":"sampling_param_max_tokens","shape":[1],"datatype":"INT32","data":[64]}, + {"name":"sampling_param_exclude_input_from_output","shape":[1],"datatype":"BOOL","data":[true]} + ], + "outputs": [{"name":"text_output"}] +}' +``` + +```json +{"model_name":"tensorrt_llm","model_version":"1","outputs":[{"name":"text_output","datatype":"BYTES","shape":[1],"data":["The bus is yellow and white, and the sign on the bus says \"Out of Service.\""]}]} +``` + ### Python client This client uses only the standard library: From 56b4521340f381804789e4770372b09ab562f481 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:16:31 -0700 Subject: [PATCH 3/6] docs: state the allowed scope of image_url access The backend now accepts only http(s) URLs and inline data URIs; local filesystem paths and file:// are rejected because the input is client-controlled. Document that and drop the local-path example. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md index 5dd71d20..a4c7a6b8 100644 --- a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -224,7 +224,7 @@ chat JSON: | Input | Datatype | Shape | Description | | ----- | -------- | ----- | ----------- | | `text_input` | `BYTES` | `[1]` | The plain question. The backend applies the chat template and inserts the per-architecture image placeholders, so do **not** add `<\|vision_start\|>` or similar tokens yourself. | -| `image_url` | `BYTES` | `[N]` | One entry per image. Accepts an `http(s)` URL, a local filesystem path readable by the server, or a `data:image/...;base64,...` URI. | +| `image_url` | `BYTES` | `[N]` | One entry per image. Accepts an `http(s)` URL the server can reach, or an inline `data:image/...;base64,...` URI. See [Allowed scope of access](#allowed-scope-of-access). | | `sampling_param_max_tokens` | `INT32` | `[1]` | Maximum number of tokens to generate. | | `sampling_param_exclude_input_from_output` | `BOOL` | `[1]` | Set to `true`; otherwise the rendered prompt is echoed back in `text_output`. | @@ -317,9 +317,27 @@ sent. ### Image source equivalence -A `data:image/...;base64,...` URI and a local file path produce the same answer -as the `http` URL for the same image, so you can pick whichever form fits your -deployment. Local paths must be readable by the server process, not the client. +A `data:image/...;base64,...` URI produces the same answer as the `http` URL for +the same image, so you can pick whichever form fits your deployment. Inline data +avoids a second network hop at the cost of a larger request body. + +### Allowed scope of access + +`image_url` is client-controlled, so the backend accepts only: + +- `http://` and `https://` URLs the server can reach +- inline `data:image/...;base64,...` URIs + +Local filesystem paths and `file://` URLs are **rejected**. Accepting them would +let any caller make the server open image files its process can read, which on a +client-accessible deployment is an arbitrary-file-read primitive. A rejected +value fails the request with an error naming the offending entry; it does not +silently fall back to a text-only answer. + +If your deployment needs to serve images that already live on the server, agree +an explicit allowlisted root with whoever owns the deployment before widening +this. Note this is deliberately narrower than `trtllm-serve`, whose media +loading is unrestricted. ### Error behavior From 8d27cdfd38b2123931dffa9ef21f3651cf640d9b Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:33:19 -0700 Subject: [PATCH 4/6] docs: note that image_url accepts web URLs only The backend rejects local paths, file:// and data: URIs, so document the http(s)-only scope and drop the data-URI examples. Signed-off-by: Faradawn Yang <73060648+faradawn@users.noreply.github.com> --- .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 27 ++++--------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md index a4c7a6b8..945faee5 100644 --- a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -224,7 +224,7 @@ chat JSON: | Input | Datatype | Shape | Description | | ----- | -------- | ----- | ----------- | | `text_input` | `BYTES` | `[1]` | The plain question. The backend applies the chat template and inserts the per-architecture image placeholders, so do **not** add `<\|vision_start\|>` or similar tokens yourself. | -| `image_url` | `BYTES` | `[N]` | One entry per image. Accepts an `http(s)` URL the server can reach, or an inline `data:image/...;base64,...` URI. See [Allowed scope of access](#allowed-scope-of-access). | +| `image_url` | `BYTES` | `[N]` | One entry per image. Accepts `http(s)` URLs the server can reach; local paths and other schemes are rejected. | | `sampling_param_max_tokens` | `INT32` | `[1]` | Maximum number of tokens to generate. | | `sampling_param_exclude_input_from_output` | `BOOL` | `[1]` | Set to `true`; otherwise the rendered prompt is echoed back in `text_output`. | @@ -315,29 +315,12 @@ ask( The model enumerates both images and describes each one in the order they were sent. -### Image source equivalence - -A `data:image/...;base64,...` URI produces the same answer as the `http` URL for -the same image, so you can pick whichever form fits your deployment. Inline data -avoids a second network hop at the cost of a larger request body. - ### Allowed scope of access -`image_url` is client-controlled, so the backend accepts only: - -- `http://` and `https://` URLs the server can reach -- inline `data:image/...;base64,...` URIs - -Local filesystem paths and `file://` URLs are **rejected**. Accepting them would -let any caller make the server open image files its process can read, which on a -client-accessible deployment is an arbitrary-file-read primitive. A rejected -value fails the request with an error naming the offending entry; it does not -silently fall back to a text-only answer. - -If your deployment needs to serve images that already live on the server, agree -an explicit allowlisted root with whoever owns the deployment before widening -this. Note this is deliberately narrower than `trtllm-serve`, whose media -loading is unrestricted. +`image_url` is client-controlled, so only `http(s)` URLs are accepted. Local +filesystem paths, `file://` and other schemes are rejected, because accepting +them would let a caller make the server read image files its process can open. +Host images the model should see on a reachable web URL. ### Error behavior From 51f26138fb785c5ecdcd063ceb3099fd56605a7f Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:07:12 -0700 Subject: [PATCH 5/6] docs: make the Qwen2.5-VL guide work on the 1.2.1 container The guide claimed validation on TensorRT-LLM 1.2.1, but following it verbatim on nvcr.io/nvidia/tritonserver:26.07-trtllm-python-py3 -- the newest published -trtllm-python-py3 tag -- fails every request that carries an image: Error generating request: cannot import name 'async_build_multimodal_prompt' from 'tensorrt_llm.inputs' The backend files readers clone call `async_build_multimodal_prompt`, which NVIDIA/TensorRT-LLM#18381 adds to `tensorrt_llm/inputs/utils.py`. That module ships inside the wheel, not in the `triton_backend/` tree, so on a 1.2.1 container the caller is present and the callee never is. The server still starts, still logs `multimodal input enabled`, and still answers text-only prompts, so the deployment looks healthy right up until the first image. Add a "Patching model.py for TensorRT-LLM 1.2.1" section carrying the replacement method and the call-site diff, gated behind a note to skip it once a container ships with #18381 in it. Say in the validation table that 1.2.1 needs that patch, and record the torch build. Also fix three things found while testing: - The multiple-images example passed `/workspace/images/second.jpg`, a local path the guide's own "Allowed scope of access" section says is rejected. That example could only ever error. Use two live http URLs and show the real two-image answer. - Show the actual rejection and connection-failure responses, including the `ssl:default [Name or service not known]` tail that was trimmed, as JSON response bodies rather than log lines. - Add troubleshooting rows for the ImportError and for a rejected scheme. Verified on 1x B200 with Qwen/Qwen2.5-VL-3B-Instruct: single image returns the answer this guide quotes, two images are described in order, and the rejection and unreachable-host paths surface as errors rather than silently degrading to text-only. Checked by applying the patch text extracted from this file to a fresh clone, so what is documented is what was run. Co-Authored-By: Claude Opus 5 --- .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 152 ++++++++++++++++-- 1 file changed, 142 insertions(+), 10 deletions(-) diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md index 945faee5..dec8f5c6 100644 --- a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -35,14 +35,23 @@ the [LLM API](https://github.com/NVIDIA/TensorRT-LLM/blob/main/examples/llm-api/ exposed by Triton's `llmapi` backend. > [!IMPORTANT] -> **This workflow depends on an unmerged TensorRT-LLM change.** +> **This workflow depends on an unmerged TensorRT-LLM change, plus a small +> patch to run it on today's container.** +> > Image support in the Triton `llmapi` backend (the optional `image_url` input > and the `triton_config.multimodal` opt-in used below) is added by > [NVIDIA/TensorRT-LLM#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381), > which has not been merged and is not present in any released TensorRT-LLM -> version or container image. Until that PR lands you must build the -> `llmapi` backend files from that branch; a stock container will not accept an +> version or container image. Until that PR lands you must take the `llmapi` +> backend files from that branch; a stock container will not accept an > `image_url` input. +> +> Those files call `async_build_multimodal_prompt`, which the same PR adds to +> the `tensorrt_llm` **wheel**. The newest published container still ships +> TensorRT-LLM 1.2.1, whose wheel does not have it, so +> [one edit to `model.py`](#patching-modelpy-for-tensorrt-llm-121) is required +> as well. Both steps go away once #18381 merges and a container ships with +> that build of TensorRT-LLM. > [!NOTE] > This guide replaces @@ -78,11 +87,16 @@ uses [`Qwen/Qwen2.5-VL-3B-Instruct`](https://huggingface.co/Qwen/Qwen2.5-VL-3B-I | ---- | ----- | | Container | `nvcr.io/nvidia/tritonserver:26.07-trtllm-python-py3` | | Triton | 2.71.0 | -| TensorRT-LLM | 1.2.1 | +| TensorRT-LLM | 1.2.1 (with the [`model.py` patch](#patching-modelpy-for-tensorrt-llm-121)) | +| torch | 2.10.0a0+b4e4ee81d3.nv25.12 | | CUDA | 13.1 | | Model | `Qwen/Qwen2.5-VL-3B-Instruct` | | Hardware | 1x NVIDIA B200 | +Every command and every response below was run on that configuration. `26.07` +is the newest `-trtllm-python-py3` tag; on it, the multimodal path does not work +without the patch. + ## Prerequisites ### Container @@ -180,6 +194,108 @@ behavior when they upgrade. The flip side is that if you forget to set it, any `image_url` values you send are **silently ignored** and you get a text-only answer, so set it explicitly for multimodal models. +## Patching `model.py` for TensorRT-LLM 1.2.1 + +> [!NOTE] +> Skip this section entirely once a `-trtllm-python-py3` container ships with a +> TensorRT-LLM build that includes +> [#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381). It is a bridge for +> today's image, not part of the design. + +The files you just cloned build the prompt by calling +`async_build_multimodal_prompt`, which #18381 adds to +`tensorrt_llm/inputs/utils.py`. That module ships **inside the `tensorrt_llm` +wheel**, not in the `triton_backend/` tree you cloned, so on a 1.2.1 container +you have the caller but never the callee. The server still starts, still reports +`multimodal input enabled`, and still answers text-only prompts — and then fails +on every request that carries an image: + +```json +{"error":"Error generating request: cannot import name 'async_build_multimodal_prompt' from 'tensorrt_llm.inputs' (/opt/venv-tritonserver/lib/python3.12/site-packages/tensorrt_llm/inputs/__init__.py)"} +``` + +1.2.1 also lacks everything that helper is built on — `MEDIA_IO_REGISTRY`, +`ContentFormat`, `MultimodalDataTracker.item_order()`, +`interleave_mm_placeholders` and `async_apply_chat_template` — so you cannot +copy the new `utils.py` across either. What does work is replacing the single +call with an equivalent written against the 1.2.1 API. Add this method to the +`TritonPythonModel` class in +`/workspace/trtllm-pr/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py`, +just above `async def _convert_request`: + +```python + async def _build_multimodal_prompt_121(self, text, media): + """Stand-in for `inputs.async_build_multimodal_prompt` on TRT-LLM 1.2.1. + + 1.2.1 has no `async_apply_chat_template` and no + `MultimodalDataTracker.item_order()`, and its + `add_multimodal_placeholders` takes three arguments rather than four. + """ + from tensorrt_llm.inputs import prompt_inputs + from tensorrt_llm.inputs.utils import (ConversationMessage, + MultimodalDataTracker, + add_multimodal_placeholders, + apply_chat_template, + async_load_image) + + mm_data_tracker = MultimodalDataTracker(self._mm_model_type) + for url in media: + mm_data_tracker.add_data("image", async_load_image(url)) + mm_placeholder_counts = mm_data_tracker.placeholder_counts() + + content = add_multimodal_placeholders(self._mm_model_type, text, + mm_placeholder_counts) + conversation = [ + ConversationMessage(role="user", content=content, media=[]) + ] + prompt_task = asyncio.to_thread( + apply_chat_template, + model_type=self._mm_model_type, + tokenizer=self._mm_tokenizer, + processor=self._mm_processor, + conversation=conversation, + add_generation_prompt=True, + mm_placeholder_counts=[mm_placeholder_counts], + ) + prompt, (mm_data, _) = await asyncio.gather( + prompt_task, mm_data_tracker.retrieve_all_async()) + + prompt = prompt_inputs(prompt) + if mm_data: + prompt["multi_modal_data"] = mm_data + return prompt +``` + +`apply_chat_template` is synchronous and does real tokenizer work, so it goes +through `asyncio.to_thread` rather than blocking the engine's event loop while +the images are still downloading. + +Then, in `_convert_request`, point the call at it: + +```diff + image_url = get_input_tensor_by_name(request, 'image_url') + if image_url is not None and image_url.size > 0: +- from tensorrt_llm.inputs import async_build_multimodal_prompt +- + media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + validate_media_urls(media) +- prompt = await async_build_multimodal_prompt( +- model_type=self._mm_model_type, +- tokenizer=self._mm_tokenizer, +- processor=self._mm_processor, +- prompt=prompt, +- media=media, +- modality="image", +- ) ++ prompt = await self._build_multimodal_prompt_121(prompt, media) +``` + +Nothing else in the backend needs touching: `validate_media_urls` and the rest +of the request path run unmodified on 1.2.1. + ## Starting the server In Slurm/MPI environments, launch through `trtllm-llmapi-launch`: @@ -300,20 +416,28 @@ The bus is yellow and white, and the sign on the bus says "Out of Service." ### Multiple images Pass more than one entry in `image_url`; the shape must match the number of -entries: +entries. Every entry must be an `http(s)` URL — see +[Allowed scope of access](#allowed-scope-of-access): ```python ask( "Describe each image.", [ "http://images.cocodataset.org/test2017/000000155781.jpg", - "/workspace/images/second.jpg", + "http://images.cocodataset.org/val2017/000000039769.jpg", ], + max_tokens=96, ) ``` The model enumerates both images and describes each one in the order they were -sent. +sent: + +``` +The first image depicts a bus on a foggy street at night. The bus has a sign on +its front that reads "OUT OF SERVICE." ... The second image shows two cats lying +on a pink couch. +``` ### Allowed scope of access @@ -322,13 +446,19 @@ filesystem paths, `file://` and other schemes are rejected, because accepting them would let a caller make the server read image files its process can open. Host images the model should see on a reachable web URL. +A rejected entry fails the whole request: + +```json +{"error":"Error generating request: Unsupported image_url '/workspace/images/second.jpg': only http, https URLs are accepted."} +``` + ### Error behavior An unreachable image URL surfaces as a Triton error rather than silently -degrading to a text-only answer, for example: +degrading to a text-only answer: -``` -[trtllm] Error generating request: Cannot connect to host example.invalid:443 +```json +{"error":"Error generating request: Cannot connect to host example.invalid:443 ssl:default [Name or service not known]"} ``` ## Troubleshooting @@ -339,6 +469,8 @@ degrading to a text-only answer, for example: | `ImportError: cannot import name 'PartReasoningText'` | The container's `openai` package is too old for `tensorrt_llm.serve`, which is imported unconditionally by the PyTorch executor. Install a newer `openai` into an overlay directory and export it on `PYTHONPATH` (see [Prerequisites](#known-issue-the-containers-openai-package-is-too-old)). | | `ConnectionRefusedError` from the client | The server is not up yet. Startup takes roughly 70 seconds; wait for `Started HTTPService` in the log, or poll until `curl -s -o /dev/null -w '%{http_code}' http://localhost:8000/v2/health/ready` returns `200`. | | Images appear to be ignored and answers are text-only | `triton_config.multimodal` is not set to `True` in `1/model.yaml`. It defaults to `False` and image inputs are silently dropped. | +| `cannot import name 'async_build_multimodal_prompt' from 'tensorrt_llm.inputs'`, only on requests carrying an image | The container's TensorRT-LLM wheel predates [#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381). The server starts and text-only requests still work, which makes this easy to miss. Apply [the 1.2.1 patch](#patching-modelpy-for-tensorrt-llm-121). | +| `Unsupported image_url '...': only http, https URLs are accepted.` | A local path, `file://` or other scheme was passed. Only `http(s)` is accepted; see [Allowed scope of access](#allowed-scope-of-access). | ## References From 9181f35a3f52b9e9b7900cc797be732bdd53bb71 Mon Sep 17 00:00:00 2001 From: Faradawn Yang <73060648+faradawn@users.noreply.github.com> Date: Wed, 9 Sep 2026 22:19:10 -0700 Subject: [PATCH 6/6] docs: ship the 1.2.1 patch as a script, and stop cloning 900 MB for four files Follow-up to the previous commit, which described the model.py change but left the reader to make it by hand. model.py is ~790 lines, and splicing a 40-line method into the right class at the right indentation is a reliable way to end up with a broken model repository. Ship the edit as trtllm_121_compat.py next to the guide instead, invoked in one command. This is how this directory already works: the Llava1.5 guide being deprecated here ships multi_modal_client.py beside it, and Llama2 ships deploy_trtllm_llama.sh. The script is defensive, because it edits a file the reader did not write: it is idempotent, it refuses to write source that does not parse, and if the call it targets is absent it explains that #18381 has probably merged and gives the one-line import check to confirm, rather than corrupting the checkout. The guide keeps the explanation -- the call-site diff and a table mapping each 1.3 API onto the 1.2.1 equivalent -- so a reader can still see what changes and why, without having to type it. Separately, replace the full clone with a blobless, LFS-skipped sparse checkout: GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --filter=blob:none --sparse ... git -C ... sparse-checkout set triton_backend/all_models/llmapi Four seconds and 9.1 MB, against roughly 900 MB of Git LFS payload for four small text files. Also state plainly that 1.2.1 is the standing target: every published -trtllm-python-py3 image ships it, so #18381 merging upstream does not remove the need for this patch. Only a new container image does. Verified end to end on 1x B200 by running the guide's own commands, including the shipped script against a fresh sparse clone with nothing hand-edited: single image returns the quoted answer, two images are described in order, a rejected scheme and an unreachable host both surface as errors. Co-Authored-By: Claude Opus 5 --- .../Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md | 142 +++++++------- .../Qwen2.5-VL/trtllm_121_compat.py | 181 ++++++++++++++++++ 2 files changed, 252 insertions(+), 71 deletions(-) create mode 100644 Popular_Models_Guide/Qwen2.5-VL/trtllm_121_compat.py diff --git a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md index dec8f5c6..13d46718 100644 --- a/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md +++ b/Popular_Models_Guide/Qwen2.5-VL/qwen2_5_vl_trtllm_guide.md @@ -47,11 +47,12 @@ exposed by Triton's `llmapi` backend. > `image_url` input. > > Those files call `async_build_multimodal_prompt`, which the same PR adds to -> the `tensorrt_llm` **wheel**. The newest published container still ships -> TensorRT-LLM 1.2.1, whose wheel does not have it, so -> [one edit to `model.py`](#patching-modelpy-for-tensorrt-llm-121) is required -> as well. Both steps go away once #18381 merges and a container ships with -> that build of TensorRT-LLM. +> the `tensorrt_llm` **wheel**. Every published `-trtllm-python-py3` image still +> ships TensorRT-LLM 1.2.1, whose wheel does not have it, so +> [a one-command patch](#patching-modelpy-for-tensorrt-llm-121) is required as +> well. This guide is written for that combination and is verified end to end on +> it; both steps go away only when a container ships a TensorRT-LLM that already +> contains #18381. > [!NOTE] > This guide replaces @@ -101,7 +102,14 @@ without the patch. ### Container +Start from a clone of this repository, so that the +[`trtllm_121_compat.py`](trtllm_121_compat.py) used below is mounted into the +container along with it: + ```bash +git clone https://github.com/triton-inference-server/tutorials.git +cd tutorials + docker run --rm -it --gpus all --network host \ -v ${PWD}:/workspace -w /workspace \ nvcr.io/nvidia/tritonserver:26.07-trtllm-python-py3 @@ -161,9 +169,15 @@ all_models/llmapi/ <- point --model-repository here > exist only on that pull request's branch. Clone the fork shown here for now; > once it lands, clone `https://github.com/NVIDIA/TensorRT-LLM.git` instead. +Only four files are needed, so skip the repository's Git LFS payload and check +out the one directory — a few seconds and about 9 MB, rather than the ~900 MB a +full clone pulls: + ```bash -git clone --depth 1 --branch feat/triton-llmapi-multimodal-image \ +GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --filter=blob:none --sparse \ + --branch feat/triton-llmapi-multimodal-image \ https://github.com/faradawn/TensorRT-LLM.git /workspace/trtllm-pr +git -C /workspace/trtllm-pr sparse-checkout set triton_backend/all_models/llmapi ``` Then point `1/model.yaml` at the model and turn on the multimodal opt-in. This @@ -196,81 +210,47 @@ answer, so set it explicitly for multimodal models. ## Patching `model.py` for TensorRT-LLM 1.2.1 -> [!NOTE] -> Skip this section entirely once a `-trtllm-python-py3` container ships with a -> TensorRT-LLM build that includes -> [#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381). It is a bridge for -> today's image, not part of the design. - -The files you just cloned build the prompt by calling -`async_build_multimodal_prompt`, which #18381 adds to +The backend files you just cloned build their prompt by calling +`async_build_multimodal_prompt`, which +[#18381](https://github.com/NVIDIA/TensorRT-LLM/pull/18381) adds to `tensorrt_llm/inputs/utils.py`. That module ships **inside the `tensorrt_llm` -wheel**, not in the `triton_backend/` tree you cloned, so on a 1.2.1 container -you have the caller but never the callee. The server still starts, still reports -`multimodal input enabled`, and still answers text-only prompts — and then fails -on every request that carries an image: +wheel**, not in the `triton_backend/` tree you cloned, and the container's wheel +is 1.2.1 — so you have the caller but never the callee. + +This is easy to miss, because nothing fails at startup. The server comes up, logs +`multimodal input enabled`, and answers text-only prompts correctly. Only +requests that actually carry an image fail: ```json {"error":"Error generating request: cannot import name 'async_build_multimodal_prompt' from 'tensorrt_llm.inputs' (/opt/venv-tritonserver/lib/python3.12/site-packages/tensorrt_llm/inputs/__init__.py)"} ``` -1.2.1 also lacks everything that helper is built on — `MEDIA_IO_REGISTRY`, -`ContentFormat`, `MultimodalDataTracker.item_order()`, -`interleave_mm_placeholders` and `async_apply_chat_template` — so you cannot -copy the new `utils.py` across either. What does work is replacing the single -call with an equivalent written against the 1.2.1 API. Add this method to the -`TritonPythonModel` class in -`/workspace/trtllm-pr/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py`, -just above `async def _convert_request`: +Copying the new `utils.py` across does not help either: 1.2.1 lacks everything +that helper is built on — `MEDIA_IO_REGISTRY`, `ContentFormat`, +`MultimodalDataTracker.item_order()`, `interleave_mm_placeholders` and +`async_apply_chat_template`. What does work is replacing that one call with an +equivalent written against the 1.2.1 API. Run the script shipped next to this +guide: -```python - async def _build_multimodal_prompt_121(self, text, media): - """Stand-in for `inputs.async_build_multimodal_prompt` on TRT-LLM 1.2.1. - - 1.2.1 has no `async_apply_chat_template` and no - `MultimodalDataTracker.item_order()`, and its - `add_multimodal_placeholders` takes three arguments rather than four. - """ - from tensorrt_llm.inputs import prompt_inputs - from tensorrt_llm.inputs.utils import (ConversationMessage, - MultimodalDataTracker, - add_multimodal_placeholders, - apply_chat_template, - async_load_image) - - mm_data_tracker = MultimodalDataTracker(self._mm_model_type) - for url in media: - mm_data_tracker.add_data("image", async_load_image(url)) - mm_placeholder_counts = mm_data_tracker.placeholder_counts() - - content = add_multimodal_placeholders(self._mm_model_type, text, - mm_placeholder_counts) - conversation = [ - ConversationMessage(role="user", content=content, media=[]) - ] - prompt_task = asyncio.to_thread( - apply_chat_template, - model_type=self._mm_model_type, - tokenizer=self._mm_tokenizer, - processor=self._mm_processor, - conversation=conversation, - add_generation_prompt=True, - mm_placeholder_counts=[mm_placeholder_counts], - ) - prompt, (mm_data, _) = await asyncio.gather( - prompt_task, mm_data_tracker.retrieve_all_async()) - - prompt = prompt_inputs(prompt) - if mm_data: - prompt["multi_modal_data"] = mm_data - return prompt +```bash +python3 /workspace/Popular_Models_Guide/Qwen2.5-VL/trtllm_121_compat.py \ + /workspace/trtllm-pr/triton_backend/all_models/llmapi/tensorrt_llm/1/model.py ``` -`apply_chat_template` is synchronous and does real tokenizer work, so it goes -through `asyncio.to_thread` rather than blocking the engine's event loop while -the images are still downloading. +``` +Patched .../llmapi/tensorrt_llm/1/model.py for TensorRT-LLM 1.2.1. +``` + +It edits nothing but that one file, refuses to write source that does not parse, +and is safe to re-run — a second invocation reports `already patched; nothing to +do`. If the call it looks for is gone, it says so and tells you how to check +whether your container already has the function, rather than corrupting the +model repository. + +### What the script changes -Then, in `_convert_request`, point the call at it: +It adds one method, `_build_multimodal_prompt_121`, and points the call site at +it: ```diff image_url = get_input_tensor_by_name(request, 'image_url') @@ -293,9 +273,29 @@ Then, in `_convert_request`, point the call at it: + prompt = await self._build_multimodal_prompt_121(prompt, media) ``` +The new method does what the 1.3 helper does, in 1.2.1's vocabulary: + +| Step | 1.3 helper | 1.2.1 equivalent used here | +| ---- | ---------- | -------------------------- | +| download images | `MEDIA_IO_REGISTRY` | `async_load_image` per URL, gathered | +| insert placeholders | `interleave_mm_placeholders`, `item_order()` | `add_multimodal_placeholders`, three-argument form | +| render chat template | `async_apply_chat_template` | `apply_chat_template` via `asyncio.to_thread` | +| build the prompt | returns `PromptInputs` | `prompt_inputs(...)` plus `multi_modal_data` | + +`apply_chat_template` is synchronous and does real tokenizer work, so it goes +through `asyncio.to_thread` rather than blocking the engine's event loop while +the images are still downloading. Read +[`trtllm_121_compat.py`](trtllm_121_compat.py) for the full method. + Nothing else in the backend needs touching: `validate_media_urls` and the rest of the request path run unmodified on 1.2.1. +> [!NOTE] +> Delete this step once a `-trtllm-python-py3` container ships a TensorRT-LLM +> that already contains #18381. Note that #18381 merging is **not** enough on its +> own — the 26.07 image's wheel stays at 1.2.1 no matter what lands upstream, so +> the patch is needed until a *new image* ships. + ## Starting the server In Slurm/MPI environments, launch through `trtllm-llmapi-launch`: diff --git a/Popular_Models_Guide/Qwen2.5-VL/trtllm_121_compat.py b/Popular_Models_Guide/Qwen2.5-VL/trtllm_121_compat.py new file mode 100644 index 00000000..b9c35ec6 --- /dev/null +++ b/Popular_Models_Guide/Qwen2.5-VL/trtllm_121_compat.py @@ -0,0 +1,181 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +"""Make the Triton ``llmapi`` backend's image path run on TensorRT-LLM 1.2.1. + +Used by ``qwen2_5_vl_trtllm_guide.md``. See the guide for the full explanation; +the short version is that the backend's ``model.py`` builds its prompt with +``tensorrt_llm.inputs.async_build_multimodal_prompt``, which is added by +NVIDIA/TensorRT-LLM#18381. That function lives in the ``tensorrt_llm`` *wheel*, +not in the ``triton_backend/`` tree you clone, so on a container whose wheel is +1.2.1 you have the caller but never the callee, and every request carrying an +``image_url`` fails with:: + + cannot import name 'async_build_multimodal_prompt' from 'tensorrt_llm.inputs' + +1.2.1 also lacks everything that helper is built on -- ``MEDIA_IO_REGISTRY``, +``ContentFormat``, ``MultimodalDataTracker.item_order()``, +``interleave_mm_placeholders`` and ``async_apply_chat_template`` -- so copying +the new ``utils.py`` across is not an option either. This script instead swaps +the single call for an equivalent written against the 1.2.1 API surface. + +Usage:: + + python3 trtllm_121_compat.py /triton_backend/all_models/llmapi/tensorrt_llm/1/model.py + +Safe to re-run: it exits cleanly if the file is already patched. Delete this +step once a ``-trtllm-python-py3`` container ships a TensorRT-LLM that already +contains #18381. +""" + +import argparse +import ast +import pathlib +import sys + +# The call this replaces, exactly as it appears in model.py. +OLD_CALL = """ from tensorrt_llm.inputs import async_build_multimodal_prompt + + media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + validate_media_urls(media) + prompt = await async_build_multimodal_prompt( + model_type=self._mm_model_type, + tokenizer=self._mm_tokenizer, + processor=self._mm_processor, + prompt=prompt, + media=media, + modality="image", + )""" + +NEW_CALL = """ media = [ + url.decode("utf-8") if isinstance(url, bytes) else str(url) + for url in image_url.reshape(-1) + ] + validate_media_urls(media) + prompt = await self._build_multimodal_prompt_121(prompt, media)""" + +# Inserted immediately above `async def _convert_request`. `asyncio` is already +# imported at module scope in model.py, so this needs no new top-level imports. +NEW_METHOD = ''' async def _build_multimodal_prompt_121(self, text, media): + """Stand-in for `inputs.async_build_multimodal_prompt` on TRT-LLM 1.2.1. + + 1.2.1 has no `async_apply_chat_template` and no + `MultimodalDataTracker.item_order()`, and its + `add_multimodal_placeholders` takes three arguments rather than four. + """ + from tensorrt_llm.inputs import prompt_inputs + from tensorrt_llm.inputs.utils import (ConversationMessage, + MultimodalDataTracker, + add_multimodal_placeholders, + apply_chat_template, + async_load_image) + + mm_data_tracker = MultimodalDataTracker(self._mm_model_type) + for url in media: + mm_data_tracker.add_data("image", async_load_image(url)) + mm_placeholder_counts = mm_data_tracker.placeholder_counts() + + content = add_multimodal_placeholders(self._mm_model_type, text, + mm_placeholder_counts) + conversation = [ + ConversationMessage(role="user", content=content, media=[]) + ] + # `apply_chat_template` is synchronous and does real tokenizer work, so + # keep it off the engine's event loop while the images download. + prompt_task = asyncio.to_thread( + apply_chat_template, + model_type=self._mm_model_type, + tokenizer=self._mm_tokenizer, + processor=self._mm_processor, + conversation=conversation, + add_generation_prompt=True, + mm_placeholder_counts=[mm_placeholder_counts], + ) + prompt, (mm_data, _) = await asyncio.gather( + prompt_task, mm_data_tracker.retrieve_all_async()) + + prompt = prompt_inputs(prompt) + if mm_data: + prompt["multi_modal_data"] = mm_data + return prompt + +''' + +ANCHOR = " async def _convert_request(self, request):" + +MOVED_ON = """{path} does not contain the call this script replaces. + +That usually means the backend has moved on -- most likely #18381 merged, in +which case check whether your container's TensorRT-LLM already provides +`async_build_multimodal_prompt` and skip this step entirely: + + python3 -c "from tensorrt_llm.inputs import async_build_multimodal_prompt" + +If that import succeeds, no patch is needed.""" + + +def main(): + parser = argparse.ArgumentParser( + description="Patch the Triton llmapi backend's model.py for " + "TensorRT-LLM 1.2.1.") + parser.add_argument( + "model_py", + type=pathlib.Path, + help="path to all_models/llmapi/tensorrt_llm/1/model.py") + args = parser.parse_args() + + path = args.model_py + if not path.is_file(): + sys.exit(f"{path} is not a file") + + source = path.read_text() + + if "_build_multimodal_prompt_121" in source: + print(f"{path} is already patched; nothing to do.") + return + + if source.count(OLD_CALL) != 1: + sys.exit(MOVED_ON.format(path=path)) + if source.count(ANCHOR) != 1: + sys.exit(f"could not locate `{ANCHOR.strip()}` in {path}") + + source = source.replace(OLD_CALL, NEW_CALL) + source = source.replace(ANCHOR, NEW_METHOD + ANCHOR, 1) + + # Fail before writing rather than leave a half-broken model repository. + try: + ast.parse(source) + except SyntaxError as exc: + sys.exit(f"patched source does not parse ({exc}); model.py left alone") + + path.write_text(source) + print(f"Patched {path} for TensorRT-LLM 1.2.1.") + + +if __name__ == "__main__": + main()