From d80d13f0900648ed473237b2cb35a3ce9b8338e2 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com> Date: Thu, 3 Sep 2026 18:05:11 -0400 Subject: [PATCH 1/2] Add Intel XPU embedding services --- docker/README.md | 77 +++--- docker/benchmarks/embedding_benchmark.py | 9 +- docker/benchmarks/gpu_preflight.py | 222 ++++++++++++++---- docker/benchmarks/qwen3_smoke_dataset.json | 2 +- docker/benchmarks/test_gpu_preflight.py | 62 +++++ .../docker-compose-local-embeddings-xpu.yaml | 139 +++++++++++ docker/docker-compose.yaml | 1 + docker/local-embeddings.config.json | 15 ++ 8 files changed, 445 insertions(+), 82 deletions(-) create mode 100644 docker/benchmarks/test_gpu_preflight.py create mode 100644 docker/docker-compose-local-embeddings-xpu.yaml diff --git a/docker/README.md b/docker/README.md index 65f209711a..eb8b7d09ba 100644 --- a/docker/README.md +++ b/docker/README.md @@ -77,25 +77,26 @@ intentional. ## Local Qwen3 embeddings -The Compose stack includes six optional Hugging Face Text Embeddings -Inference (TEI) services for the three selected Qwen3 embedding models. The -CPU and GPU services are separate so they can be benchmarked against the same -model and workload. Model files are cached in persistent, model-specific -volumes. +The Compose stack includes nine optional Hugging Face Text Embeddings +Inference (TEI) services for the three selected Qwen3 embedding models. CPU, +NVIDIA GPU, and Intel XPU services are separate so each runtime can be +benchmarked against the same model and workload. Model files are cached in +persistent, model-specific volumes. The CPU services are defined in `docker-compose-local-embeddings.yaml`; the -GPU services are defined in the separately included -`docker-compose-local-embeddings-gpu.yaml`. This keeps the GPU runtime setup -isolated while the root Compose file still exposes one consistent service -matrix. +NVIDIA services are defined in the separately included +`docker-compose-local-embeddings-gpu.yaml`; Intel services are defined in +`docker-compose-local-embeddings-xpu.yaml`. This keeps each accelerator +runtime isolated while the root Compose file still exposes one consistent +service matrix. -| Model | Vector dimensions | CPU service / host port | GPU service / host port | -|-------|-------------------:|-------------------------|-------------------------| -| Qwen3-Embedding-0.6B | 1024 | qwen3-embedding-06b-cpu / 8101 | qwen3-embedding-06b-gpu / 8201 | -| Qwen3-Embedding-4B | 2560 | qwen3-embedding-4b-cpu / 8102 | qwen3-embedding-4b-gpu / 8202 | -| Qwen3-Embedding-8B | 4096 | qwen3-embedding-8b-cpu / 8103 | qwen3-embedding-8b-gpu / 8203 | +| Model | Vector dimensions | CPU service / host port | NVIDIA GPU service / host port | Intel XPU service / host port | +|-------|-------------------:|-------------------------|--------------------------------|------------------------------| +| Qwen3-Embedding-0.6B | 1024 | qwen3-embedding-06b-cpu / 8101 | qwen3-embedding-06b-gpu / 8201 | qwen3-embedding-06b-xpu / 8301 | +| Qwen3-Embedding-4B | 2560 | qwen3-embedding-4b-cpu / 8102 | qwen3-embedding-4b-gpu / 8202 | qwen3-embedding-4b-xpu / 8302 | +| Qwen3-Embedding-8B | 4096 | qwen3-embedding-8b-cpu / 8103 | qwen3-embedding-8b-gpu / 8203 | qwen3-embedding-8b-xpu / 8303 | -Start the CPU, GPU, or complete comparison matrix from the docker directory: +Start the CPU, NVIDIA GPU, or Intel XPU profile from the docker directory: ```bash cd docker @@ -108,33 +109,40 @@ VERSION=dev docker compose -f docker-compose.yaml --profile embeddings-cpu up -d VERSION=dev docker compose -f docker-compose.yaml --profile embeddings-gpu up -d \ qwen3-embedding-06b-gpu qwen3-embedding-4b-gpu qwen3-embedding-8b-gpu -# All six services +# Intel XPU services +VERSION=dev docker compose -f docker-compose.yaml --profile embeddings-xpu up -d \ + qwen3-embedding-06b-xpu qwen3-embedding-4b-xpu qwen3-embedding-8b-xpu + +# CPU and NVIDIA GPU comparison matrix VERSION=dev docker compose -f docker-compose.yaml --profile embeddings-both up -d \ qwen3-embedding-06b-cpu qwen3-embedding-4b-cpu qwen3-embedding-8b-cpu \ qwen3-embedding-06b-gpu qwen3-embedding-4b-gpu qwen3-embedding-8b-gpu ``` -Before starting either GPU profile, verify that the host can see an NVIDIA -device and driver: +Before starting an accelerator profile, verify that the host can see a +supported device: ```bash -python3 docker/benchmarks/gpu_preflight.py --json +python3 docker/benchmarks/gpu_preflight.py --backend auto --json ``` -GPU services require the NVIDIA Container Toolkit and a compatible NVIDIA -driver. The default CUDA image targets the TEI CUDA 1.9 runtime; set -QWEN3_TEI_GPU_IMAGE when an architecture-specific image is needed. +NVIDIA services require the NVIDIA Container Toolkit and a compatible NVIDIA +driver. The default NVIDIA image targets the TEI CUDA 1.9 runtime; set +`QWEN3_TEI_GPU_IMAGE` when an architecture-specific image is needed. Intel +XPU services use the `xpu-ipex-latest` TEI image by default and pass through +`/dev/dri`; set `QWEN3_TEI_XPU_IMAGE` to use a different Intel-compatible +image. Use `--backend intel` to require Intel telemetry explicitly. The default CPU image targets x86_64; set QWEN3_TEI_CPU_IMAGE to the TEI -cpu-arm64-1.9 image on ARM64 hosts. The embeddings-both profile starts every -service in the matrix and may exceed available GPU memory if all six are -launched together. For a fair comparison, start one profile at a time or -benchmark endpoints sequentially. +cpu-arm64-1.9 image on ARM64 hosts. Accelerator profiles may exceed available +device memory when another model is already using the GPU. For a fair +comparison, start one profile at a time or benchmark endpoints sequentially. -GPU services have independent tuning variables so a GPU run does not change -the CPU run's request limits: `QWEN3_GPU_MAX_BATCH_TOKENS` (default `8192`), -`QWEN3_GPU_MAX_CLIENT_BATCH_SIZE` (default `32`), -`QWEN3_GPU_MAX_CONCURRENT_REQUESTS` (default `4`), and -`QWEN3_GPU_TOKENIZATION_WORKERS` (default `4`). +Accelerator services have independent tuning variables so an accelerator run +does not change the CPU run's request limits. NVIDIA uses +`QWEN3_GPU_MAX_*`; Intel XPU uses `QWEN3_XPU_MAX_BATCH_TOKENS` (default +`1024`), `QWEN3_XPU_MAX_CLIENT_BATCH_SIZE` (default `16`), +`QWEN3_XPU_MAX_CONCURRENT_REQUESTS` (default `4`), and +`QWEN3_XPU_TOKENIZATION_WORKERS` (default `4`). Each service exposes an OpenAI-compatible endpoint. From an Unstract container, use the internal URL; from the host, use the localhost URL: @@ -143,10 +151,13 @@ container, use the internal URL; from the host, use the localhost URL: |----------------|-------------------|---------------| | 0.6B CPU | http://qwen3-embedding-06b-cpu/v1 | http://localhost:8101/v1 | | 0.6B GPU | http://qwen3-embedding-06b-gpu/v1 | http://localhost:8201/v1 | +| 0.6B Intel XPU | http://qwen3-embedding-06b-xpu/v1 | http://localhost:8301/v1 | | 4B CPU | http://qwen3-embedding-4b-cpu/v1 | http://localhost:8102/v1 | | 4B GPU | http://qwen3-embedding-4b-gpu/v1 | http://localhost:8202/v1 | +| 4B Intel XPU | http://qwen3-embedding-4b-xpu/v1 | http://localhost:8302/v1 | | 8B CPU | http://qwen3-embedding-8b-cpu/v1 | http://localhost:8103/v1 | | 8B GPU | http://qwen3-embedding-8b-gpu/v1 | http://localhost:8203/v1 | +| 8B Intel XPU | http://qwen3-embedding-8b-xpu/v1 | http://localhost:8303/v1 | For the OpenAI Compatible Embedding adapter, set Model to the matching served-model alias, API Base to the appropriate URL above, and API Key to the @@ -172,6 +183,8 @@ python3 docker/benchmarks/embedding_benchmark.py --mode cpu \ --output /tmp/qwen3-cpu.json --strict python3 docker/benchmarks/embedding_benchmark.py --mode gpu \ --output /tmp/qwen3-gpu.json --strict +python3 docker/benchmarks/embedding_benchmark.py --mode xpu \ + --output /tmp/qwen3-xpu.json --strict python3 docker/benchmarks/embedding_benchmark.py --mode both \ --output /tmp/qwen3-both.json --strict ``` @@ -184,7 +197,7 @@ labeled queries before using quality scores to select a production model. When `--mode both` is used, the JSON report also contains one comparison per model with GPU speedups and GPU-minus-CPU retrieval-quality deltas. This makes the combined profile suitable for choosing a deployment target from one run. -This change intentionally retains all six variants; unselected services can +This change intentionally retains all nine variants; unselected services can be removed in a follow-up after the benchmark review. ## Overriding a service's config diff --git a/docker/benchmarks/embedding_benchmark.py b/docker/benchmarks/embedding_benchmark.py index b6efd0b89d..3005c27711 100644 --- a/docker/benchmarks/embedding_benchmark.py +++ b/docker/benchmarks/embedding_benchmark.py @@ -394,13 +394,16 @@ def benchmark_endpoint( def parse_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( - description="Benchmark the local Qwen3 CPU/GPU embedding matrix." + description="Benchmark the local Qwen3 CPU/GPU/Intel-XPU embedding matrix." ) parser.add_argument( "--mode", - choices=("cpu", "gpu", "both"), + choices=("cpu", "gpu", "xpu", "both"), default="both", - help="Endpoint variation to test; both tests CPU and GPU endpoints.", + help=( + "Endpoint variation to test; both tests CPU and NVIDIA GPU endpoints. " + "Use xpu for Intel XPU endpoints." + ), ) parser.add_argument( "--config", diff --git a/docker/benchmarks/gpu_preflight.py b/docker/benchmarks/gpu_preflight.py index d11684aae4..3eb44e6618 100644 --- a/docker/benchmarks/gpu_preflight.py +++ b/docker/benchmarks/gpu_preflight.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Check host NVIDIA visibility before launching GPU embedding services.""" +"""Check host accelerator visibility before launching embedding services.""" from __future__ import annotations @@ -11,67 +11,140 @@ import sys from typing import Any -GPU_QUERY = "index,name,memory.total,driver_version" -GPU_FIELDS = ("index", "name", "memory_total_mib", "driver_version") +NVIDIA_GPU_QUERY = "index,name,memory.total,driver_version" +NVIDIA_GPU_FIELDS = ("index", "name", "memory_total_mib", "driver_version") +INTEL_XPU_COMMAND = ( + "xpu-smi", + "dump", + "--json", + "--device", + "0", + "--metrics", + "MEMORY,UTILIZATION,POWER,CLOCK", + "--number", + "1", +) -def query_gpus(nvidia_smi: str) -> tuple[list[dict[str, str]], str | None]: - """Return visible GPUs or a human-readable command error.""" +def run_command( + command: list[str], +) -> tuple[subprocess.CompletedProcess[str] | None, str | None]: + """Run a short hardware query and return a useful error when it fails.""" try: completed = subprocess.run( - [ - nvidia_smi, - f"--query-gpu={GPU_QUERY}", - "--format=csv,noheader,nounits", - ], + command, capture_output=True, check=False, text=True, + timeout=10, ) + except subprocess.TimeoutExpired: + return None, f"{command[0]} timed out after 10 seconds" except OSError as error: - return [], f"could not execute {nvidia_smi}: {error}" - + return None, f"could not execute {command[0]}: {error}" if completed.returncode: detail = completed.stderr.strip() or "no diagnostic was returned" - return [], f"{nvidia_smi} failed with exit code {completed.returncode}: {detail}" + return None, f"{command[0]} failed with exit code {completed.returncode}: {detail}" + return completed, None + + +def query_nvidia_gpus(nvidia_smi: str) -> tuple[list[dict[str, str]], str | None]: + """Return visible NVIDIA GPUs or a human-readable command error.""" + completed, error = run_command( + [ + nvidia_smi, + f"--query-gpu={NVIDIA_GPU_QUERY}", + "--format=csv,noheader,nounits", + ] + ) + if error: + return [], error + assert completed is not None gpus: list[dict[str, str]] = [] for row in csv.reader(line for line in completed.stdout.splitlines() if line.strip()): - if len(row) != len(GPU_FIELDS): - return [], f"unexpected {nvidia_smi} output row: {row!r}" + if len(row) != len(NVIDIA_GPU_FIELDS): + return [], f"unexpected nvidia-smi output row: {row!r}" gpus.append( - {field: value.strip() for field, value in zip(GPU_FIELDS, row, strict=True)} + { + field: value.strip() + for field, value in zip(NVIDIA_GPU_FIELDS, row, strict=True) + } ) return gpus, None -def build_report( - minimum_gpus: int, - minimum_memory_mib: int, -) -> dict[str, Any]: - """Build a JSON-serializable preflight report.""" - nvidia_smi = shutil.which("nvidia-smi") - report: dict[str, Any] = { - "available": False, - "minimum_gpus": minimum_gpus, - "minimum_memory_mib": minimum_memory_mib, - "nvidia_smi": nvidia_smi, - "gpus": [], - } - if nvidia_smi is None: - report["error"] = "nvidia-smi was not found on PATH" - return report +def parse_json_output(output: str, command_name: str) -> Any: + """Parse JSON even when a hardware tool prefixes diagnostic text.""" + text = output.strip() + try: + return json.loads(text) + except json.JSONDecodeError: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + raise ValueError(f"{command_name} did not return JSON") from None + try: + return json.loads(text[start : end + 1]) + except json.JSONDecodeError as error: + raise ValueError(f"{command_name} returned invalid JSON: {error}") from error + - gpus, error = query_gpus(nvidia_smi) - report["gpus"] = gpus +def memory_mib(value: Any) -> str | None: + """Normalize a numeric hardware-tool memory value to integer MiB.""" + if value is None: + return None + try: + return str(int(float(str(value).replace(",", "").strip()))) + except (TypeError, ValueError): + return None + + +def query_intel_gpus(xpu_smi: str) -> tuple[list[dict[str, str]], str | None]: + """Return visible Intel XPUs using xpu-smi's machine-readable telemetry.""" + command = [xpu_smi, *INTEL_XPU_COMMAND[1:]] + completed, error = run_command(command) if error: - report["error"] = error - return report + return [], error + assert completed is not None + + try: + payload = parse_json_output(completed.stdout, xpu_smi) + except ValueError as error: + return [], str(error) + rows = payload if isinstance(payload, list) else [payload] + gpus: list[dict[str, str]] = [] + for row in rows: + if not isinstance(row, dict): + return [], f"unexpected {xpu_smi} JSON value: {row!r}" + metrics = row.get("metrics") + if not isinstance(metrics, dict): + return [], f"{xpu_smi} JSON did not contain metrics: {row!r}" + total = memory_mib(metrics.get("memory.total")) + if total is None: + return [], f"{xpu_smi} did not report memory.total: {row!r}" + device = str(row.get("device", len(gpus))) + gpu = { + "index": device, + "name": str(row.get("name", f"Intel XPU {device}")), + "memory_total_mib": total, + "driver_version": str(row.get("driver_version", "unknown")), + } + used = memory_mib(metrics.get("memory.used")) + if used is not None: + gpu["memory_used_mib"] = used + gpus.append(gpu) + return gpus, None + + +def check_requirements( + report: dict[str, Any], minimum_gpus: int, minimum_memory_mib: int +) -> None: + """Apply minimum device and memory requirements to a preflight report.""" + gpus = report["gpus"] if len(gpus) < minimum_gpus: - report["error"] = ( - f"found {len(gpus)} visible GPU(s), need at least {minimum_gpus}" - ) - return report + report["error"] = f"found {len(gpus)} visible GPU(s), need at least {minimum_gpus}" + return if minimum_memory_mib: low_memory = [ @@ -84,16 +157,67 @@ def build_report( f"GPU(s) {', '.join(low_memory)} have less than " f"{minimum_memory_mib} MiB of memory" ) + return + report["available"] = True + + +def build_report( + minimum_gpus: int, + minimum_memory_mib: int, + backend: str = "auto", +) -> dict[str, Any]: + """Build a JSON-serializable NVIDIA or Intel XPU preflight report.""" + nvidia_smi = shutil.which("nvidia-smi") + xpu_smi = shutil.which("xpu-smi") + report: dict[str, Any] = { + "available": False, + "backend": None, + "minimum_gpus": minimum_gpus, + "minimum_memory_mib": minimum_memory_mib, + "nvidia_smi": nvidia_smi, + "xpu_smi": xpu_smi, + "gpus": [], + } + candidates = { + "nvidia": (nvidia_smi, query_nvidia_gpus), + "intel": (xpu_smi, query_intel_gpus), + } + selected = ("nvidia", "intel") if backend == "auto" else (backend,) + errors: list[str] = [] + for candidate in selected: + executable, query = candidates[candidate] + if executable is None: + errors.append( + "nvidia-smi was not found on PATH" + if candidate == "nvidia" + else "xpu-smi was not found on PATH" + ) + continue + gpus, error = query(executable) + if error: + errors.append(error) + continue + report["backend"] = "nvidia-cuda" if candidate == "nvidia" else "intel-xpu" + report["gpus"] = gpus + check_requirements(report, minimum_gpus, minimum_memory_mib) + if report["available"]: return report + errors.append(str(report.get("error", "GPU requirements were not met"))) - report["available"] = True + report["error"] = "; ".join(errors) or "no supported GPU runtime was found" return report def parse_args() -> argparse.Namespace: """Parse command-line arguments.""" parser = argparse.ArgumentParser( - description="Check NVIDIA GPU availability for local TEI services." + description="Check NVIDIA or Intel XPU availability for local TEI services." + ) + parser.add_argument( + "--backend", + choices=("auto", "nvidia", "intel"), + default="auto", + help="Hardware backend to check (default: auto).", ) parser.add_argument( "--min-gpus", @@ -116,23 +240,29 @@ def parse_args() -> argparse.Namespace: def main() -> int: - """Run the GPU preflight and return a shell-friendly status.""" + """Run the preflight and return a shell-friendly status.""" args = parse_args() if args.min_gpus < 1: raise SystemExit("--min-gpus must be at least 1") if args.min_memory_mib < 0: raise SystemExit("--min-memory-mib cannot be negative") - report = build_report(args.min_gpus, args.min_memory_mib) + report = build_report(args.min_gpus, args.min_memory_mib, args.backend) if args.json: print(json.dumps(report, indent=2)) else: status = "PASS" if report["available"] else "FAIL" - print(f"NVIDIA GPU preflight: {status}") + backend = report.get("backend") or "none" + print(f"GPU preflight: {status} ({backend})") for gpu in report["gpus"]: + used = ( + f", {gpu['memory_used_mib']} MiB used" + if "memory_used_mib" in gpu + else "" + ) print( f" GPU {gpu['index']}: {gpu['name']} " - f"({gpu['memory_total_mib']} MiB, driver {gpu['driver_version']})" + f"({gpu['memory_total_mib']} MiB{used}, driver {gpu['driver_version']})" ) if report.get("error"): print(f" {report['error']}", file=sys.stderr) diff --git a/docker/benchmarks/qwen3_smoke_dataset.json b/docker/benchmarks/qwen3_smoke_dataset.json index 700bc7b1c9..e76e899ee1 100644 --- a/docker/benchmarks/qwen3_smoke_dataset.json +++ b/docker/benchmarks/qwen3_smoke_dataset.json @@ -34,7 +34,7 @@ }, { "id": "gpu", - "text": "GPU inference requires an NVIDIA driver, the NVIDIA Container Toolkit, and enough device memory for the model and its inference batches." + "text": "Accelerated inference requires a compatible device runtime, container device access, and enough device memory for the model and its inference batches." }, { "id": "benchmark", diff --git a/docker/benchmarks/test_gpu_preflight.py b/docker/benchmarks/test_gpu_preflight.py new file mode 100644 index 0000000000..7a6995e0c6 --- /dev/null +++ b/docker/benchmarks/test_gpu_preflight.py @@ -0,0 +1,62 @@ +"""Tests for cross-vendor accelerator preflight reporting.""" + +import json +import sys +from pathlib import Path +from subprocess import CompletedProcess + +sys.path.insert(0, str(Path(__file__).parent)) + +import gpu_preflight # noqa: E402 + + +def test_build_report_detects_intel_xpu_when_nvidia_is_absent(monkeypatch) -> None: + """Intel XPU telemetry is sufficient when nvidia-smi is unavailable.""" + monkeypatch.setattr( + gpu_preflight.shutil, + "which", + lambda command: "/usr/bin/xpu-smi" if command == "xpu-smi" else None, + ) + telemetry = { + "device": 0, + "metrics": { + "memory.total": "24480", + "memory.used": "21707.04", + }, + } + + def fake_run(command, **kwargs): + assert command[0] == "/usr/bin/xpu-smi" + assert kwargs["timeout"] == 10 + return CompletedProcess(command, 0, stdout=json.dumps(telemetry), stderr="") + + monkeypatch.setattr(gpu_preflight.subprocess, "run", fake_run) + + report = gpu_preflight.build_report(1, 24000) + + assert report["available"] is True + assert report["backend"] == "intel-xpu" + assert report["gpus"] == [ + { + "index": "0", + "name": "Intel XPU 0", + "memory_total_mib": "24480", + "memory_used_mib": "21707", + "driver_version": "unknown", + } + ] + + +def test_intel_preflight_reports_bad_telemetry(monkeypatch) -> None: + """Malformed Intel telemetry must fail closed with a useful error.""" + monkeypatch.setattr(gpu_preflight.shutil, "which", lambda command: "/usr/bin/xpu-smi") + monkeypatch.setattr( + gpu_preflight.subprocess, + "run", + lambda command, **kwargs: CompletedProcess(command, 0, stdout="{}", stderr=""), + ) + + report = gpu_preflight.build_report(1, 0, backend="intel") + + assert report["available"] is False + assert "metrics" in report["error"] diff --git a/docker/docker-compose-local-embeddings-xpu.yaml b/docker/docker-compose-local-embeddings-xpu.yaml new file mode 100644 index 0000000000..f96aacde7b --- /dev/null +++ b/docker/docker-compose-local-embeddings-xpu.yaml @@ -0,0 +1,139 @@ +# Optional local Qwen3 Intel XPU embedding services. +# +# Hugging Face's Intel TEI image uses Intel Extension for PyTorch and Level +# Zero. The /dev/dri mappings are intentionally kept in this separate, +# profile-gated file so the default CPU and NVIDIA profiles are unchanged. + +x-qwen3-xpu-healthcheck: &qwen3_xpu_healthcheck + test: + - CMD-SHELL + - >- + curl --fail --silent http://localhost:80/health >/dev/null || + exit 1 + interval: 15s + timeout: 10s + retries: 40 + start_period: 10m + +x-qwen3-xpu-common: &qwen3_xpu_common + restart: unless-stopped + shm_size: 1gb + expose: + - "80" + environment: + HF_HOME: /data + HF_HUB_DISABLE_TELEMETRY: "1" + DO_NOT_TRACK: "1" + HF_TOKEN: ${HF_TOKEN:-} + TOKENIZERS_PARALLELISM: "false" + labels: + traefik.enable: "false" + stop_grace_period: 30s + healthcheck: + <<: *qwen3_xpu_healthcheck + devices: + - /dev/dri:/dev/dri + volumes: + - /dev/dri/by-path:/dev/dri/by-path:ro + +x-qwen3-xpu-image: &qwen3_xpu_image + ${QWEN3_TEI_XPU_IMAGE:-ghcr.io/huggingface/text-embeddings-inference:xpu-ipex-latest} + +services: + qwen3-embedding-06b-xpu: + <<: *qwen3_xpu_common + image: *qwen3_xpu_image + container_name: unstract-qwen3-embedding-06b-xpu + profiles: + - embeddings-xpu + ports: + - "127.0.0.1:${QWEN3_06B_XPU_PORT:-8301}:80" + volumes: + - /dev/dri/by-path:/dev/dri/by-path:ro + - qwen3_embedding_06b_xpu_cache:/data + command: + - --model-id + - Qwen/Qwen3-Embedding-0.6B + - --revision + - ${QWEN3_06B_REVISION:-97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3} + - --served-model-name + - qwen3-embedding-06b + - --dtype + - float16 + - --pooling + - last-token + - --max-batch-tokens + - ${QWEN3_XPU_MAX_BATCH_TOKENS:-1024} + - --max-client-batch-size + - ${QWEN3_XPU_MAX_CLIENT_BATCH_SIZE:-16} + - --max-concurrent-requests + - ${QWEN3_XPU_MAX_CONCURRENT_REQUESTS:-4} + - --tokenization-workers + - ${QWEN3_XPU_TOKENIZATION_WORKERS:-4} + + qwen3-embedding-4b-xpu: + <<: *qwen3_xpu_common + image: *qwen3_xpu_image + container_name: unstract-qwen3-embedding-4b-xpu + profiles: + - embeddings-xpu + ports: + - "127.0.0.1:${QWEN3_4B_XPU_PORT:-8302}:80" + volumes: + - /dev/dri/by-path:/dev/dri/by-path:ro + - qwen3_embedding_4b_xpu_cache:/data + command: + - --model-id + - Qwen/Qwen3-Embedding-4B + - --revision + - ${QWEN3_4B_REVISION:-5cf2132abc99cad020ac570b19d031efec650f2b} + - --served-model-name + - qwen3-embedding-4b + - --dtype + - float16 + - --pooling + - last-token + - --max-batch-tokens + - ${QWEN3_XPU_MAX_BATCH_TOKENS:-1024} + - --max-client-batch-size + - ${QWEN3_XPU_MAX_CLIENT_BATCH_SIZE:-16} + - --max-concurrent-requests + - ${QWEN3_XPU_MAX_CONCURRENT_REQUESTS:-4} + - --tokenization-workers + - ${QWEN3_XPU_TOKENIZATION_WORKERS:-4} + + qwen3-embedding-8b-xpu: + <<: *qwen3_xpu_common + image: *qwen3_xpu_image + container_name: unstract-qwen3-embedding-8b-xpu + profiles: + - embeddings-xpu + ports: + - "127.0.0.1:${QWEN3_8B_XPU_PORT:-8303}:80" + volumes: + - /dev/dri/by-path:/dev/dri/by-path:ro + - qwen3_embedding_8b_xpu_cache:/data + command: + - --model-id + - Qwen/Qwen3-Embedding-8B + - --revision + - ${QWEN3_8B_REVISION:-1d8ad4ca9b3dd8059ad90a75d4983776a23d44af} + - --served-model-name + - qwen3-embedding-8b + - --dtype + - float16 + - --pooling + - last-token + - --max-batch-tokens + - ${QWEN3_XPU_MAX_BATCH_TOKENS:-1024} + - --max-client-batch-size + - ${QWEN3_XPU_MAX_CLIENT_BATCH_SIZE:-16} + - --max-concurrent-requests + - ${QWEN3_XPU_MAX_CONCURRENT_REQUESTS:-4} + - --tokenization-workers + - ${QWEN3_XPU_TOKENIZATION_WORKERS:-4} + +volumes: + qwen3_embedding_06b_xpu_cache: + qwen3_embedding_4b_xpu_cache: + qwen3_embedding_8b_xpu_cache: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5c444c8e51..c1738d14ff 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -3,6 +3,7 @@ include: - docker-compose-dev-essentials.yaml - docker-compose-local-embeddings.yaml - docker-compose-local-embeddings-gpu.yaml + - docker-compose-local-embeddings-xpu.yaml # Reusable host-gateway mapping so containers can reach services on the host # (e.g. host-installed Ollama at http://host.docker.internal:11434). diff --git a/docker/local-embeddings.config.json b/docker/local-embeddings.config.json index f36400772e..1c5e85c437 100644 --- a/docker/local-embeddings.config.json +++ b/docker/local-embeddings.config.json @@ -17,6 +17,11 @@ "service": "qwen3-embedding-06b-gpu", "base_url": "http://127.0.0.1:8201/v1", "internal_base_url": "http://qwen3-embedding-06b-gpu:80/v1" + }, + "xpu": { + "service": "qwen3-embedding-06b-xpu", + "base_url": "http://127.0.0.1:8301/v1", + "internal_base_url": "http://qwen3-embedding-06b-xpu:80/v1" } }, { @@ -33,6 +38,11 @@ "service": "qwen3-embedding-4b-gpu", "base_url": "http://127.0.0.1:8202/v1", "internal_base_url": "http://qwen3-embedding-4b-gpu:80/v1" + }, + "xpu": { + "service": "qwen3-embedding-4b-xpu", + "base_url": "http://127.0.0.1:8302/v1", + "internal_base_url": "http://qwen3-embedding-4b-xpu:80/v1" } }, { @@ -49,6 +59,11 @@ "service": "qwen3-embedding-8b-gpu", "base_url": "http://127.0.0.1:8203/v1", "internal_base_url": "http://qwen3-embedding-8b-gpu:80/v1" + }, + "xpu": { + "service": "qwen3-embedding-8b-xpu", + "base_url": "http://127.0.0.1:8303/v1", + "internal_base_url": "http://qwen3-embedding-8b-xpu:80/v1" } } ] From 5e4da5e7797d4087f7767eb2ec5cc8f4c50190ea Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:18:39 -0400 Subject: [PATCH 2/2] Reserve health port for local embedding services --- docker/docker-compose-local-embeddings-xpu.yaml | 2 ++ docker/docker-compose.yaml | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose-local-embeddings-xpu.yaml b/docker/docker-compose-local-embeddings-xpu.yaml index f96aacde7b..32e2d9a92a 100644 --- a/docker/docker-compose-local-embeddings-xpu.yaml +++ b/docker/docker-compose-local-embeddings-xpu.yaml @@ -26,6 +26,8 @@ x-qwen3-xpu-common: &qwen3_xpu_common DO_NOT_TRACK: "1" HF_TOKEN: ${HF_TOKEN:-} TOKENIZERS_PARALLELISM: "false" + # Allow Level Zero to use large allocations on Intel XPU devices. + UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS: "1" labels: traefik.enable: "false" stop_grace_period: 30s diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c1738d14ff..1a4ad4e212 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -403,7 +403,8 @@ services: # which is a different script and does understand roles. command: ["pg-queue-consumer"] ports: - - "8101:8090" + # 8101-8103 are reserved for the local Qwen3 CPU embedding profile. + - "8111:8090" env_file: - ../workers/.env - ./essentials.env