From 149ae386bb699d35d32c8443c383df1eb310e433 Mon Sep 17 00:00:00 2001 From: zgsu Date: Sun, 2 Aug 2026 10:04:09 +0800 Subject: [PATCH 1/2] docs(inference_guide): validate Gemma-4-31B-it on Ascend 910B3 P4+D4 --- .../gemma4-31b-it-pd-p4d4-llmisvc.yaml | 470 ++++++++++++++++++ .../plan/validated_models/gemma4-31b-it.mdx | 114 +++++ docs/en/plan/validated_models/index.mdx | 10 + 3 files changed, 594 insertions(+) create mode 100644 docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml create mode 100644 docs/en/plan/validated_models/gemma4-31b-it.mdx diff --git a/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml new file mode 100644 index 00000000..b21fdbac --- /dev/null +++ b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml @@ -0,0 +1,470 @@ +# ============================================================================= +# google/gemma-4-31B-it · P4 + D4 · KServe LLMInferenceService + InferNex Bridge +# +# Standard P/D delivery shape derived from the already validated MiniMax P8+D8 +# manifest. InferNex Bridge owns the P/D proxy; KServe owns Prefill/Decode, +# InferencePool, scheduler, route, and lifecycle resources. +# +# Decode: 100.1.69.8 · TP4 · FULL_DECODE_ONLY · Mooncake kv_consumer +# Prefill: 100.1.69.9 · TP4 · eager · Mooncake kv_producer +# ============================================================================= +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: kserve-config-llm-decode-template + namespace: zgsu-ns1 + labels: + app.kubernetes.io/part-of: infernex-bridge +spec: + template: + initContainers: [] +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: gemma4-31b-it-pd-engine + namespace: kserve + labels: + app.kubernetes.io/name: gemma4-31b-it-pd-engine + app.kubernetes.io/part-of: infernex-bridge +spec: + template: + initContainers: [] + nodeSelector: + kubernetes.io/hostname: 100.1.69.8 + hostIPC: true + tolerations: + - operator: Exists + terminationGracePeriodSeconds: 60 + containers: + - name: main + image: quay.io/ascend/vllm-ascend:nightly-main-openeuler-0721 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set +u + source /usr/local/Ascend/ascend-toolkit/set_env.sh 2>/dev/null || true + source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true + set -eo pipefail + printf '{}' > /app/mooncake.json + KV_CONFIG=$(printf '{"engine_id":"%s","kv_buffer_device":"npu","kv_connector":"MooncakeConnectorV1","kv_connector_extra_config":{"decode":{"dp_size":1,"tp_size":4},"prefill":{"dp_size":1,"tp_size":4},"use_ascend_direct":true},"kv_parallel_size":1,"kv_port":"20001","kv_rank":1,"kv_role":"kv_consumer"}' "$POD_NAME") + echo "[gemma4-pd-decode] pod=$POD_NAME ip=$POD_IP devices=$(ls /dev/davinci* 2>/dev/null | tr '\n' ' ')" + exec vllm serve /mnt/models \ + --host 0.0.0.0 \ + --port 8000 \ + --served-model-name "{{ .Spec.Model.Name }}" \ + --seed 1024 \ + --trust-remote-code \ + --api-server-count 1 \ + --enable-auto-tool-choice \ + --tool-call-parser gemma4 \ + --tensor-parallel-size 4 \ + --max-model-len -1 \ + --max-num-batched-tokens 128 \ + --max-num-seqs 96 \ + --block-size 128 \ + --gpu-memory-utilization 0.9 \ + --no-enable-prefix-caching \ + --async-scheduling \ + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' \ + --kv-transfer-config "$KV_CONFIG" + env: &engine-env + - { name: HOME, value: /home } + - { name: VLLM_LOGGING_LEVEL, value: INFO } + - { name: HF_HUB_OFFLINE, value: "1" } + - { name: TRANSFORMERS_OFFLINE, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: PYTHONHASHSEED, value: "0" } + - { name: VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES, value: "1" } + - { name: MOONCAKE_CONFIG_PATH, value: /app/mooncake.json } + - { name: PYTORCH_NPU_ALLOC_CONF, value: "expandable_segments:True" } + - { name: HCCL_OP_EXPANSION_MODE, value: AIV } + - { name: HCCL_BUFFSIZE, value: "1024" } + - { name: HCCL_INTRA_PCIE_ENABLE, value: "1" } + - { name: HCCL_INTRA_ROCE_ENABLE, value: "0" } + - { name: OMP_PROC_BIND, value: "false" } + - { name: OMP_NUM_THREADS, value: "1" } + - { name: TASK_QUEUE_ENABLE, value: "1" } + - { name: VLLM_ENGINE_READY_TIMEOUT_S, value: "3000" } + - { name: VLLM_RPC_TIMEOUT, value: "6000" } + - { name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, value: "3000" } + - { name: HCCL_EXEC_TIMEOUT, value: "600" } + - { name: HCCL_CONNECT_TIMEOUT, value: "300" } + - name: POD_NAME + valueFrom: + fieldRef: { fieldPath: metadata.name } + - name: POD_IP + valueFrom: + fieldRef: { fieldPath: status.podIP } + - name: HCCL_IF_IP + valueFrom: + fieldRef: { fieldPath: status.podIP } + - { name: GLOO_SOCKET_IFNAME, value: eth0 } + - { name: TP_SOCKET_IFNAME, value: eth0 } + - { name: HCCL_SOCKET_IFNAME, value: eth0 } + ports: + - { name: http, containerPort: 8000, protocol: TCP } + - { name: kv, containerPort: 20001, protocol: TCP } + resources: + requests: + cpu: "16" + memory: 96Gi + huawei.com/Ascend910: "4" + limits: + cpu: "48" + memory: 256Gi + huawei.com/Ascend910: "4" + securityContext: &engine-security-context + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: { type: RuntimeDefault } + startupProbe: + httpGet: { path: /health, port: http } + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 160 + readinessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 8 + volumeMounts: + - { name: model, mountPath: /mnt/models, readOnly: true } + - { name: shm, mountPath: /dev/shm } + - { name: mooncake-config, mountPath: /app } + volumes: + - name: model + persistentVolumeClaim: { claimName: gemma4-31b-n8 } + - name: shm + emptyDir: { medium: Memory, sizeLimit: 32Gi } + - name: mooncake-config + emptyDir: {} + + prefill: + replicas: 1 + template: + nodeSelector: + kubernetes.io/hostname: 100.1.69.9 + hostIPC: true + tolerations: + - operator: Exists + terminationGracePeriodSeconds: 60 + containers: + - name: main + image: quay.io/ascend/vllm-ascend:nightly-main-openeuler-0721 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set +u + source /usr/local/Ascend/ascend-toolkit/set_env.sh 2>/dev/null || true + source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true + set -eo pipefail + printf '{}' > /app/mooncake.json + KV_CONFIG=$(printf '{"engine_id":"%s","kv_buffer_device":"npu","kv_connector":"MooncakeConnectorV1","kv_connector_extra_config":{"decode":{"dp_size":1,"tp_size":4},"prefill":{"dp_size":1,"tp_size":4},"use_ascend_direct":true},"kv_parallel_size":1,"kv_port":"20000","kv_rank":0,"kv_role":"kv_producer"}' "$POD_NAME") + echo "[gemma4-pd-prefill] pod=$POD_NAME ip=$POD_IP devices=$(ls /dev/davinci* 2>/dev/null | tr '\n' ' ')" + exec vllm serve /mnt/models \ + --host 0.0.0.0 \ + --port 8000 \ + --served-model-name "{{ .Spec.Model.Name }}" \ + --seed 1024 \ + --trust-remote-code \ + --api-server-count 1 \ + --enable-auto-tool-choice \ + --tool-call-parser gemma4 \ + --tensor-parallel-size 4 \ + --max-model-len -1 \ + --max-num-batched-tokens 8192 \ + --max-num-seqs 96 \ + --block-size 128 \ + --gpu-memory-utilization 0.9 \ + --no-enable-prefix-caching \ + --enforce-eager \ + --kv-transfer-config "$KV_CONFIG" + env: *engine-env + ports: + - { name: http, containerPort: 8000, protocol: TCP } + - { name: kv, containerPort: 20000, protocol: TCP } + resources: + requests: + cpu: "16" + memory: 96Gi + huawei.com/Ascend910: "4" + limits: + cpu: "48" + memory: 256Gi + huawei.com/Ascend910: "4" + securityContext: *engine-security-context + startupProbe: + httpGet: { path: /health, port: http } + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 160 + readinessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 8 + volumeMounts: + - { name: model, mountPath: /mnt/models, readOnly: true } + - { name: shm, mountPath: /dev/shm } + - { name: mooncake-config, mountPath: /app } + volumes: + - name: model + persistentVolumeClaim: { claimName: gemma4-31b-n9 } + - name: shm + emptyDir: { medium: Memory, sizeLimit: 32Gi } + - name: mooncake-config + emptyDir: {} +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: gemma4-31b-it-pd-scheduler + namespace: kserve + labels: + app.kubernetes.io/name: gemma4-31b-it-pd-scheduler + app.kubernetes.io/part-of: infernex-bridge +spec: + router: + scheduler: + pool: + spec: + endpointPickerRef: + failureMode: FailOpen + group: "" + kind: Service + name: "{{ ChildName .ObjectMeta.Name `-epp-service` }}" + port: { number: 9002 } + selector: + matchLabels: + app.kubernetes.io/name: "{{ .ObjectMeta.Name }}" + app.kubernetes.io/part-of: llminferenceservice + kserve.io/component: workload + targetPorts: + - number: 8000 + template: + nodeSelector: + kubernetes.io/hostname: 100.1.69.8 + tolerations: + - operator: Exists + serviceAccountName: "{{ ChildName .ObjectMeta.Name `-epp-sa` }}" + initContainers: + - name: tokenizer-copy + image: 100.1.69.8:11443/mlops/guidellm:v0.7.1-1-arm64 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set -eu + for pattern in 'tokenizer*' '*.json' '*.jinja' '*.py'; do + for file in /mnt/models/$pattern; do + [ -e "$file" ] && cp -a "$file" /workspace/gemma4/ + done + done + python3 - <<'PYTOK' + import json + + path = "/workspace/gemma4/tokenizer_config.json" + data = json.load(open(path)) + data["tokenizer_class"] = "PreTrainedTokenizerFast" + for key in ("auto_map", "backend", "processor_class"): + data.pop(key, None) + json.dump(data, open(path, "w")) + PYTOK + test -f /workspace/gemma4/config.json + volumeMounts: + - { name: tokenizer-source, mountPath: /mnt/models, readOnly: true } + - { name: tokenizer-model, mountPath: /workspace/gemma4 } + containers: + - name: main + image: 100.1.69.8:11443/3rdparty/openfuyao/hermes-router:26.6.0-rc.3 + imagePullPolicy: IfNotPresent + args: + - --pool-name + - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" + - --pool-namespace + - "{{ .ObjectMeta.Namespace }}" + - --pool-group + - inference.networking.k8s.io + - --zap-encoder + - json + - --v + - "1" + - --tracing + - "false" + - --metrics-endpoint-auth + - "false" + - --config-text + - |- + # This is Hermes Router's embedded config-document schema, not + # a Kubernetes resource API group. InferNex 26.6.0-rc.3 still + # requires the x-k8s version here; the actual InferencePool and + # --pool-group above both use inference.networking.k8s.io. + apiVersion: inference.networking.x-k8s.io/v1alpha1 + kind: EndpointPickerConfig + plugins: + - type: metrics-data-source + parameters: { scheme: "http", path: "/metrics", insecureSkipVerify: true } + - type: core-metrics-extractor + - type: tokenizer + parameters: + model: /workspace/gemma4 + socketPath: /var/run/tokenizer/tokenizer.sock + - type: pd-group-producer + parameters: + pdLabelName: app.kubernetes.io/component + pdGroupLabelName: app.kubernetes.io/name + prefillValue: llminferenceservice-workload-prefill + decodeValue: llminferenceservice-workload + leaderValue: pd-proxy + - type: request-lifecycle-tracker + parameters: + storeName: gemma4-pd-routing + persistence: { enabled: false } + - type: inflight-data-producer + parameters: { storeName: gemma4-pd-routing } + - type: pd-group-filter + - type: pd-kvcache-aware-scorer + parameters: + prefillKVUsageWeight: 1 + prefillQueueWeight: 1 + prefillInflightWeight: 1 + decodeKVUsageWeight: 1 + decodeQueueWeight: 1 + decodeInflightWeight: 1 + prefillScoreWeight: 1 + decodeScoreWeight: 1 + - type: min-score-picker + - type: pd-header-handler + schedulingProfiles: + - name: default + plugins: + - pluginRef: tokenizer + - pluginRef: pd-group-producer + - pluginRef: request-lifecycle-tracker + - pluginRef: inflight-data-producer + - pluginRef: pd-group-filter + - pluginRef: pd-kvcache-aware-scorer + - pluginRef: min-score-picker + - pluginRef: pd-header-handler + ports: + - { containerPort: 9002, name: grpc, protocol: TCP } + - { containerPort: 9003, name: grpc-health, protocol: TCP } + - { containerPort: 9090, name: metrics, protocol: TCP } + startupProbe: + grpc: { port: 9003, service: liveness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + livenessProbe: + grpc: { port: 9003, service: liveness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + grpc: { port: 9003, service: readiness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: { cpu: "1", memory: 1Gi } + limits: { memory: 4Gi } + securityContext: + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: { type: RuntimeDefault } + volumeMounts: + - { name: tls-certs, mountPath: /etc/ssl/certs, readOnly: true } + - { name: tokenizer-socket-volume, mountPath: /var/run/tokenizer } + - name: tokenizer + image: 100.1.69.8:11443/3rdparty/openfuyao/hermes-router-tokenizer:26.6.0-rc.3 + imagePullPolicy: IfNotPresent + args: ["--socket-path", "/var/run/tokenizer/tokenizer.sock"] + env: + - { name: TOKENIZER_PROVIDER, value: huggingface } + - { name: TRANSFORMERS_OFFLINE, value: "1" } + volumeMounts: + - { name: tokenizer-socket-volume, mountPath: /var/run/tokenizer } + - { name: tokenizer-model, mountPath: /workspace/gemma4 } + volumes: + - name: tls-certs + secret: + secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}" + - { name: tokenizer-socket-volume, emptyDir: {} } + - { name: tokenizer-model, emptyDir: {} } + - name: tokenizer-source + persistentVolumeClaim: { claimName: gemma4-31b-n8 } + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceService +metadata: + name: gemma4-31b-it-pd + namespace: zgsu-ns1 + labels: + infernex.io/runtime: "true" + annotations: + infernex.io/disabled-components: "mooncake,cache-indexer,pd-orchestrator,eagle-eye" + model-auto.alauda.io/config-revision: "0721-p4d4-d128-capacity" +spec: + model: + name: google/gemma-4-31B-it + uri: pvc://gemma4-31b-n8/ + storageInitializer: + enabled: false + baseRefs: + - name: gemma4-31b-it-pd-engine + - name: gemma4-31b-it-pd-scheduler + router: + gateway: {} + route: + http: + spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: kserve-ingress-gateway + namespace: kserve + rules: + - matches: + - path: + type: PathPrefix + value: /zgsu-ns1/gemma4-31b-it-pd + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - group: "" + kind: Service + name: gemma4-31b-it-pd-proxy-server + port: 8000 + weight: 1 + timeouts: + request: 0s + backendRequest: 0s + replicas: 1 + prefill: + replicas: 1 diff --git a/docs/en/plan/validated_models/gemma4-31b-it.mdx b/docs/en/plan/validated_models/gemma4-31b-it.mdx new file mode 100644 index 00000000..124f78c0 --- /dev/null +++ b/docs/en/plan/validated_models/gemma4-31b-it.mdx @@ -0,0 +1,114 @@ +--- +weight: 50 +i18n: + title: + en: Gemma-4-31B-it + zh: Gemma-4-31B-it +--- + +# Gemma-4-31B-it + +`Gemma4ForConditionalGeneration` is a dense multimodal model. This validation covers +its text-generation serving path: the text backbone has 60 layers, hidden size 5,376, +32 attention heads / 16 KV heads, a 262,144-token vocabulary, and a native 262,144-token +context window. The validated deployment is **P4+D4**: one 4-card Prefill engine and one +4-card Decode engine, coordinated by KServe `LLMInferenceService` and the InferNex +Bridge. It is an **8 × Ascend 910B3 64 GB** composite service, not a 4-card replica. + +## Model identity + +| Field | Value | +|---|---| +| Publisher / model | Google / `google/gemma-4-31B-it` | +| Architecture | `gemma4` / `Gemma4ForConditionalGeneration`; dense multimodal model, validated here for text generation | +| Text backbone | 60 layers; hidden size 5,376; 32 attention heads / 16 KV heads | +| Context / vocabulary | 262,144 / 262,144 | +| Precision | BF16 | + +## Validated hardware × stack + +| Platform | Engine | Version / configuration | Status | +|---|---|---|---| +| Ascend 910B3 64 GB × 8 across 2 nodes | vLLM-Ascend | `quay.io/ascend/vllm-ascend:nightly-main-openeuler-0721` (vLLM 0.25.1, CANN 9.0.1) | P4+D4 closed-loop AIPerf validation | + +:::note +This is a dated `nightly-main` snapshot, retained because it is the exact image used by +the validated deployment. It is not a claim that the tag is the newest vLLM-Ascend build. +::: + +## Model configuration + +| Component | Setting | +|---|---| +| Prefill | TP=4 on one 910B3 node; `max-num-batched-tokens=8192`; eager execution | +| Decode | TP=4 on the second node; `max-num-batched-tokens=128`; `FULL_DECODE_ONLY` graph capture | +| P/D handoff | `MooncakeConnectorV1`, Ascend Direct KV transfer, TP4/TP4 contract | +| Capacity guardrail | `max-num-seqs=96` on both engines; `gpu-memory-utilization=0.9` | +| Prefix cache | disabled in the engine; the workload uses explicit P/D KV transfer | +| Tool calls | `--enable-auto-tool-choice --tool-call-parser gemma4` | + +## Why P4+D4 + +The earlier aggregated TP4 deployment and P4+D4 are different resource envelopes. The +aggregate service uses four NPUs and mixes long Prefill work with Decode; P4+D4 uses eight +NPUs and removes Prefill dynamic steps from the Decode process. In the earlier capacity +study, aggregate TP4 reached only 2.554 completed RPS on the 512/256 chatbot workload at +target rate 3 over 900 seconds, with `Running=104` and `Waiting=294` at the end. P4+D4 is +therefore the production candidate, but its throughput must never be presented as a +four-card aggregate result. + +## Deploy + +The deployment spec is a self-contained KServe `LLMInferenceService` template with a +TP4 Decode template, a TP4 Prefill template, InferNex router/scheduler, `InferencePool`, +and the external route. + +| Spec | File | +|---|---| +| P4+D4, KServe + InferNex Bridge | [`gemma4-31b-it-pd-p4d4-llmisvc.yaml`](https://github.com/alauda/aml-docs/tree/master/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml) | + +Before applying it, set the namespace, the two node names, the model PVCs, and registry +references for your cluster. The manifest assumes the P/D pair has equal TP size and that +the two engines can use Ascend Direct transport. + +```bash +base=https://raw.githubusercontent.com/alauda/aml-docs/master/docs/en/plan/validated_models/assets/gemma4-31b-it +kubectl apply -f $base/gemma4-31b-it-pd-p4d4-llmisvc.yaml + +# Internal KServe ingress (no MaaS API key): +curl -s http:////gemma4-31b-it-pd/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"google/gemma-4-31B-it","messages":[{"role":"user","content":"hello"}]}' +``` + +## Benchmark results + +Closed-loop `aiperf 0.7.0`, driven directly through the P/D proxy. The fixed workload is +an 8k-token prompt with 128 generated tokens; it has 120 unique prompts, each reused four +times, for 480 requests per concurrency tier. The sweep uses concurrency 8 / 16 / 32, +with a warmup equal to each tier's concurrency. TTFT / E2E are seconds, ITL is ms, Decode +is output-only token throughput, and TPS is input-plus-output token throughput. + +| Metric | Concurrency 8 | Concurrency 16 | Concurrency 32 | +|---|---:|---:|---:| +| Successful / error requests | 480 / 0 | 480 / 0 | 480 / 0 | +| Server-counted ISL / OSL (tokens) | 8,013 / 128 | 8,013 / 128 | 8,013 / 128 | +| TTFT avg / p90 / p99 (s) | 4.023 / 4.278 / 4.504 | 11.642 / 12.165 / 12.405 | 26.458 / 27.856 / 28.046 | +| ITL avg / p90 / p99 (ms) | 29.11 / 30.40 / 32.07 | 29.10 / 30.69 / 32.28 | 29.09 / 30.68 / 32.16 | +| E2E avg / p90 / p99 (s) | 7.720 / 8.044 / 8.477 | 15.338 / 15.898 / 16.409 | 30.153 / 31.554 / 32.088 | +| Decode throughput (output tokens/s) | 131.66 | 131.52 | 131.64 | +| Request throughput (requests/s) | 1.03 | 1.03 | 1.03 | +| TPS (input + output tokens/s) | 8,373.74 | 8,365.16 | 8,372.31 | + +All three tiers completed 480 requests with zero errors. Throughput and ITL are flat from +concurrency 8 onward, while TTFT grows with concurrency. For this 8k-input workload, the +P4+D4 service is therefore Prefill-capacity-bound at about 1.03 requests/s; concurrency +above 8 primarily adds queueing rather than throughput. + +:::note +The benchmark is intentionally sent to the P/D proxy instead of MaaS: at measurement time +the cluster's MaaS backend still selected the old aggregated TP4 service. This preserves a +single, unambiguous P4+D4 topology for the published figures. Server metrics and GPU +telemetry were disabled because AIPerf 0.7.0 cannot parse this vLLM metrics histogram; +the published values come from the completed request records and CSV export. +::: diff --git a/docs/en/plan/validated_models/index.mdx b/docs/en/plan/validated_models/index.mdx index 21fe3050..517a0aec 100644 --- a/docs/en/plan/validated_models/index.mdx +++ b/docs/en/plan/validated_models/index.mdx @@ -33,6 +33,7 @@ scheduling) see [Deploy](../../deploy/index.mdx). | DeepSeek-V4-Flash (W4A8) | `deepseek_v4` MoE (MLA + DSA + MTP) | 256-expert MoE | W4A8 | Ascend 910B4 ×8 | vLLM-Ascend nightly | [DeepSeek-V4-Flash (W4A8)](./deepseek-v4-flash-w4a8.mdx) | | DeepSeek-V4-Flash (W8A8) | `deepseek_v4` MoE (MLA + DSA + MTP) | 256-expert MoE | W8A8 | Ascend 910B3 ×16 | vLLM-Ascend nightly | [DeepSeek-V4-Flash (W8A8)](./deepseek-v4-flash-w8a8.mdx) | | MiniMax-M2.5 (W8A8) | `minimax_m2` MoE (full-attn, 256-expert) | 230B-A10B | W8A8 (QuaRot) | Ascend 910B3 ×16 | vLLM-Ascend v0.23.0 | [MiniMax-M2.5 (W8A8)](./minimax-m2.5-w8a8.mdx) | +| Gemma-4-31B-it | `gemma4` dense multimodal | 31B | BF16 | Ascend 910B3 ×8 | vLLM-Ascend nightly-main (0721) | [Gemma-4-31B-it](./gemma4-31b-it.mdx) | The two Qwen models and DeepSeek-V4-Flash (W4A8) were validated on Ascend 910B4 (32 GB/card), driven through KServe `LLMInferenceService` with load-aware routing (InferNex-Bridge + @@ -53,6 +54,11 @@ mooncake store and no speculative decoding. Because this model is "fast" (~10B a keeps only the decode graph + local prefix cache. It was additionally validated as a **real coding agent** (OpenCode / Pi) through the product MaaS gateway. +Gemma-4-31B-it is a dense BF16 model validated as a **P4+D4 composite service** across +two 910B3 nodes: TP4 Prefill and TP4 Decode exchange KV through Ascend Direct transport. +It uses eight NPUs in total; its figures are an operating envelope for that P/D service, +not a comparison with a four-card aggregated replica. + ## Runtime images | Engine | Device | Image (validated tag) | Used by | Notes | @@ -61,6 +67,7 @@ keeps only the decode graph + local prefix cache. It was additionally validated | vLLM-Ascend nightly (release-pinned) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-releases-v0.22.1rc-openeuler` | Qwen3.6-27B (W8A8), DeepSeek-V4-Flash (W4A8) | Carries the `qwen3_5` Gated DeltaNet hybrid + MTP **and** the `deepseek_v4` MoE stack — the stock `v0.18.0` cannot load either. Use this **release-pinned** tag, **not** the moving `nightly-main-openeuler` (it drifted to a broken build whose TP workers crash). | | vLLM-Ascend nightly-main (dated snapshot) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-main-openeuler-0709` | DeepSeek-V4-Flash (W8A8) | A **later** `nightly-main` snapshot (vLLM 0.23.0) that carries the DP2 cross-node graph + DSA-CP prefix-cache fixes the W8A8 cross-node deployment needs. Pin a **dated** snapshot like this one, not the moving `nightly-main` tag. | | vLLM-Ascend v0.23.0 (release-pinned) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:v0.23.0-openeuler` | MiniMax-M2.5 (W8A8) | The **release-pinned** `nightly-releases-v0.23.0-openeuler` build (vLLM 0.23.0), carrying the `minimax_m2` MoE stack + tool-call/reasoning parsers **and** vLLM #11505 (streaming tool-call arguments held until the closing ``). Prefer this dated **release** tag over the moving `nightly-main`. | +| vLLM-Ascend nightly-main (dated snapshot) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-main-openeuler-0721` | Gemma-4-31B-it | vLLM 0.25.1 / CANN 9.0.1 snapshot used by the validated P4+D4 service. Keep this exact image for the known-good Gemma baseline; it is not a current-version recommendation. | :::tip The Ascend CANN images are arm64. Always match the runtime image's CANN version to @@ -75,6 +82,9 @@ real serving patterns. Output is pinned to **128 tokens** and load is **closed-l concurrency 4** (4 in-flight requests, fixed); each scenario ran 240 requests. DeepSeek-V4-Flash (W8A8) and MiniMax-M2.5 (W8A8) are the exceptions — they were swept at **concurrency 8 / 16 / 32** (480 requests per tier). +Gemma-4-31B-it uses the same 8k / 128 scenario and concurrency sweep, but is sent +directly to its P4+D4 proxy because the MaaS backend was still attached to the older +aggregate service at measurement time. | Scenario | What it models | Dataset | Request shape | |---|---|---|---| From 6885cdac73165f9ac5145f63a3f66a9a5144c444 Mon Sep 17 00:00:00 2001 From: zgsu Date: Sun, 2 Aug 2026 11:33:02 +0800 Subject: [PATCH 2/2] docs(gemma4): add multi-turn AIPerf results --- .../gemma4-31b-it-pd-p4d4-llmisvc.yaml | 52 ++---------------- .../plan/validated_models/gemma4-31b-it.mdx | 55 ++++++++++++------- docs/en/plan/validated_models/index.mdx | 6 +- 3 files changed, 41 insertions(+), 72 deletions(-) diff --git a/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml index b21fdbac..41055d99 100644 --- a/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml +++ b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml @@ -264,33 +264,6 @@ spec: tolerations: - operator: Exists serviceAccountName: "{{ ChildName .ObjectMeta.Name `-epp-sa` }}" - initContainers: - - name: tokenizer-copy - image: 100.1.69.8:11443/mlops/guidellm:v0.7.1-1-arm64 - imagePullPolicy: IfNotPresent - command: ["/bin/bash", "-c"] - args: - - | - set -eu - for pattern in 'tokenizer*' '*.json' '*.jinja' '*.py'; do - for file in /mnt/models/$pattern; do - [ -e "$file" ] && cp -a "$file" /workspace/gemma4/ - done - done - python3 - <<'PYTOK' - import json - - path = "/workspace/gemma4/tokenizer_config.json" - data = json.load(open(path)) - data["tokenizer_class"] = "PreTrainedTokenizerFast" - for key in ("auto_map", "backend", "processor_class"): - data.pop(key, None) - json.dump(data, open(path, "w")) - PYTOK - test -f /workspace/gemma4/config.json - volumeMounts: - - { name: tokenizer-source, mountPath: /mnt/models, readOnly: true } - - { name: tokenizer-model, mountPath: /workspace/gemma4 } containers: - name: main image: 100.1.69.8:11443/3rdparty/openfuyao/hermes-router:26.6.0-rc.3 @@ -318,14 +291,13 @@ spec: # --pool-group above both use inference.networking.k8s.io. apiVersion: inference.networking.x-k8s.io/v1alpha1 kind: EndpointPickerConfig + # One fixed Prefill/Decode pair has no prefix-routing choice. + # Keep tokenization out of the scheduler path; the 26.6.0-rc.3 + # tokenizer RPC has a 5s init deadline while Gemma needs >17s. plugins: - type: metrics-data-source parameters: { scheme: "http", path: "/metrics", insecureSkipVerify: true } - type: core-metrics-extractor - - type: tokenizer - parameters: - model: /workspace/gemma4 - socketPath: /var/run/tokenizer/tokenizer.sock - type: pd-group-producer parameters: pdLabelName: app.kubernetes.io/component @@ -355,7 +327,6 @@ spec: schedulingProfiles: - name: default plugins: - - pluginRef: tokenizer - pluginRef: pd-group-producer - pluginRef: request-lifecycle-tracker - pluginRef: inflight-data-producer @@ -393,25 +364,10 @@ spec: seccompProfile: { type: RuntimeDefault } volumeMounts: - { name: tls-certs, mountPath: /etc/ssl/certs, readOnly: true } - - { name: tokenizer-socket-volume, mountPath: /var/run/tokenizer } - - name: tokenizer - image: 100.1.69.8:11443/3rdparty/openfuyao/hermes-router-tokenizer:26.6.0-rc.3 - imagePullPolicy: IfNotPresent - args: ["--socket-path", "/var/run/tokenizer/tokenizer.sock"] - env: - - { name: TOKENIZER_PROVIDER, value: huggingface } - - { name: TRANSFORMERS_OFFLINE, value: "1" } - volumeMounts: - - { name: tokenizer-socket-volume, mountPath: /var/run/tokenizer } - - { name: tokenizer-model, mountPath: /workspace/gemma4 } volumes: - name: tls-certs secret: secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}" - - { name: tokenizer-socket-volume, emptyDir: {} } - - { name: tokenizer-model, emptyDir: {} } - - name: tokenizer-source - persistentVolumeClaim: { claimName: gemma4-31b-n8 } dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 @@ -425,7 +381,7 @@ metadata: infernex.io/runtime: "true" annotations: infernex.io/disabled-components: "mooncake,cache-indexer,pd-orchestrator,eagle-eye" - model-auto.alauda.io/config-revision: "0721-p4d4-d128-capacity" + model-auto.alauda.io/config-revision: "0802-p4d4-aiperf-s1s2-no-tokenizer" spec: model: name: google/gemma-4-31B-it diff --git a/docs/en/plan/validated_models/gemma4-31b-it.mdx b/docs/en/plan/validated_models/gemma4-31b-it.mdx index 124f78c0..b55f0d6b 100644 --- a/docs/en/plan/validated_models/gemma4-31b-it.mdx +++ b/docs/en/plan/validated_models/gemma4-31b-it.mdx @@ -45,6 +45,7 @@ the validated deployment. It is not a claim that the tag is the newest vLLM-Asce | P/D handoff | `MooncakeConnectorV1`, Ascend Direct KV transfer, TP4/TP4 contract | | Capacity guardrail | `max-num-seqs=96` on both engines; `gpu-memory-utilization=0.9` | | Prefix cache | disabled in the engine; the workload uses explicit P/D KV transfer | +| Scheduler | one fixed P/D pair; metrics-based P/D scoring without a tokenizer sidecar | | Tool calls | `--enable-auto-tool-choice --tool-call-parser gemma4` | ## Why P4+D4 @@ -83,27 +84,32 @@ curl -s http:////gemma4-31b-it-pd/v1/chat/completions \ ## Benchmark results -Closed-loop `aiperf 0.7.0`, driven directly through the P/D proxy. The fixed workload is -an 8k-token prompt with 128 generated tokens; it has 120 unique prompts, each reused four -times, for 480 requests per concurrency tier. The sweep uses concurrency 8 / 16 / 32, -with a warmup equal to each tier's concurrency. TTFT / E2E are seconds, ITL is ms, Decode -is output-only token throughput, and TPS is input-plus-output token throughput. - -| Metric | Concurrency 8 | Concurrency 16 | Concurrency 32 | -|---|---:|---:|---:| -| Successful / error requests | 480 / 0 | 480 / 0 | 480 / 0 | -| Server-counted ISL / OSL (tokens) | 8,013 / 128 | 8,013 / 128 | 8,013 / 128 | -| TTFT avg / p90 / p99 (s) | 4.023 / 4.278 / 4.504 | 11.642 / 12.165 / 12.405 | 26.458 / 27.856 / 28.046 | -| ITL avg / p90 / p99 (ms) | 29.11 / 30.40 / 32.07 | 29.10 / 30.69 / 32.28 | 29.09 / 30.68 / 32.16 | -| E2E avg / p90 / p99 (s) | 7.720 / 8.044 / 8.477 | 15.338 / 15.898 / 16.409 | 30.153 / 31.554 / 32.088 | -| Decode throughput (output tokens/s) | 131.66 | 131.52 | 131.64 | -| Request throughput (requests/s) | 1.03 | 1.03 | 1.03 | -| TPS (input + output tokens/s) | 8,373.74 | 8,365.16 | 8,372.31 | - -All three tiers completed 480 requests with zero errors. Throughput and ITL are flat from -concurrency 8 onward, while TTFT grows with concurrency. For this 8k-input workload, the -P4+D4 service is therefore Prefill-capacity-bound at about 1.03 requests/s; concurrency -above 8 primarily adds queueing rather than throughput. +Closed-loop `aiperf 0.7.0`, driven directly through the P/D proxy. Both scenarios run +480 requests per concurrency tier at concurrency 8 / 16 / 32, with warmup equal to the +tier's concurrency. TTFT / E2E are seconds, ITL is ms, Decode is output-only token +throughput, and TPS is input-plus-output token throughput. + +**Scenario ① — fixed-length system-prompt reuse (ISL 8,013 / OSL 128)** + +| Concurrency | TTFT avg (s) | ITL avg (ms) | E2E avg (s) | Decode (tok/s) | Requests/s | TPS (in+out) | +|--:|--:|--:|--:|--:|--:|--:| +| 8 | 4.023 | 29.11 | 7.720 | 131.66 | 1.03 | 8,373.74 | +| 16 | 11.642 | 29.10 | 15.338 | 131.52 | 1.03 | 8,365.16 | +| 32 | 26.458 | 29.09 | 30.153 | 131.64 | 1.03 | 8,372.31 | + +**Scenario ② — multi-turn dialogue (average ISL 17,933 / OSL 128)** + +| Concurrency | TTFT avg (s) | ITL avg (ms) | E2E avg (s) | Decode (tok/s) | Requests/s | TPS (in+out) | +|--:|--:|--:|--:|--:|--:|--:| +| 8 | 14.377 | 28.55 | 18.003 | 56.44 | 0.44 | 7,963.44 | +| 16 | 32.026 | 28.56 | 35.654 | 56.48 | 0.44 | 7,970.08 | +| 32 | 66.372 | 28.54 | 69.996 | 56.49 | 0.44 | 7,970.40 | + +Every tier completed 480 requests with zero errors. Throughput and ITL are flat from +concurrency 8 onward in both scenarios, while TTFT grows with concurrency. The service +is Prefill-capacity-bound at roughly 8k input-dominated tokens/s: increasing average ISL +from 8,013 to 17,933 lowers request throughput from 1.03 to 0.44 requests/s, while total +token throughput changes by only about 5%. Decode ITL remains stable near 29 ms. :::note The benchmark is intentionally sent to the P/D proxy instead of MaaS: at measurement time @@ -111,4 +117,11 @@ the cluster's MaaS backend still selected the old aggregated TP4 service. This p single, unambiguous P4+D4 topology for the published figures. Server metrics and GPU telemetry were disabled because AIPerf 0.7.0 cannot parse this vLLM metrics histogram; the published values come from the completed request records and CSV export. + +The delivered scheduler intentionally omits the tokenizer plugin. This topology has only +one fixed Prefill/Decode pair and no prefix-cache producer, so tokenization adds no routing +choice. The tested Hermes Router `26.6.0-rc.3` imposes a 5-second tokenizer initialization +deadline, while this Gemma tokenizer took 17.59 seconds to cold-load. A future multi-pair, +prefix-aware deployment should first use a router build with a configurable initialization +deadline of at least 30 seconds, then restore the tokenizer plugin. ::: diff --git a/docs/en/plan/validated_models/index.mdx b/docs/en/plan/validated_models/index.mdx index 517a0aec..d3abb7f8 100644 --- a/docs/en/plan/validated_models/index.mdx +++ b/docs/en/plan/validated_models/index.mdx @@ -82,9 +82,9 @@ real serving patterns. Output is pinned to **128 tokens** and load is **closed-l concurrency 4** (4 in-flight requests, fixed); each scenario ran 240 requests. DeepSeek-V4-Flash (W8A8) and MiniMax-M2.5 (W8A8) are the exceptions — they were swept at **concurrency 8 / 16 / 32** (480 requests per tier). -Gemma-4-31B-it uses the same 8k / 128 scenario and concurrency sweep, but is sent -directly to its P4+D4 proxy because the MaaS backend was still attached to the older -aggregate service at measurement time. +Gemma-4-31B-it uses both scenarios and the same concurrency sweep, but is sent directly +to its P4+D4 proxy because the MaaS backend was still attached to the older aggregate +service at measurement time. | Scenario | What it models | Dataset | Request shape | |---|---|---|---|