WIP: Collect and normalize engine metrics with a MetricMapping

apis/metricmappings/composition.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.crossplane.io/v1
+kind: Composition
+metadata:
+  name: metricmappings.modelplane.ai
+spec:
+  compositeTypeRef:
+    apiVersion: modelplane.ai/v1alpha1
+    kind: MetricMapping
+  mode: Pipeline
+  pipeline:
+  - functionRef:
+      name: modelplane-modelplanecompose-metric-mapping
+    step: compose-metric-mapping

apis/metricmappings/definition.yaml

@@ -0,0 +1,67 @@
+apiVersion: apiextensions.crossplane.io/v2
+kind: CompositeResourceDefinition
+metadata:
+  name: metricmappings.modelplane.ai
+spec:
+  group: modelplane.ai
+  names:
+    categories: [crossplane, modelplane]
+    kind: MetricMapping
+    plural: metricmappings
+    shortNames: [mm]
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    served: true
+    referenceable: true
+    schema:
+      openAPIV3Schema:
+        type: object
+        required: [spec]
+        properties:
+          spec:
+            type: object
+            description: >-
+              How to normalize one component's Prometheus metrics onto the
+              modelplane_* surface. The metrics collector reads every
+              MetricMapping and renders the matching rename and label rules
+              into its config. Modelplane ships a MetricMapping per common
+              engine; a platform team applies one more for a new or forked
+              engine.
+            properties:
+              selector:
+                type: object
+                description: >-
+                  Selects the pods this mapping applies to, by label. An
+                  engine mapping matches the engine-type label
+                  (modelplane.ai/engine); a scheduler mapping matches the
+                  scheduler's pods.
+                properties:
+                  matchLabels:
+                    type: object
+                    additionalProperties:
+                      type: string
+              rename:
+                type: object
+                description: >-
+                  Source metric name to its modelplane_* name, e.g.
+                  vllm:time_to_first_token_seconds becomes
+                  modelplane_time_to_first_token.
+                additionalProperties:
+                  type: string
+              labels:
+                type: object
+                description: Label rewrites applied to the matched series.
+                properties:
+                  add:
+                    type: object
+                    description: Labels to add to every matched series.
+                    additionalProperties:
+                      type: string
+          status:
+            type: object
+            properties:
+              conditions:
+                type: array
+                items:
+                  type: object

apis/modeldeployments/definition.yaml

@@ -242,6 +242,22 @@ spec:
                               default: 1
                               minimum: 1
                               maximum: 64
+                            type:
+                              type: string
+                              description: >-
+                                Which engine this runs, e.g. vllm or sglang.
+                                Stamped onto the serving pods as
+                                modelplane.ai/engine, where a MetricMapping
+                                selects on it to normalize that engine's metrics
+                                onto the modelplane_* surface. Free-form rather
+                                than an enum: a platform team adds a
+                                MetricMapping for a forked or new engine without
+                                a Modelplane release, and an engine with no
+                                mapping is still scraped under its native names.
+                                Omit it and the engine's metrics are collected
+                                unnormalized.
+                              minLength: 1
+                              maxLength: 63
                             phase:
                               type: string
                               enum: [Prefill, Decode]

apis/modelreplicas/definition.yaml

@@ -137,6 +137,20 @@ spec:
                     phase:
                       type: string
                       enum: [Prefill, Decode]
+                    type:
+                      type: string
+                      description: >-
+                        Which engine this runs, e.g. vllm or sglang. Stamped
+                        onto the serving pods as modelplane.ai/engine, where a
+                        MetricMapping selects on it to normalize that engine's
+                        metrics onto the modelplane_* surface. Free-form rather
+                        than an enum: a platform team adds a MetricMapping for a
+                        forked or new engine without a Modelplane release, and an
+                        engine with no mapping is still scraped under its native
+                        names. Omit it and the engine's metrics are collected
+                        unnormalized.
+                      minLength: 1
+                      maxLength: 63
                     members:
                       type: array
                       minItems: 1

apis/servingstacks/definition.yaml

@@ -137,6 +137,15 @@ spec:
                     description: kube-prometheus-stack chart version.
                     minLength: 1
                     maxLength: 32
+                  otelCollector:
+                    type: string
+                    default: "0.116.0"
+                    description: >-
+                      OpenTelemetry Collector chart version. The collector
+                      scrapes each engine's metrics and renames them onto the
+                      modelplane_* surface from the cluster's MetricMappings.
+                    minLength: 1
+                    maxLength: 32
                   leaderWorkerSet:
                     type: string
                     default: "v0.8.0"

crossplane-project.yaml

@@ -67,6 +67,10 @@ spec:
     tarball:
       name: compose-model-service
       pathPrefix: _output/functions/compose-model-service
+  - source: Tarball
+    tarball:
+      name: compose-metric-mapping
+      pathPrefix: _output/functions/compose-metric-mapping
   - source: Tarball
     tarball:
       name: compose-usages

docs/content/examples/collecting-engine-metrics.md

@@ -1,20 +1,20 @@
 ---
 title: Collecting engine metrics
 weight: 50
-description: Scrape a vLLM engine's Prometheus metrics through the in-cluster Prometheus.
+description: Read a vLLM engine's metrics, collected and normalized onto the modelplane_* surface.
 ---
 <!-- vale write-good.Passive = NO -->
-Scraping an inference engine's Prometheus metrics, shown on the smallest serving
-shape: a 0.5B Qwen chat model on one NVIDIA L4. vLLM publishes metrics at
-`/metrics` on its serving port with no extra flag, and Modelplane runs a
-Prometheus on every workload cluster with `PodMonitor` discovery open across
-namespaces, so scraping the engine is a `PodMonitor` plus a `port-forward`. The
-model is only the subject; the same wiring fits any engine, with the SGLang,
-leader/worker, and prefill/decode differences noted at the end.
+Reading an inference engine's metrics, shown on the smallest serving shape: a
+0.5B Qwen chat model on one NVIDIA L4. Modelplane collects from every engine it
+runs, so there is nothing to wire up: a collector on each workload cluster
+discovers serving pods by label, scrapes the engine port by name, and renames the
+engine's metrics onto a common `modelplane_*` surface. The model is only the
+subject; the same applies to any engine, with the SGLang, leader/worker, and
+prefill/decode differences noted at the end.
 
 This was run end to end on GKE. The `InferenceClass` and `ModelDeployment` are the
-exact manifests from that run, and the `PodMonitor` below scraped this deployment.
-Apply the platform side first, then the ML side.
+exact manifests from that run, and the metric names below are the ones that run
+produced. Apply the platform side first, then the ML side.
 
 ## Platform
 
@@ -28,38 +28,83 @@ Apply the platform side first, then the ML side.
 
 {{< manifests "examples/collecting-engine-metrics/model-service.yaml" >}}
 
-## Scraping the metrics
+## Reading the metrics
 
-The `PodMonitor` selects engine pods by the `modelplane.ai/serving` label
-Modelplane stamps on them, and the `monitoring` namespace Prometheus discovers any
-`PodMonitor`, so this is the whole config. The engine container port is unnamed,
-so reference it by number with `targetPort`:
+Nothing needs applying for collection. Every serving pod carries
+`modelplane.ai/serving`, and its engine container's port is named `http`, which is
+what the collector's scrape config matches — so an engine is collected from as
+soon as it serves.
 
-{{< manifests "examples/collecting-engine-metrics/podmonitor.yaml" >}}
-
-The engine pods and the `PodMonitor` CRD live on the workload cluster, not the
-control plane, so apply it there. Then read the metrics from the in-cluster
-Prometheus over a `port-forward`:
+The collector re-exposes what it collected on the workload cluster, so read it
+over a `port-forward`:
 
 ```bash
-kubectl -n monitoring port-forward svc/prometheus-prometheus 9090:9090   # workload cluster
-# open http://localhost:9090, Status > Targets to confirm the scrape, then query
-# e.g. vllm:num_requests_running or vllm:gpu_cache_usage_perc
+kubectl -n monitoring port-forward svc/otel-collector 8889:8889   # workload cluster
+curl -s localhost:8889/metrics | grep '^modelplane_'
+```
+
+For the deployment above that returns the normalized names, each labelled with the
+engine that produced it:
+
 ```
+modelplane_requests_running{engine="vllm",model_name="qwen2.5-0.5b",...}
+modelplane_requests_waiting{engine="vllm",model_name="qwen2.5-0.5b",...}
+modelplane_time_to_first_token_sum{engine="vllm",model_name="qwen2.5-0.5b",...}
+modelplane_request_latency_sum{engine="vllm",model_name="qwen2.5-0.5b",...}
+```
+
+### Which names get renamed
+
+A `MetricMapping` decides. Modelplane ships one per common engine, matched to
+serving pods by the `modelplane.ai/engine` label that `engines[].type` sets, so an
+engine that declares `type: vllm` gets the vLLM mapping and no detection is
+involved. A metric with no mapping entry is not dropped — it passes through under
+its own name.
+
+Two things to know about the names that pass through. The collector's exporter
+replaces `:` with `_`, so vLLM's `vllm:gpu_cache_usage_perc` is published as
+`vllm_gpu_cache_usage_perc`. And an engine that declares no `type` matches no
+mapping, so all of its metrics pass through rather than being renamed by a guess.
+
+To normalize a new or forked engine, apply another `MetricMapping` — no Modelplane
+release is needed:
+
+```yaml
+apiVersion: modelplane.ai/v1alpha1
+kind: MetricMapping
+metadata:
+  name: my-fork
+  namespace: ml-team
+spec:
+  selector:
+    matchLabels:
+      modelplane.ai/engine: my-fork
+  rename:
+    myfork:queue_depth: modelplane_requests_waiting
+  labels:
+    add:
+      engine: my-fork
+```
+
+### Upgrading from a hand-written PodMonitor
+
+Earlier versions of this example applied a `PodMonitor` to the workload cluster by
+hand. Delete it. Collection is composed now, and leaving it in place scrapes every
+engine twice.
 
 ### Other engine shapes
 
-The `PodMonitor` above fits a single-pod vLLM engine. The selector and port shift
-by shape:
+The example above is a single-pod vLLM engine. Collection needs no changes for the
+other shapes, but what gets collected differs:
 
 - **SGLang**: exposes `/metrics` only when the engine runs with
-  `--enable-metrics`; otherwise it's identical (same selector, `targetPort: 8000`).
+  `--enable-metrics`; otherwise it is collected from the same way.
 - **Leader/worker**: only the leader serves the API and carries
-  `modelplane.ai/serving`, so the selector above already scrapes the leader alone;
-  the workers expose nothing.
+  `modelplane.ai/serving`, so only the leader is collected from; the workers serve
+  nothing and expose no metrics.
 - **prefill/decode**: two engines, labelled `llm-d.ai/role: prefill` and
-  `llm-d.ai/role: decode`. The prefill engine serves on `8000`; the decode engine
-  sits behind the routing sidecar that takes `8000` and listens on `8001`, so
-  scrape decode with `targetPort: 8001`. Select each by its role label to keep
-  them apart.
+  `llm-d.ai/role: decode`. Both are collected from without special casing, because
+  the scrape matches the engine port by name rather than by number: the decode
+  engine serves on `8001` since the routing sidecar takes `8000`, and a config
+  matching `8000` would report the sidecar's metrics as the engine's.
 <!-- vale write-good.Passive = YES -->

docs/manifests/examples/collecting-engine-metrics/model-deployment.yaml

@@ -10,7 +10,7 @@
 #   --served-model-name     the id clients pass as "model" in OpenAI requests.
 #
 # vLLM exposes Prometheus metrics at /metrics on its serving port (:8000) with no
-# extra flag, which is what the example's PodMonitor scrapes.
+# extra flag, which is what Modelplane's collector scrapes.
 #
 # No --port or --host: Modelplane's routing expects the engine on its default
 # :8000 with a /health probe, and passes args through verbatim.
@@ -26,6 +26,10 @@ spec:
     spec:
       engines:
       - name: qwen2-5-0-5b
+        # Names the engine so its serving pods carry modelplane.ai/engine, which
+        # is how the vLLM MetricMapping selects them. Without it the engine is
+        # still collected from, under vLLM's own metric names.
+        type: vllm
         members:
     # A single self-contained vLLM pod. The container named "engine" is the
     # inference server; its image and args pass through verbatim.

e2e/manifests/25-metric-mapping.yaml

@@ -0,0 +1,20 @@
+---
+# The vLLM mapping. The mock engine publishes vLLM's own metric names on
+# /metrics, and the ModelDeployment declares `type: vllm`, so this selects that
+# engine's pods by label and the collector renames only their series. An engine
+# with no matching mapping keeps its native names.
+apiVersion: modelplane.ai/v1alpha1
+kind: MetricMapping
+metadata:
+  name: vllm
+  namespace: ml-team
+spec:
+  selector:
+    matchLabels:
+      modelplane.ai/engine: vllm
+  rename:
+    vllm:num_requests_waiting: modelplane_requests_waiting
+    vllm:time_to_first_token_seconds: modelplane_time_to_first_token
+  labels:
+    add:
+      engine: vllm

e2e/manifests/40-model-deployment.yaml

@@ -15,6 +15,9 @@ spec:
     spec:
       engines:
       - name: mock
+        # Names the engine so its serving pods carry modelplane.ai/engine, which
+        # is what the vllm MetricMapping selects on.
+        type: vllm
         members:
         - role: Standalone
           nodeSelector:
@@ -44,9 +47,29 @@ spec:
                           self.send_header("content-length", str(len(b)))
                           self.end_headers()
                           self.wfile.write(b)
+                      def _text(self, body, c=200):
+                          b = body.encode()
+                          self.send_response(c)
+                          self.send_header("content-type", "text/plain; version=0.0.4")
+                          self.send_header("content-length", str(len(b)))
+                          self.end_headers()
+                          self.wfile.write(b)
                       def do_GET(self):
                           if self.path == "/health":
                               self._s({"status": "ok"})
+                          elif self.path == "/metrics":
+                              # Prometheus exposition under vLLM's own names, so the
+                              # MetricMapping has something real to rename. A real
+                              # engine publishes these on the serving port too.
+                              self._text(
+                                  "# TYPE vllm:num_requests_waiting gauge\n"
+                                  "vllm:num_requests_waiting 3\n"
+                                  "# TYPE vllm:time_to_first_token_seconds histogram\n"
+                                  'vllm:time_to_first_token_seconds_bucket{le="0.1"} 1\n'
+                                  'vllm:time_to_first_token_seconds_bucket{le="+Inf"} 2\n'
+                                  "vllm:time_to_first_token_seconds_sum 0.25\n"
+                                  "vllm:time_to_first_token_seconds_count 2\n"
+                              )
                           elif self.path.startswith("/v1/models"):
                               self._s({"object": "list", "data": [{"id": "mock", "object": "model"}]})
                           else: