From 94fa6d2dc7e6e602caf66a1fec592f2ab76624d5 Mon Sep 17 00:00:00 2001 From: Nick Nikolakakis Date: Tue, 1 Sep 2026 20:48:01 +0300 Subject: [PATCH] Bump the e2e workload node image past the DRA idle deadlock The workload cluster runs kindest/node:v1.34.0, whose kubelet deadlocks once its gRPC connection to a DRA driver has been idle for 30 minutes. The next pod referencing a ResourceClaim then blocks in ContainerCreating with a single Scheduled event, no kubelet error, and NodePrepareResources never reaching the driver. Fixed upstream in v1.34.2 (k/k#133934). CI never sees it, since those clusters are minutes old. A cluster left up overnight hits it on the first ModelDeployment of the morning, and the symptom carries nothing that points at the kubelet. The pin exists for the DRA APIs, GA in 1.34, so any 1.34 image satisfies it. v1.34.8 is the oldest published image that also clears the deadlock, and it carries containerd 2.3.1, past the 2.2.0 that installation.md pins the getting-started cluster away from. Whether Modelplane runs on containerd 2.3.x is untested here; that docs pin is a separate constraint and is left alone, since its cluster runs no DRA driver. The reuse guard now rejects a v1.34.0 or v1.34.1 cluster rather than reusing it. Long-lived dev clusters are the ones that hit this, so bumping only the image would leave them deadlocking. Fixes #419. Signed-off-by: Nick Nikolakakis --- e2e/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e/run.sh b/e2e/run.sh index 2f6fb954..03c10f27 100644 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -17,7 +17,8 @@ WL=modelplane-e2e-workload # Pinned so the workload cluster has the DRA APIs the serving stack's NVIDIA DRA # driver needs (resource.k8s.io, GA in k8s 1.34). The control-plane cluster that # project run creates needs no DRA, so its image doesn't matter here. -WL_NODE_IMAGE=kindest/node:v1.34.0@sha256:7416a61b42b1662ca6ca89f02028ac133a309a2a30ba309614e8ec94d976dc5a +# v1.34.2 or newer: older kubelets deadlock on an idle DRA connection (k/k#133934). +WL_NODE_IMAGE=kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256 METALLB_URL=https://raw.githubusercontent.com/metallb/metallb/v0.14.8/config/manifests/metallb-native.yaml # Pinned by digest (a multi-arch manifest list) so a moving :latest can't flake # the verify curl pod. @@ -50,6 +51,10 @@ if kind get clusters 2>/dev/null | grep -qx "$WL"; then # older one lacks the DRA APIs and would fail the run confusingly later. ver="$(kubectl --context "$WLCTX" get nodes -o jsonpath='{.items[0].status.nodeInfo.kubeletVersion}' 2>/dev/null || true)" case "$ver" in + v1.34.0 | v1.34.1) + echo "workload cluster $WL is $ver, whose kubelet deadlocks on an idle DRA connection (fixed in v1.34.2); recreate it with: nix run .#e2e -- --clean" >&2 + exit 1 + ;; v1.34.*) log "Reusing workload cluster $WL ($ver)" ;; *) echo "workload cluster $WL is ${ver:-unreachable}, but v1.34 is required for the DRA APIs; recreate it with: nix run .#e2e -- --clean" >&2