From 01a5d92e6bf6d7c91d6befe386d7e3afc203a5f7 Mon Sep 17 00:00:00 2001 From: Ignacio Boudgouste Date: Fri, 7 Aug 2026 12:33:35 -0300 Subject: [PATCH] fix(k8s/rollback): diagnose the failed deployment before deleting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit print_deployment_error_hints ran as the last step of the rollback workflow, after `delete deployment` had already removed the pods it inspects (kubectl get pods -l deployment_id=$DEPLOYMENT_ID). With no pods to read, the specific diagnosis was always empty, and the script's fallback for "nothing detected" printed the generic "Application Startup Issue Detected" block on every rollback — including manual ones where nothing had failed to start. Move the step ahead of the deletion. The failed deployment's pods are still present there under both strategies: scale_deployments is a no-op for blue/green, and for rolling build_context floors green replicas at MIN_REPLICAS precisely so a 0% traffic switch does not remove them. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + k8s/deployment/workflows/rollback.yaml | 11 ++++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93c48e90..5f410c55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Fix: a rollback now diagnoses the failed deployment before deleting it. The hints step ran after the deletion, so it never found the pods it inspects and every rollback ended with the same generic "Application Startup Issue Detected" text, even when nothing had failed to start. Rollbacks now surface the actual reason (image pull error, crash loop, OOM, failed health check) when there is one. ## [1.14.0] - 2026-08-03 - k8s scope deployments now report launched and healthy instance counts, so the deployment page shows live "X/Y launched" and "X/Y healthy" progress diff --git a/k8s/deployment/workflows/rollback.yaml b/k8s/deployment/workflows/rollback.yaml index 96c2b1c4..08568cf4 100644 --- a/k8s/deployment/workflows/rollback.yaml +++ b/k8s/deployment/workflows/rollback.yaml @@ -78,6 +78,14 @@ steps: - name: SERVICE_TEMPLATE_PATH type: file file: "$OUTPUT_DIR/service-$SCOPE_ID-$DEPLOYMENT_ID.yaml" + # Diagnose BEFORE deleting: the hints read the failed deployment's pods + # (kubectl get pods -l deployment_id=$DEPLOYMENT_ID). Running this after the + # delete step below always found nothing, so every rollback fell through to + # the generic "Application Startup Issue Detected" text regardless of why the + # rollback happened — or whether anything had failed at all. + - name: print_deployment_error_hints + type: script + file: "$SERVICE_PATH/deployment/print_failed_deployment_hints" - name: delete deployment type: script file: "$SERVICE_PATH/apply_templates" @@ -90,6 +98,3 @@ steps: file: "$SERVICE_PATH/deployment/delete_cluster_objects" configuration: DEPLOYMENT: green - - name: print_deployment_error_hints - type: script - file: "$SERVICE_PATH/deployment/print_failed_deployment_hints"