feat(k8s): trace-instrument the workflow — every error clear, waits live

.github/workflows/pr-checks.yml

@@ -19,6 +19,8 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true # catalog-tracing-sh — the tracing SDK the tests exercise
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y bats jq

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "vendor/catalog-tracing-sh"]
+	path = vendor/catalog-tracing-sh
+	url = https://github.com/nullplatform/catalog-tracing-sh.git
+	branch = main

azure-aro/values.yaml

@@ -1,5 +1,6 @@
 configuration:
   DNS_TYPE: azure
+  K8S_FLAVOR: aro
   USE_ACCOUNT_SLUG: false
   IMAGE_PULL_SECRETS:
     ENABLED: false

azure/values.yaml

@@ -1,5 +1,6 @@
 configuration:
   DNS_TYPE: azure
+  K8S_FLAVOR: aks
   USE_ACCOUNT_SLUG: false
   IMAGE_PULL_SECRETS:
     ENABLED: false

k8s/apply_templates

@@ -31,9 +31,39 @@ while IFS= read -r TEMPLATE_FILE; do
        IGNORE_NOT_FOUND="--ignore-not-found=true"
      fi
 
-     if ! kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND; then
-       log error "   ❌ Failed to apply"
+     TRACE_STEP_KEY=""
+     if [[ "$ACTION" == "apply" ]] && command -v np_scope_step_begin >/dev/null 2>&1; then
+       case "$FILENAME" in
+         deployment-*) TRACE_STEP_KEY="create-deployment" ;;
+         secret-*)     TRACE_STEP_KEY="create-secret" ;;
+         scaling-*)    TRACE_STEP_KEY="create-hpa" ;;
+         service-*)    TRACE_STEP_KEY="create-service" ;;
+         pdb-*)        TRACE_STEP_KEY="create-pod-disruption-budget" ;;
+         ingress-*)
+           if [[ -n "${DEPLOYMENT_ID:-}" ]]; then
+             TRACE_STEP_KEY="configure-ingress"
+           else
+             TRACE_STEP_KEY="create-ingress"
+           fi ;;
+       esac
+       [[ -n "$TRACE_STEP_KEY" ]] && np_scope_step_begin "$TRACE_STEP_KEY"
      fi
+
+     if KUBECTL_OUT=$(kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND 2>&1); then
+       [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT"
+       [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 0
+       if [[ "$ACTION" == "apply" ]] && command -v np_scope_k8s_applied >/dev/null 2>&1; then
+         np_scope_k8s_applied "${K8S_NAMESPACE:-}" "$KUBECTL_OUT"
+       fi
+       if [[ "$ACTION" == "delete" ]] && command -v np_scope_k8s_deleted >/dev/null 2>&1; then
+         np_scope_k8s_deleted "${K8S_NAMESPACE:-}" "$KUBECTL_OUT"
+       fi
+     else
+       [[ -n "$KUBECTL_OUT" ]] && echo "$KUBECTL_OUT" >&2
+       log error "   ❌ Failed to apply $FILENAME${KUBECTL_OUT:+: $KUBECTL_OUT}"
+       [[ -n "$TRACE_STEP_KEY" ]] && command -v np_scope_step_end >/dev/null 2>&1 && np_scope_step_end 1
+     fi
+     TRACE_STEP_KEY=""
    fi
 
    DEST_DIR="${BASE_DIR}/$ACTION"
@@ -52,4 +82,23 @@ if [[ "$DRY_RUN" == "true" ]]; then
   exit 1
 fi
 
+if [[ "${TRACE_TRAFFIC_SWITCH:-false}" == "true" ]] && [[ -n "${CONTEXT:-}" ]] \
+  && command -v np_scope_explain >/dev/null 2>&1; then
+  TRAFFIC_TO=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.desired_switched_traffic // 100')
+  TRAFFIC_FROM=$(echo "$CONTEXT" | jq -r '.deployment.strategy_data.switched_traffic // 0')
+  if [[ "$TRAFFIC_TO" =~ ^[0-9]+$ ]] && [[ "$TRAFFIC_FROM" =~ ^[0-9]+$ ]]; then
+    np_scope_labels "deployment.id=${DEPLOYMENT_ID:-}" "scope.id=${SCOPE_ID:-}" "action=traffic-switch"
+    np_scope_explain --title "Switch traffic for deployment ${DEPLOYMENT_ID:-}" \
+      --what "Switching blue/green traffic for deployment ${DEPLOYMENT_ID:-} from ${TRAFFIC_FROM}% to ${TRAFFIC_TO}%"
+    np_scope_affordance "{\"kind\":\"traffic-switch\",\"deployment_id\":\"${DEPLOYMENT_ID:-}\",\"current_traffic\":$TRAFFIC_TO,\"new_traffic\":$TRAFFIC_TO,\"old_traffic\":$((100 - TRAFFIC_TO)),\"target_traffic\":100}"
+    np_scope_input traffic "{\"from\":$TRAFFIC_FROM,\"desired\":$TRAFFIC_TO}"
+    np_scope_output traffic "{\"switched\":$TRAFFIC_TO}"
+    np_scope_progress "$TRAFFIC_TO" 100 percent
+  fi
+fi
+
+if command -v np_trace_flush >/dev/null 2>&1; then
+  NP_TRACE_FLUSH_TIMEOUT=5 np_trace_flush
+fi
+
 source "$SERVICE_PATH/backup/backup_templates" --action="$ACTION" --files "${APPLIED_FILES[@]}"

k8s/deployment/print_failed_deployment_hints

@@ -124,7 +124,12 @@ diagnose_failure() {
     if [[ -n "$pods_json" ]] && echo "$pods_json" | jq -e . >/dev/null 2>&1; then
         FAILURE_REASON=$(echo "$pods_json" | jq -r '
             [.items[].status.containerStatuses[]?
-                | (.state.waiting.reason // .lastState.terminated.reason // empty)
+                | (.state.waiting.reason // "") as $w
+                | (.lastState.terminated.reason // "") as $t0
+                | (if $t0 == "Completed" then "" else $t0 end) as $t
+                | (if ($w == "CrashLoopBackOff" or $w == "BackOff") and $t != "" then $t
+                   elif $w != "" then $w
+                   else $t end)
             ] | map(select(. != "" and . != "Completed")) |
             group_by(.) | max_by(length) | .[0] // empty' 2>/dev/null)
 
@@ -181,13 +186,20 @@ diagnose_failure() {
         CrashLoopBackOff|BackOff)
             HUMAN_MESSAGE="The container started and crashed repeatedly."
             SUGGESTED_FIX="Review application logs for startup errors (failed dependencies, bad config, panics)." ;;
+        Error|ContainerStatusUnknown)
+            if [[ -n "$FAILURE_EXIT_CODE" ]]; then
+                HUMAN_MESSAGE="The container started and exited with code ${FAILURE_EXIT_CODE}."
+            else
+                HUMAN_MESSAGE="The container started and crashed repeatedly."
+            fi
+            SUGGESTED_FIX="Review application logs for startup errors (failed dependencies, bad config, panics)." ;;
         OOMKilled)
             if [[ -n "$req_memory" ]]; then
                 HUMAN_MESSAGE="The container exceeded its memory limit (${req_memory}Mi) and was terminated."
             else
                 HUMAN_MESSAGE="The container exceeded its memory limit and was terminated."
             fi
-            SUGGESTED_FIX="Increase ram_memory for scope '$scope_name' or reduce application memory usage." ;;
+            SUGGESTED_FIX="Raise the RAM Memory of scope '$scope_name', or reduce how much memory the application uses." ;;
         CreateContainerConfigError)
             HUMAN_MESSAGE="The container configuration is invalid."
             SUGGESTED_FIX="Check for missing secrets or configmaps referenced by the deployment." ;;
@@ -223,18 +235,28 @@ diagnose_failure() {
             elif [[ "$UNHEALTHY_MESSAGE" =~ statuscode:[[:space:]]*([0-9]+) ]]; then
                 SUGGESTED_FIX="The app responded with HTTP ${BASH_REMATCH[1]} on $health_check_path — inspect application logs for startup errors; the process is running but $health_check_path is not returning 2xx."
             elif [[ "$UNHEALTHY_MESSAGE" == *"context deadline exceeded"* || "$UNHEALTHY_MESSAGE" == *"Client.Timeout"* || "$UNHEALTHY_MESSAGE" == *"i/o timeout"* ]]; then
-                SUGGESTED_FIX="The probe timed out — the app may be slow to start or $health_check_path is blocking. Consider increasing startup probe initialDelaySeconds/timeoutSeconds, or making $health_check_path lighter."
+                SUGGESTED_FIX="The probe timed out — the app may be slow to start, or $health_check_path is blocking. Raise the health check's Initial Delay or Timeout on scope '$scope_name', or make $health_check_path lighter."
             else
                 SUGGESTED_FIX="Ensure the app listens on port 8080 and returns 2xx on $health_check_path within the readiness window."
             fi ;;
         FailedCreate|FailedCreatePodSandBox)
             HUMAN_MESSAGE="Kubernetes could not create the pod sandbox."
             SUGGESTED_FIX="Check node health, CNI configuration, and pod security policies." ;;
+        Evicted)
+            HUMAN_MESSAGE="The pod was evicted from its node."
+            SUGGESTED_FIX="The node ran out of memory or disk. Lower the scope's resource requests, or free capacity on the cluster." ;;
+        Unschedulable)
+            HUMAN_MESSAGE="No node can accept the pod."
+            SUGGESTED_FIX="Reduce requested resources, free cluster capacity, or review nodeSelector/affinity rules." ;;
+        DeadlineExceeded)
+            HUMAN_MESSAGE="The pod ran past its active deadline and was stopped."
+            SUGGESTED_FIX="Find why the workload runs longer than the deadline it was given, or allow it more time." ;;
         "")
             HUMAN_MESSAGE=""
             SUGGESTED_FIX="" ;;
         *)
             HUMAN_MESSAGE="Pods are failing with reason: $FAILURE_REASON"
+            # Empty on purpose: an empty fix routes the reader to the generic checklist below.
             SUGGESTED_FIX="" ;;
     esac
 }

k8s/deployment/scale_deployments

@@ -7,6 +7,12 @@ GREEN_DEPLOYMENT_ID=$DEPLOYMENT_ID
 BLUE_REPLICAS=$(echo "$CONTEXT" | jq -r .blue_replicas)
 BLUE_DEPLOYMENT_ID=$(echo "$CONTEXT" | jq .scope.current_active_deployment -r)
 
+if [ "$DEPLOY_STRATEGY" != "rolling" ]; then
+  if command -v np_step_skip >/dev/null 2>&1; then
+    np_step_skip "instance counts are set by the manifests for the $DEPLOY_STRATEGY strategy — nothing to scale"
+  fi
+fi
+
 if [ "$DEPLOY_STRATEGY" = "rolling" ]; then
   GREEN_DEPLOYMENT_NAME="d-$SCOPE_ID-$GREEN_DEPLOYMENT_ID"
   BLUE_DEPLOYMENT_NAME="d-$SCOPE_ID-$BLUE_DEPLOYMENT_ID"
@@ -41,6 +47,10 @@ if [ "$DEPLOY_STRATEGY" = "rolling" ]; then
   unset TIMEOUT
   unset SKIP_DEPLOYMENT_STATUS_CHECK
 
+  if command -v np_scope_output >/dev/null 2>&1; then
+    np_scope_output replicas "{\"green\": $GREEN_REPLICAS, \"previous\": $BLUE_REPLICAS}"
+  fi
+
   log debug ""
   log info "✨ Deployments scaled successfully"
 fi

k8s/deployment/tests/print_failed_deployment_hints.bats

@@ -83,7 +83,8 @@ assert_not_contains() {
   assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi)"
   assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)"
   assert_contains "$output" "📋 Details: out of memory"
-  assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'"
+  assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app'"
+  assert_not_contains "$output" "ram_memory"
   assert_not_contains "$output" "⚠️  Application Startup Issue Detected"
 }
 
@@ -229,8 +230,9 @@ assert_not_contains() {
   run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
 
   [ "$status" -eq 0 ]
-  assert_contains "$output" "did not pass its health check at /health"
-  assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health"
+  assert_contains "$output" "📋 Reason: The application did not pass its health check at /health."
+  assert_contains "$output" "📋 Detected: Unhealthy on container api"
+  assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window."
   assert_not_contains "$output" "⚠️  Application Startup Issue Detected"
 }
 
@@ -250,13 +252,11 @@ assert_not_contains() {
   run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
 
   [ "$status" -eq 0 ]
-  # HUMAN_MESSAGE retains the base sentence and appends the translated probe failure
-  assert_contains "$output" "did not pass its health check at /health"
-  assert_contains "$output" "Detected: Startup probe"
-  assert_contains "$output" "not yet listening"
-  # SUGGESTED_FIX is targeted: tells the user the app is not binding the port
-  assert_contains "$output" "not listening on port 8080"
-  # Generic fallback fix must NOT appear
+  assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app is not yet listening on /health."
+  assert_contains "$output" "📋 Detected: Unhealthy on container api"
+  assert_contains "$output" "📋 Recent warnings:"
+  assert_contains "$output" "  • Unhealthy (×1)"
+  assert_contains "$output" "💡 Suggested fix: The container is not listening on port 8080 — verify the start command runs, the process binds to 0.0.0.0:8080, and nothing is crashing before it accepts connections."
   assert_not_contains "$output" "returns 2xx on /health within the readiness window"
 }
 
@@ -276,11 +276,9 @@ assert_not_contains() {
   run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
 
   [ "$status" -eq 0 ]
-  assert_contains "$output" "Detected: Startup probe"
-  assert_contains "$output" "HTTP 502"
-  # SUGGESTED_FIX cites the status code and points to app logs
-  assert_contains "$output" "responded with HTTP 502"
-  assert_contains "$output" "inspect application logs"
+  assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app responded with HTTP 502 (expected 2xx)."
+  assert_contains "$output" "📋 Detected: Unhealthy on container api"
+  assert_contains "$output" "💡 Suggested fix: The app responded with HTTP 502 on /health — inspect application logs for startup errors; the process is running but /health is not returning 2xx."
 }
 
 @test "print_failed_deployment_hints: enriches Unhealthy with timeout detail and targeted fix" {
@@ -299,16 +297,14 @@ assert_not_contains() {
   run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
 
   [ "$status" -eq 0 ]
-  assert_contains "$output" "Detected: Startup probe"
-  assert_contains "$output" "timed out"
-  # SUGGESTED_FIX mentions timing knobs
-  assert_contains "$output" "initialDelaySeconds"
+  assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — request timed out on /health."
+  assert_contains "$output" "📋 Detected: Unhealthy on container api"
+  assert_contains "$output" "💡 Suggested fix: The probe timed out — the app may be slow to start, or /health is blocking. Raise the health check's Initial Delay or Timeout on scope 'my-app', or make /health lighter."
+  assert_not_contains "$output" "initialDelaySeconds"
 }
 
 @test "print_failed_deployment_hints: falls back to raw Unhealthy message when translation is impossible" {
   export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1"
-  # Message does not match any known probe pattern → translate_probe_message returns non-zero.
-  # The raw text must still be surfaced in the hint instead of being silently dropped.
   export ALL_EVENTS='{"items":[{"type":"Warning","reason":"Unhealthy","lastTimestamp":"2026-05-20T13:13:42Z","message":"completely unknown probe failure format from a future K8s"}]}'
 
   kubectl() {
@@ -323,10 +319,8 @@ assert_not_contains() {
   run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
 
   [ "$status" -eq 0 ]
-  # Raw message appears verbatim in the reason line
-  assert_contains "$output" "completely unknown probe failure format from a future K8s"
-  # Base sentence is still there
-  assert_contains "$output" "did not pass its health check at /health"
+  assert_contains "$output" "📋 Reason: The application did not pass its health check at /health. Detected: completely unknown probe failure format from a future K8s"
+  assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window."
 }
 
 @test "print_failed_deployment_hints: Unhealthy picks the latest event when multiple are present" {
@@ -399,8 +393,6 @@ assert_not_contains() {
   # health_check_path default "/" must apply when CONTEXT is unset.
   assert_contains "$output" "health check at /."
   assert_contains "$output" "returns 2xx on /"
-  # Guard against the previous escape bug: a literal backslash in the message
-  # would indicate jq received {\} instead of {} and silently failed.
   assert_not_contains "$output" "{\\"
 }
 
@@ -488,3 +480,64 @@ assert_not_contains() {
   [ "$status" -eq 0 ]
   assert_contains "$output" "📊 Progress at failure: 1/3 ready, 2/3 available"
 }
+
+@test "print_failed_deployment_hints: an OOM kill inside a crash loop is diagnosed as the OOM" {
+  export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1"
+  kubectl() {
+    case "$*" in
+      "get pods"*)
+        echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=app pod=d-1_ns(abc)"}},"lastState":{"terminated":{"reason":"OOMKilled","exitCode":137}}}]}}]}'
+        ;;
+    esac
+  }
+  export -f kubectl
+
+  run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
+
+  [ "$status" -eq 0 ]
+  assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi) and was terminated."
+  assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)"
+  assert_contains "$output" "📋 Details: back-off 20s restarting failed container=app pod=d-1_ns(abc)"
+  assert_contains "$output" "💡 Suggested fix: Raise the RAM Memory of scope 'my-app', or reduce how much memory the application uses."
+  assert_not_contains "$output" "The container started and crashed repeatedly."
+}
+
+@test "print_failed_deployment_hints: a plain non-zero exit in a crash loop keeps its startup-log advice" {
+  export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1"
+  kubectl() {
+    case "$*" in
+      "get pods"*)
+        echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff","message":"back-off 20s restarting failed container=app"}},"lastState":{"terminated":{"reason":"Error","exitCode":1}}}]}}]}'
+        ;;
+    esac
+  }
+  export -f kubectl
+
+  run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
+
+  [ "$status" -eq 0 ]
+  assert_contains "$output" "📋 Reason: The container started and exited with code 1."
+  assert_contains "$output" "📋 Detected: Error on container app (exit 1)"
+  assert_contains "$output" "📋 Details: back-off 20s restarting failed container=app"
+  assert_contains "$output" "💡 Suggested fix: Review application logs for startup errors (failed dependencies, bad config, panics)."
+  assert_not_contains "$output" "Pods are failing with reason: Error"
+}
+
+@test "print_failed_deployment_hints: an evicted pod says why and what to do" {
+  export K8S_NAMESPACE="ns" DEPLOYMENT_ID="d1"
+  kubectl() {
+    case "$*" in
+      "get pods"*)
+        echo '{"items":[{"status":{"containerStatuses":[{"name":"app","state":{"waiting":{"reason":"Evicted"}}}]}}]}'
+        ;;
+    esac
+  }
+  export -f kubectl
+
+  run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints"
+
+  [ "$status" -eq 0 ]
+  assert_contains "$output" "📋 Reason: The pod was evicted from its node."
+  assert_contains "$output" "📋 Detected: Evicted on container app"
+  assert_contains "$output" "💡 Suggested fix: The node ran out of memory or disk. Lower the scope's resource requests, or free capacity on the cluster."
+}