From f145e895356a1004915efd66099799de3255665e Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Mon, 10 Aug 2026 15:27:44 -0300 Subject: [PATCH 1/5] test: assert complete translate_probe_message output --- .../tests/translate_probe_message.bats | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/k8s/deployment/tests/translate_probe_message.bats b/k8s/deployment/tests/translate_probe_message.bats index 2ff9de51..5699375a 100644 --- a/k8s/deployment/tests/translate_probe_message.bats +++ b/k8s/deployment/tests/translate_probe_message.bats @@ -18,18 +18,14 @@ setup() { run translate_probe_message 'Startup probe failed: Get "http://10.15.28.102:8080/health": dial tcp 10.15.28.102:8080: connect: connection refused' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/health" + assert_equal "$output" "Startup probe — app is not yet listening on /health" } @test "translate_probe_message: liveness probe connection refused" { run translate_probe_message 'Liveness probe failed: Get "http://10.0.0.5:8080/ping": dial tcp: connect: connection refused' [ "$status" -eq 0 ] - assert_contains "$output" "Liveness probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/ping" + assert_equal "$output" "Liveness probe — app is not yet listening on /ping" } # ----------------------------------------------------------------------------- @@ -39,16 +35,14 @@ setup() { run translate_probe_message 'Startup probe failed: HTTP probe failed with statuscode: 502' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "HTTP 502" + assert_equal "$output" "Startup probe — app responded with HTTP 502 (expected 2xx)" } @test "translate_probe_message: readiness probe HTTP 404" { run translate_probe_message 'Readiness probe failed: HTTP probe failed with statuscode: 404' [ "$status" -eq 0 ] - assert_contains "$output" "Readiness probe" - assert_contains "$output" "HTTP 404" + assert_equal "$output" "Readiness probe — app responded with HTTP 404 (expected 2xx)" } # ----------------------------------------------------------------------------- @@ -58,9 +52,7 @@ setup() { run translate_probe_message 'Startup probe failed: Get "http://10.0.0.5:8080/health": context deadline exceeded (Client.Timeout exceeded while awaiting headers)' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "timed out" - assert_contains "$output" "/health" + assert_equal "$output" "Startup probe — request timed out on /health" } # ----------------------------------------------------------------------------- @@ -86,7 +78,7 @@ setup() { run translate_probe_message 'Startup probe failed: some weird new error format' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" + assert_equal "$output" "Startup probe failed" } # ----------------------------------------------------------------------------- From 10d3661fd803f41aaa360fde45676efa50433340 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Mon, 10 Aug 2026 15:47:07 -0300 Subject: [PATCH 2/5] test: assert complete log messages in deployment tests --- .../tests/print_failed_deployment_hints.bats | 319 +++++++++++++---- .../validate_alb_target_group_capacity.bats | 331 +++++++++++++++--- .../tests/verify_ingress_reconciliation.bats | 94 +++-- 3 files changed, 585 insertions(+), 159 deletions(-) diff --git a/k8s/deployment/tests/print_failed_deployment_hints.bats b/k8s/deployment/tests/print_failed_deployment_hints.bats index aae55005..f62b463f 100644 --- a/k8s/deployment/tests/print_failed_deployment_hints.bats +++ b/k8s/deployment/tests/print_failed_deployment_hints.bats @@ -46,20 +46,25 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # Main header - assert_contains "$output" "⚠️ Application Startup Issue Detected" - # Possible causes - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Your application was unable to start" - # How to fix section - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "port 8080" - assert_contains "$output" "/health" - assert_contains "$output" "Application Logs" - assert_contains "$output" "512Mi" - assert_contains "$output" "Environment Variables" - assert_contains "$output" "my-app" - assert_contains "$output" "production" + local expected + expected=$(cat <<'EOF' + +⚠️ Application Startup Issue Detected + +💡 Possible causes: + Your application was unable to start within the expected timeframe + +🔧 How to fix: + 1. Port Configuration: Ensure your application listens on port 8080 + 2. Health Check Endpoint: Verify your app responds to: /health + 3. Application Logs: Review logs for startup errors (database connections, + missing dependencies, or initialization errors) + 4. Memory Allocation: Current allocation is 512Mi - increase if needed + 5. Environment Variables: Verify all required variables are configured in + parameters for scope 'my-app' or dimensions: production +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -80,10 +85,16 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The container exceeded its memory limit (512Mi)" - assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" - assert_contains "$output" "📋 Details: out of memory" - assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container exceeded its memory limit (512Mi) and was terminated. +📋 Detected: OOMKilled on container app (exit 137) +📋 Details: out of memory +💡 Suggested fix: Increase ram_memory for scope 'my-app' or reduce application memory usage. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -102,11 +113,17 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The container image could not be pulled." - assert_contains "$output" "📋 Detected: ImagePullBackOff on container web" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container image could not be pulled. +📋 Detected: ImagePullBackOff on container web +📋 Details: manifest unknown +💡 Suggested fix: Verify the image name, tag, and registry credentials are correct. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "exit " - assert_contains "$output" "📋 Details: manifest unknown" - assert_contains "$output" "💡 Suggested fix: Verify the image name, tag, and registry credentials" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -125,9 +142,16 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The container started and crashed repeatedly." - assert_contains "$output" "📋 Detected: CrashLoopBackOff on container worker" - assert_contains "$output" "💡 Suggested fix: Review application logs for startup errors" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container started and crashed repeatedly. +📋 Detected: CrashLoopBackOff on container worker +📋 Details: back-off 5m0s restarting failed container +💡 Suggested fix: Review application logs for startup errors (failed dependencies, bad config, panics). +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -146,8 +170,16 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The container configuration is invalid." - assert_contains "$output" "💡 Suggested fix: Check for missing secrets or configmaps" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container configuration is invalid. +📋 Detected: CreateContainerConfigError on container api +📋 Details: secret "db-creds" not found +💡 Suggested fix: Check for missing secrets or configmaps referenced by the deployment. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -166,8 +198,15 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The container failed to run its entrypoint." - assert_contains "$output" "💡 Suggested fix: Verify the start command" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container failed to run its entrypoint. +📋 Detected: RunContainerError on container app +💡 Suggested fix: Verify the start command and that required binaries exist in the image. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -186,9 +225,16 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: The startup binary is missing or not executable" - assert_contains "$output" "📋 Detected: ContainerCannotRun on container app (exit 127)" - assert_contains "$output" "💡 Suggested fix: Rebuild the image" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The startup binary is missing or not executable inside the image. +📋 Detected: ContainerCannotRun on container app (exit 127) +📋 Details: exec: "/app": no such file +💡 Suggested fix: Rebuild the image ensuring the entrypoint exists and has execute permissions. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -198,8 +244,17 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: A volume could not be mounted onto the pod." - assert_contains "$output" "💡 Suggested fix: Check that the referenced PVC, secret, or configmap exists" + local expected + expected=$(cat <<'EOF' + +📋 Reason: A volume could not be mounted onto the pod. +📋 Detected: FailedMount +📋 Recent warnings: + • FailedMount (×1) +💡 Suggested fix: Check that the referenced PVC, secret, or configmap exists and is accessible. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -209,8 +264,17 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: Kubernetes could not create the pod sandbox." - assert_contains "$output" "💡 Suggested fix: Check node health, CNI configuration" + local expected + expected=$(cat <<'EOF' + +📋 Reason: Kubernetes could not create the pod sandbox. +📋 Detected: FailedCreatePodSandBox +📋 Recent warnings: + • FailedCreatePodSandBox (×1) +💡 Suggested fix: Check node health, CNI configuration, and pod security policies. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -229,8 +293,15 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "did not pass its health check at /health" - assert_contains "$output" "💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. +📋 Detected: Unhealthy on container api +💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -250,12 +321,19 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # HUMAN_MESSAGE retains the base sentence and appends the translated probe failure - assert_contains "$output" "did not pass its health check at /health" - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "not yet listening" - # SUGGESTED_FIX is targeted: tells the user the app is not binding the port - assert_contains "$output" "not listening on port 8080" + # HUMAN_MESSAGE retains the base sentence and appends the translated probe failure; + # SUGGESTED_FIX is targeted: tells the user the app is not binding the port. + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app is not yet listening on /health. +📋 Detected: Unhealthy on container api +📋 Recent warnings: + • Unhealthy (×1) +💡 Suggested fix: The container is not listening on port 8080 — verify the start command runs, the process binds to 0.0.0.0:8080, and nothing is crashing before it accepts connections. +EOF +) + assert_equal "$output" "$expected" # Generic fallback fix must NOT appear assert_not_contains "$output" "returns 2xx on /health within the readiness window" } @@ -276,11 +354,18 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "HTTP 502" # SUGGESTED_FIX cites the status code and points to app logs - assert_contains "$output" "responded with HTTP 502" - assert_contains "$output" "inspect application logs" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app responded with HTTP 502 (expected 2xx). +📋 Detected: Unhealthy on container api +📋 Recent warnings: + • Unhealthy (×1) +💡 Suggested fix: The app responded with HTTP 502 on /health — inspect application logs for startup errors; the process is running but /health is not returning 2xx. +EOF +) + assert_equal "$output" "$expected" } @test "print_failed_deployment_hints: enriches Unhealthy with timeout detail and targeted fix" { @@ -299,10 +384,18 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "Detected: Startup probe" - assert_contains "$output" "timed out" # SUGGESTED_FIX mentions timing knobs - assert_contains "$output" "initialDelaySeconds" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — request timed out on /health. +📋 Detected: Unhealthy on container api +📋 Recent warnings: + • Unhealthy (×1) +💡 Suggested fix: The probe timed out — the app may be slow to start or /health is blocking. Consider increasing startup probe initialDelaySeconds/timeoutSeconds, or making /health lighter. +EOF +) + assert_equal "$output" "$expected" } @test "print_failed_deployment_hints: falls back to raw Unhealthy message when translation is impossible" { @@ -323,10 +416,18 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # Raw message appears verbatim in the reason line - assert_contains "$output" "completely unknown probe failure format from a future K8s" - # Base sentence is still there - assert_contains "$output" "did not pass its health check at /health" + # Raw message appears verbatim, appended to the base sentence + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. Detected: completely unknown probe failure format from a future K8s +📋 Detected: Unhealthy on container api +📋 Recent warnings: + • Unhealthy (×1) +💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on /health within the readiness window. +EOF +) + assert_equal "$output" "$expected" } @test "print_failed_deployment_hints: Unhealthy picks the latest event when multiple are present" { @@ -350,7 +451,17 @@ assert_not_contains() { [ "$status" -eq 0 ] # Latest event wins → connection-refused remediation, not the older HTTP 502 one - assert_contains "$output" "not listening on port 8080" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /health. Detected: Startup probe — app is not yet listening on /health. +📋 Detected: Unhealthy on container api +📋 Recent warnings: + • Unhealthy (×2) +💡 Suggested fix: The container is not listening on port 8080 — verify the start command runs, the process binds to 0.0.0.0:8080, and nothing is crashing before it accepts connections. +EOF +) + assert_equal "$output" "$expected" assert_not_contains "$output" "responded with HTTP 502" } @@ -374,7 +485,15 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "exceeded its memory limit" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container exceeded its memory limit and was terminated. +📋 Detected: OOMKilled on container app (exit 137) +💡 Suggested fix: Increase ram_memory for scope 'my-app' or reduce application memory usage. +EOF +) + assert_equal "$output" "$expected" # The (Mi) parenthetical must not appear empty when ram_memory is missing. assert_not_contains "$output" "(Mi)" } @@ -396,9 +515,16 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - # health_check_path default "/" must apply when CONTEXT is unset. - assert_contains "$output" "health check at /." - assert_contains "$output" "returns 2xx on /" + # health_check_path default "/" applies when CONTEXT is unset. + local expected + expected=$(cat <<'EOF' + +📋 Reason: The application did not pass its health check at /. +📋 Detected: Unhealthy on container api +💡 Suggested fix: Ensure the app listens on port 8080 and returns 2xx on / within the readiness window. +EOF +) + assert_equal "$output" "$expected" # Guard against the previous escape bug: a literal backslash in the message # would indicate jq received {\} instead of {} and silently failed. assert_not_contains "$output" "{\\" @@ -422,12 +548,31 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: Pods are failing with reason: WeirdNewError" - assert_contains "$output" "📋 Detected: WeirdNewError on container app" + # No suggested fix → fall through to generic checklist, printed alongside the specific reason. + local expected + expected=$(cat <<'EOF' + +📋 Reason: Pods are failing with reason: WeirdNewError +📋 Detected: WeirdNewError on container app + +⚠️ Application Startup Issue Detected + +💡 Possible causes: + Your application was unable to start within the expected timeframe + +🔧 How to fix: + 1. Port Configuration: Ensure your application listens on port 8080 + 2. Health Check Endpoint: Verify your app responds to: /health + 3. Application Logs: Review logs for startup errors (database connections, + missing dependencies, or initialization errors) + 4. Memory Allocation: Current allocation is 512Mi - increase if needed + 5. Environment Variables: Verify all required variables are configured in + parameters for scope 'my-app' or dimensions: production +EOF +) + assert_equal "$output" "$expected" # No suggested fix → fall through to generic checklist. assert_not_contains "$output" "💡 Suggested fix:" - assert_contains "$output" "⚠️ Application Startup Issue Detected" - assert_contains "$output" "🔧 How to fix:" } # ============================================================================= @@ -439,9 +584,20 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Reason: No node has enough resources" - assert_contains "$output" "📋 Detected: FailedScheduling" - assert_contains "$output" "💡 Suggested fix: Reduce requested resources" + # A lone apostrophe ("pod's") inside a heredoc nested in $(...) trips a bash + # command-substitution parsing quirk, so this one assertion is built with + # `read` instead of `cat` to sidestep it. + local expected + IFS= read -r -d '' expected <<'EOF' || true + +📋 Reason: No node has enough resources or matches the pod's scheduling constraints. +📋 Detected: FailedScheduling +📋 Recent warnings: + • FailedScheduling (×2) +💡 Suggested fix: Reduce requested resources, free cluster capacity, or review nodeSelector/affinity rules. +EOF + expected="${expected%$'\n'}" + assert_equal "$output" "$expected" assert_not_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -459,10 +615,20 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📋 Recent warnings:" - assert_contains "$output" "BackOff (×3)" - assert_contains "$output" "FailedMount (×2)" - assert_contains "$output" "Unhealthy (×1)" + # Normal events are not summarized; only the top 3 Warning reasons appear, most frequent first. + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container started and crashed repeatedly. +📋 Detected: BackOff +📋 Recent warnings: + • BackOff (×3) + • FailedMount (×2) + • Unhealthy (×1) +💡 Suggested fix: Review application logs for startup errors (failed dependencies, bad config, panics). +EOF +) + assert_equal "$output" "$expected" # Normal events should not be summarized assert_not_contains "$output" "Pulled (×" } @@ -486,5 +652,14 @@ assert_not_contains() { run bash "$BATS_TEST_DIRNAME/../print_failed_deployment_hints" [ "$status" -eq 0 ] - assert_contains "$output" "📊 Progress at failure: 1/3 ready, 2/3 available" + local expected + expected=$(cat <<'EOF' + +📋 Reason: The container started and crashed repeatedly. +📋 Detected: CrashLoopBackOff on container app +📊 Progress at failure: 1/3 ready, 2/3 available +💡 Suggested fix: Review application logs for startup errors (failed dependencies, bad config, panics). +EOF +) + assert_equal "$output" "$expected" } diff --git a/k8s/deployment/tests/validate_alb_target_group_capacity.bats b/k8s/deployment/tests/validate_alb_target_group_capacity.bats index 3ecd2e89..9f0119d4 100644 --- a/k8s/deployment/tests/validate_alb_target_group_capacity.bats +++ b/k8s/deployment/tests/validate_alb_target_group_capacity.bats @@ -89,15 +89,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 98/98 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 98/98" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Too many services or deployments are attached to this ALB" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Remove unused deployments or services from the ALB" - assert_contains "$output" "Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100)" - assert_contains "$output" "Request an AWS service quota increase for target groups per ALB" - assert_contains "$output" "Consider using a separate ALB for additional deployments" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when over capacity" { @@ -117,8 +125,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 100/98 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 100/98" + assert_contains "$output" "$expected" } # ============================================================================= @@ -138,8 +161,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_TARGET_GROUPS from scope-configurations provider" { @@ -148,8 +186,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_TARGET_GROUPS from container-orchestration provider" { @@ -158,8 +211,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: scope-configurations takes priority over container-orchestration" { @@ -198,13 +266,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to find load balancer 'k8s-nullplatform-internet-facing' in region 'us-east-1' + +💡 Possible causes: + The load balancer may not exist or the agent lacks permissions + +🔧 How to fix: + • Verify the ALB exists: aws elbv2 describe-load-balancers --names k8s-nullplatform-internet-facing --region us-east-1 + • Check IAM permissions for elbv2:DescribeLoadBalancers +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to find load balancer 'k8s-nullplatform-internet-facing' in region 'us-east-1'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The load balancer may not exist or the agent lacks permissions" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Verify the ALB exists: aws elbv2 describe-load-balancers --names k8s-nullplatform-internet-facing --region us-east-1" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeLoadBalancers" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB ARN is None" { @@ -220,8 +296,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1' + +💡 Possible causes: + The load balancer name may be incorrect or it was deleted + +🔧 How to fix: + • List available ALBs: aws elbv2 describe-load-balancers --region us-east-1 + • Check the balancer name in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1'" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when describe-target-groups fails" { @@ -241,12 +330,20 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to describe target groups for ALB 'k8s-nullplatform-internet-facing' + +💡 Possible causes: + The agent may lack permissions to describe target groups + +🔧 How to fix: + • Check IAM permissions for elbv2:DescribeTargetGroups +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to describe target groups for ALB 'k8s-nullplatform-internet-facing'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The agent may lack permissions to describe target groups" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeTargetGroups" + assert_contains "$output" "$expected" } # ============================================================================= @@ -320,12 +417,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Unexpected non-numeric target group count from ALB +📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 +📋 Received value: WARNING: something unexpected + +💡 Possible causes: + The AWS CLI returned an unexpected response format + +🔧 How to fix: + • Verify AWS CLI version and credentials are correct + • Run manually: aws elbv2 describe-target-groups --load-balancer-arn arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 --region us-east-1 --query 'length(TargetGroups)' +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Unexpected non-numeric target group count from ALB" - assert_contains "$output" "📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123" - assert_contains "$output" "📋 Received value: WARNING: something unexpected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The AWS CLI returned an unexpected response format" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB_MAX_TARGET_GROUPS is non-numeric" { @@ -333,10 +441,17 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB_MAX_TARGET_GROUPS must be a numeric value, got: 'abc' + +🔧 How to fix: + • Set a numeric value in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB_MAX_TARGET_GROUPS must be a numeric value, got: 'abc'" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Set a numeric value in values.yaml or scope-configurations provider" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: empty ALB ARN response triggers error" { @@ -352,8 +467,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1' + +💡 Possible causes: + The load balancer name may be incorrect or it was deleted + +🔧 How to fix: + • List available ALBs: aws elbv2 describe-load-balancers --region us-east-1 + • Check the balancer name in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1'" + assert_contains "$output" "$expected" } # ============================================================================= @@ -384,7 +512,7 @@ teardown() { run bash -c 'source "$SCRIPT"' assert_equal "$status" "0" - assert_contains "$output" "DNS type is 'external_dns', ALB target group validation only applies to route53, skipping" + assert_contains "$output" "📋 DNS type is 'external_dns', ALB target group validation only applies to route53, skipping" } @test "validate_alb_target_group_capacity: runs when DNS_TYPE is route53" { @@ -428,14 +556,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 48/48 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 48/48" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Too many scopes with additional_ports are attached to this ALB" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Reduce additional_ports across scopes sharing this ALB" - assert_contains "$output" "Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50)" - assert_contains "$output" "Request an AWS service quota increase for listeners per ALB" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when listener count is over capacity" { @@ -459,8 +596,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 50/48 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 50/48" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: passes at exactly one below listener capacity" { @@ -528,8 +680,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_LISTENERS from scope-configurations provider" { @@ -538,8 +705,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_LISTENERS from container-orchestration provider" { @@ -548,8 +730,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when describe-listeners fails" { @@ -573,9 +770,20 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to describe listeners for ALB 'k8s-nullplatform-internet-facing' + +💡 Possible causes: + The agent may lack permissions to describe listeners + +🔧 How to fix: + • Check IAM permissions for elbv2:DescribeListeners +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to describe listeners for ALB 'k8s-nullplatform-internet-facing'" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeListeners" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when listener count is non-numeric" { @@ -599,9 +807,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Unexpected non-numeric listener count from ALB +📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 +📋 Received value: WARNING: unexpected + +💡 Possible causes: + The AWS CLI returned an unexpected response format + +🔧 How to fix: + • Verify AWS CLI version and credentials are correct + • Run manually: aws elbv2 describe-listeners --load-balancer-arn arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 --region us-east-1 --query 'length(Listeners)' +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Unexpected non-numeric listener count from ALB" - assert_contains "$output" "📋 Received value: WARNING: unexpected" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB_MAX_LISTENERS is non-numeric" { @@ -609,6 +831,15 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB_MAX_LISTENERS must be a numeric value, got: 'abc' + +🔧 How to fix: + • Set a numeric value in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB_MAX_LISTENERS must be a numeric value, got: 'abc'" + assert_contains "$output" "$expected" } diff --git a/k8s/deployment/tests/verify_ingress_reconciliation.bats b/k8s/deployment/tests/verify_ingress_reconciliation.bats index f78be30b..2ddca3cf 100644 --- a/k8s/deployment/tests/verify_ingress_reconciliation.bats +++ b/k8s/deployment/tests/verify_ingress_reconciliation.bats @@ -112,14 +112,19 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Certificate error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Ingress hostname does not match any SSL/TLS certificate in ACM" - assert_contains "$output" "- Certificate does not cover the hostname (check wildcards)" - assert_contains "$output" "- Message: no certificate found for host app.example.com" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify hostname matches certificate in ACM" - assert_contains "$output" "- Ensure certificate includes exact hostname or matching wildcard" + local expected + expected=$(cat <<'EOF' +❌ Certificate error detected +💡 Possible causes: + - Ingress hostname does not match any SSL/TLS certificate in ACM + - Certificate does not cover the hostname (check wildcards) + - Message: no certificate found for host app.example.com +🔧 How to fix: + - Verify hostname matches certificate in ACM + - Ensure certificate includes exact hostname or matching wildcard +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails with full troubleshooting when ingress not found" { @@ -142,12 +147,17 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Failed to get ingress k-8-s-my-app-scope-123-internet-facing" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Ingress does not exist yet" - assert_contains "$output" "- Namespace test-namespace is incorrect" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- List ingresses: kubectl get ingress -n test-namespace" + local expected + expected=$(cat <<'EOF' +❌ Failed to get ingress k-8-s-my-app-scope-123-internet-facing +💡 Possible causes: + - Ingress does not exist yet + - Namespace test-namespace is incorrect +🔧 How to fix: + - List ingresses: kubectl get ingress -n test-namespace +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails when ALB not found" { @@ -286,8 +296,8 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "Skipping weight check on listener port 50051" - assert_contains "$output" "✅ Weights match on listener port 443" + assert_contains "$output" "⏭️ Skipping weight check on listener port 50051 (blue has no service for this port)" + assert_contains "$output" "✅ Weights match on listener port 443 (GREEN: 10, BLUE: 90)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -326,7 +336,7 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "✅ Weights match on listener port 443" + assert_contains "$output" "✅ Weights match on listener port 443 (GREEN: 10, BLUE: 90)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -406,7 +416,7 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "📝 Checking domain: app.example.com" - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: passes when the rule forwards to a single target group" { @@ -440,7 +450,7 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "✅ Single target group on listener port 443" + assert_contains "$output" "✅ Single target group on listener port 443 (traffic is no longer split between deployments)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -490,8 +500,8 @@ teardown() { [ "$status" -eq 0 ] # Both messages must appear: the one that holds, then the one that releases. - assert_contains "$output" "still has 2 target groups on listener port 443" - assert_contains "$output" "✅ Single target group on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" + assert_contains "$output" "✅ Single target group on listener port 443 (traffic is no longer split between deployments)" assert_contains "$output" "✅ ALB configuration validated successfully" # More than one read proves it retried instead of getting a single lucky hit. [ "$(wc -l < "$counter")" -gt 1 ] @@ -531,7 +541,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: skips target group check on additional port listener when blue has no service" { @@ -573,8 +583,8 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Skipping target group check on listener port 50051" - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "⏭️ Skipping target group check on listener port 50051 (blue has no service for this port)" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: treats a rule without a forward action as not converged" { @@ -610,7 +620,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 0 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 0 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: a reconciled event does not override a failed ALB check" { @@ -655,7 +665,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" assert_contains "$output" "❌ Timeout waiting for ingress reconciliation after 1s" [[ "$output" != *"✅ Ingress successfully reconciled"* ]] } @@ -716,14 +726,19 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for ingress reconciliation after 1s" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- ALB Ingress Controller not running or unhealthy" - assert_contains "$output" "- Network connectivity issues" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check controller: kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller" - assert_contains "$output" "- Check ingress: kubectl describe ingress k-8-s-my-app-scope-123-internet-facing -n test-namespace" - assert_contains "$output" "📋 Recent events:" + local expected + expected=$(cat <<'EOF' +❌ Timeout waiting for ingress reconciliation after 1s +💡 Possible causes: + - ALB Ingress Controller not running or unhealthy + - Network connectivity issues +🔧 How to fix: + - Check controller: kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller + - Check ingress: kubectl describe ingress k-8-s-my-app-scope-123-internet-facing -n test-namespace +📋 Recent events: +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails on Error event type with error messages" { @@ -749,7 +764,12 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "🔍 Verifying ingress reconciliation..." assert_contains "$output" "📋 ALB reconciliation disabled, checking cluster events only" - assert_contains "$output" "❌ Ingress reconciliation failed" - assert_contains "$output" "💡 Error messages:" - assert_contains "$output" "- Failed to sync ALB" + local expected + expected=$(cat <<'EOF' +❌ Ingress reconciliation failed +💡 Error messages: + - Failed to sync ALB +EOF +) + assert_contains "$output" "$expected" } From 48849c01a04470573c6b214d2943539685a2a69c Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Mon, 10 Aug 2026 16:01:01 -0300 Subject: [PATCH 3/5] test: assert full messages and fix kubectl mock fidelity --- .../tests/delete_cluster_objects.bats | 111 ++++++++----- k8s/deployment/tests/kill_instance.bats | 140 +++++++++++----- .../verify_http_route_reconciliation.bats | 152 ++++++++++++------ .../tests/wait_deployment_active.bats | 117 +++++++++----- 4 files changed, 355 insertions(+), 165 deletions(-) diff --git a/k8s/deployment/tests/delete_cluster_objects.bats b/k8s/deployment/tests/delete_cluster_objects.bats index 086ff5ac..684d12b5 100644 --- a/k8s/deployment/tests/delete_cluster_objects.bats +++ b/k8s/deployment/tests/delete_cluster_objects.bats @@ -23,7 +23,9 @@ setup() { kubectl() { case "$1" in delete) - echo "kubectl delete $*" + # $* already starts with "delete" (that's $1), so prefix with just + # "kubectl" — echoing "kubectl delete $*" would double the verb. + echo "kubectl $*" echo "Deleted resources" return 0 ;; @@ -52,20 +54,21 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../delete_cluster_objects" [ "$status" -eq 0 ] - # Start message - assert_contains "$output" "🔍 Starting cluster objects cleanup..." - # Strategy message - assert_contains "$output" "📋 Strategy: Deleting blue (old) deployment, keeping green (new)" - # Debug info - assert_contains "$output" "📋 Deployment to clean: deploy-old | Deployment to keep: deploy-new" - # Delete action - assert_contains "$output" "📝 Deleting resources for deployment_id=deploy-old..." - assert_contains "$output" "✅ Resources deleted for deployment_id=deploy-old" - # Verification - assert_contains "$output" "🔍 Verifying cleanup for scope_id=scope-123 in namespace=test-namespace..." - # Summary - assert_contains "$output" "✨ Cluster cleanup completed successfully" - assert_contains "$output" "📋 Only deployment_id=deploy-new remains for scope_id=scope-123" + local expected + expected=$(cat <<'EOF' +🔍 Starting cluster objects cleanup... +📋 Strategy: Deleting blue (old) deployment, keeping green (new) +📋 Deployment to clean: deploy-old | Deployment to keep: deploy-new +📝 Deleting resources for deployment_id=deploy-old... +kubectl delete deployment,service,hpa,ingress,pdb,secret,configmap -l deployment_id=deploy-old -n test-namespace --cascade=foreground --wait=true +Deleted resources +✅ Resources deleted for deployment_id=deploy-old +🔍 Verifying cleanup for scope_id=scope-123 in namespace=test-namespace... +✨ Cluster cleanup completed successfully +📋 Only deployment_id=deploy-new remains for scope_id=scope-123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -77,15 +80,21 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../delete_cluster_objects" [ "$status" -eq 0 ] - # Strategy message - assert_contains "$output" "📋 Strategy: Deleting green (new) deployment, keeping blue (old)" - # Debug info - assert_contains "$output" "📋 Deployment to clean: deploy-new | Deployment to keep: deploy-old" - # Delete action - assert_contains "$output" "📝 Deleting resources for deployment_id=deploy-new..." - assert_contains "$output" "✅ Resources deleted for deployment_id=deploy-new" - # Summary - assert_contains "$output" "📋 Only deployment_id=deploy-old remains for scope_id=scope-123" + local expected + expected=$(cat <<'EOF' +🔍 Starting cluster objects cleanup... +📋 Strategy: Deleting green (new) deployment, keeping blue (old) +📋 Deployment to clean: deploy-new | Deployment to keep: deploy-old +📝 Deleting resources for deployment_id=deploy-new... +kubectl delete deployment,service,hpa,ingress,pdb,secret,configmap -l deployment_id=deploy-new -n test-namespace --cascade=foreground --wait=true +Deleted resources +✅ Resources deleted for deployment_id=deploy-new +🔍 Verifying cleanup for scope_id=scope-123 in namespace=test-namespace... +✨ Cluster cleanup completed successfully +📋 Only deployment_id=deploy-old remains for scope_id=scope-123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -122,15 +131,24 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../delete_cluster_objects" [ "$status" -ne 0 ] - assert_contains "$output" "❌ Failed to delete resources for deployment_id=deploy-old" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Resources may have finalizers preventing deletion" - assert_contains "$output" "Network connectivity issues with Kubernetes API" - assert_contains "$output" "Insufficient permissions to delete resources" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Check for stuck finalizers" - assert_contains "$output" "Verify kubeconfig and cluster connectivity" - assert_contains "$output" "Check RBAC permissions for the service account" + local expected + expected=$(cat <<'EOF' +🔍 Starting cluster objects cleanup... +📋 Strategy: Deleting blue (old) deployment, keeping green (new) +📋 Deployment to clean: deploy-old | Deployment to keep: deploy-new +📝 Deleting resources for deployment_id=deploy-old... +❌ Failed to delete resources for deployment_id=deploy-old +💡 Possible causes: + - Resources may have finalizers preventing deletion + - Network connectivity issues with Kubernetes API + - Insufficient permissions to delete resources +🔧 How to fix: + - Check for stuck finalizers: kubectl get all -l deployment_id=deploy-old -n test-namespace -o yaml | grep finalizers + - Verify kubeconfig and cluster connectivity + - Check RBAC permissions for the service account +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -140,7 +158,9 @@ teardown() { kubectl() { case "$1" in delete) - echo "kubectl delete $*" + # $* already starts with "delete" (that's $1), so prefix with just + # "kubectl" — echoing "kubectl delete $*" would double the verb. + echo "kubectl $*" echo "Deleted resources" return 0 ;; @@ -158,7 +178,26 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../delete_cluster_objects" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Found orphaned deployment: deploy-orphan" - assert_contains "$output" "✅ Cleaned up 1 orphaned deployment(s)" + local expected + expected=$(cat <<'EOF' +🔍 Starting cluster objects cleanup... +📋 Strategy: Deleting blue (old) deployment, keeping green (new) +📋 Deployment to clean: deploy-old | Deployment to keep: deploy-new +📝 Deleting resources for deployment_id=deploy-old... +kubectl delete deployment,service,hpa,ingress,pdb,secret,configmap -l deployment_id=deploy-old -n test-namespace --cascade=foreground --wait=true +Deleted resources +✅ Resources deleted for deployment_id=deploy-old +🔍 Verifying cleanup for scope_id=scope-123 in namespace=test-namespace... +📝 Found orphaned deployment: deploy-orphan +📝 Deleting resources for deployment_id=deploy-orphan... +kubectl delete deployment,service,hpa,ingress,pdb,secret,configmap -l deployment_id=deploy-orphan -n test-namespace --cascade=foreground --wait=true +Deleted resources +✅ Resources deleted for deployment_id=deploy-orphan +✅ Cleaned up 1 orphaned deployment(s) +✨ Cluster cleanup completed successfully +📋 Only deployment_id=deploy-new remains for scope_id=scope-123 +EOF +) + assert_equal "$output" "$expected" } diff --git a/k8s/deployment/tests/kill_instance.bats b/k8s/deployment/tests/kill_instance.bats index d0699f42..bbca2ccd 100644 --- a/k8s/deployment/tests/kill_instance.bats +++ b/k8s/deployment/tests/kill_instance.bats @@ -44,8 +44,17 @@ setup() { elif [[ "$*" == *"ownerReferences"* ]]; then echo "my-replicaset-abc" fi + return 0 fi - return 0 + # Bare existence check (no -o jsonpath): the script calls this + # with identical args once before the delete (pod must exist) + # and once after (pod must be gone, since delete+wait succeed + # below). Count calls so the second one reflects a real kubectl + # after a successful deletion instead of pretending the pod is + # still there. + KUBECTL_GET_POD_CALLS=$((${KUBECTL_GET_POD_CALLS:-0} + 1)) + [ "$KUBECTL_GET_POD_CALLS" -eq 1 ] + return $? ;; replicaset) echo "d-scope-123-deploy-456" @@ -88,23 +97,28 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] - # Start message - assert_contains "$output" "🔍 Starting instance kill operation..." - # Parameter display - assert_contains "$output" "📋 Deployment ID: deploy-456" - assert_contains "$output" "📋 Instance name: my-pod-abc123" - assert_contains "$output" "📋 Scope ID: scope-123" - assert_contains "$output" "📋 Namespace: test-namespace" - # Pod verification - assert_contains "$output" "🔍 Verifying pod exists..." - assert_contains "$output" "📋 Fetching pod details..." - # Delete operation - assert_contains "$output" "📝 Deleting pod my-pod-abc123 with 30s grace period..." - assert_contains "$output" "📝 Waiting for pod termination..." - # Deployment status - assert_contains "$output" "📋 Checking deployment status after pod deletion..." - # Completion - assert_contains "$output" "✨ Instance kill operation completed for my-pod-abc123" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +📋 Deployment ID: deploy-456 +📋 Instance name: my-pod-abc123 +📋 Scope ID: scope-123 +📋 Namespace: test-namespace +🔍 Verifying pod exists... +📋 Fetching pod details... +📋 Pod: my-pod-abc123 | Status: Running | Node: node-1 | Started: 2024-01-01T00:00:00Z +📋 Pod ownership: ReplicaSet=my-replicaset-abc -> Deployment=d-scope-123-deploy-456 +📝 Deleting pod my-pod-abc123 with 30s grace period... +pod deleted +📝 Waiting for pod termination... +✅ Pod successfully terminated and removed +📋 Checking deployment status after pod deletion... +📋 Deployment d-scope-123-deploy-456: desired=3, ready=2, available=2 +📋 Kubernetes will automatically create a replacement pod +✨ Instance kill operation completed for my-pod-abc123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -120,11 +134,18 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ deployment_id parameter not found" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Parameter not provided in action request" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Ensure deployment_id is passed in the action parameters" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +❌ deployment_id parameter not found +💡 Possible causes: + - Parameter not provided in action request + - Context structure is different than expected +🔧 How to fix: + - Ensure deployment_id is passed in the action parameters +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when instance_id missing" { @@ -137,11 +158,18 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ instance_id parameter not found" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Parameter not provided in action request" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Ensure instance_id is passed in the action parameters" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +❌ instance_id parameter not found +💡 Possible causes: + - Parameter not provided in action request + - Context structure is different than expected +🔧 How to fix: + - Ensure instance_id is passed in the action parameters +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when scope_id missing" { @@ -155,11 +183,20 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ scope_id not found in context" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Context missing scope information" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Verify the action is invoked with proper scope context" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +📋 Deployment ID: deploy-456 +📋 Instance name: my-pod-abc123 +❌ scope_id not found in context +💡 Possible causes: + - Context missing scope information + - Action invoked outside of scope context +🔧 How to fix: + - Verify the action is invoked with proper scope context +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when pod not found" { @@ -178,11 +215,24 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Pod was already terminated" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "kubectl get pods" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +📋 Deployment ID: deploy-456 +📋 Instance name: my-pod-abc123 +📋 Scope ID: scope-123 +📋 Namespace: test-namespace +🔍 Verifying pod exists... +❌ Pod my-pod-abc123 not found in namespace test-namespace +💡 Possible causes: + - Pod was already terminated + - Pod name is incorrect + - Pod exists in a different namespace +🔧 How to fix: + - List pods: kubectl get pods -n test-namespace -l scope_id=scope-123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -204,8 +254,16 @@ teardown() { elif [[ "$*" == *"ownerReferences"* ]]; then echo "my-replicaset-abc" fi + return 0 fi - return 0 + # Bare existence check (no -o jsonpath): called once before the + # delete (pod must exist) and once after (pod must be gone, + # since delete+wait both succeed below). Count calls so the + # second one reflects a real kubectl after a successful + # deletion instead of pretending the pod is still there. + KUBECTL_GET_POD_CALLS=$((${KUBECTL_GET_POD_CALLS:-0} + 1)) + [ "$KUBECTL_GET_POD_CALLS" -eq 1 ] + return $? ;; replicaset) echo "d-scope-123-different-deploy" # Different deployment @@ -233,7 +291,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] - assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456" + assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456 (continuing anyway)" } @test "kill_instance: warns when pod still exists after deletion" { @@ -283,5 +341,5 @@ teardown() { [ "$status" -eq 0 ] assert_contains "$output" "⚠️ Pod deletion timeout reached" - assert_contains "$output" "⚠️ Pod still exists after deletion attempt" + assert_contains "$output" "⚠️ Pod still exists after deletion attempt (status: Terminating)" } diff --git a/k8s/deployment/tests/verify_http_route_reconciliation.bats b/k8s/deployment/tests/verify_http_route_reconciliation.bats index 6ed938d8..105c2d9a 100644 --- a/k8s/deployment/tests/verify_http_route_reconciliation.bats +++ b/k8s/deployment/tests/verify_http_route_reconciliation.bats @@ -45,9 +45,14 @@ run_with_mock() { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Route accepted"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s" - assert_contains "$output" "✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True)" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True) +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -57,63 +62,91 @@ run_with_mock() { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"False","reason":"CertificateError","message":"TLS secret not found"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ Certificate/TLS error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- TLS secret does not exist in namespace test-namespace" - assert_contains "$output" "- Certificate is invalid or expired" - assert_contains "$output" "- Gateway references incorrect certificate secret" - assert_contains "$output" "- Accepted: CertificateError - TLS secret not found" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify TLS secret: kubectl get secret -n test-namespace | grep tls" - assert_contains "$output" "- Check certificate validity" - assert_contains "$output" "- Ensure Gateway references the correct secret" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ Certificate/TLS error detected +💡 Possible causes: + - TLS secret does not exist in namespace test-namespace + - Certificate is invalid or expired + - Gateway references incorrect certificate secret + - Accepted: CertificateError - TLS secret not found +🔧 How to fix: + - Verify TLS secret: kubectl get secret -n test-namespace | grep tls + - Check certificate validity + - Ensure Gateway references the correct secret +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting on backend error" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Accepted"},{"type":"ResolvedRefs","status":"False","reason":"BackendNotFound","message":"service my-svc not found"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ Backend service error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Referenced service does not exist" - assert_contains "$output" "- Service name is misspelled in HTTPRoute" - assert_contains "$output" "- Message: service my-svc not found" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- List services: kubectl get svc -n test-namespace" - assert_contains "$output" "- Verify backend service name in HTTPRoute" - assert_contains "$output" "- Ensure service has ready endpoints" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ Backend service error detected +💡 Possible causes: + - Referenced service does not exist + - Service name is misspelled in HTTPRoute + - Message: service my-svc not found +🔧 How to fix: + - List services: kubectl get svc -n test-namespace + - Verify backend service name in HTTPRoute + - Ensure service has ready endpoints +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting when not accepted" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"False","reason":"NotAccepted","message":"Gateway not found"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ HTTPRoute not accepted by Gateway" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Reason: NotAccepted" - assert_contains "$output" "- Message: Gateway not found" - assert_contains "$output" "📋 All conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check Gateway configuration" - assert_contains "$output" "- Verify HTTPRoute spec matches Gateway requirements" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ HTTPRoute not accepted by Gateway +💡 Possible causes: + - Reason: NotAccepted + - Message: Gateway not found +📋 All conditions: + - Accepted: False (NotAccepted) - Gateway not found + - ResolvedRefs: True (ResolvedRefs) - Refs resolved +🔧 How to fix: + - Check Gateway configuration + - Verify HTTPRoute spec matches Gateway requirements +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting when refs not resolved" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Accepted"},{"type":"ResolvedRefs","status":"False","reason":"InvalidBackend","message":"Invalid backend port"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ HTTPRoute references could not be resolved" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Reason: InvalidBackend" - assert_contains "$output" "- Message: Invalid backend port" - assert_contains "$output" "📋 All conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify all referenced services exist" - assert_contains "$output" "- Check backend service ports match" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ HTTPRoute references could not be resolved +💡 Possible causes: + - Reason: InvalidBackend + - Message: Invalid backend port +📋 All conditions: + - Accepted: True (Accepted) - Accepted + - ResolvedRefs: False (InvalidBackend) - Invalid backend port +🔧 How to fix: + - Verify all referenced services exist + - Check backend service ports match +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting on timeout" { @@ -127,13 +160,30 @@ run_with_mock() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for HTTPRoute reconciliation after 1s" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Gateway controller is not running" - assert_contains "$output" "- Network policies blocking reconciliation" - assert_contains "$output" "- Resource constraints on controller" - assert_contains "$output" "📋 Current conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check Gateway controller logs" - assert_contains "$output" "- Verify Gateway and Istio configuration" + # Everything up to the dynamic "Current conditions" dump: the mock's empty + # parents list makes the jq read on it fail, so its tool-error text (not a + # product message) is deliberately excluded here. + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +📝 HTTPRoute pending sync (no parent status yet)... (0s/1s) +❌ Timeout waiting for HTTPRoute reconciliation after 1s +💡 Possible causes: + - Gateway controller is not running + - Network policies blocking reconciliation + - Resource constraints on controller +📋 Current conditions: +EOF +) + assert_contains "$output" "$expected" + + local expected_fix + expected_fix=$(cat <<'EOF' +🔧 How to fix: + - Check Gateway controller logs + - Verify Gateway and Istio configuration +EOF +) + assert_contains "$output" "$expected_fix" } diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index 52c83d39..2d9a8ffa 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -66,9 +66,14 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📋 Namespace: test-namespace" - assert_contains "$output" "📋 Timeout: 30s (max 3 iterations)" + local expected_intro + expected_intro=$(cat <<'EOF' +🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active... +📋 Namespace: test-namespace +📋 Timeout: 30s (max 3 iterations) +EOF +) + assert_contains "$output" "$expected_intro" assert_contains "$output" "📡 Checking deployment status (attempt 1/3)..." assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!" } @@ -109,11 +114,23 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📋 Namespace: test-namespace" - assert_contains "$output" "📋 Timeout: 5s (max 0 iterations)" - assert_contains "$output" "❌ Timeout waiting for deployment" - assert_contains "$output" "📋 Maximum iterations (0) reached" + local expected_intro + expected_intro=$(cat <<'EOF' +🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active... +📋 Namespace: test-namespace +📋 Timeout: 5s (max 0 iterations) +EOF +) + assert_contains "$output" "$expected_intro" + + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached +EOF +) + assert_contains "$output" "$expected_timeout" # Timeout path must source print_failed_deployment_hints; with no pod info # and no events, it falls through to the generic checklist. assert_contains "$output" "⚠️ Application Startup Issue Detected" @@ -145,11 +162,21 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for deployment" - # The hint script must read pod state and surface the user-friendly reason - assert_contains "$output" "📋 Reason: The container exceeded its memory limit" - assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" - assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'" + # The hint script must read pod state and surface the user-friendly reason, + # contiguous with the timeout header that precedes it. + local expected + expected=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached + +📋 Reason: The container exceeded its memory limit (512Mi) and was terminated. +📋 Detected: OOMKilled on container app (exit 137) +📋 Details: out of memory +💡 Suggested fix: Increase ram_memory for scope 'my-app' or reduce application memory usage. +EOF +) + assert_contains "$output" "$expected" } # ============================================================================= @@ -166,9 +193,14 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📡 Checking deployment status (attempt 1/" - assert_contains "$output" "❌ Failed to read deployment status" - assert_contains "$output" "📋 NP CLI error:" + assert_contains "$output" "📡 Checking deployment status (attempt 1/3)..." + local expected + expected=$(cat <<'EOF' + ❌ Failed to read deployment status +📋 NP CLI error: Error connecting to API +EOF +) + assert_contains "$output" "$expected" } @test "wait_deployment_active: fails when deployment status is null" { @@ -180,7 +212,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment status not found for ID deploy-456" + assert_contains "$output" " ❌ Deployment status not found for ID deploy-456" } @test "wait_deployment_active: fails when NP deployment status is not running" { @@ -194,7 +226,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment is no longer running (status: failed)" + assert_contains "$output" " ❌ Deployment is no longer running (status: failed)" # Non-running status path must also source print_failed_deployment_hints assert_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -215,7 +247,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment 'd-scope-123-deploy-456' not found in namespace 'test-namespace'" + assert_contains "$output" " ❌ Deployment 'd-scope-123-deploy-456' not found in namespace 'test-namespace'" } # ============================================================================= @@ -258,9 +290,17 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Deployment status - Available: 3/5, Updated: 4/5, Ready: 3/5" + # Leading "🔍 $(date): " is non-deterministic; assert everything after it verbatim. + assert_contains "$output" "Iteration 1 - Deployment status - Available: 3/5, Updated: 4/5, Ready: 3/5" assert_contains "$output" "⏳ Still waiting — Ready: 3/5, Available: 3/5 (attempt 1/1, 10s elapsed)" - assert_contains "$output" "❌ Timeout waiting for deployment" + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (1) reached +EOF +) + assert_contains "$output" "$expected_timeout" } @test "wait_deployment_active: handles missing status fields defaults to 0" { @@ -296,7 +336,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Available: 0/3" + assert_contains "$output" "Iteration 1 - Deployment status - Available: 0/3, Updated: 0/3, Ready: 0/3" } # ============================================================================= @@ -332,7 +372,14 @@ teardown() { # Should timeout because desired > 0 check fails [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for deployment" + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached +EOF +) + assert_contains "$output" "$expected_timeout" } # ============================================================================= @@ -505,10 +552,8 @@ teardown() { " [ "$status" -eq 1 ] - # Translated form must appear - assert_contains "$output" "Startup probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/health" + # Translated form must appear as the complete consolidated warning line + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc Startup probe failing on /health — not yet listening" # Raw connection-refused text must NOT leak through if [[ "$output" == *"connection refused"* ]]; then echo "Expected output to NOT contain raw 'connection refused' (should be translated)" @@ -554,10 +599,9 @@ teardown() { " [ "$status" -eq 1 ] - # One consolidated line with both modes joined by ', ' for natural reading - assert_contains "$output" "Startup probe failing on /health — not yet listening, responded HTTP 502 (expected 2xx)" - # Pod name must be the short form with '...' prefix marking truncation - assert_contains "$output" "Pod/...abc-hhshq" + # One consolidated line with both modes joined by ', ' for natural reading, + # using the short pod name ('...' prefix marks truncation). + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc-hhshq Startup probe failing on /health — not yet listening, responded HTTP 502 (expected 2xx)" # The long prefix must NOT appear in any logged event line if [[ "$output" == *"Pod/d-scope-123-deploy-456-abc-hhshq"* ]]; then echo "Expected output to use short pod name, not the full prefix" @@ -610,9 +654,9 @@ teardown() { " [ "$status" -eq 1 ] - # Both original messages must appear verbatim - assert_contains "$output" "some brand-new K8s probe format we cannot parse 1" - assert_contains "$output" "another unknown probe format 2" + # Both original messages must appear verbatim, each as its own complete raw warning line + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/d-scope-123-deploy-456-abc-hhshq: Unhealthy - some brand-new K8s probe format we cannot parse 1" + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/d-scope-123-deploy-456-abc-hhshq: Unhealthy - another unknown probe format 2" # The consolidated header must NOT appear because parsing failed if [[ "$output" == *"probe failing"* ]]; then echo "Expected fallback path to NOT emit the 'probe failing' header" @@ -652,8 +696,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "HTTP 502" + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc Startup probe failing — responded HTTP 502 (expected 2xx)" } # ============================================================================= @@ -854,6 +897,6 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 0 ] - assert_contains "$output" "Could not report instance counts" + assert_contains "$output" "Could not report instance counts (will retry on next change)" assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!" } From 0eb43a38d7ad2cac30db9f3f0930e1220367ae15 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Mon, 10 Aug 2026 16:12:23 -0300 Subject: [PATCH 4/5] test: assert full messages in build and gateway tests --- k8s/deployment/tests/build_context.bats | 62 +++++++++++-------- k8s/deployment/tests/build_deployment.bats | 59 +++++++++++------- .../gateway/ingress/route_traffic.bats | 59 ++++++++++++------ .../networking/gateway/rollback_traffic.bats | 48 ++++++++++---- .../networking/gateway/route_traffic.bats | 44 ++++++++----- 5 files changed, 178 insertions(+), 94 deletions(-) diff --git a/k8s/deployment/tests/build_context.bats b/k8s/deployment/tests/build_context.bats index 09605911..02a4d24d 100644 --- a/k8s/deployment/tests/build_context.bats +++ b/k8s/deployment/tests/build_context.bats @@ -33,7 +33,7 @@ teardown() { @test "validate_status: accepts valid statuses for start-initial and start-blue-green" { run validate_status "start-initial" "creating" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'start-initial' (current status: 'creating', expected: creating, waiting_for_instances or running)" + assert_equal "$output" "📝 Running action 'start-initial' (current status: 'creating', expected: creating, waiting_for_instances or running)" run validate_status "start-initial" "waiting_for_instances" [ "$status" -eq 0 ] @@ -43,7 +43,7 @@ teardown() { run validate_status "start-blue-green" "creating" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'start-blue-green' (current status: 'creating', expected: creating, waiting_for_instances or running)" + assert_equal "$output" "📝 Running action 'start-blue-green' (current status: 'creating', expected: creating, waiting_for_instances or running)" } @test "validate_status: rejects invalid statuses for start-initial" { @@ -57,7 +57,7 @@ teardown() { @test "validate_status: accepts valid statuses for switch-traffic" { run validate_status "switch-traffic" "running" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'switch-traffic' (current status: 'running', expected: running or waiting_for_instances)" + assert_equal "$output" "📝 Running action 'switch-traffic' (current status: 'running', expected: running or waiting_for_instances)" run validate_status "switch-traffic" "waiting_for_instances" [ "$status" -eq 0 ] @@ -71,7 +71,7 @@ teardown() { @test "validate_status: accepts valid statuses for rollback-deployment" { run validate_status "rollback-deployment" "rolling_back" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'rollback-deployment' (current status: 'rolling_back', expected: rolling_back or cancelling)" + assert_equal "$output" "📝 Running action 'rollback-deployment' (current status: 'rolling_back', expected: rolling_back or cancelling)" run validate_status "rollback-deployment" "cancelling" [ "$status" -eq 0 ] @@ -98,7 +98,7 @@ teardown() { @test "validate_status: accepts valid statuses for delete-deployment" { run validate_status "delete-deployment" "deleting" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'delete-deployment' (current status: 'deleting', expected: deleting, rolling_back or cancelling)" + assert_equal "$output" "📝 Running action 'delete-deployment' (current status: 'deleting', expected: deleting, rolling_back or cancelling)" run validate_status "delete-deployment" "cancelling" [ "$status" -eq 0 ] @@ -115,11 +115,11 @@ teardown() { @test "validate_status: accepts any status for unknown or empty action" { run validate_status "custom-action" "any_status" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action 'custom-action', any deployment status is accepted" + assert_equal "$output" "📝 Running action 'custom-action', any deployment status is accepted" run validate_status "" "running" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Running action '', any deployment status is accepted" + assert_equal "$output" "📝 Running action '', any deployment status is accepted" } # ============================================================================= @@ -560,15 +560,21 @@ SCRIPT run "$test_script" "$mock_service" [ "$status" -ne 0 ] - assert_contains "$output" "❌ Invalid deployment status 'failed' for action 'start-initial'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Deployment status changed during workflow execution" - assert_contains "$output" "Another action is already running on this deployment" - assert_contains "$output" "Deployment was modified externally" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Wait for any in-progress actions to complete" - assert_contains "$output" "Check the deployment status in the nullplatform dashboard" - assert_contains "$output" "Retry the action once the deployment is in the expected state" + local expected + expected=$(cat <<'EOF' +📝 Running action 'start-initial' (current status: 'failed', expected: creating, waiting_for_instances or running) +❌ Invalid deployment status 'failed' for action 'start-initial' +💡 Possible causes: + - Deployment status changed during workflow execution + - Another action is already running on this deployment + - Deployment was modified externally +🔧 How to fix: + - Wait for any in-progress actions to complete + - Check the deployment status in the nullplatform dashboard + - Retry the action once the deployment is in the expected state +EOF +) + assert_equal "$output" "$expected" } @test "error: ConfigMap not found shows full troubleshooting info" { @@ -608,15 +614,21 @@ SCRIPT run "$test_script" "$mock_service" [ "$status" -ne 0 ] - assert_contains "$output" "🔍 Validating ConfigMap 'test-config' in namespace 'test-ns'" - assert_contains "$output" "❌ ConfigMap 'test-config' does not exist in namespace 'test-ns'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "ConfigMap was not created before deployment" - assert_contains "$output" "ConfigMap name is misspelled in values.yaml" - assert_contains "$output" "ConfigMap was deleted or exists in a different namespace" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Create the ConfigMap: kubectl create configmap test-config -n test-ns --from-file=nginx.conf --from-file=default.conf" - assert_contains "$output" "Verify the ConfigMap name in your scope configuration" + local expected + expected=$(cat <<'EOF' +📝 Running action 'start-initial' (current status: 'creating', expected: creating, waiting_for_instances or running) +🔍 Validating ConfigMap 'test-config' in namespace 'test-ns' +❌ ConfigMap 'test-config' does not exist in namespace 'test-ns' +💡 Possible causes: + - ConfigMap was not created before deployment + - ConfigMap name is misspelled in values.yaml + - ConfigMap was deleted or exists in a different namespace +🔧 How to fix: + - Create the ConfigMap: kubectl create configmap test-config -n test-ns --from-file=nginx.conf --from-file=default.conf + - Verify the ConfigMap name in your scope configuration +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= diff --git a/k8s/deployment/tests/build_deployment.bats b/k8s/deployment/tests/build_deployment.bats index 086ccfb8..cfbd623a 100644 --- a/k8s/deployment/tests/build_deployment.bats +++ b/k8s/deployment/tests/build_deployment.bats @@ -53,27 +53,23 @@ teardown() { [ "$status" -eq 0 ] - # Header messages - assert_contains "$output" "📝 Building deployment templates..." - assert_contains "$output" "📋 Output directory:" - - # Deployment template - assert_contains "$output" "✅ Deployment template:" - - # Secret template - assert_contains "$output" "✅ Secret template:" - - # Scaling template - assert_contains "$output" "✅ Scaling template:" - - # Service template - assert_contains "$output" "✅ Service template:" - - # PDB template - assert_contains "$output" "✅ PDB template:" - - # Summary - assert_contains "$output" "✨ All templates built successfully" + local expected + expected=$(cat < Date: Mon, 10 Aug 2026 16:24:16 -0300 Subject: [PATCH 5/5] test: assert full messages in remaining deployment tests --- k8s/deployment/tests/apply_templates.bats | 110 ++++++++++++++++-- .../tests/delete_ingress_finalizer.bats | 22 +++- .../tests/notify_active_domains.bats | 67 ++++++++--- k8s/deployment/tests/publish_alb_metrics.bats | 26 ++--- k8s/deployment/tests/scale_deployments.bats | 26 ++++- .../verify_networking_reconciliation.bats | 33 ++++-- 6 files changed, 227 insertions(+), 57 deletions(-) diff --git a/k8s/deployment/tests/apply_templates.bats b/k8s/deployment/tests/apply_templates.bats index 610175d6..22feca5c 100644 --- a/k8s/deployment/tests/apply_templates.bats +++ b/k8s/deployment/tests/apply_templates.bats @@ -49,10 +49,18 @@ teardown() { run bash "$SERVICE_PATH/apply_templates" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Applying templates..." - assert_contains "$output" "📋 Directory:" - assert_contains "$output" "📋 Action: apply" - assert_contains "$output" "📋 Dry run: false" + local expected + expected=$(cat < 5 replicas +📋 Blue deployment: d-scope-123-deploy-old -> 3 replicas + +📝 Scaling green deployment... + ❌ Failed to scale green deployment +EOF +) + assert_equal "$output" "$expected" } @test "scale_deployments: fails when blue deployment scale fails" { @@ -184,7 +194,19 @@ run_scale_deployments() { source '$PROJECT_ROOT/k8s/deployment/scale_deployments'" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Failed to scale blue deployment" + local expected + expected=$(cat <<'EOF' +📝 Scaling deployments for rolling strategy... +📋 Green deployment: d-scope-123-deploy-new -> 5 replicas +📋 Blue deployment: d-scope-123-deploy-old -> 3 replicas + +📝 Scaling green deployment... + ✅ Green deployment scaled to 5 replicas +📝 Scaling blue deployment... + ❌ Failed to scale blue deployment +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= diff --git a/k8s/deployment/tests/verify_networking_reconciliation.bats b/k8s/deployment/tests/verify_networking_reconciliation.bats index 424a0e10..c5cc696b 100644 --- a/k8s/deployment/tests/verify_networking_reconciliation.bats +++ b/k8s/deployment/tests/verify_networking_reconciliation.bats @@ -39,9 +39,15 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: route53" - assert_contains "$output" "🔍 Verifying ingress reconciliation..." - assert_contains "$output" "⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation)" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: route53 +🔍 Verifying ingress reconciliation... +📋 Ingress: k-8-s-my-app-- | Namespace: | Timeout: 120s +⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation) +EOF +) + assert_equal "$output" "$expected" } @test "verify_networking_reconciliation: verifies HTTPRoute for external_dns without managing DNS" { @@ -65,9 +71,15 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: external_dns" - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "✅ HTTPRoute successfully reconciled" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: external_dns +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-123-public | Namespace: nullplatform | Timeout: 10s +✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True) +EOF +) + assert_equal "$output" "$expected" } @test "verify_networking_reconciliation: skips for unsupported DNS types" { @@ -77,6 +89,11 @@ teardown() { [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: unknown" - assert_contains "$output" "⚠️ Ingress reconciliation not available for DNS type: unknown, skipping" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: unknown +⚠️ Ingress reconciliation not available for DNS type: unknown, skipping +EOF +) + assert_equal "$output" "$expected" }