diff --git a/k8s/deployment/tests/apply_templates.bats b/k8s/deployment/tests/apply_templates.bats index 610175d6..22feca5c 100644 --- a/k8s/deployment/tests/apply_templates.bats +++ b/k8s/deployment/tests/apply_templates.bats @@ -49,10 +49,18 @@ teardown() { run bash "$SERVICE_PATH/apply_templates" [ "$status" -eq 0 ] - assert_contains "$output" "📝 Applying templates..." - assert_contains "$output" "📋 Directory:" - assert_contains "$output" "📋 Action: apply" - assert_contains "$output" "📋 Dry run: false" + local expected + expected=$(cat < Deployment=d-scope-123-deploy-456 +📝 Deleting pod my-pod-abc123 with 30s grace period... +pod deleted +📝 Waiting for pod termination... +✅ Pod successfully terminated and removed +📋 Checking deployment status after pod deletion... +📋 Deployment d-scope-123-deploy-456: desired=3, ready=2, available=2 +📋 Kubernetes will automatically create a replacement pod +✨ Instance kill operation completed for my-pod-abc123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -120,11 +134,18 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ deployment_id parameter not found" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Parameter not provided in action request" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Ensure deployment_id is passed in the action parameters" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +❌ deployment_id parameter not found +💡 Possible causes: + - Parameter not provided in action request + - Context structure is different than expected +🔧 How to fix: + - Ensure deployment_id is passed in the action parameters +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when instance_id missing" { @@ -137,11 +158,18 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ instance_id parameter not found" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Parameter not provided in action request" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Ensure instance_id is passed in the action parameters" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +❌ instance_id parameter not found +💡 Possible causes: + - Parameter not provided in action request + - Context structure is different than expected +🔧 How to fix: + - Ensure instance_id is passed in the action parameters +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when scope_id missing" { @@ -155,11 +183,20 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ scope_id not found in context" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Context missing scope information" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Verify the action is invoked with proper scope context" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +📋 Deployment ID: deploy-456 +📋 Instance name: my-pod-abc123 +❌ scope_id not found in context +💡 Possible causes: + - Context missing scope information + - Action invoked outside of scope context +🔧 How to fix: + - Verify the action is invoked with proper scope context +EOF +) + assert_equal "$output" "$expected" } @test "kill_instance: fails with troubleshooting when pod not found" { @@ -178,11 +215,24 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Pod was already terminated" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "kubectl get pods" + local expected + expected=$(cat <<'EOF' +🔍 Starting instance kill operation... +📋 Deployment ID: deploy-456 +📋 Instance name: my-pod-abc123 +📋 Scope ID: scope-123 +📋 Namespace: test-namespace +🔍 Verifying pod exists... +❌ Pod my-pod-abc123 not found in namespace test-namespace +💡 Possible causes: + - Pod was already terminated + - Pod name is incorrect + - Pod exists in a different namespace +🔧 How to fix: + - List pods: kubectl get pods -n test-namespace -l scope_id=scope-123 +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -204,8 +254,16 @@ teardown() { elif [[ "$*" == *"ownerReferences"* ]]; then echo "my-replicaset-abc" fi + return 0 fi - return 0 + # Bare existence check (no -o jsonpath): called once before the + # delete (pod must exist) and once after (pod must be gone, + # since delete+wait both succeed below). Count calls so the + # second one reflects a real kubectl after a successful + # deletion instead of pretending the pod is still there. + KUBECTL_GET_POD_CALLS=$((${KUBECTL_GET_POD_CALLS:-0} + 1)) + [ "$KUBECTL_GET_POD_CALLS" -eq 1 ] + return $? ;; replicaset) echo "d-scope-123-different-deploy" # Different deployment @@ -233,7 +291,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] - assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456" + assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456 (continuing anyway)" } @test "kill_instance: warns when pod still exists after deletion" { @@ -283,5 +341,5 @@ teardown() { [ "$status" -eq 0 ] assert_contains "$output" "⚠️ Pod deletion timeout reached" - assert_contains "$output" "⚠️ Pod still exists after deletion attempt" + assert_contains "$output" "⚠️ Pod still exists after deletion attempt (status: Terminating)" } diff --git a/k8s/deployment/tests/networking/gateway/ingress/route_traffic.bats b/k8s/deployment/tests/networking/gateway/ingress/route_traffic.bats index 421e58ac..d57496e1 100644 --- a/k8s/deployment/tests/networking/gateway/ingress/route_traffic.bats +++ b/k8s/deployment/tests/networking/gateway/ingress/route_traffic.bats @@ -64,12 +64,17 @@ teardown() { run bash "$PROJECT_ROOT/k8s/deployment/networking/gateway/ingress/route_traffic" --template="$MOCK_TEMPLATE" [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Creating internet-facing ingress..." - assert_contains "$output" "📋 Scope: scope-123 | Deployment: deploy-456" - assert_contains "$output" "📋 Template: $MOCK_TEMPLATE" - assert_contains "$output" "📋 Output: $OUTPUT_DIR/ingress-scope-123-deploy-456.yaml" - assert_contains "$output" "📝 Building ingress template..." - assert_contains "$output" "✅ Ingress template created: $OUTPUT_DIR/ingress-scope-123-deploy-456.yaml" + local expected + expected=$(cat < 5 replicas +📋 Blue deployment: d-scope-123-deploy-old -> 3 replicas + +📝 Scaling green deployment... + ❌ Failed to scale green deployment +EOF +) + assert_equal "$output" "$expected" } @test "scale_deployments: fails when blue deployment scale fails" { @@ -184,7 +194,19 @@ run_scale_deployments() { source '$PROJECT_ROOT/k8s/deployment/scale_deployments'" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Failed to scale blue deployment" + local expected + expected=$(cat <<'EOF' +📝 Scaling deployments for rolling strategy... +📋 Green deployment: d-scope-123-deploy-new -> 5 replicas +📋 Blue deployment: d-scope-123-deploy-old -> 3 replicas + +📝 Scaling green deployment... + ✅ Green deployment scaled to 5 replicas +📝 Scaling blue deployment... + ❌ Failed to scale blue deployment +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= diff --git a/k8s/deployment/tests/translate_probe_message.bats b/k8s/deployment/tests/translate_probe_message.bats index 2ff9de51..5699375a 100644 --- a/k8s/deployment/tests/translate_probe_message.bats +++ b/k8s/deployment/tests/translate_probe_message.bats @@ -18,18 +18,14 @@ setup() { run translate_probe_message 'Startup probe failed: Get "http://10.15.28.102:8080/health": dial tcp 10.15.28.102:8080: connect: connection refused' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/health" + assert_equal "$output" "Startup probe — app is not yet listening on /health" } @test "translate_probe_message: liveness probe connection refused" { run translate_probe_message 'Liveness probe failed: Get "http://10.0.0.5:8080/ping": dial tcp: connect: connection refused' [ "$status" -eq 0 ] - assert_contains "$output" "Liveness probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/ping" + assert_equal "$output" "Liveness probe — app is not yet listening on /ping" } # ----------------------------------------------------------------------------- @@ -39,16 +35,14 @@ setup() { run translate_probe_message 'Startup probe failed: HTTP probe failed with statuscode: 502' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "HTTP 502" + assert_equal "$output" "Startup probe — app responded with HTTP 502 (expected 2xx)" } @test "translate_probe_message: readiness probe HTTP 404" { run translate_probe_message 'Readiness probe failed: HTTP probe failed with statuscode: 404' [ "$status" -eq 0 ] - assert_contains "$output" "Readiness probe" - assert_contains "$output" "HTTP 404" + assert_equal "$output" "Readiness probe — app responded with HTTP 404 (expected 2xx)" } # ----------------------------------------------------------------------------- @@ -58,9 +52,7 @@ setup() { run translate_probe_message 'Startup probe failed: Get "http://10.0.0.5:8080/health": context deadline exceeded (Client.Timeout exceeded while awaiting headers)' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "timed out" - assert_contains "$output" "/health" + assert_equal "$output" "Startup probe — request timed out on /health" } # ----------------------------------------------------------------------------- @@ -86,7 +78,7 @@ setup() { run translate_probe_message 'Startup probe failed: some weird new error format' [ "$status" -eq 0 ] - assert_contains "$output" "Startup probe" + assert_equal "$output" "Startup probe failed" } # ----------------------------------------------------------------------------- diff --git a/k8s/deployment/tests/validate_alb_target_group_capacity.bats b/k8s/deployment/tests/validate_alb_target_group_capacity.bats index 3ecd2e89..9f0119d4 100644 --- a/k8s/deployment/tests/validate_alb_target_group_capacity.bats +++ b/k8s/deployment/tests/validate_alb_target_group_capacity.bats @@ -89,15 +89,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 98/98 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 98/98" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Too many services or deployments are attached to this ALB" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Remove unused deployments or services from the ALB" - assert_contains "$output" "Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100)" - assert_contains "$output" "Request an AWS service quota increase for target groups per ALB" - assert_contains "$output" "Consider using a separate ALB for additional deployments" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when over capacity" { @@ -117,8 +125,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 100/98 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 100/98" + assert_contains "$output" "$expected" } # ============================================================================= @@ -138,8 +161,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_TARGET_GROUPS from scope-configurations provider" { @@ -148,8 +186,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_TARGET_GROUPS from container-orchestration provider" { @@ -158,8 +211,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30 + +💡 Possible causes: + Too many services or deployments are attached to this ALB + +🔧 How to fix: + • Remove unused deployments or services from the ALB + • Increase ALB_MAX_TARGET_GROUPS in values.yaml or scope-configurations provider (AWS limit is 100) + • Request an AWS service quota increase for target groups per ALB + • Consider using a separate ALB for additional deployments +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached target group capacity: 40/30" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: scope-configurations takes priority over container-orchestration" { @@ -198,13 +266,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to find load balancer 'k8s-nullplatform-internet-facing' in region 'us-east-1' + +💡 Possible causes: + The load balancer may not exist or the agent lacks permissions + +🔧 How to fix: + • Verify the ALB exists: aws elbv2 describe-load-balancers --names k8s-nullplatform-internet-facing --region us-east-1 + • Check IAM permissions for elbv2:DescribeLoadBalancers +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to find load balancer 'k8s-nullplatform-internet-facing' in region 'us-east-1'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The load balancer may not exist or the agent lacks permissions" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Verify the ALB exists: aws elbv2 describe-load-balancers --names k8s-nullplatform-internet-facing --region us-east-1" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeLoadBalancers" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB ARN is None" { @@ -220,8 +296,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1' + +💡 Possible causes: + The load balancer name may be incorrect or it was deleted + +🔧 How to fix: + • List available ALBs: aws elbv2 describe-load-balancers --region us-east-1 + • Check the balancer name in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1'" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when describe-target-groups fails" { @@ -241,12 +330,20 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to describe target groups for ALB 'k8s-nullplatform-internet-facing' + +💡 Possible causes: + The agent may lack permissions to describe target groups + +🔧 How to fix: + • Check IAM permissions for elbv2:DescribeTargetGroups +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to describe target groups for ALB 'k8s-nullplatform-internet-facing'" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The agent may lack permissions to describe target groups" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeTargetGroups" + assert_contains "$output" "$expected" } # ============================================================================= @@ -320,12 +417,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Unexpected non-numeric target group count from ALB +📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 +📋 Received value: WARNING: something unexpected + +💡 Possible causes: + The AWS CLI returned an unexpected response format + +🔧 How to fix: + • Verify AWS CLI version and credentials are correct + • Run manually: aws elbv2 describe-target-groups --load-balancer-arn arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 --region us-east-1 --query 'length(TargetGroups)' +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Unexpected non-numeric target group count from ALB" - assert_contains "$output" "📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123" - assert_contains "$output" "📋 Received value: WARNING: something unexpected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "The AWS CLI returned an unexpected response format" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB_MAX_TARGET_GROUPS is non-numeric" { @@ -333,10 +441,17 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB_MAX_TARGET_GROUPS must be a numeric value, got: 'abc' + +🔧 How to fix: + • Set a numeric value in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB_MAX_TARGET_GROUPS must be a numeric value, got: 'abc'" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Set a numeric value in values.yaml or scope-configurations provider" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: empty ALB ARN response triggers error" { @@ -352,8 +467,21 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1' + +💡 Possible causes: + The load balancer name may be incorrect or it was deleted + +🔧 How to fix: + • List available ALBs: aws elbv2 describe-load-balancers --region us-east-1 + • Check the balancer name in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Load balancer 'k8s-nullplatform-internet-facing' not found in region 'us-east-1'" + assert_contains "$output" "$expected" } # ============================================================================= @@ -384,7 +512,7 @@ teardown() { run bash -c 'source "$SCRIPT"' assert_equal "$status" "0" - assert_contains "$output" "DNS type is 'external_dns', ALB target group validation only applies to route53, skipping" + assert_contains "$output" "📋 DNS type is 'external_dns', ALB target group validation only applies to route53, skipping" } @test "validate_alb_target_group_capacity: runs when DNS_TYPE is route53" { @@ -428,14 +556,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 48/48 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 48/48" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "Too many scopes with additional_ports are attached to this ALB" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Reduce additional_ports across scopes sharing this ALB" - assert_contains "$output" "Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50)" - assert_contains "$output" "Request an AWS service quota increase for listeners per ALB" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when listener count is over capacity" { @@ -459,8 +596,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 50/48 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 50/48" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: passes at exactly one below listener capacity" { @@ -528,8 +680,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_LISTENERS from scope-configurations provider" { @@ -538,8 +705,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: ALB_MAX_LISTENERS from container-orchestration provider" { @@ -548,8 +730,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5 + +💡 Possible causes: + Too many scopes with additional_ports are attached to this ALB. Each HTTP/GRPC additional port opens its own listener. + +🔧 How to fix: + • Reduce additional_ports across scopes sharing this ALB + • Increase ALB_MAX_LISTENERS in values.yaml or scope-configurations provider (AWS limit is 50) + • Request an AWS service quota increase for listeners per ALB + • Consider using a separate ALB for additional scopes +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB 'k8s-nullplatform-internet-facing' has reached listener capacity: 10/5" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when describe-listeners fails" { @@ -573,9 +770,20 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Failed to describe listeners for ALB 'k8s-nullplatform-internet-facing' + +💡 Possible causes: + The agent may lack permissions to describe listeners + +🔧 How to fix: + • Check IAM permissions for elbv2:DescribeListeners +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Failed to describe listeners for ALB 'k8s-nullplatform-internet-facing'" - assert_contains "$output" "Check IAM permissions for elbv2:DescribeListeners" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when listener count is non-numeric" { @@ -599,9 +807,23 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ Unexpected non-numeric listener count from ALB +📋 ALB ARN: arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 +📋 Received value: WARNING: unexpected + +💡 Possible causes: + The AWS CLI returned an unexpected response format + +🔧 How to fix: + • Verify AWS CLI version and credentials are correct + • Run manually: aws elbv2 describe-listeners --load-balancer-arn arn:aws:elasticloadbalancing:us-east-1:123456789:loadbalancer/app/alb/abc123 --region us-east-1 --query 'length(Listeners)' +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ Unexpected non-numeric listener count from ALB" - assert_contains "$output" "📋 Received value: WARNING: unexpected" + assert_contains "$output" "$expected" } @test "validate_alb_target_group_capacity: fails when ALB_MAX_LISTENERS is non-numeric" { @@ -609,6 +831,15 @@ teardown() { run bash -c 'source "$SCRIPT"' + local expected + expected=$(cat <<'EOF' +❌ ALB_MAX_LISTENERS must be a numeric value, got: 'abc' + +🔧 How to fix: + • Set a numeric value in values.yaml or scope-configurations provider +EOF +) + assert_equal "$status" "1" - assert_contains "$output" "❌ ALB_MAX_LISTENERS must be a numeric value, got: 'abc'" + assert_contains "$output" "$expected" } diff --git a/k8s/deployment/tests/verify_http_route_reconciliation.bats b/k8s/deployment/tests/verify_http_route_reconciliation.bats index 6ed938d8..105c2d9a 100644 --- a/k8s/deployment/tests/verify_http_route_reconciliation.bats +++ b/k8s/deployment/tests/verify_http_route_reconciliation.bats @@ -45,9 +45,14 @@ run_with_mock() { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Route accepted"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s" - assert_contains "$output" "✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True)" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True) +EOF +) + assert_equal "$output" "$expected" } # ============================================================================= @@ -57,63 +62,91 @@ run_with_mock() { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"False","reason":"CertificateError","message":"TLS secret not found"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ Certificate/TLS error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- TLS secret does not exist in namespace test-namespace" - assert_contains "$output" "- Certificate is invalid or expired" - assert_contains "$output" "- Gateway references incorrect certificate secret" - assert_contains "$output" "- Accepted: CertificateError - TLS secret not found" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify TLS secret: kubectl get secret -n test-namespace | grep tls" - assert_contains "$output" "- Check certificate validity" - assert_contains "$output" "- Ensure Gateway references the correct secret" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ Certificate/TLS error detected +💡 Possible causes: + - TLS secret does not exist in namespace test-namespace + - Certificate is invalid or expired + - Gateway references incorrect certificate secret + - Accepted: CertificateError - TLS secret not found +🔧 How to fix: + - Verify TLS secret: kubectl get secret -n test-namespace | grep tls + - Check certificate validity + - Ensure Gateway references the correct secret +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting on backend error" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Accepted"},{"type":"ResolvedRefs","status":"False","reason":"BackendNotFound","message":"service my-svc not found"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ Backend service error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Referenced service does not exist" - assert_contains "$output" "- Service name is misspelled in HTTPRoute" - assert_contains "$output" "- Message: service my-svc not found" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- List services: kubectl get svc -n test-namespace" - assert_contains "$output" "- Verify backend service name in HTTPRoute" - assert_contains "$output" "- Ensure service has ready endpoints" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ Backend service error detected +💡 Possible causes: + - Referenced service does not exist + - Service name is misspelled in HTTPRoute + - Message: service my-svc not found +🔧 How to fix: + - List services: kubectl get svc -n test-namespace + - Verify backend service name in HTTPRoute + - Ensure service has ready endpoints +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting when not accepted" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"False","reason":"NotAccepted","message":"Gateway not found"},{"type":"ResolvedRefs","status":"True","reason":"ResolvedRefs","message":"Refs resolved"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ HTTPRoute not accepted by Gateway" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Reason: NotAccepted" - assert_contains "$output" "- Message: Gateway not found" - assert_contains "$output" "📋 All conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check Gateway configuration" - assert_contains "$output" "- Verify HTTPRoute spec matches Gateway requirements" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ HTTPRoute not accepted by Gateway +💡 Possible causes: + - Reason: NotAccepted + - Message: Gateway not found +📋 All conditions: + - Accepted: False (NotAccepted) - Gateway not found + - ResolvedRefs: True (ResolvedRefs) - Refs resolved +🔧 How to fix: + - Check Gateway configuration + - Verify HTTPRoute spec matches Gateway requirements +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting when refs not resolved" { run_with_mock '{"status":{"parents":[{"conditions":[{"type":"Accepted","status":"True","reason":"Accepted","message":"Accepted"},{"type":"ResolvedRefs","status":"False","reason":"InvalidBackend","message":"Invalid backend port"}]}]}}' [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "❌ HTTPRoute references could not be resolved" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Reason: InvalidBackend" - assert_contains "$output" "- Message: Invalid backend port" - assert_contains "$output" "📋 All conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify all referenced services exist" - assert_contains "$output" "- Check backend service ports match" + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +❌ HTTPRoute references could not be resolved +💡 Possible causes: + - Reason: InvalidBackend + - Message: Invalid backend port +📋 All conditions: + - Accepted: True (Accepted) - Accepted + - ResolvedRefs: False (InvalidBackend) - Invalid backend port +🔧 How to fix: + - Verify all referenced services exist + - Check backend service ports match +EOF +) + assert_equal "$output" "$expected" } @test "verify_http_route_reconciliation: fails with full troubleshooting on timeout" { @@ -127,13 +160,30 @@ run_with_mock() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for HTTPRoute reconciliation after 1s" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Gateway controller is not running" - assert_contains "$output" "- Network policies blocking reconciliation" - assert_contains "$output" "- Resource constraints on controller" - assert_contains "$output" "📋 Current conditions:" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check Gateway controller logs" - assert_contains "$output" "- Verify Gateway and Istio configuration" + # Everything up to the dynamic "Current conditions" dump: the mock's empty + # parents list makes the jq read on it fail, so its tool-error text (not a + # product message) is deliberately excluded here. + local expected + expected=$(cat <<'EOF' +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-scope-123-internet-facing | Namespace: test-namespace | Timeout: 1s +📝 HTTPRoute pending sync (no parent status yet)... (0s/1s) +❌ Timeout waiting for HTTPRoute reconciliation after 1s +💡 Possible causes: + - Gateway controller is not running + - Network policies blocking reconciliation + - Resource constraints on controller +📋 Current conditions: +EOF +) + assert_contains "$output" "$expected" + + local expected_fix + expected_fix=$(cat <<'EOF' +🔧 How to fix: + - Check Gateway controller logs + - Verify Gateway and Istio configuration +EOF +) + assert_contains "$output" "$expected_fix" } diff --git a/k8s/deployment/tests/verify_ingress_reconciliation.bats b/k8s/deployment/tests/verify_ingress_reconciliation.bats index f78be30b..2ddca3cf 100644 --- a/k8s/deployment/tests/verify_ingress_reconciliation.bats +++ b/k8s/deployment/tests/verify_ingress_reconciliation.bats @@ -112,14 +112,19 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Certificate error detected" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Ingress hostname does not match any SSL/TLS certificate in ACM" - assert_contains "$output" "- Certificate does not cover the hostname (check wildcards)" - assert_contains "$output" "- Message: no certificate found for host app.example.com" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Verify hostname matches certificate in ACM" - assert_contains "$output" "- Ensure certificate includes exact hostname or matching wildcard" + local expected + expected=$(cat <<'EOF' +❌ Certificate error detected +💡 Possible causes: + - Ingress hostname does not match any SSL/TLS certificate in ACM + - Certificate does not cover the hostname (check wildcards) + - Message: no certificate found for host app.example.com +🔧 How to fix: + - Verify hostname matches certificate in ACM + - Ensure certificate includes exact hostname or matching wildcard +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails with full troubleshooting when ingress not found" { @@ -142,12 +147,17 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Failed to get ingress k-8-s-my-app-scope-123-internet-facing" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- Ingress does not exist yet" - assert_contains "$output" "- Namespace test-namespace is incorrect" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- List ingresses: kubectl get ingress -n test-namespace" + local expected + expected=$(cat <<'EOF' +❌ Failed to get ingress k-8-s-my-app-scope-123-internet-facing +💡 Possible causes: + - Ingress does not exist yet + - Namespace test-namespace is incorrect +🔧 How to fix: + - List ingresses: kubectl get ingress -n test-namespace +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails when ALB not found" { @@ -286,8 +296,8 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "Skipping weight check on listener port 50051" - assert_contains "$output" "✅ Weights match on listener port 443" + assert_contains "$output" "⏭️ Skipping weight check on listener port 50051 (blue has no service for this port)" + assert_contains "$output" "✅ Weights match on listener port 443 (GREEN: 10, BLUE: 90)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -326,7 +336,7 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "✅ Weights match on listener port 443" + assert_contains "$output" "✅ Weights match on listener port 443 (GREEN: 10, BLUE: 90)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -406,7 +416,7 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "📝 Checking domain: app.example.com" - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: passes when the rule forwards to a single target group" { @@ -440,7 +450,7 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "✅ Single target group on listener port 443" + assert_contains "$output" "✅ Single target group on listener port 443 (traffic is no longer split between deployments)" assert_contains "$output" "✅ ALB configuration validated successfully" } @@ -490,8 +500,8 @@ teardown() { [ "$status" -eq 0 ] # Both messages must appear: the one that holds, then the one that releases. - assert_contains "$output" "still has 2 target groups on listener port 443" - assert_contains "$output" "✅ Single target group on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" + assert_contains "$output" "✅ Single target group on listener port 443 (traffic is no longer split between deployments)" assert_contains "$output" "✅ ALB configuration validated successfully" # More than one read proves it retried instead of getting a single lucky hit. [ "$(wc -l < "$counter")" -gt 1 ] @@ -531,7 +541,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: skips target group check on additional port listener when blue has no service" { @@ -573,8 +583,8 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Skipping target group check on listener port 50051" - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "⏭️ Skipping target group check on listener port 50051 (blue has no service for this port)" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: treats a rule without a forward action as not converged" { @@ -610,7 +620,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 0 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 0 target groups on listener port 443: the ALB has not finished reconciling" } @test "verify_ingress_reconciliation: a reconciled event does not override a failed ALB check" { @@ -655,7 +665,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "still has 2 target groups on listener port 443" + assert_contains "$output" "❌ Rule still has 2 target groups on listener port 443: the ALB has not finished reconciling" assert_contains "$output" "❌ Timeout waiting for ingress reconciliation after 1s" [[ "$output" != *"✅ Ingress successfully reconciled"* ]] } @@ -716,14 +726,19 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for ingress reconciliation after 1s" - assert_contains "$output" "💡 Possible causes:" - assert_contains "$output" "- ALB Ingress Controller not running or unhealthy" - assert_contains "$output" "- Network connectivity issues" - assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "- Check controller: kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller" - assert_contains "$output" "- Check ingress: kubectl describe ingress k-8-s-my-app-scope-123-internet-facing -n test-namespace" - assert_contains "$output" "📋 Recent events:" + local expected + expected=$(cat <<'EOF' +❌ Timeout waiting for ingress reconciliation after 1s +💡 Possible causes: + - ALB Ingress Controller not running or unhealthy + - Network connectivity issues +🔧 How to fix: + - Check controller: kubectl logs -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller + - Check ingress: kubectl describe ingress k-8-s-my-app-scope-123-internet-facing -n test-namespace +📋 Recent events: +EOF +) + assert_contains "$output" "$expected" } @test "verify_ingress_reconciliation: fails on Error event type with error messages" { @@ -749,7 +764,12 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "🔍 Verifying ingress reconciliation..." assert_contains "$output" "📋 ALB reconciliation disabled, checking cluster events only" - assert_contains "$output" "❌ Ingress reconciliation failed" - assert_contains "$output" "💡 Error messages:" - assert_contains "$output" "- Failed to sync ALB" + local expected + expected=$(cat <<'EOF' +❌ Ingress reconciliation failed +💡 Error messages: + - Failed to sync ALB +EOF +) + assert_contains "$output" "$expected" } diff --git a/k8s/deployment/tests/verify_networking_reconciliation.bats b/k8s/deployment/tests/verify_networking_reconciliation.bats index 424a0e10..c5cc696b 100644 --- a/k8s/deployment/tests/verify_networking_reconciliation.bats +++ b/k8s/deployment/tests/verify_networking_reconciliation.bats @@ -39,9 +39,15 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: route53" - assert_contains "$output" "🔍 Verifying ingress reconciliation..." - assert_contains "$output" "⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation)" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: route53 +🔍 Verifying ingress reconciliation... +📋 Ingress: k-8-s-my-app-- | Namespace: | Timeout: 120s +⚠️ Skipping ALB verification (ALB access needed for blue-green traffic validation) +EOF +) + assert_equal "$output" "$expected" } @test "verify_networking_reconciliation: verifies HTTPRoute for external_dns without managing DNS" { @@ -65,9 +71,15 @@ teardown() { " [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: external_dns" - assert_contains "$output" "🔍 Verifying HTTPRoute reconciliation..." - assert_contains "$output" "✅ HTTPRoute successfully reconciled" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: external_dns +🔍 Verifying HTTPRoute reconciliation... +📋 HTTPRoute: k-8-s-my-app-123-public | Namespace: nullplatform | Timeout: 10s +✅ HTTPRoute successfully reconciled (Accepted: True, ResolvedRefs: True) +EOF +) + assert_equal "$output" "$expected" } @test "verify_networking_reconciliation: skips for unsupported DNS types" { @@ -77,6 +89,11 @@ teardown() { [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Verifying networking reconciliation for DNS type: unknown" - assert_contains "$output" "⚠️ Ingress reconciliation not available for DNS type: unknown, skipping" + local expected + expected=$(cat <<'EOF' +🔍 Verifying networking reconciliation for DNS type: unknown +⚠️ Ingress reconciliation not available for DNS type: unknown, skipping +EOF +) + assert_equal "$output" "$expected" } diff --git a/k8s/deployment/tests/wait_deployment_active.bats b/k8s/deployment/tests/wait_deployment_active.bats index 52c83d39..2d9a8ffa 100644 --- a/k8s/deployment/tests/wait_deployment_active.bats +++ b/k8s/deployment/tests/wait_deployment_active.bats @@ -66,9 +66,14 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 0 ] - assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📋 Namespace: test-namespace" - assert_contains "$output" "📋 Timeout: 30s (max 3 iterations)" + local expected_intro + expected_intro=$(cat <<'EOF' +🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active... +📋 Namespace: test-namespace +📋 Timeout: 30s (max 3 iterations) +EOF +) + assert_contains "$output" "$expected_intro" assert_contains "$output" "📡 Checking deployment status (attempt 1/3)..." assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!" } @@ -109,11 +114,23 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📋 Namespace: test-namespace" - assert_contains "$output" "📋 Timeout: 5s (max 0 iterations)" - assert_contains "$output" "❌ Timeout waiting for deployment" - assert_contains "$output" "📋 Maximum iterations (0) reached" + local expected_intro + expected_intro=$(cat <<'EOF' +🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active... +📋 Namespace: test-namespace +📋 Timeout: 5s (max 0 iterations) +EOF +) + assert_contains "$output" "$expected_intro" + + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached +EOF +) + assert_contains "$output" "$expected_timeout" # Timeout path must source print_failed_deployment_hints; with no pod info # and no events, it falls through to the generic checklist. assert_contains "$output" "⚠️ Application Startup Issue Detected" @@ -145,11 +162,21 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for deployment" - # The hint script must read pod state and surface the user-friendly reason - assert_contains "$output" "📋 Reason: The container exceeded its memory limit" - assert_contains "$output" "📋 Detected: OOMKilled on container app (exit 137)" - assert_contains "$output" "💡 Suggested fix: Increase ram_memory for scope 'my-app'" + # The hint script must read pod state and surface the user-friendly reason, + # contiguous with the timeout header that precedes it. + local expected + expected=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached + +📋 Reason: The container exceeded its memory limit (512Mi) and was terminated. +📋 Detected: OOMKilled on container app (exit 137) +📋 Details: out of memory +💡 Suggested fix: Increase ram_memory for scope 'my-app' or reduce application memory usage. +EOF +) + assert_contains "$output" "$expected" } # ============================================================================= @@ -166,9 +193,14 @@ teardown() { [ "$status" -eq 1 ] assert_contains "$output" "🔍 Waiting for deployment 'd-scope-123-deploy-456' to become active..." - assert_contains "$output" "📡 Checking deployment status (attempt 1/" - assert_contains "$output" "❌ Failed to read deployment status" - assert_contains "$output" "📋 NP CLI error:" + assert_contains "$output" "📡 Checking deployment status (attempt 1/3)..." + local expected + expected=$(cat <<'EOF' + ❌ Failed to read deployment status +📋 NP CLI error: Error connecting to API +EOF +) + assert_contains "$output" "$expected" } @test "wait_deployment_active: fails when deployment status is null" { @@ -180,7 +212,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment status not found for ID deploy-456" + assert_contains "$output" " ❌ Deployment status not found for ID deploy-456" } @test "wait_deployment_active: fails when NP deployment status is not running" { @@ -194,7 +226,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment is no longer running (status: failed)" + assert_contains "$output" " ❌ Deployment is no longer running (status: failed)" # Non-running status path must also source print_failed_deployment_hints assert_contains "$output" "⚠️ Application Startup Issue Detected" } @@ -215,7 +247,7 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 1 ] - assert_contains "$output" "❌ Deployment 'd-scope-123-deploy-456' not found in namespace 'test-namespace'" + assert_contains "$output" " ❌ Deployment 'd-scope-123-deploy-456' not found in namespace 'test-namespace'" } # ============================================================================= @@ -258,9 +290,17 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Deployment status - Available: 3/5, Updated: 4/5, Ready: 3/5" + # Leading "🔍 $(date): " is non-deterministic; assert everything after it verbatim. + assert_contains "$output" "Iteration 1 - Deployment status - Available: 3/5, Updated: 4/5, Ready: 3/5" assert_contains "$output" "⏳ Still waiting — Ready: 3/5, Available: 3/5 (attempt 1/1, 10s elapsed)" - assert_contains "$output" "❌ Timeout waiting for deployment" + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (1) reached +EOF +) + assert_contains "$output" "$expected_timeout" } @test "wait_deployment_active: handles missing status fields defaults to 0" { @@ -296,7 +336,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Available: 0/3" + assert_contains "$output" "Iteration 1 - Deployment status - Available: 0/3, Updated: 0/3, Ready: 0/3" } # ============================================================================= @@ -332,7 +372,14 @@ teardown() { # Should timeout because desired > 0 check fails [ "$status" -eq 1 ] - assert_contains "$output" "❌ Timeout waiting for deployment" + local expected_timeout + expected_timeout=$(cat <<'EOF' + +❌ Timeout waiting for deployment +📋 Maximum iterations (0) reached +EOF +) + assert_contains "$output" "$expected_timeout" } # ============================================================================= @@ -505,10 +552,8 @@ teardown() { " [ "$status" -eq 1 ] - # Translated form must appear - assert_contains "$output" "Startup probe" - assert_contains "$output" "not yet listening" - assert_contains "$output" "/health" + # Translated form must appear as the complete consolidated warning line + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc Startup probe failing on /health — not yet listening" # Raw connection-refused text must NOT leak through if [[ "$output" == *"connection refused"* ]]; then echo "Expected output to NOT contain raw 'connection refused' (should be translated)" @@ -554,10 +599,9 @@ teardown() { " [ "$status" -eq 1 ] - # One consolidated line with both modes joined by ', ' for natural reading - assert_contains "$output" "Startup probe failing on /health — not yet listening, responded HTTP 502 (expected 2xx)" - # Pod name must be the short form with '...' prefix marking truncation - assert_contains "$output" "Pod/...abc-hhshq" + # One consolidated line with both modes joined by ', ' for natural reading, + # using the short pod name ('...' prefix marks truncation). + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc-hhshq Startup probe failing on /health — not yet listening, responded HTTP 502 (expected 2xx)" # The long prefix must NOT appear in any logged event line if [[ "$output" == *"Pod/d-scope-123-deploy-456-abc-hhshq"* ]]; then echo "Expected output to use short pod name, not the full prefix" @@ -610,9 +654,9 @@ teardown() { " [ "$status" -eq 1 ] - # Both original messages must appear verbatim - assert_contains "$output" "some brand-new K8s probe format we cannot parse 1" - assert_contains "$output" "another unknown probe format 2" + # Both original messages must appear verbatim, each as its own complete raw warning line + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/d-scope-123-deploy-456-abc-hhshq: Unhealthy - some brand-new K8s probe format we cannot parse 1" + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/d-scope-123-deploy-456-abc-hhshq: Unhealthy - another unknown probe format 2" # The consolidated header must NOT appear because parsing failed if [[ "$output" == *"probe failing"* ]]; then echo "Expected fallback path to NOT emit the 'probe failing' header" @@ -652,8 +696,7 @@ teardown() { " [ "$status" -eq 1 ] - assert_contains "$output" "Startup probe" - assert_contains "$output" "HTTP 502" + assert_contains "$output" "9999-12-31T23:59:59Z [Warning] Pod/...abc Startup probe failing — responded HTTP 502 (expected 2xx)" } # ============================================================================= @@ -854,6 +897,6 @@ teardown() { run bash "$BATS_TEST_DIRNAME/../wait_deployment_active" [ "$status" -eq 0 ] - assert_contains "$output" "Could not report instance counts" + assert_contains "$output" "Could not report instance counts (will retry on next change)" assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!" }