From 21d59db0f0fe400896f3c9476c76c13f611de62b Mon Sep 17 00:00:00 2001 From: David Boreham Date: Mon, 3 Aug 2026 12:15:08 -0600 Subject: [PATCH] Fix remote k8s test failures --- src/stack/deploy/k8s/deploy_k8s.py | 19 ++++++++- tests/k8s-deploy/run-deploy-test.sh | 40 +++++++++++++++++-- tests/unit/test_k8s_pod_status.py | 62 +++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_k8s_pod_status.py diff --git a/src/stack/deploy/k8s/deploy_k8s.py b/src/stack/deploy/k8s/deploy_k8s.py index 14b64aa..1b79765 100644 --- a/src/stack/deploy/k8s/deploy_k8s.py +++ b/src/stack/deploy/k8s/deploy_k8s.py @@ -58,6 +58,23 @@ def _check_delete_exception(e: client.exceptions.ApiException): error_exit(f"k8s api error: {e}") +def _pod_status(pod): + # A pod enters the Running phase as soon as its containers have started, + # which is well before the application inside them is able to serve. Only + # report Running once every container also passes its readiness probe, so + # that "Running" means the same thing here as a ready pod does to k8s. + phase = pod.status.phase + if phase != "Running": + return phase + + container_statuses = pod.status.container_statuses or [] + total = len(container_statuses) + ready = len([c for c in container_statuses if c.ready]) + if not total or ready < total: + return f"Starting {ready}/{total} ready" + return f"Running {ready}/{total} ready" + + class K8sDeployer(Deployer): name: str = "k8s" type: str @@ -475,7 +492,7 @@ def status(self): if p.metadata.deletion_timestamp: output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})") else: - output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: {p.status.phase} ({p.metadata.creation_timestamp})") + output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: {_pod_status(p)} ({p.metadata.creation_timestamp})") def ps(self): self.connect_api() diff --git a/tests/k8s-deploy/run-deploy-test.sh b/tests/k8s-deploy/run-deploy-test.sh index 927d199..aae05eb 100755 --- a/tests/k8s-deploy/run-deploy-test.sh +++ b/tests/k8s-deploy/run-deploy-test.sh @@ -81,11 +81,14 @@ wait_for_log_output () { wait_for_running () { set +e - # Check that all services are running + # Check that all services are running (and ready -- "status" only reports a + # pod as Running once its containers pass their readiness probes). how_many=$1 local running=0 local check=0 - local check_limit=10 + # Against a real cluster the images are pulled from a real registry over the + # network, which can take minutes on a cold node, so allow ~5 minutes here. + local check_limit=60 while [ $running -lt $how_many ] && [ $check -lt $check_limit ]; do check=$((check + 1)) running=$($TEST_TARGET_SO manage --dir $test_deployment_dir status | grep -ic "running") @@ -147,6 +150,37 @@ add_todo() { return $rc } +# Fetch a URL until its body contains the expected text. Even once the pods are +# ready the ingress/gateway needs a moment to route to the new endpoints, so a +# single-shot fetch here is a race. +wait_for_content () { + set +e + + url=$1 + expected=$2 + + local try=0 + local rc=1 + + while [ $rc -ne 0 ] && [ $try -lt 20 ]; do + try=$((try + 1)) + curl -s "$url" | grep -q "$expected" + rc=$? + + if [ $rc -ne 0 ]; then + echo "Waiting for $expected at $url..." + sleep 5 + fi + done + + set -e + + if [ $rc -ne 0 ]; then + echo "deploy http: failed - $expected not found at $url" + exit 1 + fi +} + # Test basic stack deploy echo "Running stack deploy test" # Bit of a hack, test the most recent package @@ -218,7 +252,7 @@ if [ "$todo_title" != "$(curl -s ${TEST_SCHEME}://${TEST_HOSTNAME}/api/todos | j exit 1 fi -wget -q -O - ${TEST_SCHEME}://${TEST_HOSTNAME} | grep 'bundle.js' +wait_for_content ${TEST_SCHEME}://${TEST_HOSTNAME} 'bundle.js' echo "deploy http: passed" delete_cluster_exit diff --git a/tests/unit/test_k8s_pod_status.py b/tests/unit/test_k8s_pod_status.py new file mode 100644 index 0000000..7864ab3 --- /dev/null +++ b/tests/unit/test_k8s_pod_status.py @@ -0,0 +1,62 @@ +# Copyright © 2026 Bozeman Pass, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Tests for the pod status summary reported by "stack manage status".""" + +from types import SimpleNamespace + +import pytest + +from stack.deploy.k8s.deploy_k8s import _pod_status + + +def pod(phase, readies=None): + """Stands in for a V1Pod; _pod_status only reads phase and container_statuses.""" + container_statuses = None if readies is None else [SimpleNamespace(ready=r) for r in readies] + return SimpleNamespace(status=SimpleNamespace(phase=phase, container_statuses=container_statuses)) + + +@pytest.mark.parametrize( + "phase,readies,expected", + [ + ("Pending", None, "Pending"), + ("Failed", [False], "Failed"), + ("Succeeded", None, "Succeeded"), + # Running but not yet ready must not report as Running: the containers + # have started, but the application inside them cannot serve yet. + ("Running", None, "Starting 0/0 ready"), + ("Running", [False], "Starting 0/1 ready"), + ("Running", [True, False], "Starting 1/2 ready"), + ("Running", [True], "Running 1/1 ready"), + ("Running", [True, True], "Running 2/2 ready"), + ], +) +def test_pod_status(phase, readies, expected): + assert _pod_status(pod(phase, readies)) == expected + + +@pytest.mark.parametrize( + "phase,readies", + [ + ("Pending", None), + ("Running", None), + ("Running", [False]), + ("Running", [True, False]), + ], +) +def test_not_ready_never_says_running(phase, readies): + # The deploy tests count ready pods with a case-insensitive grep for + # "running", so no unready pod may contain that word. + assert "running" not in _pod_status(pod(phase, readies)).lower()