Fix remote k8s test failures

src/stack/deploy/k8s/deploy_k8s.py

@@ -58,6 +58,23 @@ def _check_delete_exception(e: client.exceptions.ApiException):
         error_exit(f"k8s api error: {e}")
 
 
+def _pod_status(pod):
+    # A pod enters the Running phase as soon as its containers have started,
+    # which is well before the application inside them is able to serve. Only
+    # report Running once every container also passes its readiness probe, so
+    # that "Running" means the same thing here as a ready pod does to k8s.
+    phase = pod.status.phase
+    if phase != "Running":
+        return phase
+
+    container_statuses = pod.status.container_statuses or []
+    total = len(container_statuses)
+    ready = len([c for c in container_statuses if c.ready])
+    if not total or ready < total:
+        return f"Starting {ready}/{total} ready"
+    return f"Running {ready}/{total} ready"
+
+
 class K8sDeployer(Deployer):
     name: str = "k8s"
     type: str
@@ -475,7 +492,7 @@ def status(self):
             if p.metadata.deletion_timestamp:
                 output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: Terminating ({p.metadata.deletion_timestamp})")
             else:
-                output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: {p.status.phase} ({p.metadata.creation_timestamp})")
+                output_main(f"\t{p.metadata.namespace}/{p.metadata.name}: {_pod_status(p)} ({p.metadata.creation_timestamp})")
 
     def ps(self):
         self.connect_api()

tests/k8s-deploy/run-deploy-test.sh

@@ -81,11 +81,14 @@ wait_for_log_output () {
 wait_for_running () {
   set +e
 
-  # Check that all services are running
+  # Check that all services are running (and ready -- "status" only reports a
+  # pod as Running once its containers pass their readiness probes).
   how_many=$1
   local running=0
   local check=0
-  local check_limit=10
+  # Against a real cluster the images are pulled from a real registry over the
+  # network, which can take minutes on a cold node, so allow ~5 minutes here.
+  local check_limit=60
   while [ $running -lt $how_many ] && [ $check -lt $check_limit ]; do
       check=$((check + 1))
       running=$($TEST_TARGET_SO manage --dir $test_deployment_dir status | grep -ic "running")
@@ -147,6 +150,37 @@ add_todo() {
   return $rc
 }
 
+# Fetch a URL until its body contains the expected text. Even once the pods are
+# ready the ingress/gateway needs a moment to route to the new endpoints, so a
+# single-shot fetch here is a race.
+wait_for_content () {
+  set +e
+
+  url=$1
+  expected=$2
+
+  local try=0
+  local rc=1
+
+  while [ $rc -ne 0 ] && [ $try -lt 20 ]; do
+    try=$((try + 1))
+    curl -s "$url" | grep -q "$expected"
+    rc=$?
+
+    if [ $rc -ne 0 ]; then
+      echo "Waiting for $expected at $url..."
+      sleep 5
+    fi
+  done
+
+  set -e
+
+  if [ $rc -ne 0 ]; then
+    echo "deploy http: failed - $expected not found at $url"
+    exit 1
+  fi
+}
+
 # Test basic stack deploy
 echo "Running stack deploy test"
 # Bit of a hack, test the most recent package
@@ -218,7 +252,7 @@ if [ "$todo_title" != "$(curl -s ${TEST_SCHEME}://${TEST_HOSTNAME}/api/todos | j
     exit 1
 fi
 
-wget -q -O - ${TEST_SCHEME}://${TEST_HOSTNAME} | grep 'bundle.js'
+wait_for_content ${TEST_SCHEME}://${TEST_HOSTNAME} 'bundle.js'
 echo "deploy http: passed"
 
 delete_cluster_exit

tests/unit/test_k8s_pod_status.py

@@ -0,0 +1,62 @@
+# Copyright © 2026 Bozeman Pass, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+"""Tests for the pod status summary reported by "stack manage status"."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from stack.deploy.k8s.deploy_k8s import _pod_status
+
+
+def pod(phase, readies=None):
+    """Stands in for a V1Pod; _pod_status only reads phase and container_statuses."""
+    container_statuses = None if readies is None else [SimpleNamespace(ready=r) for r in readies]
+    return SimpleNamespace(status=SimpleNamespace(phase=phase, container_statuses=container_statuses))
+
+
+@pytest.mark.parametrize(
+    "phase,readies,expected",
+    [
+        ("Pending", None, "Pending"),
+        ("Failed", [False], "Failed"),
+        ("Succeeded", None, "Succeeded"),
+        # Running but not yet ready must not report as Running: the containers
+        # have started, but the application inside them cannot serve yet.
+        ("Running", None, "Starting 0/0 ready"),
+        ("Running", [False], "Starting 0/1 ready"),
+        ("Running", [True, False], "Starting 1/2 ready"),
+        ("Running", [True], "Running 1/1 ready"),
+        ("Running", [True, True], "Running 2/2 ready"),
+    ],
+)
+def test_pod_status(phase, readies, expected):
+    assert _pod_status(pod(phase, readies)) == expected
+
+
+@pytest.mark.parametrize(
+    "phase,readies",
+    [
+        ("Pending", None),
+        ("Running", None),
+        ("Running", [False]),
+        ("Running", [True, False]),
+    ],
+)
+def test_not_ready_never_says_running(phase, readies):
+    # The deploy tests count ready pods with a case-insensitive grep for
+    # "running", so no unready pod may contain that word.
+    assert "running" not in _pod_status(pod(phase, readies)).lower()