From a94b2c48a2fdf714b42eedcd3d9b411c25a8255b Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 13:39:18 -0400
Subject: [PATCH 01/48] Add local open-source vector databases

---
 README.md                                     |   4 +
 docker/README.md                              |  43 +++++
 docker/docker-compose-dev-essentials.yaml     | 147 +++++++++++++++++-
 docker/sample.env                             |   4 +
 docker/sample.essentials.env                  |   4 -
 docker/scripts/db-setup/vector_db_setup.sh    |  18 +++
 docs/local-dev-setup-executor-migration.md    |   5 +
 .../vectordb/weaviate/src/weaviate.py         |  39 ++++-
 unstract/sdk1/tests/test_weaviate_adapter.py  |  63 ++++++++
 9 files changed, 318 insertions(+), 9 deletions(-)
 create mode 100755 docker/scripts/db-setup/vector_db_setup.sh
 create mode 100644 unstract/sdk1/tests/test_weaviate_adapter.py

diff --git a/README.md b/README.md
index 9e282b3e10..9ea5ee52e9 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,10 @@ That's it!
 - Login with username: `unstract` password: `unstract`
 - Start extracting data!
 
+The Compose quickstart also provisions local Qdrant, pgvector/PostgreSQL,
+Weaviate, and Milvus services. See [local vector database connection
+settings](docker/README.md#local-vector-databases) for the adapter values.
+
 ## 📦 Other Deployment Options
 
 ### Docker Compose
diff --git a/docker/README.md b/docker/README.md
index be71603c89..505e6c7951 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -30,6 +30,49 @@ VERSION=dev docker compose -f docker-compose.yaml --profile optional up -d
 
 Now access frontend at http://frontend.unstract.localhost
 
+## Local vector databases
+
+The default development Compose stack starts the open-source vector database
+backends supported by Unstract:
+
+- Qdrant
+- PostgreSQL with the `pgvector` extension
+- Weaviate
+- Milvus Standalone (with private etcd and MinIO dependencies)
+
+Pinecone is supported as a hosted provider and is intentionally not included
+in the local stack because it is not a self-hosted open-source service.
+
+The services use persistent named volumes and are bound to loopback on the host.
+The Unstract workers connect over the Compose network, so use the internal
+addresses below when creating an adapter in the UI:
+
+| Adapter | UI fields | Address from Unstract containers | Host address | Local credentials |
+|---------|-----------|----------------------------------|--------------|-------------------|
+| Qdrant | URL, API Key | `http://qdrant:6333` | `http://localhost:6333` | API key from `LOCAL_VECTOR_DB_API_KEY` (default: `unstract_vector_pass`) |
+| Postgres | Database, Host, Port, User, Password, Enable SSL | Host `postgres-vector`, port `5432` | Host `localhost`, port `5433` | Values from `docker/essentials.env`; set Enable SSL to `false` |
+| Weaviate | URL, API Key | `http://weaviate:8080` | `http://localhost:8084` | API key from `LOCAL_VECTOR_DB_API_KEY` (default: `unstract_vector_pass`) |
+| Milvus | URI, Token | `http://milvus:19530` | `http://localhost:19530` | Leave Token empty |
+
+For the Postgres adapter, use the `POSTGRES_USER`, `POSTGRES_PASSWORD`, and
+`POSTGRES_DB` values in `docker/essentials.env`; the vector database has its own
+container and volume even though it reuses the platform's local development
+credentials. The initialization script enables `vector` and creates the
+`POSTGRES_SCHEMA` schema on first boot.
+
+The default `run-platform.sh` flow creates `docker/.env` and
+`docker/essentials.env` before starting Compose. To start only the vector
+services during development, run:
+
+```bash
+cd docker
+VERSION=dev docker compose -f docker-compose.yaml up -d \
+  qdrant postgres-vector weaviate milvus
+```
+
+Do not use `docker compose down -v` unless deleting local vector data is
+intentional.
+
 ## Overriding a service's config
 
 By making use of the [merge compose files](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) feature its possible to override some configuration that's used by the services.
diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index 605781be15..1002ce084f 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -124,13 +124,153 @@ services:
     container_name: unstract-vector-db
     restart: unless-stopped
     ports:
-      - "6333:6333"
+      - "127.0.0.1:6333:6333"
     volumes:
       - qdrant_data:/var/lib/qdrant/data/
+    environment:
+      # This is a development-only key. The service is also bound to loopback
+      # so it is not exposed on the host's network interfaces.
+      QDRANT__SERVICE__API_KEY: "${LOCAL_VECTOR_DB_API_KEY:-unstract_vector_pass}"
     labels:
       - traefik.enable=false
+
+  # Separate pgvector instance for the Postgres adapter. The platform's `db`
+  # service is also pgvector-enabled, but keeping this volume/container
+  # independent prevents adapter connection tests from sharing platform data.
+  postgres-vector:
+    image: "pgvector/pgvector:pg15"
+    container_name: unstract-postgres-vector
+    restart: unless-stopped
+    shm_size: 128mb
+    ports:
+      - "127.0.0.1:5433:5432"
+    volumes:
+      - postgres_vector_data:/var/lib/postgresql/data/
+      - ./scripts/db-setup/vector_db_setup.sh:/docker-entrypoint-initdb.d/vector_db_setup.sh:ro
     env_file:
+      # Reuse the local platform database credentials from essentials.env while
+      # retaining a separate data directory and container.
       - ./essentials.env
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    labels:
+      - traefik.enable=false
+
+  # Weaviate is configured for API-key auth so its connection settings match
+  # the local Qdrant setup. The adapter uses the local connector for this HTTP
+  # endpoint; no cloud account is required.
+  weaviate:
+    image: "cr.weaviate.io/semitechnologies/weaviate:1.39.2"
+    container_name: unstract-weaviate
+    restart: unless-stopped
+    command:
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8080"
+      - --scheme
+      - http
+    ports:
+      - "127.0.0.1:8084:8080"
+      - "127.0.0.1:50051:50051"
+    volumes:
+      - weaviate_data:/var/lib/weaviate
+    environment:
+      QUERY_DEFAULTS_LIMIT: "25"
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "false"
+      AUTHENTICATION_APIKEY_ENABLED: "true"
+      AUTHENTICATION_APIKEY_ALLOWED_KEYS: "${LOCAL_VECTOR_DB_API_KEY:-unstract_vector_pass}"
+      AUTHENTICATION_APIKEY_USERS: "unstract-local"
+      AUTHORIZATION_ENABLE_RBAC: "true"
+      AUTHORIZATION_RBAC_ROOT_USERS: "unstract-local"
+      PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
+      DEFAULT_VECTORIZER_MODULE: "none"
+      CLUSTER_HOSTNAME: "node1"
+    labels:
+      - traefik.enable=false
+
+  # Milvus standalone needs etcd for metadata and MinIO for object storage.
+  # Keep those dependencies private to this Compose network; only Milvus's
+  # client and WebUI ports are published for local development.
+  milvus-etcd:
+    image: "quay.io/coreos/etcd:v3.5.18"
+    container_name: unstract-milvus-etcd
+    restart: unless-stopped
+    environment:
+      ETCD_AUTO_COMPACTION_MODE: revision
+      ETCD_AUTO_COMPACTION_RETENTION: "1000"
+      ETCD_QUOTA_BACKEND_BYTES: "4294967296"
+      ETCD_SNAPSHOT_COUNT: "50000"
+    volumes:
+      - milvus_etcd_data:/etcd
+    command: >-
+      etcd -advertise-client-urls=http://milvus-etcd:2379
+      -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
+    healthcheck:
+      test: ["CMD", "etcdctl", "endpoint", "health"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    labels:
+      - traefik.enable=false
+
+  milvus-minio:
+    image: "minio/minio:RELEASE.2024-05-28T17-19-04Z"
+    container_name: unstract-milvus-minio
+    restart: unless-stopped
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+      # Keep the legacy names aligned with the credentials expected by the
+      # Milvus 2.5 standalone configuration.
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - milvus_minio_data:/minio_data
+    command: minio server /minio_data --console-address ":9001"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+    labels:
+      - traefik.enable=false
+
+  milvus:
+    # Keep this aligned with the pymilvus 2.5.x dependency used by sdk1.
+    image: "milvusdb/milvus:v2.5.17"
+    container_name: unstract-milvus
+    restart: unless-stopped
+    command: ["milvus", "run", "standalone"]
+    security_opt:
+      - seccomp:unconfined
+    environment:
+      MINIO_REGION: us-east-1
+      ETCD_ENDPOINTS: milvus-etcd:2379
+      MINIO_ADDRESS: milvus-minio:9000
+      MINIO_ACCESS_KEY: minioadmin
+      MINIO_SECRET_KEY: minioadmin
+    volumes:
+      - milvus_data:/var/lib/milvus
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      interval: 30s
+      start_period: 90s
+      timeout: 20s
+      retries: 3
+    ports:
+      - "127.0.0.1:19530:19530"
+      - "127.0.0.1:9091:9091"
+    depends_on:
+      milvus-etcd:
+        condition: service_healthy
+      milvus-minio:
+        condition: service_healthy
+    labels:
+      - traefik.enable=false
 
   rabbitmq:
     image: rabbitmq:4.1.0-management
@@ -149,7 +289,12 @@ volumes:
   flipt_data:
   minio_data:
   postgres_data:
+  postgres_vector_data:
   qdrant_data:
   redis_data:
   prompt_studio_data:
   rabbitmq_data:
+  weaviate_data:
+  milvus_etcd_data:
+  milvus_minio_data:
+  milvus_data:
diff --git a/docker/sample.env b/docker/sample.env
index 47905773da..deec6a8847 100644
--- a/docker/sample.env
+++ b/docker/sample.env
@@ -2,6 +2,10 @@
 # with a YAML and JSONs
 TOOL_REGISTRY_CONFIG_SRC_PATH="${PWD}/../unstract/tool-registry/tool_registry_config"
 
+# Shared development API key for the local Qdrant and Weaviate services.
+# Keep these services bound to loopback or replace this key before exposing them.
+LOCAL_VECTOR_DB_API_KEY=unstract_vector_pass
+
 # Celery Autoscaling Configuration
 # Specify the maximum and minimum number of concurrent workers for each Celery worker.
 # Format: <max_workers>,<min_workers>
diff --git a/docker/sample.essentials.env b/docker/sample.essentials.env
index 51876fb8f9..238e61d72a 100644
--- a/docker/sample.essentials.env
+++ b/docker/sample.essentials.env
@@ -10,10 +10,6 @@ MINIO_ROOT_PASSWORD=minio123
 MINIO_ACCESS_KEY=minio
 MINIO_SECRET_KEY=minio123
 
-QDRANT_USER=unstract_vector_dev
-QDRANT_PASS=unstract_vector_pass
-QDRANT_DB=unstract_vector_db
-
 # RabbitMQ related envs
 RABBITMQ_DEFAULT_USER=admin
 RABBITMQ_DEFAULT_PASS=password
diff --git a/docker/scripts/db-setup/vector_db_setup.sh b/docker/scripts/db-setup/vector_db_setup.sh
new file mode 100755
index 0000000000..b45bce4fac
--- /dev/null
+++ b/docker/scripts/db-setup/vector_db_setup.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+: "${POSTGRES_USER:?POSTGRES_USER is required}"
+: "${POSTGRES_DB:?POSTGRES_DB is required}"
+: "${POSTGRES_SCHEMA:?POSTGRES_SCHEMA is required}"
+
+echo "Configuring pgvector database '$POSTGRES_DB' and schema '$POSTGRES_SCHEMA'"
+
+psql \
+  --username "$POSTGRES_USER" \
+  --dbname "$POSTGRES_DB" \
+  --set ON_ERROR_STOP=1 \
+  --set schema="$POSTGRES_SCHEMA" <<'SQL'
+CREATE EXTENSION IF NOT EXISTS vector;
+CREATE SCHEMA IF NOT EXISTS :"schema";
+SQL
diff --git a/docs/local-dev-setup-executor-migration.md b/docs/local-dev-setup-executor-migration.md
index 8bb6921fee..013e74b8f0 100644
--- a/docs/local-dev-setup-executor-migration.md
+++ b/docs/local-dev-setup-executor-migration.md
@@ -364,6 +364,11 @@ cd workers && ./run-worker.sh all
 | MinIO S3 API | 9000 | http://localhost:9000 |
 | MinIO Console | 9001 | http://localhost:9001 (minio/minio123) |
 | Qdrant | 6333 | http://localhost:6333 |
+| Postgres vector DB | 5433 | `psql -h localhost -p 5433 -U unstract_dev -d unstract_db` |
+| Weaviate HTTP API | 8084 | http://localhost:8084 |
+| Weaviate gRPC API | 50051 | localhost:50051 |
+| Milvus client API | 19530 | http://localhost:19530 |
+| Milvus WebUI | 9091 | http://localhost:9091/webui/ |
 | Traefik Dashboard | 8080 | http://localhost:8080 |
 
 ### Application
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py
index 1c08892f6b..7861f01c21 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/weaviate.py
@@ -1,5 +1,6 @@
 import logging
 import os
+from urllib.parse import urlparse
 
 import weaviate
 from llama_index.core.vector_stores.types import BasePydanticVectorStore
@@ -58,6 +59,39 @@ def get_doc_url() -> str:
     def get_vector_db_instance(self) -> BasePydanticVectorStore:
         return self._vector_db_instance
 
+    def _connect(self) -> weaviate.Client:
+        """Connect to either a local Weaviate server or Weaviate Cloud.
+
+        The adapter historically used the cloud connector for every URL. Local
+        Docker deployments use a different gRPC endpoint and can be reached
+        without a cloud cluster URL, so route plain HTTP URLs through
+        ``connect_to_local`` while preserving the existing HTTPS cloud path.
+        """
+        configured_url = str(self._config.get(Constants.URL, "")).strip()
+        if not configured_url:
+            raise ValueError("Weaviate URL is required")
+
+        api_key = str(self._config.get(Constants.API_KEY) or "").strip()
+        auth_credentials = Auth.api_key(api_key) if api_key else None
+        parsed_url = urlparse(
+            configured_url if "://" in configured_url else f"https://{configured_url}"
+        )
+
+        if parsed_url.scheme == "http":
+            if parsed_url.hostname is None:
+                raise ValueError("Weaviate URL must include a hostname")
+            return weaviate.connect_to_local(
+                host=parsed_url.hostname,
+                port=parsed_url.port or 8080,
+                grpc_port=50051,
+                auth_credentials=auth_credentials,
+            )
+
+        return weaviate.connect_to_weaviate_cloud(
+            cluster_url=configured_url,
+            auth_credentials=auth_credentials,
+        )
+
     def _get_vector_db_instance(self) -> BasePydanticVectorStore:
         try:
             collection_name = VectorDBHelper.get_collection_name(
@@ -68,10 +102,7 @@ def _get_vector_db_instance(self) -> BasePydanticVectorStore:
             # LLama-index throws the error if not capitalised while using
             # Weaviate
             self._collection_name = collection_name.capitalize()
-            self._client = weaviate.connect_to_weaviate_cloud(
-                cluster_url=str(self._config.get(Constants.URL)),
-                auth_credentials=Auth.api_key(str(self._config.get(Constants.API_KEY))),
-            )
+            self._client = self._connect()
 
             try:
                 # Class definition object. Weaviate's autoschema
diff --git a/unstract/sdk1/tests/test_weaviate_adapter.py b/unstract/sdk1/tests/test_weaviate_adapter.py
new file mode 100644
index 0000000000..440232b5e7
--- /dev/null
+++ b/unstract/sdk1/tests/test_weaviate_adapter.py
@@ -0,0 +1,63 @@
+from unittest.mock import MagicMock, patch
+
+from unstract.sdk1.adapters.vectordb.weaviate.src.weaviate import (
+    Constants,
+    Weaviate,
+)
+
+
+def _adapter(url: str, api_key: str = "") -> Weaviate:
+    adapter = object.__new__(Weaviate)
+    adapter._config = {
+        Constants.URL: url,
+        Constants.API_KEY: api_key,
+    }
+    return adapter
+
+
+def test_local_url_uses_local_connector_with_api_key() -> None:
+    client = MagicMock()
+
+    with patch(
+        "unstract.sdk1.adapters.vectordb.weaviate.src.weaviate.weaviate.connect_to_local",
+        return_value=client,
+    ) as connect_to_local:
+        result = _adapter("http://weaviate:8080", "local-key")._connect()
+
+    assert result is client
+    connect_to_local.assert_called_once()
+    call_kwargs = connect_to_local.call_args.kwargs
+    assert call_kwargs["host"] == "weaviate"
+    assert call_kwargs["port"] == 8080
+    assert call_kwargs["grpc_port"] == 50051
+    assert call_kwargs["auth_credentials"] is not None
+
+
+def test_local_url_allows_anonymous_connection() -> None:
+    client = MagicMock()
+
+    with patch(
+        "unstract.sdk1.adapters.vectordb.weaviate.src.weaviate.weaviate.connect_to_local",
+        return_value=client,
+    ) as connect_to_local:
+        _adapter("http://localhost:8084")._connect()
+
+    assert connect_to_local.call_args.kwargs["host"] == "localhost"
+    assert connect_to_local.call_args.kwargs["port"] == 8084
+    assert connect_to_local.call_args.kwargs["auth_credentials"] is None
+
+
+def test_https_url_preserves_cloud_connector() -> None:
+    client = MagicMock()
+
+    with patch(
+        "unstract.sdk1.adapters.vectordb.weaviate.src.weaviate.weaviate.connect_to_weaviate_cloud",
+        return_value=client,
+    ) as connect_to_cloud:
+        result = _adapter("https://example.weaviate.cloud", "cloud-key")._connect()
+
+    assert result is client
+    connect_to_cloud.assert_called_once()
+    assert connect_to_cloud.call_args.kwargs["cluster_url"] == (
+        "https://example.weaviate.cloud"
+    )

From a055a05da43a09eb4b83e3fe6a96a90dc0fac4cf Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 13:41:17 -0400
Subject: [PATCH 02/48] Keep local vector databases backward compatible

---
 docker/README.md                          | 10 ++++++----
 docker/docker-compose-dev-essentials.yaml | 17 ++++-------------
 docker/sample.env                         |  4 ----
 3 files changed, 10 insertions(+), 21 deletions(-)

diff --git a/docker/README.md b/docker/README.md
index 505e6c7951..4dbf5632f4 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -44,14 +44,16 @@ Pinecone is supported as a hosted provider and is intentionally not included
 in the local stack because it is not a self-hosted open-source service.
 
 The services use persistent named volumes and are bound to loopback on the host.
-The Unstract workers connect over the Compose network, so use the internal
-addresses below when creating an adapter in the UI:
+Qdrant and Weaviate use anonymous access because this is a local development
+stack; do not expose these ports beyond the local machine without adding
+authentication and TLS. The Unstract workers connect over the Compose network,
+so use the internal addresses below when creating an adapter in the UI:
 
 | Adapter | UI fields | Address from Unstract containers | Host address | Local credentials |
 |---------|-----------|----------------------------------|--------------|-------------------|
-| Qdrant | URL, API Key | `http://qdrant:6333` | `http://localhost:6333` | API key from `LOCAL_VECTOR_DB_API_KEY` (default: `unstract_vector_pass`) |
+| Qdrant | URL, API Key | `http://qdrant:6333` | `http://localhost:6333` | Leave API Key empty |
 | Postgres | Database, Host, Port, User, Password, Enable SSL | Host `postgres-vector`, port `5432` | Host `localhost`, port `5433` | Values from `docker/essentials.env`; set Enable SSL to `false` |
-| Weaviate | URL, API Key | `http://weaviate:8080` | `http://localhost:8084` | API key from `LOCAL_VECTOR_DB_API_KEY` (default: `unstract_vector_pass`) |
+| Weaviate | URL, API Key | `http://weaviate:8080` | `http://localhost:8084` | Leave API Key empty |
 | Milvus | URI, Token | `http://milvus:19530` | `http://localhost:19530` | Leave Token empty |
 
 For the Postgres adapter, use the `POSTGRES_USER`, `POSTGRES_PASSWORD`, and
diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index 1002ce084f..4c12f16206 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -127,10 +127,6 @@ services:
       - "127.0.0.1:6333:6333"
     volumes:
       - qdrant_data:/var/lib/qdrant/data/
-    environment:
-      # This is a development-only key. The service is also bound to loopback
-      # so it is not exposed on the host's network interfaces.
-      QDRANT__SERVICE__API_KEY: "${LOCAL_VECTOR_DB_API_KEY:-unstract_vector_pass}"
     labels:
       - traefik.enable=false
 
@@ -159,9 +155,9 @@ services:
     labels:
       - traefik.enable=false
 
-  # Weaviate is configured for API-key auth so its connection settings match
-  # the local Qdrant setup. The adapter uses the local connector for this HTTP
-  # endpoint; no cloud account is required.
+  # Weaviate uses anonymous access for this loopback-only development service.
+  # The adapter uses the local connector for this HTTP endpoint; no cloud
+  # account or API key is required.
   weaviate:
     image: "cr.weaviate.io/semitechnologies/weaviate:1.39.2"
     container_name: unstract-weaviate
@@ -180,12 +176,7 @@ services:
       - weaviate_data:/var/lib/weaviate
     environment:
       QUERY_DEFAULTS_LIMIT: "25"
-      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "false"
-      AUTHENTICATION_APIKEY_ENABLED: "true"
-      AUTHENTICATION_APIKEY_ALLOWED_KEYS: "${LOCAL_VECTOR_DB_API_KEY:-unstract_vector_pass}"
-      AUTHENTICATION_APIKEY_USERS: "unstract-local"
-      AUTHORIZATION_ENABLE_RBAC: "true"
-      AUTHORIZATION_RBAC_ROOT_USERS: "unstract-local"
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true"
       PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
       DEFAULT_VECTORIZER_MODULE: "none"
       CLUSTER_HOSTNAME: "node1"
diff --git a/docker/sample.env b/docker/sample.env
index deec6a8847..47905773da 100644
--- a/docker/sample.env
+++ b/docker/sample.env
@@ -2,10 +2,6 @@
 # with a YAML and JSONs
 TOOL_REGISTRY_CONFIG_SRC_PATH="${PWD}/../unstract/tool-registry/tool_registry_config"
 
-# Shared development API key for the local Qdrant and Weaviate services.
-# Keep these services bound to loopback or replace this key before exposing them.
-LOCAL_VECTOR_DB_API_KEY=unstract_vector_pass
-
 # Celery Autoscaling Configuration
 # Specify the maximum and minimum number of concurrent workers for each Celery worker.
 # Format: <max_workers>,<min_workers>

From e0b8591b613e43759bbb966fea9a28e823d8f3d1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 14:25:32 -0400
Subject: [PATCH 03/48] Use direct Weaviate image registry

---
 docker/docker-compose-dev-essentials.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index 4c12f16206..bc7c71fd2d 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -159,7 +159,7 @@ services:
   # The adapter uses the local connector for this HTTP endpoint; no cloud
   # account or API key is required.
   weaviate:
-    image: "cr.weaviate.io/semitechnologies/weaviate:1.39.2"
+    image: "docker.io/semitechnologies/weaviate:1.39.2"
     container_name: unstract-weaviate
     restart: unless-stopped
     command:

From 7ad31407a2735d018ef19059ecbe1138e5e776cd Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 14:45:33 -0400
Subject: [PATCH 04/48] Pin compatible Weaviate client

---
 backend/uv.lock              | 2 ++
 platform-service/uv.lock     | 8 +++++---
 unstract/sdk1/pyproject.toml | 3 +++
 unstract/sdk1/uv.lock        | 8 +++++---
 workers/uv.lock              | 8 +++++---
 5 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/backend/uv.lock b/backend/uv.lock
index 5f7fe242e7..074fbb7007 100644
--- a/backend/uv.lock
+++ b/backend/uv.lock
@@ -3943,6 +3943,7 @@ dependencies = [
     { name = "llama-index-vector-stores-postgres" },
     { name = "llama-index-vector-stores-qdrant" },
     { name = "llama-index-vector-stores-weaviate" },
+    { name = "weaviate-client" },
     { name = "llama-parse" },
     { name = "llmwhisperer-client" },
     { name = "pdfplumber" },
@@ -3982,6 +3983,7 @@ requires-dist = [
     { name = "llama-index-vector-stores-postgres", specifier = ">=0.7.3" },
     { name = "llama-index-vector-stores-qdrant", specifier = ">=0.9.1" },
     { name = "llama-index-vector-stores-weaviate", specifier = ">=1.4.1" },
+    { name = "weaviate-client", specifier = "==4.18.3" },
     { name = "llama-parse", specifier = ">=0.6.0" },
     { name = "llmwhisperer-client", specifier = ">=2.8.1" },
     { name = "pdfplumber", specifier = ">=0.11.2" },
diff --git a/platform-service/uv.lock b/platform-service/uv.lock
index 8af401acc8..898f38075a 100644
--- a/platform-service/uv.lock
+++ b/platform-service/uv.lock
@@ -2715,6 +2715,7 @@ dependencies = [
     { name = "llama-index-vector-stores-postgres" },
     { name = "llama-index-vector-stores-qdrant" },
     { name = "llama-index-vector-stores-weaviate" },
+    { name = "weaviate-client" },
     { name = "llama-parse" },
     { name = "llmwhisperer-client" },
     { name = "pdfplumber" },
@@ -2754,6 +2755,7 @@ requires-dist = [
     { name = "llama-index-vector-stores-postgres", specifier = ">=0.7.3" },
     { name = "llama-index-vector-stores-qdrant", specifier = ">=0.9.1" },
     { name = "llama-index-vector-stores-weaviate", specifier = ">=1.4.1" },
+    { name = "weaviate-client", specifier = "==4.18.3" },
     { name = "llama-parse", specifier = ">=0.6.0" },
     { name = "llmwhisperer-client", specifier = ">=2.8.1" },
     { name = "pdfplumber", specifier = ">=0.11.2" },
@@ -2818,7 +2820,7 @@ wheels = [
 
 [[package]]
 name = "weaviate-client"
-version = "4.17.0"
+version = "4.18.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "authlib" },
@@ -2829,9 +2831,9 @@ dependencies = [
     { name = "pydantic" },
     { name = "validators" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/bd/0e/e4582b007427187a9fde55fa575db4b766c81929d2b43a3dd8becce50567/weaviate_client-4.17.0.tar.gz", hash = "sha256:731d58d84b0989df4db399b686357ed285fb95971a492ccca8dec90bb2343c51", size = 769019 }
+sdist = { url = "https://files.pythonhosted.org/packages/a8/76/14e07761c5fb7e8573e3cff562e2d9073c65f266db0e67511403d10435b1/weaviate_client-4.18.3.tar.gz", hash = "sha256:9d889246d62be36641a7f2b8cedf5fb665b804d46f7a53ae37e02d297a11f119", size = 783634 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5b/c5/2da3a45866da7a935dab8ad07be05dcaee48b3ad4955144583b651929be7/weaviate_client-4.17.0-py3-none-any.whl", hash = "sha256:60e4a355b90537ee1e942ab0b76a94750897a13d9cf13c5a6decbd166d0ca8b5", size = 582763 },
+    { url = "https://files.pythonhosted.org/packages/3a/ab/f1c2bef56199505bcd07a6747e7705d84f2d40f20c757237323d13d219d0/weaviate_client-4.18.3-py3-none-any.whl", hash = "sha256:fc6ef510dd7b63ab0b673a35a7de9573abbd0626fc80de54633f0ccfd52772b7", size = 599877 },
 ]
 
 [[package]]
diff --git a/unstract/sdk1/pyproject.toml b/unstract/sdk1/pyproject.toml
index bfffb922bd..da2775d0ea 100644
--- a/unstract/sdk1/pyproject.toml
+++ b/unstract/sdk1/pyproject.toml
@@ -32,6 +32,9 @@ dependencies = [
     "llama-index-vector-stores-postgres>=0.7.3",
     "llama-index-vector-stores-milvus>=0.9.6",
     "llama-index-vector-stores-weaviate>=1.4.1",
+    # LlamaIndex imports a private Weaviate client helper that was removed in
+    # 4.20.0; keep the adapter runtime on the compatible client line.
+    "weaviate-client==4.18.3",
     "llama-index-vector-stores-pinecone>=0.7.1",
     "llama-index-vector-stores-qdrant>=0.9.1",
     # 1.17.0 has a bug: grpc.UpdateMode | None fails with protobuf's EnumTypeWrapper
diff --git a/unstract/sdk1/uv.lock b/unstract/sdk1/uv.lock
index 449ff9450b..971ec42113 100644
--- a/unstract/sdk1/uv.lock
+++ b/unstract/sdk1/uv.lock
@@ -2762,6 +2762,7 @@ dependencies = [
     { name = "singleton-decorator" },
     { name = "tiktoken" },
     { name = "unstract-core" },
+    { name = "weaviate-client" },
 ]
 
 [package.optional-dependencies]
@@ -2825,6 +2826,7 @@ requires-dist = [
     { name = "singleton-decorator", specifier = "~=1.0.0" },
     { name = "tiktoken", specifier = "~=0.12.0" },
     { name = "unstract-core", editable = "../core" },
+    { name = "weaviate-client", specifier = "==4.18.3" },
 ]
 provides-extras = ["aws", "azure", "gcs"]
 
@@ -2892,7 +2894,7 @@ wheels = [
 
 [[package]]
 name = "weaviate-client"
-version = "4.17.0"
+version = "4.18.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "authlib" },
@@ -2903,9 +2905,9 @@ dependencies = [
     { name = "pydantic" },
     { name = "validators" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/bd/0e/e4582b007427187a9fde55fa575db4b766c81929d2b43a3dd8becce50567/weaviate_client-4.17.0.tar.gz", hash = "sha256:731d58d84b0989df4db399b686357ed285fb95971a492ccca8dec90bb2343c51", size = 769019, upload-time = "2025-09-26T11:20:27.381Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a8/76/14e07761c5fb7e8573e3cff562e2d9073c65f266db0e67511403d10435b1/weaviate_client-4.18.3.tar.gz", hash = "sha256:9d889246d62be36641a7f2b8cedf5fb665b804d46f7a53ae37e02d297a11f119", size = 783634, upload-time = "2025-12-03T09:38:28.261Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5b/c5/2da3a45866da7a935dab8ad07be05dcaee48b3ad4955144583b651929be7/weaviate_client-4.17.0-py3-none-any.whl", hash = "sha256:60e4a355b90537ee1e942ab0b76a94750897a13d9cf13c5a6decbd166d0ca8b5", size = 582763, upload-time = "2025-09-26T11:20:25.864Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/ab/f1c2bef56199505bcd07a6747e7705d84f2d40f20c757237323d13d219d0/weaviate_client-4.18.3-py3-none-any.whl", hash = "sha256:fc6ef510dd7b63ab0b673a35a7de9573abbd0626fc80de54633f0ccfd52772b7", size = 599877, upload-time = "2025-12-03T09:38:26.487Z" },
 ]
 
 [[package]]
diff --git a/workers/uv.lock b/workers/uv.lock
index d80d0c223f..6c613671ae 100644
--- a/workers/uv.lock
+++ b/workers/uv.lock
@@ -4830,6 +4830,7 @@ dependencies = [
     { name = "llama-index-vector-stores-postgres" },
     { name = "llama-index-vector-stores-qdrant" },
     { name = "llama-index-vector-stores-weaviate" },
+    { name = "weaviate-client" },
     { name = "llama-parse" },
     { name = "llmwhisperer-client" },
     { name = "pdfplumber" },
@@ -4869,6 +4870,7 @@ requires-dist = [
     { name = "llama-index-vector-stores-postgres", specifier = ">=0.7.3" },
     { name = "llama-index-vector-stores-qdrant", specifier = ">=0.9.1" },
     { name = "llama-index-vector-stores-weaviate", specifier = ">=1.4.1" },
+    { name = "weaviate-client", specifier = "==4.18.3" },
     { name = "llama-parse", specifier = ">=0.6.0" },
     { name = "llmwhisperer-client", specifier = ">=2.8.1" },
     { name = "pdfplumber", specifier = ">=0.11.2" },
@@ -5117,7 +5119,7 @@ wheels = [
 
 [[package]]
 name = "weaviate-client"
-version = "4.20.1"
+version = "4.18.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "authlib" },
@@ -5128,9 +5130,9 @@ dependencies = [
     { name = "pydantic" },
     { name = "validators" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/6a/0e/450732a620dec30e5c13d0e5ba6ea81cd2b168ffb5a1e6cfa034da2fd988/weaviate_client-4.20.1.tar.gz", hash = "sha256:7d3c4835292b17d54757c3f62921e87975c01f26869db787250cebef2a17bd80", size = 807802 }
+sdist = { url = "https://files.pythonhosted.org/packages/a8/76/14e07761c5fb7e8573e3cff562e2d9073c65f266db0e67511403d10435b1/weaviate_client-4.18.3.tar.gz", hash = "sha256:9d889246d62be36641a7f2b8cedf5fb665b804d46f7a53ae37e02d297a11f119", size = 783634 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/de/3a/a2c74bfbdc5f8acbab8025205520448dcece82942206a9861ec48ccad03f/weaviate_client-4.20.1-py3-none-any.whl", hash = "sha256:6ca36bb8752c39589bfc856c1232cbcb627f69141e7a741b592e2516888f59d8", size = 618735 },
+    { url = "https://files.pythonhosted.org/packages/3a/ab/f1c2bef56199505bcd07a6747e7705d84f2d40f20c757237323d13d219d0/weaviate_client-4.18.3-py3-none-any.whl", hash = "sha256:fc6ef510dd7b63ab0b673a35a7de9573abbd0626fc80de54633f0ccfd52772b7", size = 599877 },
 ]
 
 [[package]]

From 2c2ca7b0ebd112f1676ce1e10b41ddf20a5ad31b Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 16:15:57 -0400
Subject: [PATCH 05/48] Prefill local vector database adapter settings

---
 docker/README.md                              |  9 ++++
 .../milvus/src/static/json_schema.json        |  7 +--
 .../postgres/src/static/json_schema.json      | 10 ++--
 .../qdrant/src/static/json_schema.json        |  4 +-
 .../weaviate/src/static/json_schema.json      |  9 ++--
 .../tests/test_local_vector_db_schemas.py     | 50 +++++++++++++++++++
 6 files changed, 78 insertions(+), 11 deletions(-)
 create mode 100644 unstract/sdk1/tests/test_local_vector_db_schemas.py

diff --git a/docker/README.md b/docker/README.md
index 4dbf5632f4..17cc4bf975 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -56,6 +56,15 @@ so use the internal addresses below when creating an adapter in the UI:
 | Weaviate | URL, API Key | `http://weaviate:8080` | `http://localhost:8084` | Leave API Key empty |
 | Milvus | URI, Token | `http://milvus:19530` | `http://localhost:19530` | Leave Token empty |
 
+When the adapter schema is served by the local development stack, these
+internal values are pre-filled in the form, including the sample PostgreSQL
+credentials from `docker/essentials.env` and SSL disabled for the local
+connection. If `essentials.env` has customized the PostgreSQL credentials,
+replace the pre-filled values with the customized values before testing the
+connection. Keep the container DNS names (`qdrant`, `postgres-vector`,
+`weaviate`, and `milvus`) when the adapter is used by Unstract workers; use the
+host addresses only for clients running outside the Compose network.
+
 For the Postgres adapter, use the `POSTGRES_USER`, `POSTGRES_PASSWORD`, and
 `POSTGRES_DB` values in `docker/essentials.env`; the vector database has its own
 container and volume even though it reuses the platform's local development
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/static/json_schema.json
index 22965dbbe6..82b2ec6890 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/milvus/src/static/json_schema.json
@@ -1,5 +1,6 @@
 {
   "title": "Milvus Vector DB",
+  "description": "Local Docker defaults are pre-filled for the bundled Milvus service. Replace them when connecting to Zilliz Cloud or another Milvus server.",
   "type": "object",
   "required": [
     "adapter_name",
@@ -9,15 +10,15 @@
     "adapter_name": {
       "type": "string",
       "title": "Name",
-      "default": "",
+      "default": "milvus-local",
       "description": "Provide a unique name for this adapter instance. Example: milvus-vdb-1"
     },
     "uri": {
       "type": "string",
       "title": "URI",
       "format": "uri",
-      "default": "localhost:19530",
-      "description": "Provide the URI of the Milvus server. Example: `https://<instance-id>.api.gcp-us-west1.zillizcloud.com`"
+      "default": "http://milvus:19530",
+      "description": "Provide the URI of the Milvus server. Local Docker: http://milvus:19530. Cloud example: `https://<instance-id>.api.gcp-us-west1.zillizcloud.com`"
     },
     "token": {
       "type": "string",
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
index 8ae51cc89f..ce33d286c7 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
@@ -1,5 +1,6 @@
 {
   "title": "Postgres Vector DB",
+  "description": "Local Docker defaults are pre-filled for the bundled pgvector service. Replace them when connecting to another PostgreSQL server.",
   "type": "object",
   "required": [
     "adapter_name",
@@ -13,17 +14,19 @@
     "adapter_name": {
       "type": "string",
       "title": "Name",
-      "default": "",
+      "default": "postgres-vector-local",
       "description": "Provide a unique name for this adapter instance. Example: pg-vdb-1"
     },
     "database": {
       "type": "string",
       "title": "Database",
+      "default": "unstract_db",
       "description": "Provide a name for the database"
     },
     "host": {
       "type": "string",
       "title": "Host",
+      "default": "postgres-vector",
       "description": "Hostname"
     },
     "port": {
@@ -34,19 +37,20 @@
     "user": {
       "type": "string",
       "title": "User",
+      "default": "unstract_dev",
       "description": "Username"
     },
     "password": {
       "type": "string",
       "title": "Password",
       "format": "password",
-      "default": ""
+      "default": "unstract_pass"
     },
     "enable_ssl": {
       "type": "boolean",
       "title": "Enable SSL",
       "description": "On selecting the checkbox, data encryption using SSL is enabled",
-      "default": true
+      "default": false
     }
   }
 }
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/static/json_schema.json
index 6f4b099b7b..4fbc08fea5 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/qdrant/src/static/json_schema.json
@@ -1,5 +1,6 @@
 {
   "title": "Qdrant Vector DB",
+  "description": "Local Docker defaults are pre-filled for the bundled Qdrant service. Replace them when connecting to another Qdrant server.",
   "type": "object",
   "required": [
     "adapter_name",
@@ -9,12 +10,13 @@
     "adapter_name": {
       "type": "string",
       "title": "Name",
-      "default": "",
+      "default": "qdrant-local",
       "description": "Provide a unique name for this adapter instance. Example: qdrant-vdb-1"
     },
     "url": {
       "type": "string",
       "title": "URL",
+      "default": "http://qdrant:6333",
       "pattern": "(http|https)?[\\w\\d\\-\\.:]+"
     },
     "api_key": {
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/static/json_schema.json
index 25ae48e2bb..9e72a1e889 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/weaviate/src/static/json_schema.json
@@ -1,5 +1,6 @@
 {
   "title": "Weaviate Vector DB",
+  "description": "Local Docker defaults are pre-filled for the bundled anonymous Weaviate service. Replace them when connecting to Weaviate Cloud or another server.",
   "type": "object",
   "required": [
     "adapter_name",
@@ -9,20 +10,20 @@
     "adapter_name": {
       "type": "string",
       "title": "Name",
-      "default": "",
+      "default": "weaviate-local",
       "description": "Provide a unique name for this adapter instance. Example: weaviate-vdb-1"
     },
     "url": {
       "type": "string",
       "title": "URL",
-      "default": "",
+      "default": "http://weaviate:8080",
       "format": "uri",
-      "description": "The URL of the Vector DB instance. Example: https://<your-instance>.weaviate.network"
+      "description": "The URL of the Vector DB instance. Local Docker: http://weaviate:8080. Cloud example: https://<your-instance>.weaviate.network"
     },
     "api_key": {
       "type": "string",
       "title": "Api Key",
-      "deafult": "",
+      "default": "",
       "format": "password"
     }
   }
diff --git a/unstract/sdk1/tests/test_local_vector_db_schemas.py b/unstract/sdk1/tests/test_local_vector_db_schemas.py
new file mode 100644
index 0000000000..5ec93b5ec0
--- /dev/null
+++ b/unstract/sdk1/tests/test_local_vector_db_schemas.py
@@ -0,0 +1,50 @@
+import json
+from pathlib import Path
+
+
+SCHEMA_ROOT = (
+    Path(__file__).parents[1]
+    / "src"
+    / "unstract"
+    / "sdk1"
+    / "adapters"
+    / "vectordb"
+)
+
+EXPECTED_DEFAULTS = {
+    "qdrant": {
+        "adapter_name": "qdrant-local",
+        "url": "http://qdrant:6333",
+        "api_key": "",
+    },
+    "postgres": {
+        "adapter_name": "postgres-vector-local",
+        "database": "unstract_db",
+        "host": "postgres-vector",
+        "port": 5432,
+        "user": "unstract_dev",
+        "password": "unstract_pass",
+        "enable_ssl": False,
+    },
+    "weaviate": {
+        "adapter_name": "weaviate-local",
+        "url": "http://weaviate:8080",
+        "api_key": "",
+    },
+    "milvus": {
+        "adapter_name": "milvus-local",
+        "uri": "http://milvus:19530",
+        "token": "",
+    },
+}
+
+
+def test_local_vector_db_schemas_prefill_compose_connection_values() -> None:
+    for adapter_name, expected_defaults in EXPECTED_DEFAULTS.items():
+        schema_path = SCHEMA_ROOT / adapter_name / "src" / "static" / "json_schema.json"
+        schema = json.loads(schema_path.read_text())
+        properties = schema["properties"]
+
+        for field_name, expected_value in expected_defaults.items():
+            assert properties[field_name]["default"] == expected_value
+

From 86f9c63694421c78cc90428ad308ad99471f239b Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 21:18:50 -0400
Subject: [PATCH 06/48] Clarify local Postgres SSL default

---
 .../sdk1/adapters/vectordb/postgres/src/static/json_schema.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
index ce33d286c7..fb467e914c 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
@@ -49,7 +49,7 @@
     "enable_ssl": {
       "type": "boolean",
       "title": "Enable SSL",
-      "description": "On selecting the checkbox, data encryption using SSL is enabled",
+      "description": "Disabled for the bundled local PostgreSQL service; enable only when the server is configured for SSL.",
       "default": false
     }
   }

From dbcab1b8ae19b2c528b71d6fd562edf1525a8f74 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 22:12:10 -0400
Subject: [PATCH 07/48] Enable TLS for local Postgres vector service

---
 docker/README.md                              |  9 ++-
 docker/docker-compose-dev-essentials.yaml     |  7 +++
 .../db-setup/postgres-vector-entrypoint.sh    | 56 +++++++++++++++++++
 .../vectordb/postgres/src/postgres.py         | 30 +++++++---
 .../postgres/src/static/json_schema.json      |  4 +-
 .../tests/test_local_vector_db_schemas.py     |  4 +-
 unstract/sdk1/tests/test_postgres_adapter.py  | 54 ++++++++++++++++++
 7 files changed, 147 insertions(+), 17 deletions(-)
 create mode 100644 docker/scripts/db-setup/postgres-vector-entrypoint.sh
 create mode 100644 unstract/sdk1/tests/test_postgres_adapter.py

diff --git a/docker/README.md b/docker/README.md
index 17cc4bf975..7e5b2d951d 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -52,14 +52,17 @@ so use the internal addresses below when creating an adapter in the UI:
 | Adapter | UI fields | Address from Unstract containers | Host address | Local credentials |
 |---------|-----------|----------------------------------|--------------|-------------------|
 | Qdrant | URL, API Key | `http://qdrant:6333` | `http://localhost:6333` | Leave API Key empty |
-| Postgres | Database, Host, Port, User, Password, Enable SSL | Host `postgres-vector`, port `5432` | Host `localhost`, port `5433` | Values from `docker/essentials.env`; set Enable SSL to `false` |
+| Postgres | Database, Host, Port, User, Password, Enable SSL | Host `postgres-vector`, port `5432` | Host `localhost`, port `5433` | Values from `docker/essentials.env`; SSL is enabled by default |
 | Weaviate | URL, API Key | `http://weaviate:8080` | `http://localhost:8084` | Leave API Key empty |
 | Milvus | URI, Token | `http://milvus:19530` | `http://localhost:19530` | Leave Token empty |
 
 When the adapter schema is served by the local development stack, these
 internal values are pre-filled in the form, including the sample PostgreSQL
-credentials from `docker/essentials.env` and SSL disabled for the local
-connection. If `essentials.env` has customized the PostgreSQL credentials,
+credentials from `docker/essentials.env` and SSL enabled for the local
+connection. The bundled `postgres-vector` service generates a self-signed
+certificate and starts PostgreSQL with TLS enabled. The adapter uses
+`sslmode=require` without certificate verification for this local-only
+certificate. If `essentials.env` has customized the PostgreSQL credentials,
 replace the pre-filled values with the customized values before testing the
 connection. Keep the container DNS names (`qdrant`, `postgres-vector`,
 `weaviate`, and `milvus`) when the adapter is used by Unstract workers; use the
diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index bc7c71fd2d..8e8937c93c 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -138,11 +138,17 @@ services:
     container_name: unstract-postgres-vector
     restart: unless-stopped
     shm_size: 128mb
+    # Generate a local self-signed certificate before the official Postgres
+    # entrypoint initializes or starts the server.
+    entrypoint: ["/usr/local/bin/postgres-vector-entrypoint.sh"]
+    command: ["postgres"]
     ports:
       - "127.0.0.1:5433:5432"
     volumes:
       - postgres_vector_data:/var/lib/postgresql/data/
+      - postgres_vector_ssl:/var/lib/postgresql/ssl/
       - ./scripts/db-setup/vector_db_setup.sh:/docker-entrypoint-initdb.d/vector_db_setup.sh:ro
+      - ./scripts/db-setup/postgres-vector-entrypoint.sh:/usr/local/bin/postgres-vector-entrypoint.sh:ro
     env_file:
       # Reuse the local platform database credentials from essentials.env while
       # retaining a separate data directory and container.
@@ -281,6 +287,7 @@ volumes:
   minio_data:
   postgres_data:
   postgres_vector_data:
+  postgres_vector_ssl:
   qdrant_data:
   redis_data:
   prompt_studio_data:
diff --git a/docker/scripts/db-setup/postgres-vector-entrypoint.sh b/docker/scripts/db-setup/postgres-vector-entrypoint.sh
new file mode 100644
index 0000000000..0c5b8eb619
--- /dev/null
+++ b/docker/scripts/db-setup/postgres-vector-entrypoint.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+# The local pgvector container uses a self-signed certificate because it is a
+# loopback-only development service. The adapter uses sslmode=require, which
+# encrypts the connection without requiring a separately distributed CA.
+if [[ "${1:-}" == "postgres" ]]; then
+    ssl_dir="${POSTGRES_SSL_DIR:-/var/lib/postgresql/ssl}"
+    cert_file="${POSTGRES_SSL_CERT_FILE:-${ssl_dir}/server.crt}"
+    key_file="${POSTGRES_SSL_KEY_FILE:-${ssl_dir}/server.key}"
+    common_name="${POSTGRES_SSL_COMMON_NAME:-postgres-vector}"
+    cert_days="${POSTGRES_SSL_CERT_DAYS:-3650}"
+
+    mkdir -p "${ssl_dir}"
+    if [[ "$(id -u)" -eq 0 ]]; then
+        chown postgres:postgres "${ssl_dir}"
+        chmod 700 "${ssl_dir}"
+    fi
+
+    if [[ ! -s "${cert_file}" || ! -s "${key_file}" ]]; then
+        if [[ "$(id -u)" -ne 0 ]]; then
+            echo "Postgres SSL certificate is missing and the entrypoint is not running as root" >&2
+            exit 1
+        fi
+
+        temporary_dir="$(mktemp -d "${ssl_dir}/.postgres-ssl.XXXXXX")"
+        trap 'rm -rf "${temporary_dir}"' EXIT
+
+        openssl req -new -x509 -nodes -sha256 \
+            -days "${cert_days}" \
+            -subj "/CN=${common_name}" \
+            -addext "subjectAltName=DNS:${common_name},DNS:localhost,IP:127.0.0.1" \
+            -keyout "${temporary_dir}/server.key" \
+            -out "${temporary_dir}/server.crt"
+
+        chown postgres:postgres "${temporary_dir}/server.key" "${temporary_dir}/server.crt"
+        chmod 600 "${temporary_dir}/server.key"
+        chmod 644 "${temporary_dir}/server.crt"
+        mv -f "${temporary_dir}/server.key" "${key_file}"
+        mv -f "${temporary_dir}/server.crt" "${cert_file}"
+    fi
+
+    if [[ "$(id -u)" -eq 0 ]]; then
+        chown postgres:postgres "${cert_file}" "${key_file}"
+        chmod 600 "${key_file}"
+        chmod 644 "${cert_file}"
+    fi
+
+    set -- "$@" \
+        -c "ssl=on" \
+        -c "ssl_cert_file=${cert_file}" \
+        -c "ssl_key_file=${key_file}"
+fi
+
+exec /usr/local/bin/docker-entrypoint.sh "$@"
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py
index 7182f4d42b..981280d9c0 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/postgres.py
@@ -6,6 +6,7 @@
 
 import psycopg2
 from llama_index.vector_stores.postgres import PGVectorStore
+
 from unstract.sdk1.adapters.exceptions import AdapterError
 from unstract.sdk1.adapters.vectordb.constants import VectorDbConstants
 from unstract.sdk1.adapters.vectordb.helper import VectorDBHelper
@@ -81,20 +82,31 @@ def _get_vector_db_instance(self) -> BasePydanticVectorStore:
                 Constants.SCHEMA,
                 VectorDbConstants.DEFAULT_VECTOR_DB_NAME,
             )
+            ssl_mode = (
+                "require"
+                if self._config.get(Constants.ENABLE_SSL, True)
+                else "disable"
+            )
+            connection_suffix = f"?sslmode={ssl_mode}"
+            connection_string = (
+                f"postgresql+psycopg2://"
+                f"{self._config.get(Constants.USER)}:{encoded_password}"
+                f"@{self._config.get(Constants.HOST)}:{self._config.get(Constants.PORT)}"
+                f"/{self._config.get(Constants.DATABASE)}{connection_suffix}"
+            )
+            async_connection_string = (
+                f"postgresql+asyncpg://"
+                f"{self._config.get(Constants.USER)}:{encoded_password}"
+                f"@{self._config.get(Constants.HOST)}:{self._config.get(Constants.PORT)}"
+                f"/{self._config.get(Constants.DATABASE)}{connection_suffix}"
+            )
             vector_db: BasePydanticVectorStore = PGVectorStore.from_params(
-                database=self._config.get(Constants.DATABASE),
+                connection_string=connection_string,
+                async_connection_string=async_connection_string,
                 schema_name=self._schema_name,
-                host=self._config.get(Constants.HOST),
-                password=encoded_password,
-                port=str(self._config.get(Constants.PORT)),
-                user=self._config.get(Constants.USER),
                 table_name=self._collection_name,
                 embed_dim=dimension,
             )
-            if self._config.get(Constants.ENABLE_SSL, True):
-                ssl_mode = "require"
-            else:
-                ssl_mode = "disable"
             self._client = psycopg2.connect(
                 database=self._config.get(Constants.DATABASE),
                 host=self._config.get(Constants.HOST),
diff --git a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
index fb467e914c..5d9baafec0 100644
--- a/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
+++ b/unstract/sdk1/src/unstract/sdk1/adapters/vectordb/postgres/src/static/json_schema.json
@@ -49,8 +49,8 @@
     "enable_ssl": {
       "type": "boolean",
       "title": "Enable SSL",
-      "description": "Disabled for the bundled local PostgreSQL service; enable only when the server is configured for SSL.",
-      "default": false
+      "description": "TLS is enabled for the bundled local PostgreSQL service; disable only when connecting to a server without SSL.",
+      "default": true
     }
   }
 }
diff --git a/unstract/sdk1/tests/test_local_vector_db_schemas.py b/unstract/sdk1/tests/test_local_vector_db_schemas.py
index 5ec93b5ec0..adba7a76a4 100644
--- a/unstract/sdk1/tests/test_local_vector_db_schemas.py
+++ b/unstract/sdk1/tests/test_local_vector_db_schemas.py
@@ -1,7 +1,6 @@
 import json
 from pathlib import Path
 
-
 SCHEMA_ROOT = (
     Path(__file__).parents[1]
     / "src"
@@ -24,7 +23,7 @@
         "port": 5432,
         "user": "unstract_dev",
         "password": "unstract_pass",
-        "enable_ssl": False,
+        "enable_ssl": True,
     },
     "weaviate": {
         "adapter_name": "weaviate-local",
@@ -47,4 +46,3 @@ def test_local_vector_db_schemas_prefill_compose_connection_values() -> None:
 
         for field_name, expected_value in expected_defaults.items():
             assert properties[field_name]["default"] == expected_value
-
diff --git a/unstract/sdk1/tests/test_postgres_adapter.py b/unstract/sdk1/tests/test_postgres_adapter.py
new file mode 100644
index 0000000000..a91dec9abb
--- /dev/null
+++ b/unstract/sdk1/tests/test_postgres_adapter.py
@@ -0,0 +1,54 @@
+import pytest
+
+from unstract.sdk1.adapters.vectordb.postgres.src import postgres as postgres_module
+
+
+@pytest.mark.parametrize(
+    ("enable_ssl", "sslmode"),
+    ((True, "require"), (False, "disable")),
+)
+def test_postgres_adapter_applies_sslmode_to_all_connections(
+    monkeypatch: pytest.MonkeyPatch,
+    enable_ssl: bool,
+    sslmode: str,
+) -> None:
+    captured: dict[str, object] = {}
+
+    def fake_from_params(**kwargs: object) -> object:
+        captured["vector_store"] = kwargs
+        return object()
+
+    def fake_connect(**kwargs: object) -> object:
+        captured["probe"] = kwargs
+        return object()
+
+    monkeypatch.setattr(
+        postgres_module.PGVectorStore,
+        "from_params",
+        fake_from_params,
+    )
+    monkeypatch.setattr(postgres_module.psycopg2, "connect", fake_connect)
+
+    postgres_module.Postgres(
+        {
+            "database": "unstract_db",
+            "host": "postgres-vector",
+            "port": 5432,
+            "user": "unstract_dev",
+            "password": "unstract_pass",
+            "enable_ssl": enable_ssl,
+        }
+    )
+
+    vector_store_kwargs = captured["vector_store"]
+    assert isinstance(vector_store_kwargs, dict)
+    assert vector_store_kwargs["connection_string"].endswith(
+        f"/unstract_db?sslmode={sslmode}"
+    )
+    assert vector_store_kwargs["async_connection_string"].endswith(
+        f"/unstract_db?sslmode={sslmode}"
+    )
+
+    probe_kwargs = captured["probe"]
+    assert isinstance(probe_kwargs, dict)
+    assert probe_kwargs["sslmode"] == sslmode

From c8d9e9c3b2a145db947fea893e87f0e905c94442 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 3 Sep 2026 22:15:39 -0400
Subject: [PATCH 08/48] Make Postgres TLS entrypoint executable

---
 docker/scripts/db-setup/postgres-vector-entrypoint.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 docker/scripts/db-setup/postgres-vector-entrypoint.sh

diff --git a/docker/scripts/db-setup/postgres-vector-entrypoint.sh b/docker/scripts/db-setup/postgres-vector-entrypoint.sh
old mode 100644
new mode 100755

From 790435e4aa353982b8157d506ad62abf1957dccc Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <Timothy.Gregg@complete.tech>
Date: Mon, 7 Sep 2026 02:54:01 -0400
Subject: [PATCH 09/48] Sync local changes from domains/etl.home.complete.tech
 (2026-09-07)

---
 Caddyfile.snippet          | 26 +++++++++++++
 DOMAIN_CONTEXT.md          | 77 ++++++++++++++++++++++++++++++++++++++
 docker/frontend-nginx.conf | 71 +++++++++++++++++++++++++++++++++++
 3 files changed, 174 insertions(+)
 create mode 100644 Caddyfile.snippet
 create mode 100644 DOMAIN_CONTEXT.md
 create mode 100644 docker/frontend-nginx.conf

diff --git a/Caddyfile.snippet b/Caddyfile.snippet
new file mode 100644
index 0000000000..2e0f7123b4
--- /dev/null
+++ b/Caddyfile.snippet
@@ -0,0 +1,26 @@
+(etl_home_access_log) {
+    log {
+        level INFO
+        format json
+        output file /var/log/caddy/etl-home-access.jsonl {
+            roll_size 50MiB
+            roll_keep 14
+            roll_keep_for 336h
+        }
+    }
+}
+
+etl.home.complete.tech:80 {
+    redir https://etl.home.complete.tech{uri} permanent
+}
+
+etl.home.complete.tech:443 {
+    import etl_home_access_log
+    tls /certs/fullchain.pem /certs/privkey.pem
+
+    reverse_proxy 172.30.88.1:13110 {
+        header_up X-Forwarded-For {remote_host}
+        header_up X-Forwarded-Proto https
+        header_up X-Forwarded-Host {host}
+    }
+}
diff --git a/DOMAIN_CONTEXT.md b/DOMAIN_CONTEXT.md
new file mode 100644
index 0000000000..ac5856b6dc
--- /dev/null
+++ b/DOMAIN_CONTEXT.md
@@ -0,0 +1,77 @@
+# Domain Context: etl.home.complete.tech
+
+Local home: `C:\Users\timot\Documents\projects\domains\etl.home.complete.tech`
+
+Public URL: `https://etl.home.complete.tech/`
+
+Source: `https://github.com/Zipstack/unstract`
+
+Service host: `completetrain@train.home.complete.tech`
+
+Remote root: `/home/completetrain/etl.home.complete.tech`
+
+Runtime: rootless Podman under `completetrain`, fronted by the shared rootful
+Caddy container `home-complete-tech-domain-proxy`.
+
+## Deployment shape
+
+- Upstream source is pinned to the release tag `v0.187.2`.
+- Unstract is run with the upstream Docker Compose files plus
+  `docker/compose.train.yaml`.
+- The Unstract Traefik service is published only on host port `13110`; all
+  database, broker, storage, and worker ports remain internal to
+  `unstract-network`.
+- The runner and Unstract Traefik use the rootless Podman API socket at
+  `/run/user/1000/podman/podman.sock`, mounted at `/var/run/docker.sock` in
+  their containers. This preserves Unstract's Docker-compatible container
+  spawning path without exposing the rootful host socket.
+- The frontend is mounted with `docker/frontend-nginx.conf` and listens on
+  container port `8080`, because the published frontend image runs Nginx as a
+  non-root user under rootless Podman.
+- Shared Caddy proxies `etl.home.complete.tech` to `172.30.88.1:13110` and
+  uses the existing wildcard certificate mounted at `/certs`.
+- LAN DNS maps `etl.home.complete.tech` to `192.168.1.146` on
+  `router.complete.tech`.
+
+## Persistent data
+
+The Compose named volumes are owned by the rootless project and must not be
+removed during upgrades:
+
+- `unstract-etl-home-complete-tech_postgres_data`
+- `unstract-etl-home-complete-tech_redis_data`
+- `unstract-etl-home-complete-tech_minio_data`
+- `unstract-etl-home-complete-tech_qdrant_data`
+- `unstract-etl-home-complete-tech_prompt_studio_data`
+- `unstract-etl-home-complete-tech_rabbitmq_data`
+- `unstract-etl-home-complete-tech_flipt_data`
+- `/home/completetrain/etl.home.complete.tech/docker/workflow_data`
+
+The backend and platform-service `ENCRYPTION_KEY` values are generated once
+and must be backed up securely. Losing or changing that key makes encrypted
+adapter credentials inaccessible.
+
+## Operational commands
+
+```sh
+cd /home/completetrain/etl.home.complete.tech
+export DOCKER_HOST=unix:///run/user/1000/podman/podman.sock
+export VERSION=v0.187.2
+docker compose -f docker/docker-compose.yaml -f docker/compose.train.yaml ps
+docker compose -f docker/docker-compose.yaml -f docker/compose.train.yaml logs --tail=100 backend
+```
+
+The rootless Podman user socket and the shared rootful Caddy proxy are separate
+ownership domains. Inspect both before changing either one.
+
+## Verification
+
+```sh
+curl -fsSI https://etl.home.complete.tech/
+curl -ksS -o /dev/null -w '%{http_code}\n' https://etl.home.complete.tech/api/v1/health
+ssh router.complete.tech "nslookup etl.home.complete.tech 127.0.0.1"
+```
+
+The unauthenticated health request should return `401`; an authenticated health
+request should return `200`. The public root should return the Unstract
+frontend, with API and WebSocket paths routed by Traefik to the backend.
diff --git a/docker/frontend-nginx.conf b/docker/frontend-nginx.conf
new file mode 100644
index 0000000000..8a3f283b3b
--- /dev/null
+++ b/docker/frontend-nginx.conf
@@ -0,0 +1,71 @@
+# Rootless Train deployment variant of frontend/nginx.conf.
+# The image runs Nginx as the non-root nginx user, so use an unprivileged port.
+worker_processes auto;
+
+error_log /var/log/nginx/error.log notice;
+pid /tmp/nginx.pid;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log /var/log/nginx/access.log main;
+    sendfile on;
+    server_tokens off;
+    keepalive_timeout 65;
+
+    gzip on;
+    gzip_types text/css application/javascript application/json image/svg+xml;
+    gzip_min_length 1024;
+    gzip_vary on;
+    gzip_proxied any;
+
+    client_body_temp_path /tmp/client_temp 1 2;
+    proxy_temp_path /tmp/proxy_temp 1 2;
+    fastcgi_temp_path /tmp/fastcgi_temp 1 2;
+    uwsgi_temp_path /tmp/uwsgi_temp 1 2;
+    scgi_temp_path /tmp/scgi_temp 1 2;
+
+    map $uri $cache_control {
+        default "no-cache";
+        ~^/assets/ "public, max-age=31536000, immutable";
+    }
+
+    server {
+        listen 8080;
+        root /usr/share/nginx/html;
+        include /etc/nginx/mime.types;
+
+        add_header X-Content-Type-Options "nosniff" always;
+        add_header X-Frame-Options "SAMEORIGIN" always;
+        add_header Referrer-Policy "strict-origin-when-cross-origin" always;
+        add_header Content-Security-Policy-Report-Only "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://cdn.jsdelivr.net https://unpkg.com https://eu.i.posthog.com https://eu-assets.i.posthog.com https://www.googletagmanager.com https://www.google.com/recaptcha/ https://www.gstatic.com/recaptcha/ https://js.stripe.com https://app.productfruits.com; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://eu.i.posthog.com https://eu-assets.i.posthog.com; font-src 'self' data:; connect-src 'self' blob: wss: https://cdn.jsdelivr.net https://eu.i.posthog.com https://eu-assets.i.posthog.com https://www.google-analytics.com https://api.stripe.com https://app.productfruits.com; frame-src 'self' https://www.google.com/recaptcha/ https://recaptcha.google.com https://js.stripe.com https://hooks.stripe.com; worker-src 'self' blob: https://unpkg.com https://cdn.jsdelivr.net; object-src 'none'; base-uri 'self'; form-action 'self' https://checkout.stripe.com; frame-ancestors 'self'" always;
+        add_header Cache-Control $cache_control always;
+
+        if ($request_method ~ ^(TRACE|TRACK)$) {
+            return 405;
+        }
+
+        location /assets/ {
+            limit_except GET HEAD {
+                deny all;
+            }
+            try_files $uri =404;
+        }
+
+        location / {
+            limit_except GET HEAD {
+                deny all;
+            }
+            try_files $uri /index.html;
+        }
+    }
+}

From de593263bd5dbf1ea754fda965b9556d165c8910 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Mon, 7 Sep 2026 23:44:05 -0400
Subject: [PATCH 10/48] Document disk exhaustion recovery without destructive
 reset

Record the PostgreSQL WAL recovery failure, retained-container readiness
checks, and guardrails against WAL reset, volume removal, or recreation.
Keep the public recovery record free of deployment-specific host and container
identifiers.
---
 docs/train-disk-recovery-20260907.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 docs/train-disk-recovery-20260907.md

diff --git a/docs/train-disk-recovery-20260907.md b/docs/train-disk-recovery-20260907.md
new file mode 100644
index 0000000000..e4e655f51b
--- /dev/null
+++ b/docs/train-disk-recovery-20260907.md
@@ -0,0 +1,16 @@
+# Disk exhaustion recovery
+
+On September 7, the deployment host's root filesystem had zero available bytes.
+The PostgreSQL database failed WAL recovery because it could not extend a data
+file. After host log recovery freed capacity, the existing stopped database
+container was started without replacement or volume changes. `pg_isready`
+returned accepting connections and the public application route returned HTTP
+200. This verifies database readiness and frontend reachability, not document
+processing.
+
+For recurrence, first restore host capacity and inspect the database logs.
+Resolve the exact database container ID with `podman inspect <database-container>`,
+verify that it is stopped, and start that existing ID. Check
+`podman exec <database-container> pg_isready` before checking the public route.
+Do not reset WAL, remove volumes, or recreate the database as a disk-space
+remedy. Keep unrelated one-shot bootstrap jobs stopped during recovery.

From a65bad96a54f106538d9e3d6602c166138103f6c Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 05:56:28 -0400
Subject: [PATCH 11/48] Add bounded Train Unstract readiness probes

---
 docker/healthchecks/README.md                |  30 +++
 docker/healthchecks/unstract-services.sh     | 178 +++++++++++++
 tests/healthchecks/test_unstract_services.py | 265 +++++++++++++++++++
 3 files changed, 473 insertions(+)
 create mode 100644 docker/healthchecks/README.md
 create mode 100644 docker/healthchecks/unstract-services.sh
 create mode 100644 tests/healthchecks/test_unstract_services.py

diff --git a/docker/healthchecks/README.md b/docker/healthchecks/README.md
new file mode 100644
index 0000000000..523ee5bf10
--- /dev/null
+++ b/docker/healthchecks/README.md
@@ -0,0 +1,30 @@
+# Unstract service health probes
+
+`unstract-services.sh` is a bounded, read-only probe entrypoint for the core
+services in the Train Compose deployment.  It is kept separate from the base
+Compose file so a Train-specific deployment can bind-mount it read-only and
+select the service name in that service's native Podman healthcheck.
+
+The probe returns zero only after the service-specific readiness contract has
+passed.  It never prints response bodies, environment values, or credentials;
+failure output names only the service whose probe failed.  The helper uses the
+clients already present in the target images.  In particular, Qdrant's minimal
+image has no curl/wget, so its check uses the Bash runtime and the official
+`/healthz` endpoint with a 512-byte response bound.
+
+The Train deployment should mount this file at
+`/usr/local/bin/unstract-services.sh` with `:ro`, then use for example:
+
+```yaml
+healthcheck:
+  test: ["CMD-SHELL", "/usr/local/bin/unstract-services.sh backend"]
+  interval: 30s
+  timeout: 5s
+  start_period: 120s
+  retries: 3
+```
+
+The exact service contracts and a prepared Train fragment are kept in the
+private integration evidence bundle for the deployment owner.  The script's
+endpoint and command paths can be overridden with environment variables for
+isolated contract tests; production defaults target service-local listeners.
diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
new file mode 100644
index 0000000000..fa02737e2a
--- /dev/null
+++ b/docker/healthchecks/unstract-services.sh
@@ -0,0 +1,178 @@
+#!/bin/sh
+# Read-only readiness probes for the core Train Unstract services.
+#
+# The script is intentionally dependency-light: each probe uses a client that is
+# already shipped in the service image.  It emits only a short, stable failure
+# reason so Podman health logs never contain response bodies or credentials.
+set -eu
+
+service=${1:-}
+timeout_seconds=${HEALTHCHECK_TIMEOUT_SECONDS:-3}
+timeout_bin=${TIMEOUT_BIN:-timeout}
+
+fail() {
+    printf 'unstract-health: %s probe failed\n' "$service" >&2
+    exit 1
+}
+
+usage() {
+    printf '%s\n' \
+        'usage: unstract-services.sh {weaviate|vector-db|redis|proxy|rabbitmq|minio|db|x2text-service|platform-service|backend|frontend}' \
+        >&2
+    exit 2
+}
+
+[ -n "$service" ] || usage
+
+# Every endpoint is overridable for contract tests and disposable local checks;
+# production defaults are the service-local listeners used by compose.train.yaml.
+weaviate_url=${WEAVIATE_META_URL:-http://127.0.0.1:8080/v1/meta}
+qdrant_host=${QDRANT_HOST:-127.0.0.1}
+qdrant_port=${QDRANT_PORT:-6333}
+proxy_url=${TRAEFIK_OVERVIEW_URL:-http://127.0.0.1:8080/api/overview}
+x2text_url=${X2TEXT_HEALTH_URL:-http://127.0.0.1:3004/api/v1/x2text/health}
+platform_url=${PLATFORM_HEALTH_URL:-http://127.0.0.1:3001/health}
+backend_url=${BACKEND_HEALTH_URL:-http://127.0.0.1:8000/internal/v1/health/}
+frontend_url=${FRONTEND_INDEX_URL:-http://127.0.0.1:8080/}
+
+wget_bin=${WGET_BIN:-wget}
+curl_bin=${CURL_BIN:-curl}
+python_bin=${PYTHON_BIN:-.venv/bin/python}
+qdrant_bash_bin=${QDRANT_BASH_BIN:-bash}
+redis_cli_bin=${REDIS_CLI_BIN:-redis-cli}
+rabbitmq_diagnostics_bin=${RABBITMQ_DIAGNOSTICS_BIN:-rabbitmq-diagnostics}
+pg_isready_bin=${PG_ISREADY_BIN:-pg_isready}
+psql_bin=${PSQL_BIN:-psql}
+
+probe_weaviate() {
+    body=$("$wget_bin" -qO- --timeout="$timeout_seconds" "$weaviate_url" 2>/dev/null) || fail
+    # /v1/meta is a bounded, application-level response.  It proves that the
+    # Weaviate HTTP API is serving its metadata, rather than only accepting TCP.
+    printf '%s' "$body" | grep -Eq '"version"[[:space:]]*:[[:space:]]*"[^"[:space:]]+"' || fail
+    # Also require the official readiness endpoint.  It intentionally has an
+    # empty body, so its HTTP status is the contract here.
+    ready_url=${WEAVIATE_READY_URL:-http://127.0.0.1:8080/v1/.well-known/ready}
+    "$wget_bin" -qO /dev/null --timeout="$timeout_seconds" "$ready_url" 2>/dev/null || fail
+}
+
+probe_vector_db() {
+    # The Qdrant image does not ship curl/wget.  Its Debian base does ship Bash,
+    # so use Bash's TCP client to exercise the real REST health endpoint.  The
+    # response is bounded and matched on both HTTP status and body semantics.
+    "$timeout_bin" "$timeout_seconds" "$qdrant_bash_bin" -ec '
+        exec 3<>/dev/tcp/'"$qdrant_host"'/'"$qdrant_port"'
+        printf "GET /healthz HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n" >&3
+        response=$(head -c 512 <&3)
+        case "$response" in
+            *"HTTP/1.1 200"*"healthz check passed"*) exit 0 ;;
+            *) exit 1 ;;
+        esac
+    ' 2>/dev/null || fail
+}
+
+probe_redis() {
+    # PING is read-only and is authenticated automatically when the image's
+    # REDISCLI_AUTH/ACL environment is supplied by Compose.
+    response=$("$timeout_bin" "$timeout_seconds" "$redis_cli_bin" --raw ping 2>/dev/null) || fail
+    [ "$response" = PONG ] || fail
+}
+
+probe_proxy() {
+    body=$("$wget_bin" -qO- --timeout="$timeout_seconds" "$proxy_url" 2>/dev/null) || fail
+    # Traefik's overview is its own control-plane readiness contract.  Require
+    # at least one router and service, with no reported warnings or errors.
+    printf '%s' "$body" | grep -Eq '"routers":\{"total":[1-9][0-9]*,"warnings":0,"errors":0\}' || fail
+    printf '%s' "$body" | grep -Eq '"services":\{"total":[1-9][0-9]*,"warnings":0,"errors":0\}' || fail
+}
+
+probe_rabbitmq() {
+    "$timeout_bin" "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_running >/dev/null 2>&1 || fail
+    "$timeout_bin" "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_local_alarms >/dev/null 2>&1 || fail
+}
+
+probe_minio() {
+    # MinIO's unauthenticated readiness endpoint reports cluster readiness and
+    # avoids a mutating S3 operation or a dependency on an mc alias file.
+    "$curl_bin" -fsS --max-time "$timeout_seconds" \
+        "${MINIO_READY_URL:-http://127.0.0.1:9000/minio/health/ready}" \
+        >/dev/null 2>&1 || fail
+}
+
+probe_db() {
+    db_user=${POSTGRES_USER:-postgres}
+    db_name=${POSTGRES_DB:-postgres}
+    "$timeout_bin" "$timeout_seconds" "$pg_isready_bin" -t "$timeout_seconds" -U "$db_user" -d "$db_name" >/dev/null 2>&1 || fail
+    result=$("$timeout_bin" "$timeout_seconds" "$psql_bin" -XAtqc 'SELECT 1' -U "$db_user" -d "$db_name" 2>/dev/null) || fail
+    [ "$result" = 1 ] || fail
+}
+
+probe_python_body() {
+    url=$1
+    expected=$2
+    "$python_bin" -c '
+import sys
+import urllib.request
+
+with urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[3])) as response:
+    if response.status != 200 or response.read(128).decode("utf-8") != sys.argv[2]:
+        raise SystemExit(1)
+' "$url" "$expected" "$timeout_seconds" >/dev/null 2>&1 || fail
+}
+
+probe_x2text() {
+    probe_python_body "$x2text_url" OK
+}
+
+probe_platform() {
+    # platform-service opens its configured PostgreSQL connection in the
+    # before-request hook, so this endpoint validates both HTTP and that local
+    # dependency initialization completed.
+    probe_python_body "$platform_url" OK
+}
+
+probe_backend() {
+    "$python_bin" -c '
+import json
+import os
+import sys
+import urllib.request
+
+token = os.environ.get("INTERNAL_SERVICE_API_KEY")
+if not token:
+    raise SystemExit(1)
+request = urllib.request.Request(
+    sys.argv[1], headers={"Authorization": "Bearer " + token}
+)
+with urllib.request.urlopen(request, timeout=float(sys.argv[2])) as response:
+    if response.status != 200:
+        raise SystemExit(1)
+    payload = json.loads(response.read(1024).decode("utf-8"))
+    if payload.get("status") != "healthy" or payload.get("authenticated") is not True:
+        raise SystemExit(1)
+' "$backend_url" "$timeout_seconds" >/dev/null 2>&1 || fail
+}
+
+probe_frontend() {
+    body=$("$curl_bin" -fsS --max-time "$timeout_seconds" "$frontend_url" 2>/dev/null) || fail
+    case "$body" in
+        *'<title>Unstract</title>'*) : ;;
+        *) fail ;;
+    esac
+}
+
+case "$service" in
+    weaviate) probe_weaviate ;;
+    vector-db) probe_vector_db ;;
+    redis) probe_redis ;;
+    proxy) probe_proxy ;;
+    rabbitmq) probe_rabbitmq ;;
+    minio) probe_minio ;;
+    db) probe_db ;;
+    x2text-service) probe_x2text ;;
+    platform-service) probe_platform ;;
+    backend) probe_backend ;;
+    frontend) probe_frontend ;;
+    *) usage ;;
+esac
+
+exit 0
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
new file mode 100644
index 0000000000..2ba4f85af9
--- /dev/null
+++ b/tests/healthchecks/test_unstract_services.py
@@ -0,0 +1,265 @@
+"""Contract tests for the bounded Unstract service probes."""
+
+from __future__ import annotations
+
+import http.server
+import json
+import os
+import socketserver
+import subprocess
+import threading
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).parents[2]
+SCRIPT = ROOT / "docker" / "healthchecks" / "unstract-services.sh"
+
+
+def run_probe(service: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
+    probe_env = os.environ.copy()
+    if env:
+        probe_env.update(env)
+    return subprocess.run(
+        ["sh", str(SCRIPT), service],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=probe_env,
+        timeout=8,
+    )
+
+
+def write_fake(tmp_path: Path, name: str, body: str) -> Path:
+    path = tmp_path / name
+    path.write_text("#!/bin/sh\n" + body, encoding="utf-8")
+    path.chmod(0o755)
+    return path
+
+
+def test_probe_script_is_valid_posix_shell() -> None:
+    result = subprocess.run(
+        ["sh", "-n", str(SCRIPT)], check=False, capture_output=True, text=True
+    )
+    assert result.returncode == 0, result.stderr
+
+
+def test_weaviate_requires_metadata_and_ready_status(tmp_path: Path) -> None:
+    wget = write_fake(
+        tmp_path,
+        "wget",
+        """
+case "$*" in
+  *v1/meta*) printf '%s' "$FAKE_META" ;;
+  *well-known/ready*) : ;;
+esac
+exit "${FAKE_EXIT:-0}"
+""",
+    )
+    base = {"WGET_BIN": str(wget), "FAKE_META": '{"version":"1.39.2"}'}
+    assert run_probe("weaviate", base).returncode == 0
+
+    failed = run_probe("weaviate", {**base, "FAKE_META": '{"modules":{}}'})
+    assert failed.returncode != 0
+    assert "modules" not in failed.stderr
+
+
+def test_traefik_requires_nonempty_error_free_overview(tmp_path: Path) -> None:
+    wget = write_fake(
+        tmp_path,
+        "wget",
+        """
+case "$*" in
+  *api/overview*) printf '%s' "$FAKE_OVERVIEW" ;;
+esac
+""",
+    )
+    healthy = (
+        '{"http":{"routers":{"total":9,"warnings":0,"errors":0},'
+        '"services":{"total":8,"warnings":0,"errors":0}}}'
+    )
+    result = run_probe(
+        "proxy", {"WGET_BIN": str(wget), "FAKE_OVERVIEW": healthy}
+    )
+    assert result.returncode == 0, result.stderr
+
+    unhealthy = healthy.replace('"errors":0', '"errors":1', 1)
+    failed = run_probe(
+        "proxy", {"WGET_BIN": str(wget), "FAKE_OVERVIEW": unhealthy}
+    )
+    assert failed.returncode != 0
+
+
+class _QdrantHandler(socketserver.BaseRequestHandler):
+    response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nhealthz check passed"
+
+    def handle(self) -> None:
+        self.request.recv(1024)
+        self.request.sendall(self.response)
+
+
+class _QdrantServer(socketserver.TCPServer):
+    allow_reuse_address = True
+
+
+def qdrant_server(response: bytes) -> tuple[_QdrantServer, int, threading.Thread]:
+    handler = type("ResponseHandler", (_QdrantHandler,), {"response": response})
+    server = _QdrantServer(("127.0.0.1", 0), handler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    return server, server.server_address[1], thread
+
+
+def test_qdrant_probe_checks_status_and_body() -> None:
+    server, port, _ = qdrant_server(_QdrantHandler.response)
+    try:
+        result = run_probe(
+            "vector-db",
+            {"QDRANT_HOST": "127.0.0.1", "QDRANT_PORT": str(port)},
+        )
+        assert result.returncode == 0, result.stderr
+    finally:
+        server.shutdown()
+        server.server_close()
+
+    server, port, _ = qdrant_server(
+        b"HTTP/1.1 503 Service Unavailable\r\nConnection: close\r\n\r\nnot ready"
+    )
+    try:
+        result = run_probe(
+            "vector-db",
+            {"QDRANT_HOST": "127.0.0.1", "QDRANT_PORT": str(port)},
+        )
+        assert result.returncode != 0
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+@pytest.mark.parametrize(
+    ("service", "command_env", "command_body", "failure_env"),
+    [
+        ("redis", "REDIS_CLI_BIN", 'printf "%s\\n" "${FAKE_REPLY}"', {"FAKE_REPLY": "NOPE"}),
+        (
+            "rabbitmq",
+            "RABBITMQ_DIAGNOSTICS_BIN",
+            'test "${FAKE_FAIL:-0}" = 1 && exit 1 || exit 0',
+            {"FAKE_FAIL": "1"},
+        ),
+        (
+            "minio",
+            "CURL_BIN",
+            'test "${FAKE_FAIL:-0}" = 1 && exit 1 || exit 0',
+            {"FAKE_FAIL": "1"},
+        ),
+    ],
+)
+def test_native_datastore_probes_propagate_failures(
+    tmp_path: Path,
+    service: str,
+    command_env: str,
+    command_body: str,
+    failure_env: dict[str, str],
+) -> None:
+    name = command_env.lower().replace("_bin", "")
+    fake = write_fake(tmp_path, name, command_body)
+    env = {command_env: str(fake)}
+    if service == "redis":
+        env["FAKE_REPLY"] = "PONG"
+    assert run_probe(service, env).returncode == 0
+    assert run_probe(service, {**env, **failure_env}).returncode != 0
+
+
+def test_postgres_probe_requires_read_only_query_result(tmp_path: Path) -> None:
+    pg_isready = write_fake(tmp_path, "pg_isready", "exit 0")
+    psql = write_fake(
+        tmp_path,
+        "psql",
+        'printf "%s" "${FAKE_RESULT:-1}"',
+    )
+    env = {
+        "PG_ISREADY_BIN": str(pg_isready),
+        "PSQL_BIN": str(psql),
+        "POSTGRES_USER": "probe-user",
+        "POSTGRES_DB": "probe-db",
+    }
+    result = run_probe("db", env)
+    assert result.returncode == 0, result.stderr
+    assert run_probe("db", {**env, "FAKE_RESULT": "0"}).returncode != 0
+
+
+class _AppHandler(http.server.BaseHTTPRequestHandler):
+    mode = "healthy"
+
+    def do_GET(self) -> None:  # noqa: N802 - stdlib protocol hook
+        if self.path == "/backend" and self.headers.get("Authorization") != "Bearer test-token":
+            self.send_response(401)
+            self.end_headers()
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        if self.path == "/backend":
+            body = {"status": "healthy", "authenticated": True}
+            self.wfile.write(json.dumps(body).encode())
+        elif self.path == "/frontend":
+            self.wfile.write(b"<html><title>Unstract</title></html>")
+        elif self.mode == "healthy":
+            self.wfile.write(b"OK")
+        else:
+            self.wfile.write(b"BROKEN")
+
+    def log_message(self, *_args: object) -> None:
+        return
+
+
+class _AppServer(http.server.ThreadingHTTPServer):
+    allow_reuse_address = True
+
+
+@pytest.fixture()
+def app_server() -> tuple[_AppServer, str]:
+    server = _AppServer(("127.0.0.1", 0), _AppHandler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield server, f"http://127.0.0.1:{server.server_address[1]}"
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+@pytest.mark.parametrize(
+    ("service", "path", "variable"),
+    [
+        ("x2text-service", "/x2text", "X2TEXT_HEALTH_URL"),
+        ("platform-service", "/platform", "PLATFORM_HEALTH_URL"),
+        ("backend", "/backend", "BACKEND_HEALTH_URL"),
+        ("frontend", "/frontend", "FRONTEND_INDEX_URL"),
+    ],
+)
+def test_application_probes_validate_response_contract(
+    app_server: tuple[_AppServer, str], service: str, path: str, variable: str
+) -> None:
+    _server, base_url = app_server
+    env = {variable: base_url + path, "PYTHON_BIN": os.environ.get("PYTHON", "python3")}
+    if service == "backend":
+        env["INTERNAL_SERVICE_API_KEY"] = "test-token"
+    result = run_probe(service, env)
+    assert result.returncode == 0, result.stderr
+
+
+def test_application_probe_rejects_wrong_body(app_server: tuple[_AppServer, str]) -> None:
+    server, base_url = app_server
+    _AppHandler.mode = "broken"
+    try:
+        result = run_probe(
+            "platform-service",
+            {
+                "PLATFORM_HEALTH_URL": base_url + "/platform",
+                "PYTHON_BIN": os.environ.get("PYTHON", "python3"),
+            },
+        )
+        assert result.returncode != 0
+    finally:
+        _AppHandler.mode = "healthy"

From f5fc87d3d62f34de67a068b32e53dfae502b6420 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 06:30:44 -0400
Subject: [PATCH 12/48] Harden bounded Unstract probe transports

---
 docker/healthchecks/unstract-services.sh     | 206 +++++++++++++++++--
 tests/healthchecks/test_unstract_services.py |  84 ++++++++
 2 files changed, 269 insertions(+), 21 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index fa02737e2a..44b7e1e74b 100644
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -2,13 +2,30 @@
 # Read-only readiness probes for the core Train Unstract services.
 #
 # The script is intentionally dependency-light: each probe uses a client that is
-# already shipped in the service image.  It emits only a short, stable failure
+# already shipped in the service image. It emits only a short, stable failure
 # reason so Podman health logs never contain response bodies or credentials.
 set -eu
 
 service=${1:-}
-timeout_seconds=${HEALTHCHECK_TIMEOUT_SECONDS:-3}
+timeout_seconds=${HEALTHCHECK_TIMEOUT_SECONDS-3}
 timeout_bin=${TIMEOUT_BIN:-timeout}
+head_bin=${HEAD_BIN:-head}
+wc_bin=${WC_BIN:-wc}
+rm_bin=${RM_BIN:-rm}
+mktemp_bin=${MKTEMP_BIN:-mktemp}
+
+case "$timeout_seconds" in
+    ''|*[!0-9]*|0*)
+        printf 'unstract-health: invalid timeout configuration\n' >&2
+        exit 2
+        ;;
+    [1-9]|1[0-9]|2[0-9]|30)
+        :
+        ;;
+    *)
+        timeout_seconds=30
+        ;;
+esac
 
 fail() {
     printf 'unstract-health: %s probe failed\n' "$service" >&2
@@ -44,30 +61,164 @@ rabbitmq_diagnostics_bin=${RABBITMQ_DIAGNOSTICS_BIN:-rabbitmq-diagnostics}
 pg_isready_bin=${PG_ISREADY_BIN:-pg_isready}
 psql_bin=${PSQL_BIN:-psql}
 
+# BusyBox wget has no max-filesize or max-redirect option. Stream through a
+# bounded head process, while capturing response headers so redirects can be
+# rejected even when the client follows them internally. The status file keeps
+# the upstream client status visible without relying on non-POSIX pipefail.
+bounded_wget() {
+    bounded_wget_url=$1
+    bounded_wget_limit=$2
+    bounded_wget_headers=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-headers.XXXXXX" 2>/dev/null) || return 1
+    bounded_wget_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_wget_headers" >/dev/null 2>&1 || :
+        return 1
+    }
+    bounded_wget_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    if (
+        if "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" 2>"$bounded_wget_headers"; then
+            bounded_wget_status=0
+        else
+            bounded_wget_status=$?
+        fi
+        printf '%s\n' "$bounded_wget_status" >"$bounded_wget_status_file"
+        exit "$bounded_wget_status"
+    ) | "$head_bin" -c "$((bounded_wget_limit + 1))" >"$bounded_wget_body_file"; then
+        bounded_wget_pipeline_status=0
+    else
+        bounded_wget_pipeline_status=$?
+    fi
+    if [ "$bounded_wget_pipeline_status" -ne 0 ]; then
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    bounded_wget_status=$("$head_bin" -c 16 "$bounded_wget_status_file" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    case "$bounded_wget_status" in
+        0) ;;
+        *)
+            "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+            return 1
+            ;;
+    esac
+    bounded_wget_header_size=$("$wc_bin" -c <"$bounded_wget_headers" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    if [ "$bounded_wget_header_size" -gt 16384 ]; then
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    if grep -Eq '(^|[[:space:]])HTTP/[0-9.]+[[:space:]]+3[0-9][0-9]([[:space:]]|$)|^.*Location:' "$bounded_wget_headers"; then
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    bounded_wget_body_size=$("$wc_bin" -c <"$bounded_wget_body_file") || {
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    if [ "$bounded_wget_body_size" -gt "$bounded_wget_limit" ]; then
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    bounded_wget_body=$("$head_bin" -c "$bounded_wget_limit" "$bounded_wget_body_file") || {
+        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
+    printf '%s' "$bounded_wget_body"
+}
+
+bounded_curl() {
+    bounded_curl_url=$1
+    bounded_curl_limit=$2
+    bounded_curl_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || return 1
+    bounded_curl_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_curl_body_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    if (
+        if "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
+            --max-time "$timeout_seconds" "$bounded_curl_url"; then
+            bounded_curl_status=0
+        else
+            bounded_curl_status=$?
+        fi
+        printf '%s\n' "$bounded_curl_status" >"$bounded_curl_status_file"
+        exit "$bounded_curl_status"
+    ) | "$head_bin" -c "$((bounded_curl_limit + 1))" >"$bounded_curl_body_file"; then
+        bounded_curl_pipeline_status=0
+    else
+        bounded_curl_pipeline_status=$?
+    fi
+    if [ "$bounded_curl_pipeline_status" -ne 0 ]; then
+        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    bounded_curl_status=$("$head_bin" -c 16 "$bounded_curl_status_file" 2>/dev/null) || {
+        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    case "$bounded_curl_status" in
+        0) ;;
+        *)
+            "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+            return 1
+            ;;
+    esac
+    bounded_curl_body_size=$("$wc_bin" -c <"$bounded_curl_body_file") || {
+        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    if [ "$bounded_curl_body_size" -gt "$bounded_curl_limit" ]; then
+        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+        return 1
+    fi
+    bounded_curl_body=$("$head_bin" -c "$bounded_curl_limit" "$bounded_curl_body_file") || {
+        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+        return 1
+    }
+    "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
+    printf '%s' "$bounded_curl_body"
+}
+
 probe_weaviate() {
-    body=$("$wget_bin" -qO- --timeout="$timeout_seconds" "$weaviate_url" 2>/dev/null) || fail
-    # /v1/meta is a bounded, application-level response.  It proves that the
-    # Weaviate HTTP API is serving its metadata, rather than only accepting TCP.
+    body=$(bounded_wget "$weaviate_url" 65536 2>/dev/null) || fail
+    # /v1/meta is a bounded, application-level response. It proves that the
+    # Weaviate HTTP API is serving metadata, rather than only accepting TCP.
     printf '%s' "$body" | grep -Eq '"version"[[:space:]]*:[[:space:]]*"[^"[:space:]]+"' || fail
-    # Also require the official readiness endpoint.  It intentionally has an
+    # Also require the official readiness endpoint. It intentionally has an
     # empty body, so its HTTP status is the contract here.
     ready_url=${WEAVIATE_READY_URL:-http://127.0.0.1:8080/v1/.well-known/ready}
-    "$wget_bin" -qO /dev/null --timeout="$timeout_seconds" "$ready_url" 2>/dev/null || fail
+    bounded_wget "$ready_url" 1024 >/dev/null 2>&1 || fail
 }
 
 probe_vector_db() {
-    # The Qdrant image does not ship curl/wget.  Its Debian base does ship Bash,
-    # so use Bash's TCP client to exercise the real REST health endpoint.  The
+    # The Qdrant image does not ship curl/wget. Its Debian base does ship Bash,
+    # so use Bash's TCP client to exercise the real REST health endpoint. The
     # response is bounded and matched on both HTTP status and body semantics.
     "$timeout_bin" "$timeout_seconds" "$qdrant_bash_bin" -ec '
-        exec 3<>/dev/tcp/'"$qdrant_host"'/'"$qdrant_port"'
+        host=$1
+        port=$2
+        case "$host" in
+            ""|*[!A-Za-z0-9_.:-]*) exit 1 ;;
+        esac
+        case "$port" in
+            ""|*[!0-9]*) exit 1 ;;
+        esac
+        [ "$port" -ge 1 ] && [ "$port" -le 65535 ] || exit 1
+        exec 3<>/dev/tcp/"$host"/"$port"
         printf "GET /healthz HTTP/1.1\\r\\nHost: localhost\\r\\nConnection: close\\r\\n\\r\\n" >&3
         response=$(head -c 512 <&3)
         case "$response" in
             *"HTTP/1.1 200"*"healthz check passed"*) exit 0 ;;
             *) exit 1 ;;
         esac
-    ' 2>/dev/null || fail
+    ' -- "$qdrant_host" "$qdrant_port" 2>/dev/null || fail
 }
 
 probe_redis() {
@@ -78,8 +229,8 @@ probe_redis() {
 }
 
 probe_proxy() {
-    body=$("$wget_bin" -qO- --timeout="$timeout_seconds" "$proxy_url" 2>/dev/null) || fail
-    # Traefik's overview is its own control-plane readiness contract.  Require
+    body=$(bounded_wget "$proxy_url" 65536 2>/dev/null) || fail
+    # Traefik's overview is its own control-plane readiness contract. Require
     # at least one router and service, with no reported warnings or errors.
     printf '%s' "$body" | grep -Eq '"routers":\{"total":[1-9][0-9]*,"warnings":0,"errors":0\}' || fail
     printf '%s' "$body" | grep -Eq '"services":\{"total":[1-9][0-9]*,"warnings":0,"errors":0\}' || fail
@@ -93,7 +244,7 @@ probe_rabbitmq() {
 probe_minio() {
     # MinIO's unauthenticated readiness endpoint reports cluster readiness and
     # avoids a mutating S3 operation or a dependency on an mc alias file.
-    "$curl_bin" -fsS --max-time "$timeout_seconds" \
+    "$curl_bin" -fsS --location --max-redirs 0 --max-time "$timeout_seconds" \
         "${MINIO_READY_URL:-http://127.0.0.1:9000/minio/health/ready}" \
         >/dev/null 2>&1 || fail
 }
@@ -113,8 +264,13 @@ probe_python_body() {
 import sys
 import urllib.request
 
-with urllib.request.urlopen(sys.argv[1], timeout=float(sys.argv[3])) as response:
-    if response.status != 200 or response.read(128).decode("utf-8") != sys.argv[2]:
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+opener = urllib.request.build_opener(NoRedirect)
+with opener.open(sys.argv[1], timeout=float(sys.argv[3])) as response:
+    if response.status != 200 or response.read(129).decode("utf-8") != sys.argv[2]:
         raise SystemExit(1)
 ' "$url" "$expected" "$timeout_seconds" >/dev/null 2>&1 || fail
 }
@@ -125,8 +281,8 @@ probe_x2text() {
 
 probe_platform() {
     # platform-service opens its configured PostgreSQL connection in the
-    # before-request hook, so this endpoint validates both HTTP and that local
-    # dependency initialization completed.
+    # before-request hook, so this endpoint validates HTTP and local dependency
+    # initialization together.
     probe_python_body "$platform_url" OK
 }
 
@@ -137,23 +293,31 @@ import os
 import sys
 import urllib.request
 
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
 token = os.environ.get("INTERNAL_SERVICE_API_KEY")
 if not token:
     raise SystemExit(1)
 request = urllib.request.Request(
     sys.argv[1], headers={"Authorization": "Bearer " + token}
 )
-with urllib.request.urlopen(request, timeout=float(sys.argv[2])) as response:
+opener = urllib.request.build_opener(NoRedirect)
+with opener.open(request, timeout=float(sys.argv[2])) as response:
     if response.status != 200:
         raise SystemExit(1)
-    payload = json.loads(response.read(1024).decode("utf-8"))
+    raw = response.read(1025)
+    if len(raw) > 1024:
+        raise SystemExit(1)
+    payload = json.loads(raw.decode("utf-8"))
     if payload.get("status") != "healthy" or payload.get("authenticated") is not True:
         raise SystemExit(1)
 ' "$backend_url" "$timeout_seconds" >/dev/null 2>&1 || fail
 }
 
 probe_frontend() {
-    body=$("$curl_bin" -fsS --max-time "$timeout_seconds" "$frontend_url" 2>/dev/null) || fail
+    body=$(bounded_curl "$frontend_url" 65536 2>/dev/null) || fail
     case "$body" in
         *'<title>Unstract</title>'*) : ;;
         *) fail ;;
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index 2ba4f85af9..a715e74dd3 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -64,6 +64,72 @@ def test_weaviate_requires_metadata_and_ready_status(tmp_path: Path) -> None:
     assert "modules" not in failed.stderr
 
 
+@pytest.mark.parametrize("timeout", ["", "0", "00", "-1", "abc"])
+def test_invalid_timeout_configuration_is_rejected(timeout: str) -> None:
+    result = run_probe(
+        "redis",
+        {"HEALTHCHECK_TIMEOUT_SECONDS": timeout},
+    )
+    assert result.returncode == 2
+    assert "invalid timeout configuration" in result.stderr
+
+
+def test_timeout_configuration_is_capped(tmp_path: Path) -> None:
+    timeout_record = tmp_path / "timeout"
+    timeout = write_fake(
+        tmp_path,
+        "timeout",
+        f'printf "%s" "$1" > "{timeout_record}"; shift; "$@"',
+    )
+    redis_cli = write_fake(tmp_path, "redis-cli", 'printf "PONG\\n"')
+    result = run_probe(
+        "redis",
+        {
+            "HEALTHCHECK_TIMEOUT_SECONDS": "999999999999999999999999",
+            "TIMEOUT_BIN": str(timeout),
+            "REDIS_CLI_BIN": str(redis_cli),
+        },
+    )
+    assert result.returncode == 0, result.stderr
+    assert timeout_record.read_text(encoding="utf-8") == "30"
+
+
+def test_qdrant_host_and_port_are_data_not_shell_source(tmp_path: Path) -> None:
+    marker = tmp_path / "injected"
+    result = run_probe(
+        "vector-db",
+        {
+            "QDRANT_HOST": f"host; touch {marker}; #",
+            "QDRANT_PORT": f"6333; touch {marker}; #",
+        },
+    )
+    assert result.returncode != 0
+    assert not marker.exists()
+
+
+def test_weaviate_response_is_bounded_and_client_failures_propagate(
+    tmp_path: Path,
+) -> None:
+    wget = write_fake(
+        tmp_path,
+        "wget",
+        """
+printf 'HTTP/1.1 200 OK\\r\\n' >&2
+case "${FAKE_MODE:-ok}" in
+  big) i=0; while [ "$i" -lt 70000 ]; do printf x; i=$((i + 1)); done ;;
+  fail) printf '%s' '{\"version\":\"1\"}'; exit 7 ;;
+  redirect) printf 'Location: http://example.test/\\r\\n' >&2; printf '%s' '{\"version\":\"1\"}' ;;
+  *) printf '%s' '{\"version\":\"1\"}' ;;
+esac
+""",
+    )
+    base = {"WGET_BIN": str(wget)}
+    assert run_probe("weaviate", base).returncode == 0
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "big"}).returncode != 0
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "fail"}).returncode != 0
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "redirect"}).returncode != 0
+
+
 def test_traefik_requires_nonempty_error_free_overview(tmp_path: Path) -> None:
     wget = write_fake(
         tmp_path,
@@ -90,6 +156,24 @@ def test_traefik_requires_nonempty_error_free_overview(tmp_path: Path) -> None:
     assert failed.returncode != 0
 
 
+def test_frontend_response_is_bounded(tmp_path: Path) -> None:
+    curl = write_fake(
+        tmp_path,
+        "curl",
+        """
+case "${FAKE_MODE:-ok}" in
+  big) i=0; while [ "$i" -lt 70000 ]; do printf x; i=$((i + 1)); done ;;
+  fail) exit 7 ;;
+  *) printf '%s' '<html><title>Unstract</title></html>' ;;
+esac
+""",
+    )
+    base = {"CURL_BIN": str(curl)}
+    assert run_probe("frontend", base).returncode == 0
+    assert run_probe("frontend", {**base, "FAKE_MODE": "big"}).returncode != 0
+    assert run_probe("frontend", {**base, "FAKE_MODE": "fail"}).returncode != 0
+
+
 class _QdrantHandler(socketserver.BaseRequestHandler):
     response = b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nhealthz check passed"
 

From 309754ee4253acd73fd4c56960a3df3193e2aabd Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 06:50:32 -0400
Subject: [PATCH 13/48] Bound probe headers and process lifetime

---
 docker/healthchecks/unstract-services.sh     | 275 ++++++++++++-------
 tests/healthchecks/test_unstract_services.py |  79 ++++++
 2 files changed, 250 insertions(+), 104 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index 44b7e1e74b..2ec8be638b 100644
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -13,6 +13,7 @@ head_bin=${HEAD_BIN:-head}
 wc_bin=${WC_BIN:-wc}
 rm_bin=${RM_BIN:-rm}
 mktemp_bin=${MKTEMP_BIN:-mktemp}
+mkfifo_bin=${MKFIFO_BIN:-mkfifo}
 
 case "$timeout_seconds" in
     ''|*[!0-9]*|0*)
@@ -61,133 +62,188 @@ rabbitmq_diagnostics_bin=${RABBITMQ_DIAGNOSTICS_BIN:-rabbitmq-diagnostics}
 pg_isready_bin=${PG_ISREADY_BIN:-pg_isready}
 psql_bin=${PSQL_BIN:-psql}
 
-# BusyBox wget has no max-filesize or max-redirect option. Stream through a
-# bounded head process, while capturing response headers so redirects can be
+# BusyBox wget has no max-filesize or max-redirect option. Stream through
+# bounded head processes, while capturing response headers so redirects can be
 # rejected even when the client follows them internally. The status file keeps
 # the upstream client status visible without relying on non-POSIX pipefail.
+bounded_wget_cleanup() {
+    for bounded_wget_cleanup_pid in \
+        "${bounded_wget_client_pid-}" \
+        "${bounded_wget_body_reader_pid-}" \
+        "${bounded_wget_header_reader_pid-}"; do
+        if [ -n "$bounded_wget_cleanup_pid" ]; then
+            kill "$bounded_wget_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_wget_cleanup_pid in \
+        "${bounded_wget_client_pid-}" \
+        "${bounded_wget_body_reader_pid-}" \
+        "${bounded_wget_header_reader_pid-}"; do
+        if [ -n "$bounded_wget_cleanup_pid" ]; then
+            wait "$bounded_wget_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_wget_cleanup_file in \
+        "${bounded_wget_headers-}" \
+        "${bounded_wget_body_file-}" \
+        "${bounded_wget_status_file-}" \
+        "${bounded_wget_body_fifo-}" \
+        "${bounded_wget_header_fifo-}"; do
+        if [ -n "$bounded_wget_cleanup_file" ]; then
+            "$rm_bin" -f "$bounded_wget_cleanup_file" >/dev/null 2>&1 || :
+        fi
+    done
+}
+
+bounded_wget_finish() {
+    trap - HUP INT TERM EXIT
+    bounded_wget_cleanup
+}
+
 bounded_wget() {
     bounded_wget_url=$1
     bounded_wget_limit=$2
+    bounded_wget_result_file=
+    bounded_wget_headers=
+    bounded_wget_body_file=
+    bounded_wget_status_file=
+    bounded_wget_body_fifo=
+    bounded_wget_header_fifo=
+    bounded_wget_client_pid=
+    bounded_wget_body_reader_pid=
+    bounded_wget_header_reader_pid=
+    trap bounded_wget_cleanup HUP INT TERM EXIT
+
     bounded_wget_headers=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-headers.XXXXXX" 2>/dev/null) || return 1
-    bounded_wget_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_wget_headers" >/dev/null 2>&1 || :
-        return 1
-    }
-    bounded_wget_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    if (
-        if "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" 2>"$bounded_wget_headers"; then
-            bounded_wget_status=0
-        else
-            bounded_wget_status=$?
-        fi
-        printf '%s\n' "$bounded_wget_status" >"$bounded_wget_status_file"
-        exit "$bounded_wget_status"
-    ) | "$head_bin" -c "$((bounded_wget_limit + 1))" >"$bounded_wget_body_file"; then
-        bounded_wget_pipeline_status=0
+    bounded_wget_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || return 1
+    bounded_wget_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || return 1
+    bounded_wget_body_fifo=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body-fifo.XXXXXX" 2>/dev/null) || return 1
+    bounded_wget_header_fifo=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-header-fifo.XXXXXX" 2>/dev/null) || return 1
+    "$rm_bin" -f "$bounded_wget_body_fifo" "$bounded_wget_header_fifo" >/dev/null 2>&1 || return 1
+    "$mkfifo_bin" "$bounded_wget_body_fifo" >/dev/null 2>&1 || return 1
+    "$mkfifo_bin" "$bounded_wget_header_fifo" >/dev/null 2>&1 || return 1
+    "$head_bin" -c "$((bounded_wget_limit + 1))" <"$bounded_wget_body_fifo" >"$bounded_wget_body_file" &
+    bounded_wget_body_reader_pid=$!
+    "$head_bin" -c 16385 <"$bounded_wget_header_fifo" >"$bounded_wget_headers" &
+    bounded_wget_header_reader_pid=$!
+    "$timeout_bin" "$timeout_seconds" "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" \
+        >"$bounded_wget_body_fifo" 2>"$bounded_wget_header_fifo" &
+    bounded_wget_client_pid=$!
+    if wait "$bounded_wget_client_pid"; then
+        bounded_wget_status=0
     else
-        bounded_wget_pipeline_status=$?
+        bounded_wget_status=$?
     fi
-    if [ "$bounded_wget_pipeline_status" -ne 0 ]; then
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
+    bounded_wget_client_pid=
+    printf '%s\n' "$bounded_wget_status" >"$bounded_wget_status_file" || return 1
+    if wait "$bounded_wget_body_reader_pid"; then
+        bounded_wget_body_reader_pid=
+    else
+        bounded_wget_body_reader_status=$?
+        bounded_wget_body_reader_pid=
+        return "$bounded_wget_body_reader_status"
     fi
-    bounded_wget_status=$("$head_bin" -c 16 "$bounded_wget_status_file" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
+    if wait "$bounded_wget_header_reader_pid"; then
+        bounded_wget_header_reader_pid=
+    else
+        bounded_wget_header_reader_status=$?
+        bounded_wget_header_reader_pid=
+        return "$bounded_wget_header_reader_status"
+    fi
+    bounded_wget_status=$("$head_bin" -c 16 "$bounded_wget_status_file" 2>/dev/null) || return 1
     case "$bounded_wget_status" in
         0) ;;
-        *)
-            "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-            return 1
-            ;;
+        *) return 1 ;;
     esac
-    bounded_wget_header_size=$("$wc_bin" -c <"$bounded_wget_headers" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    if [ "$bounded_wget_header_size" -gt 16384 ]; then
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    fi
-    if grep -Eq '(^|[[:space:]])HTTP/[0-9.]+[[:space:]]+3[0-9][0-9]([[:space:]]|$)|^.*Location:' "$bounded_wget_headers"; then
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    fi
-    bounded_wget_body_size=$("$wc_bin" -c <"$bounded_wget_body_file") || {
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    if [ "$bounded_wget_body_size" -gt "$bounded_wget_limit" ]; then
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    fi
-    bounded_wget_body=$("$head_bin" -c "$bounded_wget_limit" "$bounded_wget_body_file") || {
-        "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    "$rm_bin" -f "$bounded_wget_headers" "$bounded_wget_body_file" "$bounded_wget_status_file" >/dev/null 2>&1 || :
-    printf '%s' "$bounded_wget_body"
+    bounded_wget_header_size=$("$wc_bin" -c <"$bounded_wget_headers" 2>/dev/null) || return 1
+    [ "$bounded_wget_header_size" -le 16384 ] || return 1
+    grep -Eiq '(^|[[:space:]])HTTP/[0-9.]+[[:space:]]+3[0-9][0-9]([[:space:]]|$)|^.*Location:' "$bounded_wget_headers" && return 1
+    bounded_wget_body_size=$("$wc_bin" -c <"$bounded_wget_body_file") || return 1
+    [ "$bounded_wget_body_size" -le "$bounded_wget_limit" ] || return 1
+    bounded_wget_result_file=$bounded_wget_body_file
+}
+
+bounded_curl_cleanup() {
+    for bounded_curl_cleanup_pid in \
+        "${bounded_curl_client_pid-}" \
+        "${bounded_curl_body_reader_pid-}"; do
+        if [ -n "$bounded_curl_cleanup_pid" ]; then
+            kill "$bounded_curl_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_curl_cleanup_pid in \
+        "${bounded_curl_client_pid-}" \
+        "${bounded_curl_body_reader_pid-}"; do
+        if [ -n "$bounded_curl_cleanup_pid" ]; then
+            wait "$bounded_curl_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_curl_cleanup_file in \
+        "${bounded_curl_body_file-}" \
+        "${bounded_curl_status_file-}" \
+        "${bounded_curl_body_fifo-}"; do
+        if [ -n "$bounded_curl_cleanup_file" ]; then
+            "$rm_bin" -f "$bounded_curl_cleanup_file" >/dev/null 2>&1 || :
+        fi
+    done
+}
+
+bounded_curl_finish() {
+    trap - HUP INT TERM EXIT
+    bounded_curl_cleanup
 }
 
 bounded_curl() {
     bounded_curl_url=$1
     bounded_curl_limit=$2
+    bounded_curl_result_file=
+    bounded_curl_body_file=
+    bounded_curl_status_file=
+    bounded_curl_body_fifo=
+    bounded_curl_client_pid=
+    bounded_curl_body_reader_pid=
+    trap bounded_curl_cleanup HUP INT TERM EXIT
     bounded_curl_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || return 1
-    bounded_curl_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_curl_body_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    if (
-        if "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
-            --max-time "$timeout_seconds" "$bounded_curl_url"; then
-            bounded_curl_status=0
-        else
-            bounded_curl_status=$?
-        fi
-        printf '%s\n' "$bounded_curl_status" >"$bounded_curl_status_file"
-        exit "$bounded_curl_status"
-    ) | "$head_bin" -c "$((bounded_curl_limit + 1))" >"$bounded_curl_body_file"; then
-        bounded_curl_pipeline_status=0
+    bounded_curl_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || return 1
+    bounded_curl_body_fifo=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body-fifo.XXXXXX" 2>/dev/null) || return 1
+    "$rm_bin" -f "$bounded_curl_body_fifo" >/dev/null 2>&1 || return 1
+    "$mkfifo_bin" "$bounded_curl_body_fifo" >/dev/null 2>&1 || return 1
+    "$head_bin" -c "$((bounded_curl_limit + 1))" <"$bounded_curl_body_fifo" >"$bounded_curl_body_file" &
+    bounded_curl_body_reader_pid=$!
+    "$timeout_bin" "$timeout_seconds" "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
+        --max-time "$timeout_seconds" "$bounded_curl_url" >"$bounded_curl_body_fifo" &
+    bounded_curl_client_pid=$!
+    if wait "$bounded_curl_client_pid"; then
+        bounded_curl_status=0
     else
-        bounded_curl_pipeline_status=$?
+        bounded_curl_status=$?
     fi
-    if [ "$bounded_curl_pipeline_status" -ne 0 ]; then
-        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-        return 1
+    bounded_curl_client_pid=
+    printf '%s\n' "$bounded_curl_status" >"$bounded_curl_status_file" || return 1
+    if wait "$bounded_curl_body_reader_pid"; then
+        bounded_curl_body_reader_pid=
+    else
+        bounded_curl_body_reader_status=$?
+        bounded_curl_body_reader_pid=
+        return "$bounded_curl_body_reader_status"
     fi
-    bounded_curl_status=$("$head_bin" -c 16 "$bounded_curl_status_file" 2>/dev/null) || {
-        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
+    bounded_curl_status=$("$head_bin" -c 16 "$bounded_curl_status_file" 2>/dev/null) || return 1
     case "$bounded_curl_status" in
         0) ;;
-        *)
-            "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-            return 1
-            ;;
+        *) return 1 ;;
     esac
-    bounded_curl_body_size=$("$wc_bin" -c <"$bounded_curl_body_file") || {
-        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    if [ "$bounded_curl_body_size" -gt "$bounded_curl_limit" ]; then
-        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-        return 1
-    fi
-    bounded_curl_body=$("$head_bin" -c "$bounded_curl_limit" "$bounded_curl_body_file") || {
-        "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-        return 1
-    }
-    "$rm_bin" -f "$bounded_curl_body_file" "$bounded_curl_status_file" >/dev/null 2>&1 || :
-    printf '%s' "$bounded_curl_body"
+    bounded_curl_body_size=$("$wc_bin" -c <"$bounded_curl_body_file") || return 1
+    [ "$bounded_curl_body_size" -le "$bounded_curl_limit" ] || return 1
+    bounded_curl_result_file=$bounded_curl_body_file
 }
 
 probe_weaviate() {
-    body=$(bounded_wget "$weaviate_url" 65536 2>/dev/null) || fail
+    bounded_wget "$weaviate_url" 65536 2>/dev/null || fail
+    body=$("$head_bin" -c 65536 "$bounded_wget_result_file") || {
+        bounded_wget_finish
+        fail
+    }
+    bounded_wget_finish
     # /v1/meta is a bounded, application-level response. It proves that the
     # Weaviate HTTP API is serving metadata, rather than only accepting TCP.
     printf '%s' "$body" | grep -Eq '"version"[[:space:]]*:[[:space:]]*"[^"[:space:]]+"' || fail
@@ -195,6 +251,7 @@ probe_weaviate() {
     # empty body, so its HTTP status is the contract here.
     ready_url=${WEAVIATE_READY_URL:-http://127.0.0.1:8080/v1/.well-known/ready}
     bounded_wget "$ready_url" 1024 >/dev/null 2>&1 || fail
+    bounded_wget_finish
 }
 
 probe_vector_db() {
@@ -229,7 +286,12 @@ probe_redis() {
 }
 
 probe_proxy() {
-    body=$(bounded_wget "$proxy_url" 65536 2>/dev/null) || fail
+    bounded_wget "$proxy_url" 65536 2>/dev/null || fail
+    body=$("$head_bin" -c 65536 "$bounded_wget_result_file") || {
+        bounded_wget_finish
+        fail
+    }
+    bounded_wget_finish
     # Traefik's overview is its own control-plane readiness contract. Require
     # at least one router and service, with no reported warnings or errors.
     printf '%s' "$body" | grep -Eq '"routers":\{"total":[1-9][0-9]*,"warnings":0,"errors":0\}' || fail
@@ -244,9 +306,9 @@ probe_rabbitmq() {
 probe_minio() {
     # MinIO's unauthenticated readiness endpoint reports cluster readiness and
     # avoids a mutating S3 operation or a dependency on an mc alias file.
-    "$curl_bin" -fsS --location --max-redirs 0 --max-time "$timeout_seconds" \
-        "${MINIO_READY_URL:-http://127.0.0.1:9000/minio/health/ready}" \
+    bounded_curl "${MINIO_READY_URL:-http://127.0.0.1:9000/minio/health/ready}" 1024 \
         >/dev/null 2>&1 || fail
+    bounded_curl_finish
 }
 
 probe_db() {
@@ -317,7 +379,12 @@ with opener.open(request, timeout=float(sys.argv[2])) as response:
 }
 
 probe_frontend() {
-    body=$(bounded_curl "$frontend_url" 65536 2>/dev/null) || fail
+    bounded_curl "$frontend_url" 65536 2>/dev/null || fail
+    body=$("$head_bin" -c 65536 "$bounded_curl_result_file") || {
+        bounded_curl_finish
+        fail
+    }
+    bounded_curl_finish
     case "$body" in
         *'<title>Unstract</title>'*) : ;;
         *) fail ;;
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index a715e74dd3..d0cbea5ff2 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -8,6 +8,7 @@
 import socketserver
 import subprocess
 import threading
+import time
 from pathlib import Path
 
 import pytest
@@ -130,6 +131,84 @@ def test_weaviate_response_is_bounded_and_client_failures_propagate(
     assert run_probe("weaviate", {**base, "FAKE_MODE": "redirect"}).returncode != 0
 
 
+def test_wget_headers_and_total_deadline_are_bounded(tmp_path: Path) -> None:
+    wget = write_fake(
+        tmp_path,
+        "wget",
+        """
+case "${FAKE_MODE:-ok}" in
+  big_headers)
+    i=0
+    while [ "$i" -lt 20000000 ]; do printf x >&2; i=$((i + 1)); done
+    printf '%s' '{\"version\":\"1\"}'
+    ;;
+  lower_redirect)
+    printf 'HTTP/1.1 200 OK\\r\\nlocation: http://example.test/\\r\\n' >&2
+    printf '%s' '{\"version\":\"1\"}'
+    ;;
+  slow)
+    sleep 5
+    ;;
+  *)
+    printf 'HTTP/1.1 200 OK\\r\\n' >&2
+    printf '%s' '{\"version\":\"1\"}'
+    ;;
+esac
+""",
+    )
+    base = {"WGET_BIN": str(wget), "HEALTHCHECK_TIMEOUT_SECONDS": "1"}
+    assert run_probe("weaviate", base).returncode == 0
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "big_headers"}).returncode != 0
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "lower_redirect"}).returncode != 0
+    started = time.monotonic()
+    assert run_probe("weaviate", {**base, "FAKE_MODE": "slow"}).returncode != 0
+    assert time.monotonic() - started < 4
+
+
+def test_minio_response_is_bounded(tmp_path: Path) -> None:
+    curl = write_fake(
+        tmp_path,
+        "curl",
+        """
+case "${FAKE_MODE:-ok}" in
+  big) i=0; while [ "$i" -lt 70000 ]; do printf x; i=$((i + 1)); done ;;
+  *) : ;;
+esac
+""",
+    )
+    base = {"CURL_BIN": str(curl)}
+    assert run_probe("minio", base).returncode == 0
+    assert run_probe("minio", {**base, "FAKE_MODE": "big"}).returncode != 0
+
+
+def test_probe_signal_cleans_temporary_files(tmp_path: Path) -> None:
+    wget = write_fake(tmp_path, "wget", "sleep 10")
+    env = os.environ.copy()
+    env.update(
+        {
+            "TMPDIR": str(tmp_path),
+            "WGET_BIN": str(wget),
+            "HEALTHCHECK_TIMEOUT_SECONDS": "30",
+        }
+    )
+    process = subprocess.Popen(
+        ["sh", str(SCRIPT), "weaviate"],
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        time.sleep(0.2)
+        process.terminate()
+        process.wait(timeout=4)
+    finally:
+        if process.poll() is None:
+            process.kill()
+            process.wait(timeout=4)
+    assert not list(tmp_path.glob("unstract-health-*"))
+
+
 def test_traefik_requires_nonempty_error_free_overview(tmp_path: Path) -> None:
     wget = write_fake(
         tmp_path,

From 6236fe3b6ee1f870ab5fe65964aef92d3ab25a57 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 07:28:31 -0400
Subject: [PATCH 14/48] Bound native probe output and source guards

---
 docker/healthchecks/unstract-services.sh     | 107 ++++++++++++++++++-
 tests/healthchecks/test_unstract_services.py |  64 +++++++++++
 2 files changed, 167 insertions(+), 4 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index 2ec8be638b..cbcfab1073 100644
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -237,6 +237,95 @@ bounded_curl() {
     bounded_curl_result_file=$bounded_curl_body_file
 }
 
+# Native command clients can also return an unexpectedly large response before
+# exiting.  Keep their stdout in a bounded FIFO capture and apply the same
+# outer deadline used by HTTP clients.  This is used for the Redis and
+# PostgreSQL probes, whose contracts only need a tiny scalar response.
+bounded_exec_cleanup() {
+    for bounded_exec_cleanup_pid in \
+        "${bounded_exec_client_pid-}" \
+        "${bounded_exec_reader_pid-}"; do
+        if [ -n "$bounded_exec_cleanup_pid" ]; then
+            kill "$bounded_exec_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_exec_cleanup_pid in \
+        "${bounded_exec_client_pid-}" \
+        "${bounded_exec_reader_pid-}"; do
+        if [ -n "$bounded_exec_cleanup_pid" ]; then
+            wait "$bounded_exec_cleanup_pid" >/dev/null 2>&1 || :
+        fi
+    done
+    for bounded_exec_cleanup_file in \
+        "${bounded_exec_body_file-}" \
+        "${bounded_exec_status_file-}" \
+        "${bounded_exec_body_fifo-}"; do
+        if [ -n "$bounded_exec_cleanup_file" ]; then
+            "$rm_bin" -f "$bounded_exec_cleanup_file" >/dev/null 2>&1 || :
+        fi
+    done
+}
+
+bounded_exec_finish() {
+    trap - HUP INT TERM EXIT
+    bounded_exec_cleanup
+}
+
+bounded_exec() {
+    bounded_exec_limit=$1
+    shift
+    bounded_exec_result_file=
+    bounded_exec_body_file=
+    bounded_exec_status_file=
+    bounded_exec_body_fifo=
+    bounded_exec_client_pid=
+    bounded_exec_reader_pid=
+    trap bounded_exec_cleanup HUP INT TERM EXIT
+
+    bounded_exec_body_file=$(
+        "$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null
+    ) || return 1
+    bounded_exec_status_file=$(
+        "$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null
+    ) || return 1
+    bounded_exec_body_fifo=$(
+        "$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body-fifo.XXXXXX" 2>/dev/null
+    ) || return 1
+    "$rm_bin" -f "$bounded_exec_body_fifo" >/dev/null 2>&1 || return 1
+    "$mkfifo_bin" "$bounded_exec_body_fifo" >/dev/null 2>&1 || return 1
+    "$head_bin" -c "$((bounded_exec_limit + 1))" <"$bounded_exec_body_fifo" \
+        >"$bounded_exec_body_file" &
+    bounded_exec_reader_pid=$!
+    "$timeout_bin" "$timeout_seconds" "$@" >"$bounded_exec_body_fifo" 2>/dev/null &
+    bounded_exec_client_pid=$!
+    if wait "$bounded_exec_client_pid"; then
+        bounded_exec_status=0
+    else
+        bounded_exec_status=$?
+    fi
+    bounded_exec_client_pid=
+    printf '%s\n' "$bounded_exec_status" >"$bounded_exec_status_file" || return 1
+    if wait "$bounded_exec_reader_pid"; then
+        bounded_exec_reader_pid=
+    else
+        bounded_exec_reader_status=$?
+        bounded_exec_reader_pid=
+        return "$bounded_exec_reader_status"
+    fi
+    bounded_exec_status=$(
+        "$head_bin" -c 16 "$bounded_exec_status_file" 2>/dev/null
+    ) || return 1
+    case "$bounded_exec_status" in
+        0) ;;
+        *) return 1 ;;
+    esac
+    bounded_exec_body_size=$(
+        "$wc_bin" -c <"$bounded_exec_body_file"
+    ) || return 1
+    [ "$bounded_exec_body_size" -le "$bounded_exec_limit" ] || return 1
+    bounded_exec_result_file=$bounded_exec_body_file
+}
+
 probe_weaviate() {
     bounded_wget "$weaviate_url" 65536 2>/dev/null || fail
     body=$("$head_bin" -c 65536 "$bounded_wget_result_file") || {
@@ -281,7 +370,12 @@ probe_vector_db() {
 probe_redis() {
     # PING is read-only and is authenticated automatically when the image's
     # REDISCLI_AUTH/ACL environment is supplied by Compose.
-    response=$("$timeout_bin" "$timeout_seconds" "$redis_cli_bin" --raw ping 2>/dev/null) || fail
+    bounded_exec 16 "$redis_cli_bin" --raw ping || fail
+    response=$("$head_bin" -c 16 "$bounded_exec_result_file") || {
+        bounded_exec_finish
+        fail
+    }
+    bounded_exec_finish
     [ "$response" = PONG ] || fail
 }
 
@@ -315,14 +409,19 @@ probe_db() {
     db_user=${POSTGRES_USER:-postgres}
     db_name=${POSTGRES_DB:-postgres}
     "$timeout_bin" "$timeout_seconds" "$pg_isready_bin" -t "$timeout_seconds" -U "$db_user" -d "$db_name" >/dev/null 2>&1 || fail
-    result=$("$timeout_bin" "$timeout_seconds" "$psql_bin" -XAtqc 'SELECT 1' -U "$db_user" -d "$db_name" 2>/dev/null) || fail
+    bounded_exec 16 "$psql_bin" -XAtqc 'SELECT 1' -U "$db_user" -d "$db_name" || fail
+    result=$("$head_bin" -c 16 "$bounded_exec_result_file") || {
+        bounded_exec_finish
+        fail
+    }
+    bounded_exec_finish
     [ "$result" = 1 ] || fail
 }
 
 probe_python_body() {
     url=$1
     expected=$2
-    "$python_bin" -c '
+    "$timeout_bin" "$timeout_seconds" "$python_bin" -c '
 import sys
 import urllib.request
 
@@ -349,7 +448,7 @@ probe_platform() {
 }
 
 probe_backend() {
-    "$python_bin" -c '
+    "$timeout_bin" "$timeout_seconds" "$python_bin" -c '
 import json
 import os
 import sys
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index d0cbea5ff2..29331db0b8 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -95,6 +95,29 @@ def test_timeout_configuration_is_capped(tmp_path: Path) -> None:
     assert timeout_record.read_text(encoding="utf-8") == "30"
 
 
+def test_redis_response_and_total_deadline_are_bounded(tmp_path: Path) -> None:
+    redis_cli = write_fake(
+        tmp_path,
+        "redis-cli",
+        """
+case "${FAKE_MODE:-ok}" in
+  big) i=0; while [ "$i" -lt 70000 ]; do printf x; i=$((i + 1)); done ;;
+  slow) sleep 5 ;;
+  *) printf 'PONG\n' ;;
+esac
+""",
+    )
+    base = {
+        "REDIS_CLI_BIN": str(redis_cli),
+        "HEALTHCHECK_TIMEOUT_SECONDS": "1",
+    }
+    assert run_probe("redis", base).returncode == 0
+    assert run_probe("redis", {**base, "FAKE_MODE": "big"}).returncode != 0
+    started = time.monotonic()
+    assert run_probe("redis", {**base, "FAKE_MODE": "slow"}).returncode != 0
+    assert time.monotonic() - started < 4
+
+
 def test_qdrant_host_and_port_are_data_not_shell_source(tmp_path: Path) -> None:
     marker = tmp_path / "injected"
     result = run_probe(
@@ -351,6 +374,33 @@ def test_postgres_probe_requires_read_only_query_result(tmp_path: Path) -> None:
     assert run_probe("db", {**env, "FAKE_RESULT": "0"}).returncode != 0
 
 
+def test_postgres_response_and_total_deadline_are_bounded(tmp_path: Path) -> None:
+    pg_isready = write_fake(tmp_path, "pg_isready", "exit 0")
+    psql = write_fake(
+        tmp_path,
+        "psql",
+        """
+case "${FAKE_MODE:-ok}" in
+  big) i=0; while [ "$i" -lt 70000 ]; do printf x; i=$((i + 1)); done ;;
+  slow) sleep 5 ;;
+  *) printf '1\n' ;;
+esac
+""",
+    )
+    base = {
+        "PG_ISREADY_BIN": str(pg_isready),
+        "PSQL_BIN": str(psql),
+        "POSTGRES_USER": "probe-user",
+        "POSTGRES_DB": "probe-db",
+        "HEALTHCHECK_TIMEOUT_SECONDS": "1",
+    }
+    assert run_probe("db", base).returncode == 0
+    assert run_probe("db", {**base, "FAKE_MODE": "big"}).returncode != 0
+    started = time.monotonic()
+    assert run_probe("db", {**base, "FAKE_MODE": "slow"}).returncode != 0
+    assert time.monotonic() - started < 4
+
+
 class _AppHandler(http.server.BaseHTTPRequestHandler):
     mode = "healthy"
 
@@ -426,3 +476,17 @@ def test_application_probe_rejects_wrong_body(app_server: tuple[_AppServer, str]
         assert result.returncode != 0
     finally:
         _AppHandler.mode = "healthy"
+
+
+def test_python_probe_has_outer_deadline(tmp_path: Path) -> None:
+    python_bin = write_fake(tmp_path, "python", "sleep 5")
+    started = time.monotonic()
+    result = run_probe(
+        "x2text-service",
+        {
+            "PYTHON_BIN": str(python_bin),
+            "HEALTHCHECK_TIMEOUT_SECONDS": "1",
+        },
+    )
+    assert result.returncode != 0
+    assert time.monotonic() - started < 4

From 70c918b05bd1744f318ffaaa1ca97a58656cdbb3 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 07:30:32 -0400
Subject: [PATCH 15/48] Cover native probe cleanup on signals

---
 tests/healthchecks/test_unstract_services.py | 28 ++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index 29331db0b8..107cfb14cf 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -232,6 +232,34 @@ def test_probe_signal_cleans_temporary_files(tmp_path: Path) -> None:
     assert not list(tmp_path.glob("unstract-health-*"))
 
 
+def test_native_probe_signal_cleans_temporary_files(tmp_path: Path) -> None:
+    redis_cli = write_fake(tmp_path, "redis-cli", "sleep 10")
+    env = os.environ.copy()
+    env.update(
+        {
+            "TMPDIR": str(tmp_path),
+            "REDIS_CLI_BIN": str(redis_cli),
+            "HEALTHCHECK_TIMEOUT_SECONDS": "30",
+        }
+    )
+    process = subprocess.Popen(
+        ["sh", str(SCRIPT), "redis"],
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        time.sleep(0.2)
+        process.terminate()
+        process.wait(timeout=4)
+    finally:
+        if process.poll() is None:
+            process.kill()
+            process.wait(timeout=4)
+    assert not list(tmp_path.glob("unstract-health-*"))
+
+
 def test_traefik_requires_nonempty_error_free_overview(tmp_path: Path) -> None:
     wget = write_fake(
         tmp_path,

From fe5dfb0012409a51db1d6b9951ce1ff2f72a38f8 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Mon, 7 Sep 2026 23:44:05 -0400
Subject: [PATCH 16/48] Document disk exhaustion recovery without destructive
 reset

Record the PostgreSQL WAL recovery failure, retained-container readiness
checks, and guardrails against WAL reset, volume removal, or recreation.
Keep the public recovery record free of deployment-specific host and container
identifiers.
---
 docs/train-disk-recovery-20260907.md | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 docs/train-disk-recovery-20260907.md

diff --git a/docs/train-disk-recovery-20260907.md b/docs/train-disk-recovery-20260907.md
new file mode 100644
index 0000000000..e4e655f51b
--- /dev/null
+++ b/docs/train-disk-recovery-20260907.md
@@ -0,0 +1,16 @@
+# Disk exhaustion recovery
+
+On September 7, the deployment host's root filesystem had zero available bytes.
+The PostgreSQL database failed WAL recovery because it could not extend a data
+file. After host log recovery freed capacity, the existing stopped database
+container was started without replacement or volume changes. `pg_isready`
+returned accepting connections and the public application route returned HTTP
+200. This verifies database readiness and frontend reachability, not document
+processing.
+
+For recurrence, first restore host capacity and inspect the database logs.
+Resolve the exact database container ID with `podman inspect <database-container>`,
+verify that it is stopped, and start that existing ID. Check
+`podman exec <database-container> pg_isready` before checking the public route.
+Do not reset WAL, remove volumes, or recreate the database as a disk-space
+remedy. Keep unrelated one-shot bootstrap jobs stopped during recovery.

From c7da8e5607a421d8bfaf1180f9cd719c7f7a8e78 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 06:01:21 -0400
Subject: [PATCH 17/48] Add read-only worker readiness probes

---
 .../src/unstract/runner/controller/health.py  |  40 ++-
 runner/tests/test_health.py                   |  39 +++
 workers/log_consumer/redis_stream_consumer.py | 211 ++++++++++++---
 workers/log_consumer/scheduler.sh             |  52 ++++
 workers/log_consumer/scheduler_health.py      | 240 ++++++++++++++++++
 workers/log_consumer/worker.py                |   4 +-
 .../infrastructure/monitoring/health.py       | 148 +++++++++--
 workers/tests/test_log_stream_consumer.py     |  37 +++
 workers/tests/test_scheduler_health.py        |  84 ++++++
 workers/tests/test_worker_health.py           |  52 ++++
 workers/worker.py                             |  52 ++++
 11 files changed, 893 insertions(+), 66 deletions(-)
 create mode 100644 runner/tests/test_health.py
 create mode 100644 workers/log_consumer/scheduler_health.py
 create mode 100644 workers/tests/test_scheduler_health.py
 create mode 100644 workers/tests/test_worker_health.py

diff --git a/runner/src/unstract/runner/controller/health.py b/runner/src/unstract/runner/controller/health.py
index 8c4349448c..1fbefaf4b5 100644
--- a/runner/src/unstract/runner/controller/health.py
+++ b/runner/src/unstract/runner/controller/health.py
@@ -1,6 +1,7 @@
 import logging
+from typing import Any
 
-from flask import Blueprint
+from flask import Blueprint, jsonify
 
 logger = logging.getLogger(__name__)
 
@@ -8,7 +9,40 @@
 health_bp = Blueprint("health", __name__)
 
 
-# Define a route to ping test
+def _container_runtime_ready() -> tuple[bool, str | None]:
+    """Perform a bounded, read-only ping against the mounted container socket."""
+    try:
+        # Import lazily so importing the Flask blueprint does not create a Docker
+        # client or touch the socket. ``ping`` only asks the daemon for liveness;
+        # it does not list, create, publish, or remove a tool container.
+        from docker import DockerClient
+
+        client = DockerClient.from_env(timeout=2)
+        try:
+            client.ping()
+        finally:
+            client.close()
+    except Exception as exc:
+        # Keep credentials, socket paths, and daemon error text out of the HTTP
+        # body. The exception class is enough for an operator to identify the
+        # failed dependency while logs retain only the same sanitized class name.
+        logger.warning("Runner container runtime probe failed: %s", type(exc).__name__)
+        return False, type(exc).__name__
+    return True, None
+
+
 @health_bp.route("/health", methods=["GET"])
-def health_check() -> str:
+def health_check() -> str | tuple[Any, int]:
+    runtime_ready, error_type = _container_runtime_ready()
+    if not runtime_ready:
+        return (
+            jsonify(
+                {
+                    "status": "unhealthy",
+                    "dependency": "container_runtime",
+                    "error": error_type or "unavailable",
+                }
+            ),
+            503,
+        )
     return "OK"
diff --git a/runner/tests/test_health.py b/runner/tests/test_health.py
new file mode 100644
index 0000000000..846281b155
--- /dev/null
+++ b/runner/tests/test_health.py
@@ -0,0 +1,39 @@
+"""Runner readiness must include the mounted container runtime."""
+
+from __future__ import annotations
+
+from flask import Flask
+
+from unstract.runner.controller import health as health_module
+
+
+def _client():
+    app = Flask(__name__)
+    app.register_blueprint(health_module.health_bp, url_prefix="/v1/api")
+    return app.test_client()
+
+
+def test_health_returns_ok_when_container_runtime_ping_passes(monkeypatch):
+    monkeypatch.setattr(health_module, "_container_runtime_ready", lambda: (True, None))
+
+    response = _client().get("/v1/api/health")
+
+    assert response.status_code == 200
+    assert response.get_data(as_text=True) == "OK"
+
+
+def test_health_returns_503_without_runtime_readiness(monkeypatch):
+    monkeypatch.setattr(
+        health_module,
+        "_container_runtime_ready",
+        lambda: (False, "PermissionError"),
+    )
+
+    response = _client().get("/v1/api/health")
+
+    assert response.status_code == 503
+    assert response.get_json() == {
+        "status": "unhealthy",
+        "dependency": "container_runtime",
+        "error": "PermissionError",
+    }
diff --git a/workers/log_consumer/redis_stream_consumer.py b/workers/log_consumer/redis_stream_consumer.py
index a2242f11f2..50e6c7a41f 100644
--- a/workers/log_consumer/redis_stream_consumer.py
+++ b/workers/log_consumer/redis_stream_consumer.py
@@ -31,6 +31,8 @@
 import signal
 import socket
 import sys
+import threading
+import time
 from types import FrameType
 from typing import Any
 
@@ -66,6 +68,122 @@
 _shutdown = False
 
 
+class _RedisStreamHealth:
+    """Freshness state for the Redis stream poll loop.
+
+    A process check cannot tell whether this consumer can still reach Redis. The
+    loop records a successful ``BLMOVE`` return (including an empty-list timeout),
+    which proves that the Redis read completed without publishing or consuming
+    anything as a side effect of the health request. A failed or hung read leaves
+    the previous success timestamp untouched, so the endpoint eventually returns
+    503 after the configured bound.
+    """
+
+    def __init__(self, queue_name: str) -> None:
+        self._queue_name = queue_name
+        self._lock = threading.Lock()
+        self._last_success: float | None = None
+        self._poll_failures = 0
+
+    def mark_success(self) -> None:
+        with self._lock:
+            self._last_success = time.monotonic()
+
+    def mark_failure(self) -> None:
+        with self._lock:
+            self._poll_failures += 1
+
+    def seconds_since_last_success(self) -> float:
+        with self._lock:
+            last_success = self._last_success
+        # A finite, large value keeps the JSON response valid before the first
+        # completed read while making the probe unambiguously stale.
+        return 1_000_000.0 if last_success is None else max(
+            0.0, time.monotonic() - last_success
+        )
+
+    def status(self) -> dict[str, object]:
+        with self._lock:
+            failures = self._poll_failures
+        return {
+            "queue": self._queue_name,
+            "redis_poll_failures": failures,
+        }
+
+
+def _health_port_from_env() -> int | None:
+    """Return the opt-in health port, validating bad deployment input early."""
+    raw = os.getenv("LOG_STREAM_CONSUMER_HEALTH_PORT")
+    if raw is None or raw == "":
+        return None
+    try:
+        port = int(raw)
+    except ValueError as exc:
+        raise ValueError(
+            f"Invalid LOG_STREAM_CONSUMER_HEALTH_PORT={raw!r}: {exc}"
+        ) from exc
+    if not 0 <= port <= 65535:
+        raise ValueError(f"LOG_STREAM_CONSUMER_HEALTH_PORT={port} out of range")
+    return port
+
+
+def _health_stale_seconds() -> float:
+    """Resolve a bound that outlives the blocking read and its socket timeout."""
+    default = max(15.0, float(_SOCKET_TIMEOUT_SECONDS * 2))
+    raw = os.getenv("LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS")
+    if raw is None or raw == "":
+        return default
+    try:
+        stale_after = float(raw)
+    except ValueError as exc:
+        raise ValueError(
+            f"Invalid LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS={raw!r}: {exc}"
+        ) from exc
+    if stale_after <= 0:
+        raise ValueError(
+            "LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS must be positive"
+        )
+    return stale_after
+
+
+def _maybe_start_health_server(
+    health: _RedisStreamHealth,
+) -> Any | None:
+    """Start the opt-in loop probe without making it a process dependency."""
+    port = _health_port_from_env()
+    if port is None:
+        return None
+
+    from queue_backend.pg_queue.liveness import LivenessServer
+
+    stale_after = _health_stale_seconds()
+    server = LivenessServer(
+        freshness_fn=health.seconds_since_last_success,
+        stale_after=stale_after,
+        port=port,
+        check_name="redis_stream_loop",
+        age_key="seconds_since_last_successful_poll",
+        extra_status_fn=health.status,
+        thread_name="redis-stream-liveness",
+        log_label="redis stream consumer",
+    )
+    try:
+        server.start()
+    except OSError:
+        logger.exception(
+            "Redis stream consumer: liveness could not bind :%s; continuing "
+            "without a probe",
+            port,
+        )
+        return None
+    logger.info(
+        "Redis stream consumer: liveness on :%s/health (stale after %ss)",
+        server.bound_port,
+        stale_after,
+    )
+    return server
+
+
 def _processing_list_name() -> str:
     """Per-pod parking list, so one pod never reclaims another's in-flight envelope."""
     return f"{_QUEUE_NAME}:processing:{os.getenv('HOSTNAME') or socket.gethostname()}"
@@ -114,46 +232,59 @@ def run() -> int:
     signal.signal(signal.SIGTERM, _handle_signal)
     signal.signal(signal.SIGINT, _handle_signal)
 
-    # Not ``RedisQueueClient.from_env()``: that hard-codes the 5s socket timeout, which
-    # cannot outlive this loop's block. Built directly so the two stay related by
-    # construction — see _SOCKET_TIMEOUT_SECONDS.
-    redis_client = create_redis_client(
-        decode_responses=True,
-        socket_timeout=_SOCKET_TIMEOUT_SECONDS,
-    )
-    processing = _processing_list_name()
-    logger.info(
-        "Log stream consumer starting: queue='%s' processing='%s'",
-        _QUEUE_NAME,
-        processing,
-    )
-    _recover_in_flight(redis_client, processing)
-
-    while not _shutdown:
-        try:
-            raw = redis_client.blmove(
-                _QUEUE_NAME, processing, _BLOCK_TIMEOUT_SECONDS, "LEFT", "RIGHT"
-            )
-        except Exception:
-            # Connection blips must not kill the pod — the next iteration reconnects via
-            # the client's own retry. Sleeping is unnecessary: BLMOVE already blocks.
-            logger.error("Log stream read failed; retrying", exc_info=True)
-            continue
-
-        if raw is None:  # timeout, no work — loop so shutdown can be observed
-            continue
-
-        try:
-            _dispatch(raw)
-        except Exception:
-            # Match the Celery consumer's posture: a poison envelope is logged and
-            # dropped, never retried forever. logs_consumer already swallows its own
-            # sink failures, so reaching here means a malformed envelope.
-            logger.error("Discarding unprocessable log envelope", exc_info=True)
-        finally:
-            # Remove exactly one copy, whether it succeeded or was discarded — leaving it
-            # parked would have it re-queued on the next restart and replayed forever.
-            redis_client.lrem(processing, 1, raw)
+    health = _RedisStreamHealth(_QUEUE_NAME)
+    health_server = _maybe_start_health_server(health)
+    try:
+        # Not ``RedisQueueClient.from_env()``: that hard-codes the 5s socket timeout,
+        # which cannot outlive this loop's block. Built directly so the two stay
+        # related by construction — see _SOCKET_TIMEOUT_SECONDS.
+        redis_client = create_redis_client(
+            decode_responses=True,
+            socket_timeout=_SOCKET_TIMEOUT_SECONDS,
+        )
+        processing = _processing_list_name()
+        logger.info(
+            "Log stream consumer starting: queue='%s' processing='%s'",
+            _QUEUE_NAME,
+            processing,
+        )
+        _recover_in_flight(redis_client, processing)
+
+        while not _shutdown:
+            try:
+                raw = redis_client.blmove(
+                    _QUEUE_NAME, processing, _BLOCK_TIMEOUT_SECONDS, "LEFT", "RIGHT"
+                )
+                # A nil result is still a successful Redis round trip and proves
+                # that an idle queue is reachable. The health GET itself never
+                # calls BLMOVE and therefore never consumes an envelope.
+                health.mark_success()
+            except Exception:
+                health.mark_failure()
+                # Connection blips must not kill the pod — the next iteration
+                # reconnects via the client's own retry. Sleeping is unnecessary:
+                # BLMOVE already blocks.
+                logger.error("Log stream read failed; retrying", exc_info=True)
+                continue
+
+            if raw is None:  # timeout, so the loop can observe shutdown
+                continue
+
+            try:
+                _dispatch(raw)
+            except Exception:
+                # Match the Celery consumer's posture: a poison envelope is logged
+                # and dropped, never retried forever. logs_consumer already swallows
+                # its own sink failures, so reaching here means a malformed envelope.
+                logger.error("Discarding unprocessable log envelope", exc_info=True)
+            finally:
+                # Remove exactly one copy, whether it succeeded or was discarded —
+                # leaving it parked would have it re-queued on the next restart and
+                # replayed forever.
+                redis_client.lrem(processing, 1, raw)
+    finally:
+        if health_server is not None:
+            health_server.stop()
 
     logger.info("Log stream consumer stopped")
     return 0
diff --git a/workers/log_consumer/scheduler.sh b/workers/log_consumer/scheduler.sh
index e09a82dbd7..94574bca58 100755
--- a/workers/log_consumer/scheduler.sh
+++ b/workers/log_consumer/scheduler.sh
@@ -18,6 +18,16 @@ NOTIFICATION_BUFFER_INTERVAL="${NOTIFICATION_BUFFER_POLL_INTERVAL:-10}"
 DEFAULT_BUFFER_FLUSH_CMD="/app/.venv/bin/python /app/log_consumer/process_notification_buffer.py"
 BUFFER_FLUSH_CMD="${NOTIFICATION_BUFFER_TASK_COMMAND:-$DEFAULT_BUFFER_FLUSH_CMD}"
 
+# Optional local readiness endpoint. The probe reads this state file; it never
+# invokes either task. A missing port preserves the historical process-only mode
+# for deployments that have not yet wired a Compose healthcheck.
+HEALTH_STATE_FILE="${LOG_HISTORY_SCHEDULER_HEALTH_STATE:-/tmp/log-history-scheduler-health.json}"
+HEALTH_PID=""
+LAST_LOG_SUCCESS=""
+LAST_BUFFER_SUCCESS=""
+LAST_LOG_FAILURE=""
+LAST_BUFFER_FAILURE=""
+
 # Loop wakes at the finer of the two cadences (min, floored at 1s); each task
 # fires independently once its own interval has elapsed.
 if [[ "${LOG_HISTORY_INTERVAL}" -lt "${NOTIFICATION_BUFFER_INTERVAL}" ]]; then
@@ -35,18 +45,50 @@ echo "Task 1 (log history): ${LOG_HISTORY_CMD}"
 echo "Task 2 (notification buffer flush): ${BUFFER_FLUSH_CMD}"
 echo "=========================================="
 
+write_health_state() {
+    local temp_state="${HEALTH_STATE_FILE}.$$"
+    if ! printf '{"parent_pid":%s,"last_log_success":%s,"last_buffer_success":%s,"last_log_failure":%s,"last_buffer_failure":%s}\n' \
+        "$$" \
+        "${LAST_LOG_SUCCESS:-null}" \
+        "${LAST_BUFFER_SUCCESS:-null}" \
+        "${LAST_LOG_FAILURE:-null}" \
+        "${LAST_BUFFER_FAILURE:-null}" >"${temp_state}"; then
+        echo "Warning: scheduler health state could not be written" >&2
+        return 0
+    fi
+    if ! mv -f -- "${temp_state}" "${HEALTH_STATE_FILE}"; then
+        echo "Warning: scheduler health state could not be published" >&2
+    fi
+}
+
+start_health_probe() {
+    if [[ -z "${LOG_HISTORY_SCHEDULER_HEALTH_PORT:-}" ]]; then
+        return 0
+    fi
+    export LOG_HISTORY_SCHEDULER_HEALTH_PARENT_PID="$$"
+    export LOG_HISTORY_SCHEDULER_HEALTH_STATE="${HEALTH_STATE_FILE}"
+    write_health_state
+    /app/.venv/bin/python /app/log_consumer/scheduler_health.py &
+    HEALTH_PID="$!"
+    echo "Scheduler health endpoint starting on :${LOG_HISTORY_SCHEDULER_HEALTH_PORT}/health"
+}
+
 cleanup() {
     echo ""
     echo "=========================================="
     echo "Scheduler received shutdown signal"
     echo "Exiting gracefully..."
     echo "=========================================="
+    if [[ -n "${HEALTH_PID}" ]]; then
+        kill "${HEALTH_PID}" 2>/dev/null || true
+    fi
     return 0
 }
 
 # The trap exits after cleanup runs; cleanup itself returns so the function
 # has an explicit terminal return (no unreachable code after exit).
 trap 'cleanup; exit 0' SIGTERM SIGINT
+start_health_probe
 
 run_task() {
     # $1 = display name, $2 = command, $3 = run number. Returns the command's
@@ -57,9 +99,19 @@ run_task() {
     local exit_code=0
     echo "[$(date '+%Y-%m-%d %H:%M:%S')] [Run #${run_num}] Triggering ${task_name}..."
     if eval "${cmd}" 2>&1; then
+        case "${task_name}" in
+            process_log_history) LAST_LOG_SUCCESS="$(date '+%s')" ;;
+            process_notification_buffer) LAST_BUFFER_SUCCESS="$(date '+%s')" ;;
+        esac
+        write_health_state
         echo "[$(date '+%Y-%m-%d %H:%M:%S')] [Run #${run_num}] ✓ ${task_name} OK"
     else
         exit_code=$?
+        case "${task_name}" in
+            process_log_history) LAST_LOG_FAILURE="$(date '+%s')" ;;
+            process_notification_buffer) LAST_BUFFER_FAILURE="$(date '+%s')" ;;
+        esac
+        write_health_state
         echo "[$(date '+%Y-%m-%d %H:%M:%S')] [Run #${run_num}] ✗ ${task_name} failed with exit code ${exit_code}"
     fi
     return "${exit_code}"
diff --git a/workers/log_consumer/scheduler_health.py b/workers/log_consumer/scheduler_health.py
new file mode 100644
index 0000000000..983bcff3e5
--- /dev/null
+++ b/workers/log_consumer/scheduler_health.py
@@ -0,0 +1,240 @@
+"""Read-only readiness endpoint for the log-history scheduler shell loop.
+
+The scheduler intentionally keeps running when one periodic task fails so a
+transient backend error does not kill the container. That makes a process check
+misleading: this probe reads the scheduler's small local state file and returns
+503 when either task has never succeeded, when a task failure is newer than its
+last success, or when either success is stale. A GET never invokes either task.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import signal
+import sys
+import time
+from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+from typing import Any, Callable
+from urllib.parse import urlsplit
+
+
+@dataclass(frozen=True)
+class SchedulerHealth:
+    status: str
+    message: str
+    details: dict[str, Any]
+
+    @property
+    def http_status(self) -> int:
+        return 200 if self.status == "healthy" else 503
+
+
+def read_state(path: str | os.PathLike[str]) -> dict[str, Any] | None:
+    """Read a bounded JSON state document, treating malformed state as missing."""
+    try:
+        # The shell writer keeps this document tiny. Refuse unexpectedly large
+        # input so a damaged bind mount cannot make the probe unbounded.
+        if Path(path).stat().st_size > 16 * 1024:
+            return None
+        with Path(path).open(encoding="utf-8") as state_file:
+            value = json.load(state_file)
+    except (OSError, ValueError, TypeError):
+        return None
+    return value if isinstance(value, dict) else None
+
+
+def _parent_is_scheduler(parent_pid: int) -> bool:
+    """Confirm the recorded PID is still the scheduler shell process."""
+    try:
+        os.kill(parent_pid, 0)
+    except (OSError, TypeError, ValueError):
+        return False
+
+    try:
+        command_line = Path(f"/proc/{parent_pid}/cmdline").read_bytes()
+    except OSError:
+        return False
+    return b"scheduler.sh" in command_line
+
+
+def evaluate_state(
+    state: dict[str, Any] | None,
+    *,
+    now: float | None = None,
+    stale_after: float,
+    parent_alive: Callable[[int], bool] = _parent_is_scheduler,
+) -> SchedulerHealth:
+    """Evaluate scheduler task freshness without executing application work."""
+    if stale_after <= 0:
+        raise ValueError("stale_after must be positive")
+    if state is None:
+        return SchedulerHealth("unhealthy", "scheduler state unavailable", {})
+
+    current_time = time.time() if now is None else now
+    try:
+        parent_pid = int(state["parent_pid"])
+    except (KeyError, TypeError, ValueError):
+        return SchedulerHealth("unhealthy", "scheduler parent identity unavailable", {})
+    if not parent_alive(parent_pid):
+        return SchedulerHealth(
+            "unhealthy",
+            "scheduler loop is not running",
+            {"parent_pid": parent_pid},
+        )
+
+    ages: dict[str, float] = {}
+    failure_fields = {
+        "log_history": "last_log_success",
+        "notification_buffer": "last_buffer_success",
+    }
+    for task_name, success_key in failure_fields.items():
+        success_value = state.get(success_key)
+        failure_key = success_key.replace("success", "failure")
+        failure_value = state.get(failure_key)
+        if success_value is None:
+            return SchedulerHealth(
+                "starting",
+                f"{task_name} task has not completed successfully",
+                {"task": task_name},
+            )
+        try:
+            success_time = float(success_value)
+            ages[task_name] = max(0.0, current_time - success_time)
+        except (TypeError, ValueError):
+            return SchedulerHealth(
+                "unhealthy",
+                f"{task_name} task success timestamp is invalid",
+                {"task": task_name},
+            )
+        try:
+            if failure_value is not None and float(failure_value) > success_time:
+                return SchedulerHealth(
+                    "unhealthy",
+                    f"{task_name} task failed after its last success",
+                    {"task": task_name, "age_seconds": round(ages[task_name], 3)},
+                )
+        except (TypeError, ValueError):
+            return SchedulerHealth(
+                "unhealthy",
+                f"{task_name} task failure timestamp is invalid",
+                {"task": task_name},
+            )
+        if ages[task_name] > stale_after:
+            return SchedulerHealth(
+                "unhealthy",
+                f"{task_name} task success is stale",
+                {
+                    "task": task_name,
+                    "age_seconds": round(ages[task_name], 3),
+                    "stale_after_seconds": stale_after,
+                },
+            )
+
+    return SchedulerHealth(
+        "healthy",
+        "scheduler loop and both periodic tasks are fresh",
+        {
+            "parent_pid": parent_pid,
+            "log_history_age_seconds": round(ages["log_history"], 3),
+            "notification_buffer_age_seconds": round(
+                ages["notification_buffer"], 3
+            ),
+            "stale_after_seconds": stale_after,
+        },
+    )
+
+
+def serve(
+    *,
+    port: int,
+    state_path: str,
+    stale_after: float,
+    parent_pid: int,
+) -> None:
+    """Serve the scheduler probe until the shell parent terminates this process."""
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_GET(self) -> None:
+            if urlsplit(self.path).path not in {"/health", "/healthz", "/livez"}:
+                self.send_response(404)
+                self.end_headers()
+                return
+            state = read_state(state_path)
+            result = evaluate_state(
+                state,
+                stale_after=stale_after,
+                parent_alive=lambda pid: (
+                    pid == parent_pid and _parent_is_scheduler(pid)
+                ),
+            )
+            body = json.dumps(
+                {
+                    "status": result.status,
+                    "check": "log_history_scheduler",
+                    "message": result.message,
+                    **result.details,
+                },
+                separators=(",", ":"),
+            ).encode("utf-8")
+            self.send_response(result.http_status)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            try:
+                self.wfile.write(body)
+            except (BrokenPipeError, ConnectionResetError):
+                pass
+
+        def log_message(self, *_: object) -> None:
+            pass
+
+    server = HTTPServer(("0.0.0.0", port), Handler)
+
+    def _stop(_signum: int, _frame: object) -> None:
+        raise SystemExit(0)
+
+    signal.signal(signal.SIGTERM, _stop)
+    signal.signal(signal.SIGINT, _stop)
+    try:
+        server.serve_forever(poll_interval=0.5)
+    finally:
+        server.server_close()
+
+
+def _positive_float(raw: str, name: str) -> float:
+    try:
+        value = float(raw)
+    except ValueError as exc:
+        raise ValueError(f"{name} must be numeric") from exc
+    if value <= 0:
+        raise ValueError(f"{name} must be positive")
+    return value
+
+
+def main() -> None:
+    port = int(os.environ["LOG_HISTORY_SCHEDULER_HEALTH_PORT"])
+    state_path = os.getenv(
+        "LOG_HISTORY_SCHEDULER_HEALTH_STATE", "/tmp/log-history-scheduler-health.json"
+    )
+    stale_after = _positive_float(
+        os.getenv("LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS", "120"),
+        "LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS",
+    )
+    parent_pid = int(os.environ["LOG_HISTORY_SCHEDULER_HEALTH_PARENT_PID"])
+    serve(
+        port=port,
+        state_path=state_path,
+        stale_after=stale_after,
+        parent_pid=parent_pid,
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (KeyError, ValueError) as exc:
+        print(f"scheduler health configuration error: {exc}", file=sys.stderr)
+        raise SystemExit(2) from exc
diff --git a/workers/log_consumer/worker.py b/workers/log_consumer/worker.py
index 5b56391637..f05f5a83bf 100644
--- a/workers/log_consumer/worker.py
+++ b/workers/log_consumer/worker.py
@@ -47,8 +47,8 @@ def check_log_consumer_health():
         return HealthCheckResult(
             name="log_consumer_health",
             status=HealthStatus.DEGRADED,
-            message=f"Health check failed: {e}",
-            details={"error": str(e)},
+            message="Log consumer health check failed",
+            details={"error": type(e).__name__},
         )
 
 
diff --git a/workers/shared/infrastructure/monitoring/health.py b/workers/shared/infrastructure/monitoring/health.py
index 3a9b90ff51..bf7711b31a 100644
--- a/workers/shared/infrastructure/monitoring/health.py
+++ b/workers/shared/infrastructure/monitoring/health.py
@@ -42,6 +42,91 @@ class HealthCheckResult:
     timestamp: datetime | None = None
 
 
+class WorkerHeartbeat:
+    """Readiness state bound to a real Celery worker heartbeat.
+
+    A worker process can remain alive after its broker connection is gone. This
+    state is marked ready only by the ``worker_ready`` signal and refreshed by
+    Celery's ``heartbeat_sent`` signal, so the HTTP endpoint cannot turn process
+    existence or merely constructed client configuration into readiness.
+    """
+
+    def __init__(self, stale_after_seconds: float) -> None:
+        if stale_after_seconds <= 0:
+            raise ValueError("stale_after_seconds must be positive")
+        self._stale_after_seconds = stale_after_seconds
+        self._lock = threading.Lock()
+        self._ready = False
+        self._stopped = False
+        self._last_heartbeat: float | None = None
+
+    def mark_ready(self) -> None:
+        with self._lock:
+            self._ready = True
+            self._stopped = False
+            # worker_ready follows the broker connection handshake. Treat it as
+            # the first heartbeat so the endpoint is useful immediately while
+            # subsequent heartbeat_sent signals continue to refresh freshness.
+            self._last_heartbeat = time.monotonic()
+
+    def mark_heartbeat(self) -> None:
+        with self._lock:
+            if self._ready and not self._stopped:
+                self._last_heartbeat = time.monotonic()
+
+    def mark_stopped(self) -> None:
+        with self._lock:
+            self._stopped = True
+
+    def check(self) -> HealthCheckResult:
+        with self._lock:
+            ready = self._ready
+            stopped = self._stopped
+            last_heartbeat = self._last_heartbeat
+
+        now = time.monotonic()
+        age = None if last_heartbeat is None else max(0.0, now - last_heartbeat)
+        details: dict[str, Any] = {
+            "ready": ready,
+            "stopped": stopped,
+            "stale_after_seconds": self._stale_after_seconds,
+        }
+        if age is not None:
+            details["heartbeat_age_seconds"] = round(age, 3)
+
+        if stopped:
+            return HealthCheckResult(
+                name="worker_heartbeat",
+                status=HealthStatus.UNHEALTHY,
+                message="Worker shutdown has started",
+                details=details,
+                timestamp=datetime.now(UTC),
+            )
+        if not ready or age is None:
+            return HealthCheckResult(
+                name="worker_heartbeat",
+                status=HealthStatus.UNHEALTHY,
+                message="Worker broker readiness has not been established",
+                details=details,
+                timestamp=datetime.now(UTC),
+            )
+        if age > self._stale_after_seconds:
+            return HealthCheckResult(
+                name="worker_heartbeat",
+                status=HealthStatus.UNHEALTHY,
+                message="Worker heartbeat is stale",
+                details=details,
+                timestamp=datetime.now(UTC),
+            )
+        return HealthCheckResult(
+            name="worker_heartbeat",
+            status=HealthStatus.HEALTHY,
+            message="Worker broker heartbeat is fresh",
+            details=details,
+            timestamp=datetime.now(UTC),
+        )
+
+
 @dataclass
 class SystemMetrics:
     """System metrics for health monitoring."""
@@ -102,25 +187,44 @@ def check_api_connectivity(self) -> HealthCheckResult:
         start_time = time.time()
 
         try:
+            # This check is intentionally configuration-only. A health GET must
+            # not publish work or invoke an unknown application endpoint, but it
+            # must still reject a worker that has no internal API target/key.
+            api_base_url = getattr(self.config, "internal_api_base_url", "")
+            api_key = getattr(self.config, "internal_api_key", "")
+            if not api_base_url or not api_key:
+                return HealthCheckResult(
+                    name="api_connectivity",
+                    status=HealthStatus.UNHEALTHY,
+                    message="Internal API configuration is incomplete",
+                    details={
+                        "base_url_configured": bool(api_base_url),
+                        "service_key_configured": bool(api_key),
+                    },
+                    execution_time=time.time() - start_time,
+                    timestamp=datetime.now(UTC),
+                )
+
             if not self.api_client:
-                # Use singleton API client to reduce initialization noise
-                from .api_client_singleton import get_singleton_api_client
+                # Use singleton API client to reduce initialization noise. This
+                # only constructs the local client; no task-producing API call is
+                # made by the health endpoint.
+                from ...utils.api_client_singleton import get_singleton_api_client
 
                 self.api_client = get_singleton_api_client(self.config)
 
-            # Simply check if API client can be configured properly
-            # Avoid making actual API calls that might hit non-existent endpoints
             execution_time = time.time() - start_time
 
-            # If we can create the client without errors, consider API connectivity healthy
+            # Client construction proves only that local configuration can be
+            # parsed; the broker heartbeat and task-specific checks provide the
+            # runtime readiness signal without making a task-producing request.
             return HealthCheckResult(
                 name="api_connectivity",
                 status=HealthStatus.HEALTHY,
                 message="API client configuration successful",
                 details={
-                    "api_base_url": getattr(
-                        self.config, "internal_api_base_url", "unknown"
-                    )
+                    "base_url_configured": True,
+                    "service_key_configured": True,
                 },
                 execution_time=execution_time,
                 timestamp=datetime.now(UTC),
@@ -131,8 +235,8 @@ def check_api_connectivity(self) -> HealthCheckResult:
             return HealthCheckResult(
                 name="api_connectivity",
                 status=HealthStatus.UNHEALTHY,
-                message=f"API request failed: {str(e)}",
-                details={"error": str(e)},
+                message="API client configuration failed",
+                details={"error": type(e).__name__},
                 execution_time=execution_time,
                 timestamp=datetime.now(UTC),
             )
@@ -141,8 +245,8 @@ def check_api_connectivity(self) -> HealthCheckResult:
             return HealthCheckResult(
                 name="api_connectivity",
                 status=HealthStatus.UNHEALTHY,
-                message=f"Unexpected error: {str(e)}",
-                details={"error": str(e)},
+                message="API client configuration failed",
+                details={"error": type(e).__name__},
                 execution_time=execution_time,
                 timestamp=datetime.now(UTC),
             )
@@ -206,8 +310,8 @@ def check_system_resources(self) -> HealthCheckResult:
             return HealthCheckResult(
                 name="system_resources",
                 status=HealthStatus.UNHEALTHY,
-                message=f"Failed to check system resources: {str(e)}",
-                details={"error": str(e)},
+                message="Failed to check system resources",
+                details={"error": type(e).__name__},
                 execution_time=execution_time,
                 timestamp=datetime.now(UTC),
             )
@@ -255,8 +359,8 @@ def check_worker_process(self) -> HealthCheckResult:
             return HealthCheckResult(
                 name="worker_process",
                 status=HealthStatus.UNHEALTHY,
-                message=f"Failed to check worker process: {str(e)}",
-                details={"error": str(e)},
+                message="Failed to check worker process",
+                details={"error": type(e).__name__},
                 execution_time=execution_time,
                 timestamp=datetime.now(UTC),
             )
@@ -286,8 +390,8 @@ def run_all_checks(self) -> dict[str, Any]:
                     HealthCheckResult(
                         name=name,
                         status=HealthStatus.UNHEALTHY,
-                        message=f"Custom check failed: {str(e)}",
-                        details={"error": str(e)},
+                        message="Custom check failed",
+                        details={"error": type(e).__name__},
                         timestamp=datetime.now(UTC),
                     )
                 )
@@ -425,9 +529,9 @@ def do_GET(self):
                 self._send_json_response({"error": "Not found"}, 404)
 
         except Exception as e:
-            logger.error(f"Health check endpoint error: {e}")
+            logger.error("Health check endpoint error: %s", type(e).__name__)
             self._send_json_response(
-                {"error": "Internal server error", "detail": str(e)}, 500
+                {"error": "Internal server error", "detail": type(e).__name__}, 500
             )
 
     def _send_json_response(self, data: dict[str, Any], status_code: int):
@@ -477,7 +581,9 @@ def handler_factory(*args, **kwargs):
             logger.debug(f"Health check server started on port {self.port}")
 
         except Exception as e:
-            logger.error(f"Failed to start health check server: {e}")
+            logger.error(
+                "Failed to start health check server: %s", type(e).__name__
+            )
             raise
 
     def stop(self):
diff --git a/workers/tests/test_log_stream_consumer.py b/workers/tests/test_log_stream_consumer.py
index 8c3f3091e6..9856dd3249 100644
--- a/workers/tests/test_log_stream_consumer.py
+++ b/workers/tests/test_log_stream_consumer.py
@@ -210,3 +210,40 @@ def _blmove(*_a, **_k):
             consumer.run()
 
         assert redis.blmove.call_args[0][2] == consumer._BLOCK_TIMEOUT_SECONDS
+
+
+class TestRedisStreamHealth:
+    def test_is_stale_before_the_first_completed_read(self, consumer):
+        health = consumer._RedisStreamHealth("log_stream_queue")
+
+        assert health.seconds_since_last_success() > 100_000
+        assert health.status() == {
+            "queue": "log_stream_queue",
+            "redis_poll_failures": 0,
+        }
+
+    def test_successful_empty_poll_is_a_real_readiness_signal(self, consumer):
+        health = consumer._RedisStreamHealth("log_stream_queue")
+        health.mark_success()
+
+        assert health.seconds_since_last_success() < 1
+
+    def test_failures_do_not_refresh_the_success_timestamp(self, consumer):
+        health = consumer._RedisStreamHealth("log_stream_queue")
+        health.mark_success()
+        health.mark_failure()
+        health.mark_failure()
+
+        assert health.seconds_since_last_success() < 1
+        assert health.status()["redis_poll_failures"] == 2
+
+    def test_health_port_is_opt_in_and_validated(self, consumer, monkeypatch):
+        monkeypatch.delenv("LOG_STREAM_CONSUMER_HEALTH_PORT", raising=False)
+        assert consumer._health_port_from_env() is None
+
+        monkeypatch.setenv("LOG_STREAM_CONSUMER_HEALTH_PORT", "8091")
+        assert consumer._health_port_from_env() == 8091
+
+        monkeypatch.setenv("LOG_STREAM_CONSUMER_HEALTH_PORT", "not-a-port")
+        with pytest.raises(ValueError, match="LOG_STREAM_CONSUMER_HEALTH_PORT"):
+            consumer._health_port_from_env()
diff --git a/workers/tests/test_scheduler_health.py b/workers/tests/test_scheduler_health.py
new file mode 100644
index 0000000000..4d2c61ac60
--- /dev/null
+++ b/workers/tests/test_scheduler_health.py
@@ -0,0 +1,84 @@
+"""Characterize the shell scheduler's read-only readiness contract."""
+
+from __future__ import annotations
+
+from log_consumer.scheduler_health import evaluate_state
+
+
+def _state(**overrides):
+    state = {
+        "parent_pid": 42,
+        "last_log_success": 90.0,
+        "last_buffer_success": 90.0,
+        "last_log_failure": None,
+        "last_buffer_failure": None,
+    }
+    state.update(overrides)
+    return state
+
+
+def _alive(_pid: int) -> bool:
+    return True
+
+
+def test_requires_both_periodic_tasks_to_succeed():
+    result = evaluate_state(
+        _state(last_buffer_success=None),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "starting"
+    assert result.http_status == 503
+    assert "notification_buffer" in result.message
+
+
+def test_recent_successes_are_healthy():
+    result = evaluate_state(
+        _state(),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "healthy"
+    assert result.http_status == 200
+    assert result.details["log_history_age_seconds"] == 10.0
+
+
+def test_new_failure_is_unhealthy_until_a_new_success():
+    result = evaluate_state(
+        _state(last_log_failure=95.0),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "unhealthy"
+    assert result.http_status == 503
+    assert result.details["task"] == "log_history"
+
+
+def test_old_success_becomes_stale():
+    result = evaluate_state(
+        _state(last_buffer_success=0.0),
+        now=100.0,
+        stale_after=30.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "unhealthy"
+    assert "stale" in result.message
+
+
+def test_dead_parent_is_not_healthy_even_with_fresh_state():
+    result = evaluate_state(
+        _state(),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=lambda _pid: False,
+    )
+
+    assert result.status == "unhealthy"
+    assert result.message == "scheduler loop is not running"
diff --git a/workers/tests/test_worker_health.py b/workers/tests/test_worker_health.py
new file mode 100644
index 0000000000..d16ce1d78f
--- /dev/null
+++ b/workers/tests/test_worker_health.py
@@ -0,0 +1,52 @@
+"""Tests for readiness bound to the Celery broker heartbeat lifecycle."""
+
+from __future__ import annotations
+
+from shared.infrastructure.monitoring.health import HealthStatus, WorkerHeartbeat
+
+
+def test_worker_heartbeat_is_unhealthy_before_worker_ready():
+    heartbeat = WorkerHeartbeat(stale_after_seconds=30)
+
+    result = heartbeat.check()
+
+    assert result.status is HealthStatus.UNHEALTHY
+    assert result.message == "Worker broker readiness has not been established"
+
+
+def test_worker_ready_establishes_initial_freshness():
+    heartbeat = WorkerHeartbeat(stale_after_seconds=30)
+    heartbeat.mark_ready()
+
+    result = heartbeat.check()
+
+    assert result.status is HealthStatus.HEALTHY
+    assert result.details["ready"] is True
+    assert result.details["heartbeat_age_seconds"] < 1
+
+
+def test_heartbeat_refresh_is_ignored_after_shutdown():
+    heartbeat = WorkerHeartbeat(stale_after_seconds=30)
+    heartbeat.mark_ready()
+    heartbeat.mark_stopped()
+    heartbeat.mark_heartbeat()
+
+    result = heartbeat.check()
+
+    assert result.status is HealthStatus.UNHEALTHY
+    assert result.message == "Worker shutdown has started"
+
+
+def test_stale_heartbeat_is_unhealthy(monkeypatch):
+    clock = iter((100.0, 140.0))
+    monkeypatch.setattr(
+        "shared.infrastructure.monitoring.health.time.monotonic",
+        lambda: next(clock),
+    )
+    heartbeat = WorkerHeartbeat(stale_after_seconds=30)
+    heartbeat.mark_ready()
+
+    result = heartbeat.check()
+
+    assert result.status is HealthStatus.UNHEALTHY
+    assert result.message == "Worker heartbeat is stale"
diff --git a/workers/worker.py b/workers/worker.py
index 077fbd9741..29afd67afd 100755
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -28,6 +28,7 @@
 from shared.enums.worker_enums import WorkerType  # noqa: E402
 from shared.infrastructure import initialize_worker_infrastructure  # noqa: E402
 from shared.infrastructure.config.builder import WorkerBuilder  # noqa: E402
+from shared.infrastructure.monitoring.health import WorkerHeartbeat  # noqa: E402
 from shared.models.worker_models import get_celery_setting  # noqa: E402
 from shared.patterns.factory.client_factory import ClientFactory  # noqa: E402
 
@@ -533,6 +534,57 @@ def load_worker_tasks(worker_type: WorkerType) -> None:
             "check the worker task registration."
         )
 
+
+def _worker_health_stale_seconds() -> float:
+    """Resolve the broker-heartbeat freshness bound from deployment settings."""
+    default = max(30.0, float(config.health_check_interval * 3))
+    raw = os.getenv("WORKER_HEALTH_STALE_SECONDS")
+    if raw is None or raw == "":
+        return default
+    try:
+        value = float(raw)
+    except ValueError as exc:
+        raise ValueError(f"WORKER_HEALTH_STALE_SECONDS={raw!r} is not numeric") from exc
+    if value <= 0:
+        raise ValueError(f"WORKER_HEALTH_STALE_SECONDS={value} must be positive")
+    return value
+
+
+# The generic HealthServer was previously only constructed by the unused builder
+# convenience path. Wire it to the actual Celery lifecycle here, after task loading
+# has registered all worker-specific checks. The server starts only after
+# worker_ready, so an importing process or a pre-broker worker cannot report green.
+_worker_heartbeat = WorkerHeartbeat(_worker_health_stale_seconds())
+_worker_health_checker, _worker_health_server = WorkerBuilder.setup_health_monitoring(
+    worker_type, config
+)
+_worker_health_checker.add_custom_check("worker_heartbeat", _worker_heartbeat.check)
+
+
+@signals.worker_ready.connect
+def on_worker_ready(**_kwargs):
+    """Start the probe after Celery has completed its broker readiness handshake."""
+    _worker_heartbeat.mark_ready()
+    try:
+        _worker_health_server.start()
+    except OSError:
+        # A health bind failure must be visible as probe absence, but must not
+        # kill a worker that can still drain its queue.
+        logger.exception("Worker health server could not bind; continuing without it")
+
+
+@signals.heartbeat_sent.connect
+def on_heartbeat_sent(**_kwargs):
+    """Refresh the readiness signal only when Celery emits a broker heartbeat."""
+    _worker_heartbeat.mark_heartbeat()
+
+
+@signals.worker_shutdown.connect
+def on_worker_shutdown(**_kwargs):
+    """Stop the local endpoint before the worker process exits."""
+    _worker_heartbeat.mark_stopped()
+    _worker_health_server.stop()
+
 # Log successful configuration
 logger.info(f"✅ Successfully loaded {worker_type} worker using WorkerBuilder")
 logger.info(

From 337f377aad6e98d8a2b75999e63f55ee8c90b81d Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 06:44:05 -0400
Subject: [PATCH 18/48] Harden worker probe freshness and HTTP bounds

---
 workers/log_consumer/redis_stream_consumer.py |  36 ++-
 workers/log_consumer/scheduler_health.py      |  97 +++++--
 workers/pg_queue_consumer/supervisor.py       |  36 +--
 workers/queue_backend/pg_queue/consumer.py    |  38 ++-
 workers/queue_backend/pg_queue/liveness.py    | 124 ++++++++-
 workers/queue_backend/pg_queue/reaper.py      | 245 ++++++++++--------
 .../infrastructure/monitoring/health.py       |  27 +-
 workers/tests/test_log_stream_consumer.py     |  18 ++
 workers/tests/test_pg_consumer_supervisor.py  |   8 +
 workers/tests/test_pg_queue_consumer.py       |  11 +-
 workers/tests/test_pg_reaper.py               |   1 +
 workers/tests/test_scheduler_health.py        |  57 ++++
 workers/worker.py                             |   3 +-
 13 files changed, 525 insertions(+), 176 deletions(-)

diff --git a/workers/log_consumer/redis_stream_consumer.py b/workers/log_consumer/redis_stream_consumer.py
index 50e6c7a41f..34fc7749e2 100644
--- a/workers/log_consumer/redis_stream_consumer.py
+++ b/workers/log_consumer/redis_stream_consumer.py
@@ -27,6 +27,7 @@
 from __future__ import annotations
 
 import json
+import math
 import os
 import signal
 import socket
@@ -52,9 +53,25 @@
 from log_consumer.tasks import logs_consumer  # noqa: E402
 
 _QUEUE_NAME = os.getenv("LOG_STREAM_QUEUE_NAME", "log_stream_queue")
+
+
+def _positive_int_env(name: str, default: int) -> int:
+    """Read a positive integer setting and fail with its deployment name."""
+    raw = os.getenv(name)
+    if raw is None or raw == "":
+        return default
+    try:
+        value = int(raw)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"Invalid {name}={raw!r}: {exc}") from exc
+    if value <= 0:
+        raise ValueError(f"Invalid {name}={value}: value must be positive")
+    return value
+
+
 # BLMOVE blocks up to this long before returning None, which is the loop's only chance to
 # notice a shutdown signal. Keep it well under the pod's terminationGracePeriodSeconds.
-_BLOCK_TIMEOUT_SECONDS = int(os.getenv("LOG_STREAM_BLOCK_TIMEOUT", "5"))
+_BLOCK_TIMEOUT_SECONDS = _positive_int_env("LOG_STREAM_BLOCK_TIMEOUT", 5)
 # redis-py enforces ``socket_timeout`` on the BLMOVE read itself, so it MUST exceed the
 # server-side block or every call aborts mid-block with ``redis.TimeoutError``.
 # ``create_redis_client`` defaults it to 5s — exactly ``_BLOCK_TIMEOUT_SECONDS`` — and the
@@ -96,17 +113,20 @@ def mark_failure(self) -> None:
     def seconds_since_last_success(self) -> float:
         with self._lock:
             last_success = self._last_success
-        # A finite, large value keeps the JSON response valid before the first
-        # completed read while making the probe unambiguously stale.
-        return 1_000_000.0 if last_success is None else max(
+        # No completed Redis round trip means readiness has not been established.
+        # Keep this non-finite so the shared probe cannot turn green merely
+        # because an operator chose a very large stale bound.
+        return float("inf") if last_success is None else max(
             0.0, time.monotonic() - last_success
         )
 
     def status(self) -> dict[str, object]:
         with self._lock:
+            ready = self._last_success is not None
             failures = self._poll_failures
         return {
             "queue": self._queue_name,
+            "redis_poll_ready": ready,
             "redis_poll_failures": failures,
         }
 
@@ -122,8 +142,10 @@ def _health_port_from_env() -> int | None:
         raise ValueError(
             f"Invalid LOG_STREAM_CONSUMER_HEALTH_PORT={raw!r}: {exc}"
         ) from exc
-    if not 0 <= port <= 65535:
-        raise ValueError(f"LOG_STREAM_CONSUMER_HEALTH_PORT={port} out of range")
+    if not 1 <= port <= 65535:
+        raise ValueError(
+            f"LOG_STREAM_CONSUMER_HEALTH_PORT={port} must be between 1 and 65535"
+        )
     return port
 
 
@@ -139,7 +161,7 @@ def _health_stale_seconds() -> float:
         raise ValueError(
             f"Invalid LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS={raw!r}: {exc}"
         ) from exc
-    if stale_after <= 0:
+    if not math.isfinite(stale_after) or stale_after <= 0:
         raise ValueError(
             "LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS must be positive"
         )
diff --git a/workers/log_consumer/scheduler_health.py b/workers/log_consumer/scheduler_health.py
index 983bcff3e5..ff3eb5621e 100644
--- a/workers/log_consumer/scheduler_health.py
+++ b/workers/log_consumer/scheduler_health.py
@@ -10,12 +10,13 @@
 from __future__ import annotations
 
 import json
+import math
 import os
 import signal
 import sys
 import time
 from dataclasses import dataclass
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Any, Callable
 from urllib.parse import urlsplit
@@ -68,12 +69,22 @@ def evaluate_state(
     parent_alive: Callable[[int], bool] = _parent_is_scheduler,
 ) -> SchedulerHealth:
     """Evaluate scheduler task freshness without executing application work."""
-    if stale_after <= 0:
+    if not math.isfinite(stale_after) or stale_after <= 0:
         raise ValueError("stale_after must be positive")
     if state is None:
         return SchedulerHealth("unhealthy", "scheduler state unavailable", {})
 
     current_time = time.time() if now is None else now
+    try:
+        current_time = float(current_time)
+    except (TypeError, ValueError):
+        return SchedulerHealth(
+            "unhealthy", "scheduler evaluation time is invalid", {}
+        )
+    if not math.isfinite(current_time):
+        return SchedulerHealth(
+            "unhealthy", "scheduler evaluation time is invalid", {}
+        )
     try:
         parent_pid = int(state["parent_pid"])
     except (KeyError, TypeError, ValueError):
@@ -102,20 +113,42 @@ def evaluate_state(
             )
         try:
             success_time = float(success_value)
-            ages[task_name] = max(0.0, current_time - success_time)
         except (TypeError, ValueError):
             return SchedulerHealth(
                 "unhealthy",
                 f"{task_name} task success timestamp is invalid",
                 {"task": task_name},
             )
+        if not math.isfinite(success_time):
+            return SchedulerHealth(
+                "unhealthy",
+                f"{task_name} task success timestamp is invalid",
+                {"task": task_name},
+            )
+        if success_time > current_time:
+            return SchedulerHealth(
+                "unhealthy",
+                f"{task_name} task success timestamp is in the future",
+                {"task": task_name},
+            )
+        ages[task_name] = current_time - success_time
         try:
-            if failure_value is not None and float(failure_value) > success_time:
-                return SchedulerHealth(
-                    "unhealthy",
-                    f"{task_name} task failed after its last success",
-                    {"task": task_name, "age_seconds": round(ages[task_name], 3)},
-                )
+            if failure_value is not None:
+                failure_time = float(failure_value)
+                if not math.isfinite(failure_time):
+                    raise ValueError
+                if failure_time > current_time:
+                    return SchedulerHealth(
+                        "unhealthy",
+                        f"{task_name} task failure timestamp is in the future",
+                        {"task": task_name},
+                    )
+                if failure_time > success_time:
+                    return SchedulerHealth(
+                        "unhealthy",
+                        f"{task_name} task failed after its last success",
+                        {"task": task_name, "age_seconds": round(ages[task_name], 3)},
+                    )
         except (TypeError, ValueError):
             return SchedulerHealth(
                 "unhealthy",
@@ -157,10 +190,19 @@ def serve(
     """Serve the scheduler probe until the shell parent terminates this process."""
 
     class Handler(BaseHTTPRequestHandler):
+        def setup(self) -> None:
+            super().setup()
+            # A health client that connects and never finishes its request must
+            # not pin a server thread indefinitely.
+            self.connection.settimeout(2.0)
+
         def do_GET(self) -> None:
             if urlsplit(self.path).path not in {"/health", "/healthz", "/livez"}:
-                self.send_response(404)
-                self.end_headers()
+                try:
+                    self.send_response(404)
+                    self.end_headers()
+                except (BrokenPipeError, ConnectionResetError, TimeoutError):
+                    pass
                 return
             state = read_state(state_path)
             result = evaluate_state(
@@ -179,19 +221,22 @@ def do_GET(self) -> None:
                 },
                 separators=(",", ":"),
             ).encode("utf-8")
-            self.send_response(result.http_status)
-            self.send_header("Content-Type", "application/json")
-            self.send_header("Content-Length", str(len(body)))
-            self.end_headers()
             try:
+                self.send_response(result.http_status)
+                self.send_header("Content-Type", "application/json")
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
                 self.wfile.write(body)
-            except (BrokenPipeError, ConnectionResetError):
+            except (BrokenPipeError, ConnectionResetError, TimeoutError):
                 pass
 
         def log_message(self, *_: object) -> None:
             pass
 
-    server = HTTPServer(("0.0.0.0", port), Handler)
+    port = _positive_port(str(port), "LOG_HISTORY_SCHEDULER_HEALTH_PORT")
+    server = ThreadingHTTPServer(("0.0.0.0", port), Handler)
+    server.daemon_threads = True
+    server.block_on_close = False
 
     def _stop(_signum: int, _frame: object) -> None:
         raise SystemExit(0)
@@ -209,13 +254,27 @@ def _positive_float(raw: str, name: str) -> float:
         value = float(raw)
     except ValueError as exc:
         raise ValueError(f"{name} must be numeric") from exc
-    if value <= 0:
+    if not math.isfinite(value) or value <= 0:
         raise ValueError(f"{name} must be positive")
     return value
 
 
+def _positive_port(raw: str, name: str) -> int:
+    """Parse a concrete listening port; zero is not a deployable health port."""
+    try:
+        value = int(raw)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"{name} must be an integer port") from exc
+    if not 1 <= value <= 65535:
+        raise ValueError(f"{name} must be between 1 and 65535")
+    return value
+
+
 def main() -> None:
-    port = int(os.environ["LOG_HISTORY_SCHEDULER_HEALTH_PORT"])
+    port = _positive_port(
+        os.environ["LOG_HISTORY_SCHEDULER_HEALTH_PORT"],
+        "LOG_HISTORY_SCHEDULER_HEALTH_PORT",
+    )
     state_path = os.getenv(
         "LOG_HISTORY_SCHEDULER_HEALTH_STATE", "/tmp/log-history-scheduler-health.json"
     )
diff --git a/workers/pg_queue_consumer/supervisor.py b/workers/pg_queue_consumer/supervisor.py
index 62f39a6836..d2759ce140 100644
--- a/workers/pg_queue_consumer/supervisor.py
+++ b/workers/pg_queue_consumer/supervisor.py
@@ -143,15 +143,15 @@ class _Fleet:
 
     def __init__(self, concurrency: int) -> None:
         self._n = concurrency
-        # Shared, fork-inherited heartbeat slots (one last-poll wall-time per
-        # child). lock=False is safe: a slot is written either by the parent
-        # (seed, at construction, while no child owns it) OR by that child's
-        # heartbeat thread — never concurrently — and only read by the parent, so
-        # a torn double read just yields one stale sample that self-corrects.
+        # Shared, fork-inherited heartbeat slots (one last-success wall-time per
+        # child). Zero means that no child has completed a dependency read yet;
+        # the probe must stay unhealthy until each live slot earns a real sample.
+        # lock=False is safe: a slot is written by its child heartbeat thread and
+        # only read by the parent, so a torn double read just yields one stale
+        # sample that self-corrects.
         self._heartbeats = multiprocessing.Array("d", concurrency, lock=False)
-        now = time.time()
         for i in range(concurrency):
-            self._heartbeats[i] = now
+            self._heartbeats[i] = 0.0
         self._pids: dict[int, int] = {}
         self._last_fork: dict[int, float] = {}
         self._consecutive_crashes: dict[int, int] = {}
@@ -233,7 +233,13 @@ def is_crash_looping(self) -> bool:
 
     def oldest_age(self) -> float:
         now = time.time()
-        return max((now - hb for hb in self._heartbeats), default=0.0)
+        ages = (
+            float("inf")
+            if not math.isfinite(hb) or hb <= 0
+            else max(0.0, now - hb)
+            for hb in self._heartbeats
+        )
+        return max(ages, default=0.0)
 
     def freshness(self) -> float:
         """Liveness verdict source: a crash-looping fleet is force-stale (``inf``)
@@ -261,15 +267,15 @@ def _run_child(slot: int, heartbeats) -> None:  # noqa: ANN001 (ctypes array)
     consumer = build_consumer_from_env()
 
     def _publish_heartbeat() -> None:
-        # last-poll wall-time = now − (seconds since last poll). Frozen while a
-        # task runs (the consumer stamps its heartbeat at the top of poll_once),
-        # so a child stuck on a too-long task goes stale exactly as the single
-        # consumer does. Guarded so a transient error (e.g. teardown during
-        # shutdown) logs loudly and the loop continues instead of dying silently
-        # and false-staling a healthy child.
+        # Dependency-aware wall-time. A child that keeps looping while every PG
+        # read fails publishes 0, which the parent treats as infinitely stale;
+        # a task stuck after a poll still ages from the poll start.
         while True:
             try:
-                heartbeats[slot] = time.time() - consumer.seconds_since_last_poll()
+                age = consumer.seconds_since_dependency_progress()
+                heartbeats[slot] = (
+                    0.0 if not math.isfinite(age) else time.time() - age
+                )
             except Exception:
                 logger.exception(
                     "PG-queue consumer: heartbeat publish failed for slot=%s", slot
diff --git a/workers/queue_backend/pg_queue/consumer.py b/workers/queue_backend/pg_queue/consumer.py
index 241a3d291e..88e3ac64e9 100644
--- a/workers/queue_backend/pg_queue/consumer.py
+++ b/workers/queue_backend/pg_queue/consumer.py
@@ -42,6 +42,7 @@
 from ..fairness import FAIRNESS_HEADER_NAME
 from .client import PgQueueClient
 from .connection import CONN_DEAD_ERRORS
+from .liveness import DependencyHeartbeat
 from .liveness import LivenessServer as _BaseLivenessServer
 from .result_backend import PgResultBackend
 from .task_payload import to_payload
@@ -361,6 +362,10 @@ def __init__(
         # long-running task (poll_once not returning) goes stale and is caught —
         # something pgrep-based --status and the launch-time check cannot see.
         self._last_poll_monotonic = time.monotonic()
+        # Dependency-aware health state. The poll-loop timestamp above remains
+        # useful for metrics and task-stall detection; this tracker only
+        # succeeds after every queue read in a cycle has returned.
+        self._dependency_health = DependencyHeartbeat()
 
     def poll_once(self) -> int:
         """Claim + process one batch per queue (read once each, in list order);
@@ -372,21 +377,39 @@ def poll_once(self) -> int:
         after a partial failure).
         """
         self._last_poll_monotonic = time.monotonic()
+        self._dependency_health.begin()
         total = 0
+        db_cycle_failed = False
         for queue_name in self.queue_names:
             try:
                 messages = self._client.read(
                     queue_name, vt_seconds=self.lease_seconds, qty=self.batch_size
                 )
+            except Exception:
+                db_cycle_failed = True
+                logger.exception(
+                    "PG-queue consumer: poll failed for queue %r; "
+                    "continuing with the other queues",
+                    queue_name,
+                )
+                continue
+            try:
                 for message in messages:
                     self._handle(message)
                 total += len(messages)
             except Exception:
+                # Task/ack failures are handled by _handle where possible. Keep
+                # them separate from the read dependency: a successful read
+                # proves PG progress even when a customer task is retried.
                 logger.exception(
-                    "PG-queue consumer: poll failed for queue %r; "
+                    "PG-queue consumer: task handling failed for queue %r; "
                     "continuing with the other queues",
                     queue_name,
                 )
+        if db_cycle_failed:
+            self._dependency_health.fail()
+        else:
+            self._dependency_health.succeed()
         return total
 
     @contextlib.contextmanager
@@ -1016,6 +1039,14 @@ def seconds_since_last_poll(self) -> float:
         """Seconds since the last poll attempt (for the liveness heartbeat)."""
         return time.monotonic() - self._last_poll_monotonic
 
+    def seconds_since_dependency_progress(self) -> float:
+        """Age of the last successful PG read, or infinity before one exists."""
+        return self._dependency_health.age()
+
+    def dependency_health_status(self) -> dict[str, Any]:
+        """Machine-readable dependency state for the liveness response."""
+        return self._dependency_health.status()
+
     def run(self, *, install_signals: bool = True, require_tasks: bool = True) -> None:
         """Poll loop with empty-queue backoff and graceful shutdown.
 
@@ -1211,13 +1242,14 @@ def __init__(
     ) -> None:
         from .metrics import ConsumerMetrics
 
-        metrics = ConsumerMetrics(freshness_fn=consumer.seconds_since_last_poll)
+        metrics = ConsumerMetrics(freshness_fn=consumer.seconds_since_dependency_progress)
         super().__init__(
-            freshness_fn=consumer.seconds_since_last_poll,
+            freshness_fn=consumer.seconds_since_dependency_progress,
             stale_after=stale_after,
             port=port,
             check_name="pg_queue_poll",
             age_key="seconds_since_last_poll",
+            extra_status_fn=consumer.dependency_health_status,
             metrics_fn=metrics.render,
             thread_name="pg-consumer-liveness",
             log_label="pg-queue consumer",
diff --git a/workers/queue_backend/pg_queue/liveness.py b/workers/queue_backend/pg_queue/liveness.py
index 07f41f6ceb..781c83b1fe 100644
--- a/workers/queue_backend/pg_queue/liveness.py
+++ b/workers/queue_backend/pg_queue/liveness.py
@@ -23,15 +23,96 @@
 
 import contextlib
 import logging
+import math
+import threading
+import time
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
-    from http.server import HTTPServer
+    from http.server import ThreadingHTTPServer
     from threading import Thread
 
 logger = logging.getLogger(__name__)
 
+_HEALTH_FAILURE_THRESHOLD = 3
+
+
+class DependencyHeartbeat:
+    """Track loop progress together with the dependency operation it drives.
+
+    A loop timestamp alone is insufficient: a PG consumer can stamp the top of
+    every cycle while every database read fails, and a reaper can tick while
+    its lease renewal is broken. begin() is called immediately before
+    dependency work, succeed() only after the required operation returns, and
+    fail() on an exception. The health age is stale when no operation has
+    succeeded, when the loop is stuck in a cycle, or after a short repeated
+    failure streak.
+    """
+
+    def __init__(self, *, failure_threshold: int = _HEALTH_FAILURE_THRESHOLD) -> None:
+        if failure_threshold <= 0:
+            raise ValueError("failure_threshold must be positive")
+        self._failure_threshold = failure_threshold
+        self._lock = threading.Lock()
+        self._last_started: float | None = None
+        self._last_success: float | None = None
+        self._failure_streak = 0
+        self._total_failures = 0
+        self._total_successes = 0
+
+    def begin(self) -> None:
+        with self._lock:
+            self._last_started = time.monotonic()
+
+    def succeed(self) -> None:
+        with self._lock:
+            self._last_success = time.monotonic()
+            self._failure_streak = 0
+            self._total_successes += 1
+
+    def fail(self) -> None:
+        with self._lock:
+            self._failure_streak += 1
+            self._total_failures += 1
+
+    def age(self) -> float:
+        with self._lock:
+            last_started = self._last_started
+            last_success = self._last_success
+            failure_streak = self._failure_streak
+        if last_success is None or failure_streak >= self._failure_threshold:
+            return float("inf")
+        now = time.monotonic()
+        success_age = max(0.0, now - last_success)
+        started_age = (
+            0.0
+            if last_started is None
+            else max(0.0, now - last_started)
+        )
+        # Either a wedged cycle or a missing dependency success is unhealthy.
+        return max(success_age, started_age)
+
+    def status(self) -> dict[str, Any]:
+        with self._lock:
+            last_success = self._last_success
+            failure_streak = self._failure_streak
+            total_failures = self._total_failures
+            total_successes = self._total_successes
+        age = self.age()
+        return {
+            "dependency_ready": last_success is not None
+            and failure_streak < self._failure_threshold
+            and math.isfinite(age),
+            "dependency_failure_streak": failure_streak,
+            "dependency_failures_total": total_failures,
+            "dependency_successes_total": total_successes,
+            "dependency_health_age_seconds": (
+                round(age, 3) if math.isfinite(age) else None
+            ),
+            "dependency_failure_threshold": self._failure_threshold,
+        }
+
 
 class LivenessServer:
     """Generic liveness probe: 200 while ``freshness_fn()`` is within
@@ -70,8 +151,10 @@ def __init__(
         # Re-validate here (not only at the env boundary): a direct caller could
         # otherwise build an always-503 probe that crash-loops the pod. Mirrors
         # the codebase's load-bearing re-validation convention (PgReaper.__init__).
-        if stale_after <= 0:
+        if not math.isfinite(stale_after) or stale_after <= 0:
             raise ValueError(f"stale_after must be positive, got {stale_after!r}")
+        if not isinstance(port, int) or not 0 <= port <= 65535:
+            raise ValueError(f"port must be an integer in range 0-65535, got {port!r}")
         self._freshness_fn = freshness_fn
         self._stale_after = stale_after
         self._port = port
@@ -84,13 +167,13 @@ def __init__(
         # source process after the consumer/reaper extraction (e.g. "pg-queue
         # consumer" / "pg-queue reaper") — they all log via this module's logger.
         self._log_label = log_label
-        self._httpd: HTTPServer | None = None
+        self._httpd: ThreadingHTTPServer | None = None
         self._thread: Thread | None = None
 
     def start(self) -> None:
         import json
         import threading
-        from http.server import BaseHTTPRequestHandler, HTTPServer
+        from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
         from urllib.parse import urlsplit
 
         if self._httpd is not None:
@@ -108,6 +191,12 @@ def start(self) -> None:
         log_label = self._log_label
 
         class _Handler(BaseHTTPRequestHandler):
+            def setup(self) -> None:
+                super().setup()
+                # A client that connects and never finishes its request must
+                # not pin a probe thread forever.
+                self.connection.settimeout(2.0)
+
             def do_GET(self) -> None:
                 # Strip any query string — a probe like /health?foo=bar must match.
                 path = urlsplit(self.path).path
@@ -115,14 +204,17 @@ def do_GET(self) -> None:
                     self._serve_metrics()
                     return
                 if path not in paths:
-                    self.send_response(404)
-                    self.end_headers()
+                    try:
+                        self.send_response(404)
+                        self.end_headers()
+                    except (BrokenPipeError, ConnectionResetError, TimeoutError):
+                        pass
                     return
                 # One clock read so age and the healthy/stale verdict share an
                 # instant. The verdict is purely freshness — extra_status_fn
                 # fields are informational and never flip it.
                 age = freshness_fn()
-                stale = age > stale_after
+                stale = not math.isfinite(age) or age > stale_after
                 # Extra fields first, then overlay the core fields — so a caller's
                 # extra_status_fn can NEVER clobber status/check/age_key/
                 # stale_after_seconds (which a monitor reads): core always wins.
@@ -133,7 +225,7 @@ def do_GET(self) -> None:
                     {
                         "status": "unhealthy" if stale else "healthy",
                         "check": check_name,
-                        age_key: round(age, 3),
+                        age_key: round(age, 3) if math.isfinite(age) else None,
                         "stale_after_seconds": stale_after,
                     }
                 )
@@ -146,7 +238,7 @@ def do_GET(self) -> None:
                     self.send_header("Content-Type", "application/json")
                     self.end_headers()
                     self.wfile.write(body)
-                except (BrokenPipeError, ConnectionResetError):
+                except (BrokenPipeError, ConnectionResetError, TimeoutError):
                     pass  # client (probe) hung up mid-response — not our problem
 
             def _serve_metrics(self) -> None:
@@ -156,7 +248,9 @@ def _serve_metrics(self) -> None:
                     body = metrics_fn()  # type: ignore[misc]  # guarded by caller
                 except Exception:
                     logger.exception("%s: /metrics render failed", log_label)
-                    with contextlib.suppress(BrokenPipeError, ConnectionResetError):
+                    with contextlib.suppress(
+                        BrokenPipeError, ConnectionResetError, TimeoutError
+                    ):
                         self.send_response(500)
                         self.end_headers()
                     return
@@ -166,7 +260,7 @@ def _serve_metrics(self) -> None:
                     self.send_header("Content-Type", metrics_content_type)
                     self.end_headers()
                     self.wfile.write(body)
-                except (BrokenPipeError, ConnectionResetError):
+                except (BrokenPipeError, ConnectionResetError, TimeoutError):
                     pass  # scraper hung up mid-response — not our problem
 
             def log_message(self, *_: object) -> None:
@@ -177,7 +271,7 @@ def log_error(self, fmt: str, *args: object) -> None:
                 # don't let the pass above swallow them — surface to our logger.
                 logger.warning(f"{log_label} liveness handler: " + fmt, *args)
 
-        def _serve(httpd: HTTPServer) -> None:
+        def _serve(httpd: ThreadingHTTPServer) -> None:
             try:
                 httpd.serve_forever()
             except Exception:
@@ -185,7 +279,11 @@ def _serve(httpd: HTTPServer) -> None:
                 # answering (connection refused) with no breadcrumb.
                 logger.exception("%s liveness server thread crashed", log_label)
 
-        httpd = HTTPServer(("0.0.0.0", self._port), _Handler)
+        # Thread each request so one slow or abandoned client cannot block the
+        # orchestrator's next health GET behind socketserver's serial handler.
+        httpd = ThreadingHTTPServer(("0.0.0.0", self._port), _Handler)
+        httpd.daemon_threads = True
+        httpd.block_on_close = False
         self._httpd = httpd
         self._thread = threading.Thread(
             target=_serve, args=(httpd,), daemon=True, name=self._thread_name
diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py
index b43438c44f..8d5fe82c35 100644
--- a/workers/queue_backend/pg_queue/reaper.py
+++ b/workers/queue_backend/pg_queue/reaper.py
@@ -68,6 +68,7 @@
 from ..barrier import barrier_stuck_timeout_seconds
 from .connection import create_pg_connection
 from .leader_election import LeaderLease, default_worker_id
+from .liveness import DependencyHeartbeat
 from .liveness import LivenessServer as _BaseLivenessServer
 from .metrics import ReaperMetrics
 from .pg_scheduler import dispatch_due_periodic_tasks, dispatch_due_schedules
@@ -1147,6 +1148,11 @@ def __init__(
         # standby tick counts as progress too (the loop is alive), so this tracks
         # loop liveness, not leadership.
         self._last_tick_monotonic = time.monotonic()
+        # A tick timestamp alone can stay fresh through an unavailable DB or
+        # lease. This tracker is advanced only after the lease operation and
+        # required leader work complete successfully.
+        self._dependency_health = DependencyHeartbeat()
+        self._tick_health_failed = False
         # Queue-wide metrics snapshot cadence (same None-sentinel pattern as the
         # sweep gate: first leader tick refreshes immediately).
         self._last_gauge_refresh_monotonic: float | None = None
@@ -1169,6 +1175,17 @@ def seconds_since_last_tick(self) -> float:
         """Seconds since the last tick started — the liveness heartbeat age."""
         return time.monotonic() - self._last_tick_monotonic
 
+    def seconds_since_dependency_progress(self) -> float:
+        """Age of the last successful lease/recovery cycle."""
+        return self._dependency_health.age()
+
+    def dependency_health_status(self) -> dict[str, object]:
+        """Machine-readable DB/lease state for the liveness response."""
+        return self._dependency_health.status()
+
+    def _mark_tick_health_failure(self) -> None:
+        self._tick_health_failed = True
+
     def _get_sweep_conn(self) -> PgConnection:
         # Recreate only an OWNED missing/closed connection; an injected one is the
         # caller's and is never swapped (mirrors LeaderLease / PgQueueClient).
@@ -1202,122 +1219,127 @@ def _discard_owned_sweep_conn(self) -> None:
 
     def tick(self) -> TickOutcome:
         """One cycle: maintain leadership, then sweep iff leader."""
-        # Heartbeat at the START of the cycle: a tick that begins but then errors
-        # still proves the loop is running (the error path is caught by run()).
+        # Record the loop start before any lease or DB work. A cycle that later
+        # blocks is stale even if the previous cycle was healthy.
         self._last_tick_monotonic = time.monotonic()
-        if self._is_leader:
+        self._dependency_health.begin()
+        self._tick_health_failed = False
+        try:
+            if self._is_leader:
+                try:
+                    still_leader = self._lease.renew()
+                except Exception:
+                    # A raised renew means leadership is unknown: stop acting
+                    # before letting it propagate.
+                    self._is_leader = False
+                    self._step_down_metrics()
+                    self._mark_tick_health_failure()
+                    raise
+                if not still_leader:
+                    logger.warning(
+                        "Reaper: lost leadership (lease taken over) — stepping "
+                        "down to standby"
+                    )
+                    self._is_leader = False
+                    self._step_down_metrics()
+            if not self._is_leader:
+                try:
+                    acquired = self._lease.try_acquire()
+                except Exception:
+                    self._mark_tick_health_failure()
+                    raise
+                if acquired:
+                    self._is_leader = True
+                    logger.info("Reaper: acquired leadership")
+            if not self._is_leader:
+                # A completed lease read still proves the DB path is alive for a
+                # standby; leadership is surfaced separately in the payload.
+                self._dependency_health.succeed()
+                return TickOutcome(was_leader=False, reclaimed=0)
+
             try:
-                still_leader = self._lease.renew()
+                reclaimed = len(
+                    recover_expired_barriers(
+                        self._get_sweep_conn(),
+                        self._get_api_client(),
+                        self._stuck_timeout_seconds,
+                        metrics=self._metrics,
+                    )
+                )
             except Exception:
-                # A raised renew == "leadership unknown": stop acting (honour the
-                # lease's documented contract) before letting it propagate.
-                self._is_leader = False
-                self._step_down_metrics()
+                self._mark_tick_health_failure()
+                self._discard_owned_sweep_conn()
                 raise
-            if not still_leader:
-                logger.warning(
-                    "Reaper: lost leadership (lease taken over) — stepping down "
-                    "to standby"
-                )
-                self._is_leader = False
-                self._step_down_metrics()
-        if not self._is_leader and self._lease.try_acquire():
-            self._is_leader = True
-            logger.info("Reaper: acquired leadership")
-        if not self._is_leader:
-            return TickOutcome(was_leader=False, reclaimed=0)
-        try:
-            reclaimed = len(
-                recover_expired_barriers(
-                    self._get_sweep_conn(),
-                    self._get_api_client(),
-                    self._stuck_timeout_seconds,
-                    metrics=self._metrics,
-                )
-            )
-        except Exception:
-            self._discard_owned_sweep_conn()
-            raise
-        # Crash-redelivery: re-arm queue messages whose owning worker
-        # died (state='claimed', vt expired) back to 'ready'. Runs EVERY leader tick
-        # (the redelivery cadence), like barrier recovery above and NOT the
-        # retention sweep — a crashed batch must not wait the 5-min sweep interval.
-        # Cheap (partial claimed-index scoped). On failure it increments a DEDICATED
-        # counter (so a persistent redelivery outage is distinguishable from a
-        # barrier/scheduler fault) then re-raises + discards the conn — SAME
-        # semantics as barrier recovery above (recovery work is critical, not
-        # swallow-and-continue like the retention sweeps). A re-arm fault therefore
-        # also defers this tick's schedule dispatch; both recover next tick.
-        try:
-            rearmed = rearm_expired_claims(self._get_sweep_conn())
-            if rearmed:
-                self._metrics.queue_rearmed.inc(rearmed)
-                logger.info(
-                    "Reaper: re-armed %s expired in-flight queue message(s) "
-                    "to 'ready' (crashed-worker redelivery)",
-                    rearmed,
+
+            # Crash-redelivery: re-arm queue messages whose owning worker died.
+            try:
+                rearmed = rearm_expired_claims(self._get_sweep_conn())
+                if rearmed:
+                    self._metrics.queue_rearmed.inc(rearmed)
+                    logger.info(
+                        "Reaper: re-armed %s expired in-flight queue message(s) "
+                        "to 'ready' (crashed-worker redelivery)",
+                        rearmed,
+                    )
+            except Exception:
+                self._mark_tick_health_failure()
+                self._metrics.queue_rearm_failures.inc()
+                logger.exception(
+                    "Reaper: re-arm sweep failed — crashed-worker queue redelivery "
+                    "is stalled this tick (see pg_reaper_queue_rearm_failures_total)"
                 )
-        except Exception:
-            self._metrics.queue_rearm_failures.inc()
-            logger.exception(
-                "Reaper: re-arm sweep failed — crashed-worker queue redelivery "
-                "is stalled this tick (see pg_reaper_queue_rearm_failures_total)"
-            )
-            self._discard_owned_sweep_conn()
-            raise
-        # Delayed-visibility delivery (UN-3843): promote due 'scheduled' rows so
-        # consumers can claim them. Placed with the re-arm sweep because it shares
-        # its cadence requirement — this is the DELIVERY path for delayed messages,
-        # so a slower interval would directly add latency to every countdown/eta
-        # dispatch. Same failure posture as the re-arm above (dedicated counter,
-        # re-raise, discard the conn): a stalled promotion sweep means delayed
-        # messages silently never fire, which must not be swallowed.
-        try:
-            promoted = promote_due_scheduled(self._get_sweep_conn())
-            if promoted:
-                self._metrics.queue_promoted.inc(promoted)
-                logger.info(
-                    "Reaper: promoted %s due scheduled queue message(s) to 'ready' "
-                    "(delayed-visibility delivery)",
-                    promoted,
+                self._discard_owned_sweep_conn()
+                raise
+
+            # Delayed-visibility delivery: promote due scheduled rows.
+            try:
+                promoted = promote_due_scheduled(self._get_sweep_conn())
+                if promoted:
+                    self._metrics.queue_promoted.inc(promoted)
+                    logger.info(
+                        "Reaper: promoted %s due scheduled queue message(s) to "
+                        "'ready' (delayed-visibility delivery)",
+                        promoted,
+                    )
+            except Exception:
+                self._mark_tick_health_failure()
+                self._metrics.queue_promote_failures.inc()
+                logger.exception(
+                    "Reaper: promotion sweep failed — delayed messages will not "
+                    "become claimable this tick "
+                    "(see pg_reaper_queue_promote_failures_total)"
                 )
+                self._discard_owned_sweep_conn()
+                raise
+
+            # Fire due PG-owned schedules.
+            try:
+                dispatch_due_schedules(self._get_sweep_conn())
+            except Exception:
+                self._mark_tick_health_failure()
+                self._discard_owned_sweep_conn()
+                raise
+
+            # Fire non-pipeline periodics.
+            try:
+                dispatch_due_periodic_tasks(self._get_sweep_conn())
+            except Exception:
+                self._mark_tick_health_failure()
+                self._discard_owned_sweep_conn()
+                raise
+
+            # Retention and gauge sweeps are best-effort and intentionally do
+            # not gate the required lease/recovery readiness signal.
+            self._maybe_sweep()
+            self._maybe_refresh_gauges()
+            if self._tick_health_failed:
+                self._dependency_health.fail()
+            else:
+                self._dependency_health.succeed()
+            return TickOutcome(was_leader=True, reclaimed=reclaimed)
         except Exception:
-            self._metrics.queue_promote_failures.inc()
-            logger.exception(
-                "Reaper: promotion sweep failed — delayed (countdown/eta) queue "
-                "messages will not become claimable this tick "
-                "(see pg_reaper_queue_promote_failures_total)"
-            )
-            self._discard_owned_sweep_conn()
-            raise
-        # Orchestrator's second job: fire due PG-owned schedules (Beat
-        # replacement). Ordered AFTER recovery so this cycle's recovery has
-        # already completed before any scheduler error can propagate (the except
-        # below still re-raises + discards the conn). Dark by default — fires
-        # nothing until rows are pg_owned.
-        try:
-            dispatch_due_schedules(self._get_sweep_conn())
-        except Exception:
-            self._discard_owned_sweep_conn()
-            raise
-        # ...and the non-pipeline periodics (UN-3796): dashboard_metrics.*,
-        # log-history, audit, anything an operator adds. Separate call because each
-        # row carries its own task/args/queue rather than the pipeline trigger's one
-        # fixed shape; same leader gating, same dark-by-default posture (nothing
-        # fires until a row is pg_owned). Ordered after the pipeline dispatch so a
-        # fault here cannot stop pipelines, which are the customer-visible ones.
-        try:
-            dispatch_due_periodic_tasks(self._get_sweep_conn())
-        except Exception:
-            self._discard_owned_sweep_conn()
+            self._dependency_health.fail()
             raise
-        # Orchestrator's third job: retention cleanup (cadence-gated, so it does
-        # NOT run every tick). Last so a sweep error can't skip recovery/schedules.
-        self._maybe_sweep()
-        # Queue-wide metrics snapshot (cadence-gated, best-effort — a metrics
-        # failure must never fail the tick). After all real work.
-        self._maybe_refresh_gauges()
-        return TickOutcome(was_leader=True, reclaimed=reclaimed)
 
     def _maybe_sweep(self) -> None:
         """Run the retention sweep at most once per ``_sweep_interval``.
@@ -1621,12 +1643,15 @@ class ReaperLivenessServer(_BaseLivenessServer):
 
     def __init__(self, reaper: PgReaper, *, port: int, stale_after: float) -> None:
         super().__init__(
-            freshness_fn=reaper.seconds_since_last_tick,
+            freshness_fn=reaper.seconds_since_dependency_progress,
             stale_after=stale_after,
             port=port,
             check_name="pg_reaper_tick",
             age_key="seconds_since_last_tick",
-            extra_status_fn=lambda: {"is_leader": reaper.is_leader},
+            extra_status_fn=lambda: {
+                "is_leader": reaper.is_leader,
+                **reaper.dependency_health_status(),
+            },
             metrics_fn=reaper.metrics.render,
             thread_name="pg-reaper-liveness",
             log_label="pg-queue reaper",
diff --git a/workers/shared/infrastructure/monitoring/health.py b/workers/shared/infrastructure/monitoring/health.py
index bf7711b31a..ffd206b282 100644
--- a/workers/shared/infrastructure/monitoring/health.py
+++ b/workers/shared/infrastructure/monitoring/health.py
@@ -10,7 +10,7 @@
 from dataclasses import asdict, dataclass
 from datetime import UTC, datetime
 from enum import Enum
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import Any
 
 import psutil
@@ -491,6 +491,12 @@ def __init__(self, health_checker: HealthChecker, *args, **kwargs):
         self.health_checker = health_checker
         super().__init__(*args, **kwargs)
 
+    def setup(self) -> None:
+        super().setup()
+        # A client that opens a health connection and then stops sending must
+        # not consume the only server thread or keep shutdown waiting forever.
+        self.connection.settimeout(2.0)
+
     def do_GET(self):
         """Handle GET requests."""
         try:
@@ -536,11 +542,18 @@ def do_GET(self):
 
     def _send_json_response(self, data: dict[str, Any], status_code: int):
         """Send JSON response."""
-        self.send_response(status_code)
-        self.send_header("Content-Type", "application/json")
-        self.end_headers()
         response_data = json.dumps(data, default=str, indent=2)
-        self.wfile.write(response_data.encode("utf-8"))
+        body = response_data.encode("utf-8")
+        try:
+            self.send_response(status_code)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+        except (BrokenPipeError, ConnectionResetError, TimeoutError):
+            # Probe clients may disappear after a timeout. This is not a
+            # server failure and should not produce a secondary traceback.
+            pass
 
     def log_message(self, format, *args):
         """Override to suppress routine health check request logs."""
@@ -570,7 +583,9 @@ def start(self):
             def handler_factory(*args, **kwargs):
                 return HealthHTTPHandler(self.health_checker, *args, **kwargs)
 
-            self.server = HTTPServer(("0.0.0.0", self.port), handler_factory)
+            self.server = ThreadingHTTPServer(("0.0.0.0", self.port), handler_factory)
+            self.server.daemon_threads = True
+            self.server.block_on_close = False
 
             # Start server in background thread
             self.server_thread = threading.Thread(
diff --git a/workers/tests/test_log_stream_consumer.py b/workers/tests/test_log_stream_consumer.py
index 9856dd3249..d91fc62ae5 100644
--- a/workers/tests/test_log_stream_consumer.py
+++ b/workers/tests/test_log_stream_consumer.py
@@ -219,6 +219,7 @@ def test_is_stale_before_the_first_completed_read(self, consumer):
         assert health.seconds_since_last_success() > 100_000
         assert health.status() == {
             "queue": "log_stream_queue",
+            "redis_poll_ready": False,
             "redis_poll_failures": 0,
         }
 
@@ -227,6 +228,7 @@ def test_successful_empty_poll_is_a_real_readiness_signal(self, consumer):
         health.mark_success()
 
         assert health.seconds_since_last_success() < 1
+        assert health.status()["redis_poll_ready"] is True
 
     def test_failures_do_not_refresh_the_success_timestamp(self, consumer):
         health = consumer._RedisStreamHealth("log_stream_queue")
@@ -247,3 +249,19 @@ def test_health_port_is_opt_in_and_validated(self, consumer, monkeypatch):
         monkeypatch.setenv("LOG_STREAM_CONSUMER_HEALTH_PORT", "not-a-port")
         with pytest.raises(ValueError, match="LOG_STREAM_CONSUMER_HEALTH_PORT"):
             consumer._health_port_from_env()
+
+        monkeypatch.setenv("LOG_STREAM_CONSUMER_HEALTH_PORT", "0")
+        with pytest.raises(ValueError, match="between 1 and 65535"):
+            consumer._health_port_from_env()
+
+    @pytest.mark.parametrize("value", ["0", "-1", "not-a-timeout"])
+    def test_block_timeout_is_positive_and_named(self, consumer, monkeypatch, value):
+        monkeypatch.setenv("LOG_STREAM_BLOCK_TIMEOUT", value)
+        with pytest.raises(ValueError, match="LOG_STREAM_BLOCK_TIMEOUT"):
+            consumer._positive_int_env("LOG_STREAM_BLOCK_TIMEOUT", 5)
+
+    @pytest.mark.parametrize("value", ["nan", "inf", "-inf"])
+    def test_health_stale_bound_is_finite(self, consumer, monkeypatch, value):
+        monkeypatch.setenv("LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS", value)
+        with pytest.raises(ValueError, match="must be positive"):
+            consumer._health_stale_seconds()
diff --git a/workers/tests/test_pg_consumer_supervisor.py b/workers/tests/test_pg_consumer_supervisor.py
index aaee2ba447..7d12573ff2 100644
--- a/workers/tests/test_pg_consumer_supervisor.py
+++ b/workers/tests/test_pg_consumer_supervisor.py
@@ -170,8 +170,16 @@ def test_freshness_is_inf_when_crash_looping(self):
             f.schedule_restart(0, uptime=0.1)
         assert math.isinf(f.freshness())
 
+    def test_freshness_is_inf_until_each_child_reports_dependency_progress(self):
+        import math
+
+        f = _Fleet(2)
+        f._heartbeats[0] = time.time()
+        assert math.isinf(f.freshness())
+
     def test_freshness_is_oldest_age_when_healthy(self):
         f = _Fleet(2)
+        f._heartbeats[0] = time.time()  # slot 0 completed a successful read
         f._heartbeats[1] = time.time() - 100
         assert 99 < f.freshness() < 102
 
diff --git a/workers/tests/test_pg_queue_consumer.py b/workers/tests/test_pg_queue_consumer.py
index 3e5d836dce..94c8d4c70a 100644
--- a/workers/tests/test_pg_queue_consumer.py
+++ b/workers/tests/test_pg_queue_consumer.py
@@ -697,7 +697,10 @@ def test_liveness_server_reports_200_then_503(self):
 
         from queue_backend.pg_queue.consumer import LivenessServer
 
-        consumer = PgQueueConsumer(["q"], client=MagicMock())
+        client = MagicMock()
+        client.read.return_value = []
+        consumer = PgQueueConsumer(["q"], client=client)
+        consumer.poll_once()  # readiness requires one completed PG read
         server = LivenessServer(consumer, port=0, stale_after=60)
         server.start()
         try:
@@ -707,6 +710,7 @@ def test_liveness_server_reports_200_then_503(self):
                 assert json.loads(resp.read())["status"] == "healthy"
 
             consumer._last_poll_monotonic -= 120  # force the loop stale
+            consumer._dependency_health._last_success -= 120  # force PG age stale
             with pytest.raises(urllib.error.HTTPError) as ei:
                 urllib.request.urlopen(url, timeout=5)
             assert ei.value.code == 503
@@ -723,7 +727,10 @@ def test_liveness_aliases_and_unknown_path(self):
 
         from queue_backend.pg_queue.consumer import LivenessServer
 
-        consumer = PgQueueConsumer(["q"], client=MagicMock())
+        client = MagicMock()
+        client.read.return_value = []
+        consumer = PgQueueConsumer(["q"], client=client)
+        consumer.poll_once()  # readiness requires one completed PG read
         server = LivenessServer(consumer, port=0, stale_after=60)
         server.start()
         try:
diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py
index add6d8c5ce..e05cd66fa2 100644
--- a/workers/tests/test_pg_reaper.py
+++ b/workers/tests/test_pg_reaper.py
@@ -1185,6 +1185,7 @@ def test_fresh_returns_200(self):
         reaper = PgReaper(
             _FakeLease(acquires=False), interval_seconds=0.01, sweep_conn=object()
         )
+        reaper.tick()  # readiness requires one completed lease operation
         server = self._server(reaper)
         try:
             status, body = _http_get(server)
diff --git a/workers/tests/test_scheduler_health.py b/workers/tests/test_scheduler_health.py
index 4d2c61ac60..cc0c6552dc 100644
--- a/workers/tests/test_scheduler_health.py
+++ b/workers/tests/test_scheduler_health.py
@@ -2,7 +2,12 @@
 
 from __future__ import annotations
 
+import math
+
+import pytest
+
 from log_consumer.scheduler_health import evaluate_state
+from log_consumer.scheduler_health import _positive_float, _positive_port
 
 
 def _state(**overrides):
@@ -82,3 +87,55 @@ def test_dead_parent_is_not_healthy_even_with_fresh_state():
 
     assert result.status == "unhealthy"
     assert result.message == "scheduler loop is not running"
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("last_log_success", math.nan),
+        ("last_log_success", math.inf),
+        ("last_buffer_failure", math.nan),
+        ("last_buffer_failure", math.inf),
+    ],
+)
+def test_nonfinite_state_timestamps_are_unhealthy(field, value):
+    result = evaluate_state(
+        _state(**{field: value}),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "unhealthy"
+    assert "timestamp is invalid" in result.message
+
+
+@pytest.mark.parametrize(
+    "field,message",
+    [
+        ("last_log_success", "success timestamp is in the future"),
+        ("last_buffer_failure", "failure timestamp is in the future"),
+    ],
+)
+def test_future_state_timestamps_are_unhealthy(field, message):
+    result = evaluate_state(
+        _state(**{field: 101.0}),
+        now=100.0,
+        stale_after=120.0,
+        parent_alive=_alive,
+    )
+
+    assert result.status == "unhealthy"
+    assert message in result.message
+
+
+@pytest.mark.parametrize("value", [0, -1, 65536])
+def test_scheduler_health_port_must_be_concrete(value):
+    with pytest.raises(ValueError, match="between 1 and 65535"):
+        _positive_port(str(value), "PORT")
+
+
+@pytest.mark.parametrize("value", ["nan", "inf", "0", "-1"])
+def test_scheduler_stale_bound_must_be_finite_and_positive(value):
+    with pytest.raises(ValueError, match="must be positive"):
+        _positive_float(value, "STALE")
diff --git a/workers/worker.py b/workers/worker.py
index 29afd67afd..80d55d443b 100755
--- a/workers/worker.py
+++ b/workers/worker.py
@@ -7,6 +7,7 @@
 
 import importlib.util
 import logging
+import math
 import os
 import sys
 import threading
@@ -545,7 +546,7 @@ def _worker_health_stale_seconds() -> float:
         value = float(raw)
     except ValueError as exc:
         raise ValueError(f"WORKER_HEALTH_STALE_SECONDS={raw!r} is not numeric") from exc
-    if value <= 0:
+    if not math.isfinite(value) or value <= 0:
         raise ValueError(f"WORKER_HEALTH_STALE_SECONDS={value} must be positive")
     return value
 

From bf0193bbd86c88feb30a867ed84cc36e880936e1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 07:22:41 -0400
Subject: [PATCH 19/48] Reset replacement worker heartbeats

---
 workers/pg_queue_consumer/supervisor.py      | 15 +++++++++----
 workers/queue_backend/pg_queue/metrics.py    |  6 ++++--
 workers/queue_backend/pg_queue/reaper.py     | 18 +++++++++++-----
 workers/tests/test_pg_consumer_supervisor.py | 22 +++++++++++++-------
 workers/tests/test_pg_metrics.py             |  4 ++--
 workers/tests/test_pg_reaper.py              |  2 ++
 6 files changed, 47 insertions(+), 20 deletions(-)

diff --git a/workers/pg_queue_consumer/supervisor.py b/workers/pg_queue_consumer/supervisor.py
index d2759ce140..dd9b157cba 100644
--- a/workers/pg_queue_consumer/supervisor.py
+++ b/workers/pg_queue_consumer/supervisor.py
@@ -173,18 +173,25 @@ def _validate(self, slot: int) -> None:
             raise IndexError(f"slot {slot} out of range [0, {self._n})")
 
     def record_fork(self, slot: int, pid: int) -> None:
-        """Mark ``slot`` alive under ``pid``; clears any pending restart. Note the
-        heartbeat is deliberately NOT reseeded here — a re-forked child must earn
-        freshness by actually polling, so a crash-looping slot ages instead of
-        looking perpetually fresh.
+        """Mark ``slot`` alive under ``pid``; clears any pending restart.
+
+        A replacement child must earn readiness with its own completed
+        dependency read. Reset the shared timestamp here because a child can be
+        forked after a prior child left a fresh sample in the same slot.
         """
         self._validate(slot)
         self._pids[slot] = pid
         self._last_fork[slot] = time.monotonic()
+        self._heartbeats[slot] = 0.0
         self._restart_due.pop(slot, None)
 
     def reap(self, slot: int) -> float:
         """Drop the slot's pid + last-fork together; return the child's uptime (s)."""
+        self._validate(slot)
+        # Keep the slot stale during the gap between reaping the old process and
+        # recording its replacement. This also prevents a failed fork from
+        # inheriting the old child's dependency-ready timestamp.
+        self._heartbeats[slot] = 0.0
         forked_at = self._last_fork.pop(slot, time.monotonic())
         self._pids.pop(slot, None)
         return time.monotonic() - forked_at
diff --git a/workers/queue_backend/pg_queue/metrics.py b/workers/queue_backend/pg_queue/metrics.py
index d34c7d885d..4ce0c6260e 100644
--- a/workers/queue_backend/pg_queue/metrics.py
+++ b/workers/queue_backend/pg_queue/metrics.py
@@ -219,7 +219,8 @@ def __init__(
         super().__init__()
         self._function_gauge(
             "pg_reaper_heartbeat_age_seconds",
-            "Seconds since the reaper tick loop last ran (liveness heartbeat)",
+            "Seconds since the reaper last completed required lease/recovery "
+            "work (liveness heartbeat)",
             heartbeat_fn,
         )
         self._function_gauge(
@@ -306,7 +307,8 @@ def __init__(
         self.tick_failures = Counter(
             "pg_reaper_tick_failures_total",
             "Reaper cycles that raised (recovery/scheduler SELECT failures — the "
-            "heartbeat stays fresh through these, so alert on this counter)",
+            "dependency heartbeat becomes stale after repeated failures; alert on "
+            "this counter too)",
             registry=self.registry,
         )
         self.gauge_refresh_failures = Counter(
diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py
index 8d5fe82c35..ac143e21c8 100644
--- a/workers/queue_backend/pg_queue/reaper.py
+++ b/workers/queue_backend/pg_queue/reaper.py
@@ -1157,7 +1157,11 @@ def __init__(
         # sweep gate: first leader tick refreshes immediately).
         self._last_gauge_refresh_monotonic: float | None = None
         self._metrics = ReaperMetrics(
-            heartbeat_fn=self.seconds_since_last_tick,
+            # Export the same dependency-aware age used by /health. The loop
+            # start timestamp remains useful for diagnostics, but publishing it
+            # here would let /metrics report fresh while /health is stale after
+            # repeated lease/DB failures.
+            heartbeat_fn=self.seconds_since_dependency_progress,
             is_leader_fn=lambda: self._is_leader,
         )
 
@@ -1636,9 +1640,10 @@ def _install_signal_handlers(self) -> None:
 class ReaperLivenessServer(_BaseLivenessServer):
     """Reaper tick-loop liveness — a thin wrapper over the shared
     :class:`queue_backend.pg_queue.liveness.LivenessServer`, bound to the reaper's
-    heartbeat (``seconds_since_last_tick``) and surfacing ``is_leader`` (which pod
-    holds the lease — informational; the 200/503 verdict is purely the heartbeat,
-    so a standby is healthy).
+    dependency heartbeat (``seconds_since_dependency_progress``) and surfacing
+    ``is_leader`` (which pod holds the lease — informational; the 200/503 verdict
+    is based on completed lease/recovery progress, so a standby is healthy after a
+    successful lease operation).
     """
 
     def __init__(self, reaper: PgReaper, *, port: int, stale_after: float) -> None:
@@ -1647,9 +1652,12 @@ def __init__(self, reaper: PgReaper, *, port: int, stale_after: float) -> None:
             stale_after=stale_after,
             port=port,
             check_name="pg_reaper_tick",
-            age_key="seconds_since_last_tick",
+            age_key="seconds_since_dependency_progress",
             extra_status_fn=lambda: {
                 "is_leader": reaper.is_leader,
+                "seconds_since_last_tick": round(
+                    reaper.seconds_since_last_tick(), 3
+                ),
                 **reaper.dependency_health_status(),
             },
             metrics_fn=reaper.metrics.render,
diff --git a/workers/tests/test_pg_consumer_supervisor.py b/workers/tests/test_pg_consumer_supervisor.py
index 7d12573ff2..7b11bb2172 100644
--- a/workers/tests/test_pg_consumer_supervisor.py
+++ b/workers/tests/test_pg_consumer_supervisor.py
@@ -7,6 +7,7 @@
 """
 
 import errno
+import math
 import threading
 import time
 from unittest.mock import MagicMock, patch
@@ -141,12 +142,23 @@ def test_slot_out_of_range_raises(self):
             f.record_fork(5, 111)
 
     def test_record_fork_does_not_reseed_heartbeat(self):
-        # The crash-loop fix: a re-fork must NOT refresh the slot, or a child that
-        # never polls looks perpetually fresh.
+        # A replacement child must earn readiness with its own dependency read;
+        # retaining the prior child's timestamp would report a crash-looping
+        # replacement as healthy.
         f = _Fleet(1)
         f._heartbeats[0] = time.time() - 500  # an aged slot
         f.record_fork(0, 111)
-        assert f.oldest_age() > 400  # still aged, not reset to ~0
+        assert f._heartbeats[0] == 0.0
+        assert math.isinf(f.freshness())
+
+    def test_reap_clears_heartbeat_before_replacement(self):
+        f = _Fleet(1)
+        f._heartbeats[0] = time.time()
+        f.record_fork(0, 111)
+        f._heartbeats[0] = time.time()
+        f.reap(0)
+        assert f._heartbeats[0] == 0.0
+        assert math.isinf(f.freshness())
 
     def test_immediate_crash_increments_then_loops(self):
         f = _Fleet(1)
@@ -163,16 +175,12 @@ def test_healthy_uptime_resets_crash_counter(self):
         assert n == 0 and f.is_crash_looping() is False
 
     def test_freshness_is_inf_when_crash_looping(self):
-        import math
-
         f = _Fleet(1)
         for _ in range(_CRASH_LOOP_THRESHOLD):
             f.schedule_restart(0, uptime=0.1)
         assert math.isinf(f.freshness())
 
     def test_freshness_is_inf_until_each_child_reports_dependency_progress(self):
-        import math
-
         f = _Fleet(2)
         f._heartbeats[0] = time.time()
         assert math.isinf(f.freshness())
diff --git a/workers/tests/test_pg_metrics.py b/workers/tests/test_pg_metrics.py
index e21bc2b67f..52b54df2c2 100644
--- a/workers/tests/test_pg_metrics.py
+++ b/workers/tests/test_pg_metrics.py
@@ -535,8 +535,8 @@ def test_sweep_failure_increments_labeled_counter(self, monkeypatch):
         ) == pytest.approx(1.0)
 
     def test_run_counts_tick_failures(self):
-        # The heartbeat is stamped at tick START, so /health stays 200 through
-        # every-tick failures — this counter is the only machine-readable signal.
+        # Tick failures remain counted separately from the dependency-aware
+        # heartbeat, which now makes both /health and /metrics stale.
         reaper = self._reaper(_FakeLease(acquires=True))
         with patch.object(
             reaper_mod,
diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py
index e05cd66fa2..67a7c6fe18 100644
--- a/workers/tests/test_pg_reaper.py
+++ b/workers/tests/test_pg_reaper.py
@@ -1195,6 +1195,8 @@ def test_fresh_returns_200(self):
         assert body["status"] == "healthy"
         assert body["check"] == "pg_reaper_tick"
         assert body["is_leader"] is False
+        assert "seconds_since_dependency_progress" in body
+        assert "seconds_since_last_tick" in body
 
     def test_stale_returns_503(self):
         reaper = PgReaper(_FakeLease(), interval_seconds=0.01, sweep_conn=object())

From 61fcb4fa7fb69f424199494d0ce10444b8a4fe3d Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 07:49:58 -0400
Subject: [PATCH 20/48] Reject non-finite reaper stale windows

---
 workers/queue_backend/pg_queue/reaper.py | 6 ++++--
 workers/tests/test_pg_reaper.py          | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py
index ac143e21c8..7d57eaf493 100644
--- a/workers/queue_backend/pg_queue/reaper.py
+++ b/workers/queue_backend/pg_queue/reaper.py
@@ -56,6 +56,7 @@
 
 import contextlib
 import logging
+import math
 import os
 import signal
 import threading
@@ -1676,9 +1677,10 @@ def _reaper_health_stale_from_env() -> float:
         raise ValueError(
             f"WORKER_PG_REAPER_HEALTH_STALE_SECONDS={raw!r} is not a number."
         ) from exc
-    if value <= 0:
+    if not math.isfinite(value) or value <= 0:
         raise ValueError(
-            f"WORKER_PG_REAPER_HEALTH_STALE_SECONDS={value} must be positive."
+            "WORKER_PG_REAPER_HEALTH_STALE_SECONDS="
+            f"{value} must be finite and positive."
         )
     return value
 
diff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.py
index 67a7c6fe18..ac62caec25 100644
--- a/workers/tests/test_pg_reaper.py
+++ b/workers/tests/test_pg_reaper.py
@@ -1275,7 +1275,7 @@ def test_stale_overridable(self, monkeypatch):
         monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS", "10")
         assert reaper_mod._reaper_health_stale_from_env() == pytest.approx(10.0)
 
-    @pytest.mark.parametrize("bad", ["0", "-1", "x"])
+    @pytest.mark.parametrize("bad", ["0", "-1", "x", "nan", "inf", "-inf"])
     def test_stale_invalid_raises(self, monkeypatch, bad):
         monkeypatch.setenv("WORKER_PG_REAPER_HEALTH_STALE_SECONDS", bad)
         with pytest.raises(ValueError):

From dca5a81ade3477eada687ac7984032215a33e8d1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 07:11:29 -0400
Subject: [PATCH 21/48] Add Compose healthchecks for worker services

---
 docker/docker-compose.yaml | 95 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 95 insertions(+)

diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 9f6ca7789d..18a55c0239 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -160,6 +160,13 @@ services:
       - ./workflow_data:/data
       # Docker socket bind mount to spawn tool containers
       - /var/run/docker.sock:/var/run/docker.sock
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:5002/v1/api/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     depends_on:
       - redis
       - rabbitmq
@@ -186,8 +193,17 @@ services:
       - LOG_TRANSPORT=redis
       # Scheduler interval in seconds
       - LOG_HISTORY_CONSUMER_INTERVAL=${LOG_HISTORY_CONSUMER_INTERVAL:-5}
+      - LOG_HISTORY_SCHEDULER_HEALTH_PORT=8092
+      - LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS=${LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS:-120}
       # Override example: TASK_TRIGGER_COMMAND=/custom/trigger/script.sh
       - TASK_TRIGGER_COMMAND=${TASK_TRIGGER_COMMAND:-}
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8092/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     labels:
       - traefik.enable=false
 
@@ -204,6 +220,13 @@ services:
     depends_on:
       - db
       - redis
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-orchestrator-api
@@ -240,6 +263,13 @@ services:
     depends_on:
       - db
       - redis
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-orchestrator-general
@@ -274,6 +304,13 @@ services:
       - db
       - redis
       - rabbitmq   # still dispatches the tool-execution RPC to the Celery executor (until the executor/tool-RPC moves to PG)
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-fileproc
@@ -315,6 +352,13 @@ services:
       - db
       - redis
       - rabbitmq   # may dispatch Celery notifications when not PG-routed (until notifications move to PG)
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-callback
@@ -354,6 +398,13 @@ services:
       - db
       - redis
       - rabbitmq   # the trigger dispatches async_execute_bin (Celery path when not PG-routed)
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-scheduler
@@ -407,6 +458,13 @@ services:
     depends_on:
       - db
       - redis
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-metrics
@@ -459,6 +517,15 @@ services:
       # Downstream sinks are unchanged — same durable buffer the Celery worker wrote to.
       - LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}
       - ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}
+      - LOG_STREAM_CONSUMER_HEALTH_PORT=8091
+      - LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS=${LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS:-15}
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8091/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     labels:
       - traefik.enable=false
 
@@ -482,6 +549,13 @@ services:
       - db
       - redis
       - platform-service
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-executor
@@ -534,6 +608,13 @@ services:
       - db
       - redis
       - platform-service
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-ide-callback
@@ -575,6 +656,13 @@ services:
       - db
       - redis
       - platform-service
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-notification
@@ -614,6 +702,13 @@ services:
     depends_on:
       - db
       - redis
+    healthcheck:
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8086/health"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
     environment:
       - ENVIRONMENT=development
       - APPLICATION_NAME=unstract-worker-pg-reaper

From 5046fd603d15169bdbda0feceac738a8ad786ce0 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:09:34 -0400
Subject: [PATCH 22/48] Add Train core service health overlay

---
 docker/compose.train.healthchecks.yaml       | 98 ++++++++++++++++++++
 docker/healthchecks/README.md                |  9 +-
 docker/healthchecks/unstract-services.sh     |  0
 tests/healthchecks/test_unstract_services.py | 36 +++++++
 4 files changed, 139 insertions(+), 4 deletions(-)
 create mode 100644 docker/compose.train.healthchecks.yaml
 mode change 100644 => 100755 docker/healthchecks/unstract-services.sh

diff --git a/docker/compose.train.healthchecks.yaml b/docker/compose.train.healthchecks.yaml
new file mode 100644
index 0000000000..9d9bf5e67d
--- /dev/null
+++ b/docker/compose.train.healthchecks.yaml
@@ -0,0 +1,98 @@
+# Train health coverage overlay.
+#
+# Apply this file after docker/compose.train.yaml:
+#
+#   docker compose -f docker/docker-compose.yaml \
+#     -f docker/compose.train.yaml \
+#     -f docker/compose.train.healthchecks.yaml up -d
+#
+# The overlay only adds a read-only probe mount and a bounded application
+# healthcheck.  It does not replace ports, volumes, images, environment, or
+# container names from the owner Compose files.  The existing worker and runner
+# checks remain defined in docker/docker-compose.yaml.
+
+x-unstract-service-probe: &unstract_service_probe
+  - ./healthchecks/unstract-services.sh:/usr/local/bin/unstract-services.sh:ro
+
+x-unstract-healthcheck: &unstract_healthcheck
+  interval: 30s
+  timeout: 10s
+  retries: 3
+
+services:
+  db:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 30s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "db"]
+
+  redis:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 15s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "redis"]
+
+  minio:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 60s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "minio"]
+
+  reverse-proxy:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 60s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "proxy"]
+
+  qdrant:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 60s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "vector-db"]
+
+  rabbitmq:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 60s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "rabbitmq"]
+
+  weaviate:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 120s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "weaviate"]
+
+  x2text-service:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 120s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "x2text-service"]
+
+  platform-service:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 120s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "platform-service"]
+
+  backend:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 180s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "backend"]
+
+  frontend:
+    volumes: *unstract_service_probe
+    healthcheck:
+      <<: *unstract_healthcheck
+      start_period: 60s
+      test: ["CMD", "/usr/local/bin/unstract-services.sh", "frontend"]
diff --git a/docker/healthchecks/README.md b/docker/healthchecks/README.md
index 523ee5bf10..a6b796d4fd 100644
--- a/docker/healthchecks/README.md
+++ b/docker/healthchecks/README.md
@@ -24,7 +24,8 @@ healthcheck:
   retries: 3
 ```
 
-The exact service contracts and a prepared Train fragment are kept in the
-private integration evidence bundle for the deployment owner.  The script's
-endpoint and command paths can be overridden with environment variables for
-isolated contract tests; production defaults target service-local listeners.
+The tracked `docker/compose.train.healthchecks.yaml` overlay mounts this probe
+read-only and supplies the eleven core service checks.  Apply it after the
+Train-only `docker/compose.train.yaml` file.  The script's endpoint and command
+paths can be overridden with environment variables for isolated contract
+tests; production defaults target service-local listeners.
diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
old mode 100644
new mode 100755
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index 107cfb14cf..73a5bc31f3 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import pytest
+import yaml
 
 ROOT = Path(__file__).parents[2]
 SCRIPT = ROOT / "docker" / "healthchecks" / "unstract-services.sh"
@@ -518,3 +519,38 @@ def test_python_probe_has_outer_deadline(tmp_path: Path) -> None:
     )
     assert result.returncode != 0
     assert time.monotonic() - started < 4
+
+
+def test_train_overlay_mounts_read_only_probe_and_sets_core_checks() -> None:
+    overlay_path = ROOT / "docker" / "compose.train.healthchecks.yaml"
+    overlay = yaml.safe_load(overlay_path.read_text(encoding="utf-8"))
+    expected = {
+        "db": "db",
+        "redis": "redis",
+        "minio": "minio",
+        "reverse-proxy": "proxy",
+        "qdrant": "vector-db",
+        "rabbitmq": "rabbitmq",
+        "weaviate": "weaviate",
+        "x2text-service": "x2text-service",
+        "platform-service": "platform-service",
+        "backend": "backend",
+        "frontend": "frontend",
+    }
+
+    assert set(overlay["services"]) == set(expected)
+    for service, probe_name in expected.items():
+        config = overlay["services"][service]
+        assert config["volumes"] == [
+            "./healthchecks/unstract-services.sh:/usr/local/bin/unstract-services.sh:ro"
+        ]
+        healthcheck = config["healthcheck"]
+        assert healthcheck["test"] == [
+            "CMD",
+            "/usr/local/bin/unstract-services.sh",
+            probe_name,
+        ]
+        assert healthcheck["interval"] == "30s"
+        assert healthcheck["timeout"] == "10s"
+        assert healthcheck["retries"] == 3
+        assert isinstance(healthcheck["start_period"], str)

From ca6074e15f20dada579e825c606a4455f37c8634 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:06:23 -0400
Subject: [PATCH 23/48] Add guarded Train health deployment artifacts

---
 docker/compose.train.healthchecks.yaml        |    6 +-
 docker/docker-compose-dev-essentials.yaml     |   11 +-
 docker/healthchecks/README.md                 |   11 +-
 docker/healthchecks/http-readiness.sh         |  119 ++
 docker/healthchecks/postgres-readiness.sh     |  152 ++
 .../scripts/train_health_deployment_guard.py  | 1856 +++++++++++++++++
 docs/train-unstract-health-deployment.md      |  205 ++
 tests/healthchecks/test_unstract_services.py  |   78 +-
 8 files changed, 2426 insertions(+), 12 deletions(-)
 create mode 100644 docker/healthchecks/http-readiness.sh
 create mode 100644 docker/healthchecks/postgres-readiness.sh
 create mode 100644 docker/scripts/train_health_deployment_guard.py
 create mode 100644 docs/train-unstract-health-deployment.md

diff --git a/docker/compose.train.healthchecks.yaml b/docker/compose.train.healthchecks.yaml
index 9d9bf5e67d..17f0b00e39 100644
--- a/docker/compose.train.healthchecks.yaml
+++ b/docker/compose.train.healthchecks.yaml
@@ -8,11 +8,11 @@
 #
 # The overlay only adds a read-only probe mount and a bounded application
 # healthcheck.  It does not replace ports, volumes, images, environment, or
-# container names from the owner Compose files.  The existing worker and runner
-# checks remain defined in docker/docker-compose.yaml.
+# container names from the owner Compose files.  Worker and runner checks are
+# staged separately in docker/compose.train.worker-healthchecks.yaml.
 
 x-unstract-service-probe: &unstract_service_probe
-  - ./healthchecks/unstract-services.sh:/usr/local/bin/unstract-services.sh:ro
+  - ${UNSTRACT_HEALTHCHECK_SOURCE:-./healthchecks/unstract-services.sh}:/usr/local/bin/unstract-services.sh:ro
 
 x-unstract-healthcheck: &unstract_healthcheck
   interval: 30s
diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index 8e8937c93c..53438f7b6e 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -149,14 +149,15 @@ services:
       - postgres_vector_ssl:/var/lib/postgresql/ssl/
       - ./scripts/db-setup/vector_db_setup.sh:/docker-entrypoint-initdb.d/vector_db_setup.sh:ro
       - ./scripts/db-setup/postgres-vector-entrypoint.sh:/usr/local/bin/postgres-vector-entrypoint.sh:ro
+      - ./healthchecks/postgres-readiness.sh:/usr/local/bin/postgres-readiness.sh:ro
     env_file:
       # Reuse the local platform database credentials from essentials.env while
       # retaining a separate data directory and container.
       - ./essentials.env
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      test: ["CMD", "/usr/local/bin/postgres-readiness.sh"]
       interval: 10s
-      timeout: 5s
+      timeout: 6s
       retries: 5
     labels:
       - traefik.enable=false
@@ -227,9 +228,10 @@ services:
       MINIO_SECRET_KEY: minioadmin
     volumes:
       - milvus_minio_data:/minio_data
+      - ./healthchecks/http-readiness.sh:/usr/local/bin/http-readiness.sh:ro
     command: minio server /minio_data --console-address ":9001"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      test: ["CMD", "/usr/local/bin/http-readiness.sh", "http://127.0.0.1:9000/minio/health/ready"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -252,8 +254,9 @@ services:
       MINIO_SECRET_KEY: minioadmin
     volumes:
       - milvus_data:/var/lib/milvus
+      - ./healthchecks/http-readiness.sh:/usr/local/bin/http-readiness.sh:ro
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
+      test: ["CMD", "/usr/local/bin/http-readiness.sh", "http://127.0.0.1:9091/healthz"]
       interval: 30s
       start_period: 90s
       timeout: 20s
diff --git a/docker/healthchecks/README.md b/docker/healthchecks/README.md
index a6b796d4fd..6780394192 100644
--- a/docker/healthchecks/README.md
+++ b/docker/healthchecks/README.md
@@ -25,7 +25,10 @@ healthcheck:
 ```
 
 The tracked `docker/compose.train.healthchecks.yaml` overlay mounts this probe
-read-only and supplies the eleven core service checks.  Apply it after the
-Train-only `docker/compose.train.yaml` file.  The script's endpoint and command
-paths can be overridden with environment variables for isolated contract
-tests; production defaults target service-local listeners.
+read-only and supplies the eleven core service checks.  The companion
+`docker/compose.train.worker-healthchecks.yaml` overlay supplies the runner and
+twelve worker checks while preserving the Train checkout's local Compose
+changes.  Apply both after the Train-only `docker/compose.train.yaml` file.  The
+script's endpoint and command paths can be overridden with environment
+variables for isolated contract tests; production defaults target service-local
+listeners.
diff --git a/docker/healthchecks/http-readiness.sh b/docker/healthchecks/http-readiness.sh
new file mode 100644
index 0000000000..a9d6ddcab8
--- /dev/null
+++ b/docker/healthchecks/http-readiness.sh
@@ -0,0 +1,119 @@
+#!/bin/sh
+# Bounded, body-suppressing HTTP readiness for the existing Milvus/MinIO rows.
+# The URL is a fixed Compose healthcheck argument, never a credential source.
+set -eu
+
+url=${1:?health URL is required}
+
+probe_timeout=3s
+probe_max_output_bytes=4096
+probe_tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/http-readiness.XXXXXX") || exit 1
+
+runner_pid=
+reader_pid=
+
+# Stop the FIFO reader and curl together. The reader is separate from the
+# timeout/curl group, which may contain descendants that ignore TERM. Send
+# TERM first, then KILL after a bounded grace period, and reap every process.
+cleanup_processes() {
+  active=0
+  if [ -n "${reader_pid:-}" ]; then
+    kill -TERM "$reader_pid" 2>/dev/null || true
+    active=1
+  fi
+  if [ -n "${runner_pid:-}" ]; then
+    kill -TERM -"$runner_pid" 2>/dev/null || true
+    active=1
+  fi
+  if [ "$active" -eq 1 ]; then
+    sleep 1
+    if [ -n "${reader_pid:-}" ]; then
+      kill -KILL "$reader_pid" 2>/dev/null || true
+    fi
+    if [ -n "${runner_pid:-}" ]; then
+      kill -KILL -"$runner_pid" 2>/dev/null || true
+    fi
+    if [ -n "${reader_pid:-}" ]; then
+      wait "$reader_pid" 2>/dev/null || true
+      reader_pid=
+    fi
+    if [ -n "${runner_pid:-}" ]; then
+      wait "$runner_pid" 2>/dev/null || true
+      runner_pid=
+    fi
+  fi
+}
+
+# curl may exit successfully while a descendant still has the FIFO open.
+# Close the whole timeout/curl group before waiting for the reader so a direct
+# success cannot leak the reader indefinitely.
+stop_runner_group() {
+  pid=$1
+  [ -n "$pid" ] || return 0
+  kill -TERM -"$pid" 2>/dev/null || true
+  kill -KILL -"$pid" 2>/dev/null || true
+}
+
+wait_reader_bounded() {
+  polls=0
+  while kill -0 "$reader_pid" 2>/dev/null; do
+    if [ "$polls" -ge 5 ]; then
+      kill -TERM "$reader_pid" 2>/dev/null || true
+      kill -KILL "$reader_pid" 2>/dev/null || true
+      break
+    fi
+    sleep 0.05
+    polls=$((polls + 1))
+  done
+  reader_status=0
+  wait "$reader_pid" 2>/dev/null || reader_status=$?
+  reader_pid=
+}
+
+cleanup_exit() {
+  status=$?
+  trap - EXIT
+  trap ':' HUP INT TERM
+  cleanup_processes
+  rm -rf "$probe_tmp_dir"
+  exit "$status"
+}
+
+cleanup_signal() {
+  status=$1
+  trap - EXIT
+  trap ':' HUP INT TERM
+  cleanup_processes
+  rm -rf "$probe_tmp_dir"
+  exit "$status"
+}
+
+trap cleanup_exit EXIT
+trap 'cleanup_signal 129' HUP
+trap 'cleanup_signal 130' INT
+trap 'cleanup_signal 143' TERM
+
+fifo="$probe_tmp_dir/body.fifo"
+mkfifo "$fifo" || exit 1
+timeout --signal=TERM --kill-after=1s "$probe_timeout" curl \
+  --fail --silent --show-error --max-time 3 "$url" \
+  >"$fifo" 2>/dev/null &
+runner_pid=$!
+dd if="$fifo" bs=1 count=$((probe_max_output_bytes + 1)) \
+  >"$probe_tmp_dir/body" 2>/dev/null &
+reader_pid=$!
+status=0
+wait "$runner_pid" 2>/dev/null || status=$?
+stop_runner_group "$runner_pid"
+runner_pid=
+wait_reader_bounded
+rm -f "$fifo"
+[ "$status" -eq 0 ] || exit 1
+[ "$reader_status" -eq 0 ] || exit 1
+output_bytes=$(wc -c <"$probe_tmp_dir/body") || exit 1
+case "$output_bytes" in
+  ''|*[!0-9]*) exit 1 ;;
+esac
+[ "$output_bytes" -le "$probe_max_output_bytes" ] || exit 1
+
+exit 0
diff --git a/docker/healthchecks/postgres-readiness.sh b/docker/healthchecks/postgres-readiness.sh
new file mode 100644
index 0000000000..c9469c16af
--- /dev/null
+++ b/docker/healthchecks/postgres-readiness.sh
@@ -0,0 +1,152 @@
+#!/bin/sh
+# Native PostgreSQL liveness plus an authenticated, read-only SELECT 1.
+# Credentials stay in the container environment and are never printed.
+set -eu
+
+: "${POSTGRES_USER:?POSTGRES_USER is required}"
+: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}"
+: "${POSTGRES_DB:?POSTGRES_DB is required}"
+
+host=${PGHOST:-127.0.0.1}
+port=${PGPORT:-5432}
+case "$port" in
+  ''|*[!0-9]*) exit 1 ;;
+esac
+
+# Keep each native client bounded even when the healthcheck runner itself is
+# misconfigured. GNU timeout starts the client in its own process group; its
+# TERM/KILL sequence therefore also cleans up descendants. A bounded FIFO
+# reader prevents an unhealthy client from making command substitution retain
+# unbounded output in memory.
+probe_timeout=3s
+probe_max_output_bytes=4096
+probe_tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/postgres-readiness.XXXXXX") || exit 1
+
+runner_pid=
+reader_pid=
+
+# Stop the FIFO reader and the native client together. The reader is separate
+# from the client group, while the timeout/client group may contain children
+# that ignore TERM. Send TERM first, then KILL after a bounded grace period,
+# and reap every process before returning the probe's failure status.
+cleanup_processes() {
+  active=0
+  if [ -n "${reader_pid:-}" ]; then
+    kill -TERM "$reader_pid" 2>/dev/null || true
+    active=1
+  fi
+  if [ -n "${runner_pid:-}" ]; then
+    kill -TERM -"$runner_pid" 2>/dev/null || true
+    active=1
+  fi
+  if [ "$active" -eq 1 ]; then
+    sleep 1
+    if [ -n "${reader_pid:-}" ]; then
+      kill -KILL "$reader_pid" 2>/dev/null || true
+    fi
+    if [ -n "${runner_pid:-}" ]; then
+      kill -KILL -"$runner_pid" 2>/dev/null || true
+    fi
+    if [ -n "${reader_pid:-}" ]; then
+      wait "$reader_pid" 2>/dev/null || true
+      reader_pid=
+    fi
+    if [ -n "${runner_pid:-}" ]; then
+      wait "$runner_pid" 2>/dev/null || true
+      runner_pid=
+    fi
+  fi
+}
+
+# A client can exit successfully while a descendant still has the FIFO open.
+# Once the timeout/client status is known, close that whole process group
+# before waiting for the reader so a successful direct exit cannot leak the
+# reader indefinitely.
+stop_runner_group() {
+  pid=$1
+  [ -n "$pid" ] || return 0
+  kill -TERM -"$pid" 2>/dev/null || true
+  kill -KILL -"$pid" 2>/dev/null || true
+}
+
+wait_reader_bounded() {
+  polls=0
+  while kill -0 "$reader_pid" 2>/dev/null; do
+    if [ "$polls" -ge 5 ]; then
+      kill -TERM "$reader_pid" 2>/dev/null || true
+      kill -KILL "$reader_pid" 2>/dev/null || true
+      break
+    fi
+    sleep 0.05
+    polls=$((polls + 1))
+  done
+  reader_status=0
+  wait "$reader_pid" 2>/dev/null || reader_status=$?
+  reader_pid=
+}
+
+cleanup_exit() {
+  status=$?
+  trap - EXIT
+  trap ':' HUP INT TERM
+  cleanup_processes
+  rm -rf "$probe_tmp_dir"
+  exit "$status"
+}
+
+cleanup_signal() {
+  status=$1
+  trap - EXIT
+  trap ':' HUP INT TERM
+  cleanup_processes
+  rm -rf "$probe_tmp_dir"
+  exit "$status"
+}
+
+trap cleanup_exit EXIT
+trap 'cleanup_signal 129' HUP
+trap 'cleanup_signal 130' INT
+trap 'cleanup_signal 143' TERM
+
+run_bounded() {
+  output_file=$1
+  shift
+  fifo="$output_file.fifo"
+  rm -f "$fifo"
+  mkfifo "$fifo" || return 1
+  timeout --signal=TERM --kill-after=1s "$probe_timeout" "$@" \
+    >"$fifo" 2>/dev/null &
+  runner_pid=$!
+  dd if="$fifo" bs=1 count=$((probe_max_output_bytes + 1)) \
+    >"$output_file" 2>/dev/null &
+  reader_pid=$!
+  status=0
+  wait "$runner_pid" 2>/dev/null || status=$?
+  stop_runner_group "$runner_pid"
+  runner_pid=
+  wait_reader_bounded
+  rm -f "$fifo"
+  [ "$status" -eq 0 ] || return 1
+  [ "$reader_status" -eq 0 ] || return 1
+  output_bytes=$(wc -c <"$output_file") || return 1
+  case "$output_bytes" in
+    ''|*[!0-9]*) return 1 ;;
+  esac
+  [ "$output_bytes" -le "$probe_max_output_bytes" ]
+}
+
+# Keep server liveness distinct from authentication/query readiness. A server
+# can answer pg_isready while the application identity is rejected.
+run_bounded "$probe_tmp_dir/liveness" pg_isready -q -h "$host" -p "$port" \
+  -U "$POSTGRES_USER" -d "$POSTGRES_DB" || exit 1
+
+PGPASSWORD="$POSTGRES_PASSWORD" run_bounded "$probe_tmp_dir/query" psql \
+  --no-psqlrc --no-password --quiet --no-align --tuples-only \
+  --set=ON_ERROR_STOP=1 \
+  --host="$host" --port="$port" \
+  --username="$POSTGRES_USER" --dbname="$POSTGRES_DB" \
+  --command='SELECT 1' || exit 1
+
+result=$(cat "$probe_tmp_dir/query") || exit 1
+
+[ "$result" = 1 ]
diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
new file mode 100644
index 0000000000..538db3d6bc
--- /dev/null
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -0,0 +1,1856 @@
+#!/usr/bin/env python3
+"""Guarded, targeted deployment helper for the Train Unstract health checks.
+
+The default operation is read-only.  ``capture`` records a sanitized runtime
+snapshot and ``preflight`` refuses to continue when the dirty live checkout,
+container identity, mounts, networks, or environment hashes drift.  The
+mutating ``apply`` and ``rollback`` phases require an explicit confirmation
+token and an external candidate image lock.  They recreate only the 24
+health-covered workloads in two bounded batches; they never build, pull,
+delete source files, reset the checkout, or run project-wide Compose commands.
+
+The script deliberately keeps secret values out of all output.  Environment
+values are represented by length and SHA-256 digest only, and health log
+outputs are not retained.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import datetime as dt
+import fcntl
+import hashlib
+import json
+import os
+import select
+import shlex
+import subprocess
+import sys
+import tempfile
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+PROJECT = "unstract-etl-home-complete-tech"
+DEFAULT_PROJECT_DIR = Path("/home/completetrain/etl.home.complete.tech")
+DEFAULT_COMPOSE_FILES = (
+    "docker/docker-compose.yaml",
+    "docker/compose.train.worker-healthchecks.yaml",
+    "docker/compose.train.healthchecks.yaml",
+)
+CONFIRM_TOKEN = "APPLY_UNSTRACT_HEALTH"
+DEFAULT_COMMAND_TIMEOUT_SECONDS = 300
+DEFAULT_LOCK_TIMEOUT_SECONDS = 30
+DEFAULT_BATCH_TIMEOUT_SECONDS = 900
+DEFAULT_APPLY_TIMEOUT_SECONDS = 2400
+DEFAULT_ROLLBACK_TIMEOUT_SECONDS = 1200
+ADVISORY_LOCK_SQL = "SELECT pg_try_advisory_lock(hashtextextended('train-unstract-health-deploy', 0));"
+ADVISORY_UNLOCK_SQL = "SELECT pg_advisory_unlock(hashtextextended('train-unstract-health-deploy', 0));"
+
+# Compose service names.  Keep this explicit so a typo or a newly added service
+# cannot silently turn a targeted deployment into a project-wide update.
+WORKER_SERVICES = (
+    "runner",
+    "worker-log-history-scheduler-v2",
+    "worker-pg-orchestrator-api",
+    "worker-pg-orchestrator-general",
+    "worker-pg-fileproc",
+    "worker-pg-callback",
+    "worker-pg-scheduler",
+    "worker-pg-metrics",
+    "worker-log-stream-consumer",
+    "worker-pg-executor",
+    "worker-pg-ide-callback",
+    "worker-pg-notification",
+    "worker-pg-reaper",
+)
+CORE_SERVICES = (
+    "db",
+    "redis",
+    "minio",
+    "reverse-proxy",
+    "qdrant",
+    "rabbitmq",
+    "weaviate",
+    "x2text-service",
+    "platform-service",
+    "backend",
+    "frontend",
+)
+TARGET_SERVICES = WORKER_SERVICES + CORE_SERVICES
+PROBE_MOUNT_TARGET = "/usr/local/bin/unstract-services.sh"
+EXPECTED_NETWORK = "unstract-network"
+SOURCE_STATE_SCHEMA = "unstract-source-state/v2"
+BACKUP_SCHEMA = "unstract-health-backup/v2"
+REPLACEMENT_SCHEMA = "unstract-health-replacements/v1"
+
+# The two log consumers need these values to expose their source-level
+# heartbeat endpoints.  Every other environment value must survive a
+# replacement byte-for-byte (represented by length and SHA-256 in captures).
+ALLOWED_ENV_ADDITIONS = {
+    "worker-log-history-scheduler-v2": {
+        "LOG_HISTORY_SCHEDULER_HEALTH_PORT",
+        "LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS",
+    },
+    "worker-log-stream-consumer": {
+        "LOG_STREAM_CONSUMER_HEALTH_PORT",
+        "LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS",
+    },
+}
+
+
+class OperationDeadline:
+    """Monotonic deadline shared by every command in one guarded operation."""
+
+    def __init__(self, seconds: float) -> None:
+        if seconds <= 0 or seconds > 24 * 60 * 60:
+            raise GuardError("operation timeout must be between 1 second and 24 hours")
+        self.ends_at = time.monotonic() + seconds
+
+    def remaining(self, requested: float | None = None) -> float:
+        left = self.ends_at - time.monotonic()
+        if left <= 0:
+            raise GuardError("guarded operation exceeded its total deadline")
+        if requested is None:
+            return left
+        if requested <= 0:
+            raise GuardError("command timeout must be positive")
+        return min(left, requested)
+
+
+class GuardError(RuntimeError):
+    """A precondition or postcondition failed."""
+
+
+def utc_now() -> str:
+    return dt.datetime.now(dt.UTC).isoformat()
+
+
+def sha256_bytes(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
+
+
+def sha256_file(path: Path) -> str:
+    try:
+        with path.open("rb") as handle:
+            digest = hashlib.sha256()
+            for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                digest.update(chunk)
+            return digest.hexdigest()
+    except OSError as exc:
+        raise GuardError(f"cannot hash required artifact {path}: {exc}") from exc
+
+
+def run(
+    args: list[str],
+    *,
+    cwd: Path | None = None,
+    env: dict[str, str] | None = None,
+    check: bool = True,
+    input_text: str | None = None,
+    timeout_seconds: float = DEFAULT_COMMAND_TIMEOUT_SECONDS,
+    deadline: OperationDeadline | None = None,
+) -> subprocess.CompletedProcess[str]:
+    timeout = deadline.remaining(timeout_seconds) if deadline else timeout_seconds
+    try:
+        result = subprocess.run(
+            args,
+            cwd=str(cwd) if cwd else None,
+            env=env,
+            input=input_text,
+            text=True,
+            capture_output=True,
+            check=False,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise GuardError(f"command timed out: {shlex.join(args)}") from exc
+    except OSError as exc:
+        raise GuardError(f"cannot execute {shlex.join(args)}: {exc}") from exc
+    if check and result.returncode != 0:
+        raise GuardError(f"command failed ({result.returncode}): {shlex.join(args)}")
+    return result
+
+
+def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
+    try:
+        return json.loads(result.stdout)
+    except json.JSONDecodeError as exc:
+        raise GuardError(f"{description} did not return JSON") from exc
+
+
+def image_digest(image: dict[str, Any]) -> str | None:
+    return image.get("Digest") or next(iter(image.get("RepoDigests") or []), None)
+
+
+def env_hashes(values: list[str] | None) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = {}
+    for item in values or []:
+        key, separator, value = item.partition("=")
+        if not separator:
+            value = ""
+        result[key] = {"length": len(value), "sha256": sha256_bytes(value.encode())}
+    return dict(sorted(result.items()))
+
+
+def selected_labels(labels: dict[str, str]) -> dict[str, str]:
+    keys = (
+        "com.docker.compose.project",
+        "com.docker.compose.service",
+        "com.docker.compose.config-hash",
+        "com.docker.compose.config_files",
+        "com.docker.compose.project.working_dir",
+        "com.docker.compose.project.environment_file",
+        "com.docker.compose.version",
+        "com.docker.compose.oneoff",
+        "com.docker.compose.container-number",
+    )
+    return {key: labels[key] for key in keys if key in labels}
+
+
+def health_config(value: dict[str, Any] | None) -> dict[str, Any]:
+    if not value:
+        return {"configured": False}
+    test = value.get("Test") or []
+    return {
+        "configured": True,
+        "test_sha256": sha256_bytes(json.dumps(test, separators=(",", ":")).encode()),
+        "test_argv_count": len(test),
+        "interval": value.get("Interval"),
+        "timeout": value.get("Timeout"),
+        "start_period": value.get("StartPeriod"),
+        "retries": value.get("Retries"),
+    }
+
+
+def health_runtime(value: dict[str, Any] | None) -> dict[str, Any]:
+    value = value or {}
+    logs = value.get("Log") or []
+    last = logs[-1] if logs else None
+    return {
+        "status": value.get("Status") or "none",
+        "failing_streak": value.get("FailingStreak", 0),
+        "log_count": len(logs),
+        "last": {
+            "start": last.get("Start"),
+            "end": last.get("End"),
+            "exit_code": last.get("ExitCode"),
+        }
+        if last
+        else None,
+    }
+
+
+def normalize_mount(mount: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "type": mount.get("Type"),
+        "name": mount.get("Name"),
+        "source": mount.get("Source"),
+        "destination": mount.get("Destination"),
+        "rw": mount.get("RW"),
+        "options": sorted(mount.get("Options") or []),
+    }
+
+
+def normalize_networks(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    """Keep stable network identity while omitting replacement-specific IPs."""
+    result: dict[str, dict[str, Any]] = {}
+    for name, network in sorted(value.items()):
+        result[name] = {
+            "aliases": sorted(network.get("Aliases") or []),
+            "network_mode": network.get("NetworkID") or None,
+            "driver_opts": network.get("DriverOpts") or {},
+        }
+    return result
+
+
+def normalize_option(value: Any) -> Any:
+    """Canonicalize Podman inspect fields without exposing command secrets."""
+    if isinstance(value, dict):
+        return {str(key): normalize_option(item) for key, item in sorted(value.items())}
+    if isinstance(value, (list, tuple)):
+        return [normalize_option(item) for item in value]
+    return value
+
+
+def runtime_options(item: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]:
+    host_config = item.get("HostConfig") or {}
+    return normalize_option(
+        {
+            "command": config.get("Cmd"),
+            "entrypoint": config.get("Entrypoint"),
+            "user": config.get("User"),
+            "working_dir": config.get("WorkingDir"),
+            "stop_signal": config.get("StopSignal"),
+            "stop_timeout": config.get("StopTimeout"),
+            "tty": config.get("Tty"),
+            "open_stdin": config.get("OpenStdin"),
+            "read_only": host_config.get("ReadonlyRootfs"),
+            "privileged": host_config.get("Privileged"),
+            "cap_add": host_config.get("CapAdd"),
+            "cap_drop": host_config.get("CapDrop"),
+            "security_opt": host_config.get("SecurityOpt"),
+            "restart_policy": host_config.get("RestartPolicy"),
+            "shm_size": host_config.get("ShmSize"),
+            "dns": host_config.get("Dns"),
+            "extra_hosts": host_config.get("ExtraHosts"),
+            "devices": host_config.get("Devices"),
+            "ulimits": host_config.get("Ulimits"),
+            "port_bindings": (item.get("NetworkSettings") or {}).get("Ports"),
+        }
+    )
+
+
+def inspect_project(
+    project: str = PROJECT, *, deadline: OperationDeadline | None = None
+) -> list[dict[str, Any]]:
+    ids_result = run(
+        ["podman", "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}"],
+        deadline=deadline,
+    )
+    ids = ids_result.stdout.split()
+    if not ids:
+        return []
+    raw = parse_json_output(
+        run(["podman", "inspect", *ids], deadline=deadline), "podman inspect"
+    )
+    containers: list[dict[str, Any]] = []
+    for item in sorted(raw, key=lambda value: value.get("Name", "")):
+        config = item.get("Config") or {}
+        labels = config.get("Labels") or {}
+        state = item.get("State") or {}
+        network_settings = item.get("NetworkSettings") or {}
+        networks = network_settings.get("Networks") or {}
+        containers.append(
+            {
+                "id": item.get("Id"),
+                "name": (item.get("Name") or "").lstrip("/"),
+                "compose": selected_labels(labels),
+                "image": {
+                    "name": item.get("ImageName"),
+                    "id": item.get("Image"),
+                    "digest": item.get("ImageDigest"),
+                    "driver": item.get("Driver"),
+                },
+                "state": {
+                    "status": state.get("Status"),
+                    "running": state.get("Running"),
+                    "started_at": state.get("StartedAt"),
+                    "finished_at": state.get("FinishedAt"),
+                    "exit_code": state.get("ExitCode"),
+                    "error_present": bool(state.get("Error")),
+                    "oom_killed": state.get("OOMKilled"),
+                    "restarting": state.get("Restarting"),
+                    "restart_count": item.get("RestartCount", 0),
+                },
+                "health": {
+                    "configured": health_config(config.get("Healthcheck")),
+                    "runtime": health_runtime(state.get("Health")),
+                },
+                "env_hashes": env_hashes(config.get("Env")),
+                "mounts": [normalize_mount(mount) for mount in item.get("Mounts") or []],
+                "options": runtime_options(item, config),
+                "graphdriver": {
+                    "driver": item.get("Driver"),
+                    "upper_dir": (item.get("GraphDriver") or {}).get("Data", {}).get("UpperDir"),
+                    "work_dir": (item.get("GraphDriver") or {}).get("Data", {}).get("WorkDir"),
+                },
+                "networks": sorted(networks),
+                "network_details": normalize_networks(networks),
+                "user": config.get("User"),
+                "working_dir": config.get("WorkingDir"),
+                "rootless_runtime": item.get("OCIRuntime"),
+            }
+        )
+    return containers
+
+
+def source_state(
+    project_dir: Path, *, deadline: OperationDeadline | None = None
+) -> dict[str, Any]:
+    head = run(
+        ["git", "-C", str(project_dir), "rev-parse", "HEAD"], deadline=deadline
+    ).stdout.strip()
+    status = run(
+        [
+            "git",
+            "-C",
+            str(project_dir),
+            "status",
+            "--porcelain=v1",
+            "--untracked-files=all",
+            "-z",
+        ],
+        deadline=deadline,
+    ).stdout.split("\0")
+    entries = [entry for entry in status if entry]
+    paths: list[str] = []
+    status_hashes: dict[str, dict[str, Any]] = {}
+    for entry in entries:
+        if len(entry) < 4:
+            continue
+        path = entry[3:]
+        # Porcelain v1 uses a second NUL record for rename/copy destinations.
+        # The destination is the only path that can be written by a concurrent
+        # checkout, so retain both names when present and hash each separately.
+        paths.append(path)
+        candidate = project_dir / path
+        try:
+            if candidate.is_file() and not candidate.is_symlink():
+                status_hashes[path] = {
+                    "bytes": candidate.stat().st_size,
+                    "sha256": sha256_file(candidate),
+                }
+            elif candidate.is_symlink():
+                status_hashes[path] = {
+                    "symlink": os.readlink(candidate),
+                    "sha256": sha256_bytes(os.readlink(candidate).encode()),
+                }
+            else:
+                status_hashes[path] = {"missing": True}
+        except OSError as exc:
+            raise GuardError(f"cannot hash dirty source path {path}: {exc}") from exc
+    return {
+        "schema": SOURCE_STATE_SCHEMA,
+        "path": str(project_dir),
+        "head": head,
+        "status_paths": paths,
+        "status_hashes": status_hashes,
+        "tracked_dirty_or_untracked_count": len(paths),
+    }
+
+
+def queue_snapshot(deadline: OperationDeadline | None = None) -> dict[str, Any]:
+    """Read queue and in-flight job state without claiming or consuming work."""
+    rabbit = run(
+        [
+            "podman",
+            "exec",
+            "unstract-rabbitmq",
+            "rabbitmqctl",
+            "list_queues",
+            "name",
+            "messages",
+            "messages_ready",
+            "messages_unacknowledged",
+            "consumers",
+            "--formatter=json",
+        ],
+        check=False,
+        deadline=deadline,
+    )
+    rabbit_queues: list[dict[str, Any]] = []
+    rabbit_parse_ok = False
+    if rabbit.returncode == 0:
+        try:
+            for row in json.loads(rabbit.stdout or "[]"):
+                rabbit_queues.append(
+                    {
+                        "name_present": bool(row.get("name")),
+                        "messages": row.get("messages"),
+                        "messages_ready": row.get("messages_ready"),
+                        "messages_unacknowledged": row.get("messages_unacknowledged"),
+                        "consumers": row.get("consumers"),
+                    }
+                )
+            rabbit_parse_ok = True
+        except json.JSONDecodeError:
+            pass
+
+    sql = (
+        "SELECT count(*) AS queue_rows, "
+        "count(*) FILTER (WHERE state = 'claimed') AS claimed_rows, "
+        "count(*) FILTER (WHERE state = 'scheduled') AS scheduled_rows "
+        "FROM unstract.pg_queue_message; "
+        "SELECT count(*) FILTER (WHERE remaining > 0) AS active_barriers, "
+        "count(*) AS barrier_rows FROM unstract.pg_barrier_state; "
+        "SELECT count(*) AS orchestration_claims "
+        "FROM unstract.pg_orchestration_claim; "
+        "SELECT count(*) AS task_result_rows FROM unstract.pg_task_result; "
+        "SELECT count(*) AS batch_dedup_rows FROM unstract.pg_batch_dedup"
+    )
+    pg = run(
+        [
+            "podman",
+            "exec",
+            "unstract-db",
+            "sh",
+            "-c",
+            "psql -XAtq -F '|' -U \"$POSTGRES_USER\" -d \"$POSTGRES_DB\" -c "
+            + shlex.quote(sql),
+        ],
+        check=False,
+        deadline=deadline,
+    )
+    pg_counts: dict[str, int] | None = None
+    values = [line.strip() for line in pg.stdout.splitlines() if line.strip()]
+    if pg.returncode == 0:
+        parsed_rows: list[list[int]] = []
+        try:
+            for value in values:
+                parts = value.split("|")
+                if not parts or not all(part.isdigit() for part in parts):
+                    raise ValueError
+                parsed_rows.append([int(part) for part in parts])
+        except ValueError:
+            parsed_rows = []
+        if (
+            len(parsed_rows) == 5
+            and len(parsed_rows[0]) == 3
+            and len(parsed_rows[1]) == 2
+            and all(len(row) == 1 for row in parsed_rows[2:])
+        ):
+            pg_counts = {
+                "pg_queue_message": parsed_rows[0][0],
+                "pg_queue_claimed": parsed_rows[0][1],
+                "pg_queue_scheduled": parsed_rows[0][2],
+                "pg_active_barriers": parsed_rows[1][0],
+                "pg_barrier_rows": parsed_rows[1][1],
+                "pg_orchestration_claims": parsed_rows[2][0],
+                "pg_task_result": parsed_rows[3][0],
+                "pg_batch_dedup": parsed_rows[4][0],
+            }
+
+    active_jobs = None
+    if pg_counts is not None:
+        active_jobs = {
+            "pg_queue_claimed": pg_counts["pg_queue_claimed"],
+            "pg_active_barriers": pg_counts["pg_active_barriers"],
+            "pg_orchestration_claims": pg_counts["pg_orchestration_claims"],
+            "total": (
+                pg_counts["pg_queue_claimed"]
+                + pg_counts["pg_active_barriers"]
+                + pg_counts["pg_orchestration_claims"]
+            ),
+        }
+    rabbit_empty = (
+        rabbit.returncode == 0
+        and rabbit_parse_ok
+        and all(
+            row.get("name_present")
+            and row.get("messages") == 0
+            and row.get("messages_ready") == 0
+            and row.get("messages_unacknowledged") == 0
+            for row in rabbit_queues
+        )
+    )
+    quiescent = (
+        rabbit_empty
+        and pg_counts is not None
+        and pg_counts["pg_queue_message"] == 0
+        and active_jobs is not None
+        and active_jobs["total"] == 0
+    )
+    return {
+        "observed_at": utc_now(),
+        "mode": "read_only_snapshot",
+        "rabbitmq": {
+            "command_succeeded": rabbit.returncode == 0,
+            "parsed": rabbit_parse_ok,
+            "empty": rabbit_empty,
+            "queue_count": len(rabbit_queues),
+            "queues": rabbit_queues,
+        },
+        "postgres": {
+            "command_succeeded": pg.returncode == 0,
+            "counts": pg_counts,
+            "active_jobs": active_jobs,
+        },
+        "active_jobs": active_jobs,
+        "quiescent": quiescent,
+    }
+
+
+def capture(
+    project_dir: Path, *, deadline: OperationDeadline | None = None
+) -> dict[str, Any]:
+    uid = os.getuid() if hasattr(os, "getuid") else None
+    return {
+        "schema": "unstract-deployment-prep/v2",
+        "captured_at": utc_now(),
+        "host": {
+            "uid": uid,
+            "hostname": os.uname().nodename,
+            "rootless_project": PROJECT,
+        },
+        "source": source_state(project_dir, deadline=deadline),
+        "job_quiescence": queue_snapshot(deadline),
+        "containers": inspect_project(deadline=deadline),
+    }
+
+
+def write_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def service_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = {}
+    for container in snapshot.get("containers", []):
+        service = container.get("compose", {}).get("com.docker.compose.service")
+        if service:
+            result[service] = container
+    return result
+
+
+def container_name_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
+    return {
+        container["name"]: container
+        for container in snapshot.get("containers", [])
+        if container.get("name")
+    }
+
+
+def compare_preserved_runtime(
+    previous: dict[str, Any], actual: dict[str, Any], service: str
+) -> None:
+    """Compare the identity/data contract while permitting a new container ID."""
+    for field in (
+        "name",
+        "env_hashes",
+        "networks",
+        "network_details",
+        "options",
+        "user",
+        "working_dir",
+        "rootless_runtime",
+    ):
+        if previous.get(field) != actual.get(field):
+            raise GuardError(f"runtime {field} changed for {service}")
+    previous_mounts = {
+        mount["destination"]: mount
+        for mount in previous.get("mounts", [])
+        if mount.get("destination") != PROBE_MOUNT_TARGET
+    }
+    actual_mounts = {
+        mount["destination"]: mount
+        for mount in actual.get("mounts", [])
+        if mount.get("destination") != PROBE_MOUNT_TARGET
+    }
+    if previous_mounts != actual_mounts:
+        raise GuardError(f"data mounts/options changed for {service}")
+    if not actual.get("state", {}).get("running"):
+        raise GuardError(f"service is not running: {service}")
+
+
+def compare_untargeted_runtime(
+    baseline: dict[str, Any], current: dict[str, Any]
+) -> None:
+    """Refuse to proceed if a non-target container changed during the apply."""
+    old = container_name_map(baseline)
+    new = container_name_map(current)
+    target_names = {
+        service_map(baseline)[service]["name"]
+        for service in TARGET_SERVICES
+        if service in service_map(baseline)
+    }
+    old_other = set(old) - target_names
+    new_other = set(new) - target_names
+    if old_other != new_other:
+        raise GuardError("untargeted container set changed")
+    for name in sorted(old_other):
+        previous, actual = old[name], new[name]
+        for field in (
+            "id",
+            "image",
+            "env_hashes",
+            "mounts",
+            "options",
+            "networks",
+            "network_details",
+        ):
+            if previous.get(field) != actual.get(field):
+                raise GuardError(f"untargeted container changed: {name}")
+
+
+def compare_source_and_quiescence(
+    baseline: dict[str, Any], current: dict[str, Any]
+) -> None:
+    if baseline.get("source") != current.get("source"):
+        raise GuardError("dirty live source state changed since baseline capture")
+    if not current.get("job_quiescence", {}).get("quiescent"):
+        raise GuardError("fresh queue or active-job quiescence check failed")
+
+
+def verify_untouched_targets(
+    baseline: dict[str, Any],
+    current: dict[str, Any],
+    untouched_services: tuple[str, ...],
+) -> None:
+    old, new = service_map(baseline), service_map(current)
+    for service in untouched_services:
+        if service not in old or service not in new:
+            raise GuardError(f"service missing during guarded apply: {service}")
+        if old[service].get("id") != new[service].get("id"):
+            raise GuardError(f"untouched target was recreated: {service}")
+        compare_preserved_runtime(old[service], new[service], service)
+
+
+def artifact_hashes(root: Path) -> dict[str, str]:
+    files = (
+        "docker/healthchecks/unstract-services.sh",
+        "docker/healthchecks/http-readiness.sh",
+        "docker/healthchecks/postgres-readiness.sh",
+        "docker/docker-compose-dev-essentials.yaml",
+        "docker/compose.train.healthchecks.yaml",
+        "docker/compose.train.worker-healthchecks.yaml",
+    )
+    return {file: sha256_file(root / file) for file in files}
+
+
+def source_tree_hash(root: Path) -> str:
+    return run(["git", "-C", str(root), "rev-parse", "HEAD^{tree}"]).stdout.strip()
+
+
+def load_lock(path: Path) -> dict[str, Any]:
+    try:
+        lock = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read candidate lock {path}: {exc}") from exc
+    if lock.get("schema") != "unstract-health-candidate/v1":
+        raise GuardError("candidate lock has an unsupported schema")
+    if not lock.get("source_commit") or not lock.get("candidate_version"):
+        raise GuardError("candidate lock must pin source_commit and candidate_version")
+    if not lock.get("source_tree"):
+        raise GuardError("candidate lock must pin the candidate source tree")
+    images = lock.get("images")
+    if not isinstance(images, dict) or set(images) != set(TARGET_SERVICES):
+        raise GuardError("candidate lock must pin exactly the targeted service images")
+    for service in TARGET_SERVICES:
+        image = images[service]
+        if (
+            not isinstance(image, dict)
+            or not image.get("reference")
+            or not image.get("id")
+            or not image.get("digest")
+        ):
+            raise GuardError(f"candidate image lock is incomplete for {service}")
+    return lock
+
+
+def load_baseline(path: Path) -> dict[str, Any]:
+    try:
+        baseline = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read baseline {path}: {exc}") from exc
+    if baseline.get("schema") != "unstract-deployment-prep/v2":
+        raise GuardError("baseline has an unsupported schema; capture a fresh baseline")
+    if baseline.get("source", {}).get("schema") != SOURCE_STATE_SCHEMA:
+        raise GuardError("baseline source state is incomplete; capture a fresh baseline")
+    if not isinstance(baseline.get("containers"), list):
+        raise GuardError("baseline container snapshot is missing")
+    return baseline
+
+
+def command_lock(args: argparse.Namespace) -> int:
+    """Write a complete lock only from already-present candidate images."""
+    source = Path(args.candidate_source)
+    source_commit = run(["git", "-C", str(source), "rev-parse", "HEAD"]).stdout.strip()
+    require_clean_candidate_source(source, source_commit)
+    references: dict[str, str] = {}
+    for item in args.image:
+        service, separator, reference = item.partition("=")
+        if not separator or service not in TARGET_SERVICES or not reference:
+            raise GuardError(
+                "--image must be repeated as service=image-reference for every target service"
+            )
+        if service in references:
+            raise GuardError(f"duplicate candidate image mapping for {service}")
+        references[service] = reference
+    missing = set(TARGET_SERVICES) - set(references)
+    if missing:
+        raise GuardError(f"candidate image mappings are missing: {sorted(missing)}")
+    images: dict[str, dict[str, str]] = {}
+    for service, reference in sorted(references.items()):
+        rows = parse_json_output(
+            run(["podman", "image", "inspect", reference]),
+            f"candidate image {reference}",
+        )
+        if not rows:
+            raise GuardError(f"candidate image inspect returned no rows for {service}")
+        row = rows[0]
+        digest = image_digest(row)
+        if not row.get("Id") or not digest:
+            raise GuardError(f"candidate image has no immutable identity for {service}")
+        images[service] = {"reference": reference, "id": row["Id"], "digest": digest}
+    lock = {
+        "schema": "unstract-health-candidate/v1",
+        "created_at": utc_now(),
+        "source_commit": source_commit,
+        "source_tree": source_tree_hash(source),
+        "candidate_version": args.candidate_version,
+        "artifacts": artifact_hashes(source),
+        "images": images,
+    }
+    write_json(Path(args.output), lock)
+    print(f"lock: wrote immutable source and image lock for {len(images)} services")
+    return 0
+
+
+def candidate_image_snapshot(
+    lock: dict[str, Any], *, deadline: OperationDeadline | None = None
+) -> dict[str, dict[str, Any]]:
+    result: dict[str, dict[str, Any]] = {}
+    for service, expected in lock["images"].items():
+        inspected = run(
+            ["podman", "image", "inspect", expected["reference"]],
+            check=False,
+            deadline=deadline,
+        )
+        if inspected.returncode != 0:
+            raise GuardError(f"candidate image is absent: {expected['reference']}")
+        rows = parse_json_output(inspected, f"candidate image {expected['reference']}")
+        if not rows:
+            raise GuardError(f"candidate image inspect returned no rows for {service}")
+        row = rows[0]
+        actual = {
+            "reference": expected["reference"],
+            "id": row.get("Id") or row.get("ID"),
+            "digest": image_digest(row),
+        }
+        if actual["id"] != expected["id"] or actual["digest"] != expected["digest"]:
+            raise GuardError(
+                f"candidate image identity changed for {service}: "
+                f"expected {expected['id']} / {expected['digest']}, "
+                f"found {actual['id']} / {actual['digest']}"
+            )
+        result[service] = actual
+    return result
+
+
+def compose_config(
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    *,
+    candidate_version: str,
+    probe_source: Path,
+    image_override: Path | None = None,
+    deadline: OperationDeadline | None = None,
+) -> dict[str, Any]:
+    env = os.environ.copy()
+    env["VERSION"] = candidate_version
+    env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+    files = compose_files + ((str(image_override),) if image_override else ())
+    args = ["docker", "compose"]
+    for compose_file in files:
+        args.extend(["-f", compose_file])
+    args.extend(["config", "--format", "json"])
+    return parse_json_output(
+        run(args, cwd=project_dir, env=env, deadline=deadline), "Compose config"
+    )
+
+
+def check_candidate_config(
+    config: dict[str, Any],
+    baseline: dict[str, Any],
+    lock: dict[str, Any],
+) -> None:
+    services = config.get("services") or {}
+    missing = set(TARGET_SERVICES) - set(services)
+    if missing:
+        raise GuardError(f"candidate Compose config is missing services: {sorted(missing)}")
+    if (config.get("networks") or {}).get("default", {}).get("name") != EXPECTED_NETWORK:
+        raise GuardError("candidate Compose config changes the unstract-network name")
+    old = service_map(baseline)
+    for service in TARGET_SERVICES:
+        candidate = services[service]
+        expected_image = lock["images"][service]["reference"]
+        if candidate.get("image") != expected_image:
+            raise GuardError(
+                f"candidate Compose image mismatch for {service}: "
+                f"{candidate.get('image')} != {expected_image}"
+            )
+        if service not in old:
+            raise GuardError(f"baseline has no existing container for {service}")
+        old_name = old[service]["name"]
+        if candidate.get("container_name") != old_name:
+            raise GuardError(
+                f"container identity changed for {service}: "
+                f"{candidate.get('container_name')} != {old_name}"
+            )
+        networks = candidate.get("networks") or {}
+        if EXPECTED_NETWORK not in networks and "default" not in networks:
+            raise GuardError(f"candidate service {service} leaves {EXPECTED_NETWORK}")
+        healthcheck = candidate.get("healthcheck") or {}
+        health_test = healthcheck.get("test") or []
+        if not health_test or health_test == ["NONE"]:
+            raise GuardError(f"candidate healthcheck is missing for {service}")
+        old_mounts = {
+            mount["destination"]: mount
+            for mount in old[service].get("mounts", [])
+            if mount.get("destination") != PROBE_MOUNT_TARGET
+        }
+        candidate_mounts = {
+            mount.get("target"): mount
+            for mount in candidate.get("volumes", [])
+            if mount.get("target") != PROBE_MOUNT_TARGET
+        }
+        for destination, old_mount in old_mounts.items():
+            new_mount = candidate_mounts.get(destination)
+            if not new_mount:
+                raise GuardError(f"candidate removed {service} mount {destination}")
+            old_source = (
+                old_mount.get("name") or old_mount.get("source")
+                if old_mount.get("type") == "volume"
+                else old_mount.get("source")
+            )
+            new_source = new_mount.get("source")
+            if old_source and new_source and old_source != new_source:
+                raise GuardError(
+                    f"candidate changed {service} mount source for {destination}: "
+                    f"{new_source} != {old_source}"
+                )
+            if old_mount.get("rw") is not None and new_mount.get("read_only") is not None:
+                if bool(old_mount["rw"]) == bool(new_mount["read_only"]):
+                    raise GuardError(f"candidate changed mount access for {service} {destination}")
+        if set(candidate_mounts) != set(old_mounts):
+            raise GuardError(f"candidate changed data mounts for {service}")
+
+        candidate_environment = candidate.get("environment") or {}
+        if isinstance(candidate_environment, list):
+            candidate_environment = {
+                item.partition("=")[0]: item.partition("=")[2]
+                for item in candidate_environment
+                if isinstance(item, str)
+            }
+        if not isinstance(candidate_environment, dict):
+            raise GuardError(f"candidate environment is not a mapping for {service}")
+        old_environment = old[service].get("env_hashes") or {}
+        allowed_additions = ALLOWED_ENV_ADDITIONS.get(service, set())
+        for key, value in candidate_environment.items():
+            if value is None:
+                # Compose's null means "inherit from the host".  It is not a
+                # deterministic deployment contract and cannot be preflighted.
+                raise GuardError(f"candidate environment is host-inherited for {service}")
+            value_hash = {
+                "length": len(str(value)),
+                "sha256": sha256_bytes(str(value).encode()),
+            }
+            if key in old_environment and old_environment[key] != value_hash:
+                raise GuardError(f"candidate environment changed for {service}: {key}")
+            if key not in old_environment and key not in allowed_additions:
+                raise GuardError(f"candidate added environment for {service}: {key}")
+        # Compose config does not include image-provided defaults such as PATH,
+        # while the runtime snapshot does.  The post-recreation inspect still
+        # compares the complete environment contract; only the two declared
+        # heartbeat additions are permitted.
+
+
+def compare_baseline_current(
+    baseline: dict[str, Any], current: dict[str, Any], *, allow_new_probe: bool
+) -> None:
+    old = service_map(baseline)
+    new = service_map(current)
+    for service in TARGET_SERVICES:
+        if service not in old or service not in new:
+            raise GuardError(f"service {service} is missing from baseline or current runtime")
+        previous, actual = old[service], new[service]
+        for field in (
+            "name",
+            "env_hashes",
+            "networks",
+            "network_details",
+            "options",
+            "user",
+            "working_dir",
+            "rootless_runtime",
+        ):
+            if previous.get(field) != actual.get(field):
+                raise GuardError(f"runtime {field} drifted for {service}")
+        previous_mounts = {
+            mount["destination"]: mount
+            for mount in previous.get("mounts", [])
+            if allow_new_probe or mount.get("destination") != PROBE_MOUNT_TARGET
+        }
+        actual_mounts = {
+            mount["destination"]: mount
+            for mount in actual.get("mounts", [])
+            if allow_new_probe or mount.get("destination") != PROBE_MOUNT_TARGET
+        }
+        if previous_mounts != actual_mounts:
+            raise GuardError(f"runtime mounts/options drifted for {service}")
+        if not previous.get("state", {}).get("running"):
+            raise GuardError(f"baseline service {service} was not running")
+    if not current.get("job_quiescence", {}).get("quiescent"):
+        raise GuardError("fresh job quiescence check failed")
+
+
+def compare_post_apply(
+    baseline: dict[str, Any],
+    current: dict[str, Any],
+    lock: dict[str, Any],
+    expected_services: tuple[str, ...],
+) -> None:
+    old, new = service_map(baseline), service_map(current)
+    for service in expected_services:
+        previous, actual = old[service], new[service]
+        candidate = lock["images"][service]
+        if actual["image"]["id"] != candidate["id"]:
+            raise GuardError(f"post-apply image ID mismatch for {service}")
+        if actual["image"].get("digest") != candidate["digest"]:
+            raise GuardError(f"post-apply image digest mismatch for {service}")
+        if actual["name"] != previous["name"]:
+            raise GuardError(f"post-apply container name changed for {service}")
+        if actual["networks"] != previous["networks"]:
+            raise GuardError(f"post-apply networks changed for {service}")
+        if actual.get("network_details") != previous.get("network_details"):
+            raise GuardError(f"post-apply network options changed for {service}")
+        if actual.get("options") != previous.get("options"):
+            raise GuardError(f"post-apply container options changed for {service}")
+        old_mounts = {
+            mount["destination"]: mount
+            for mount in previous.get("mounts", [])
+            if mount.get("destination") != PROBE_MOUNT_TARGET
+        }
+        new_mounts = {
+            mount["destination"]: mount
+            for mount in actual.get("mounts", [])
+            if mount.get("destination") != PROBE_MOUNT_TARGET
+        }
+        if old_mounts != new_mounts:
+            raise GuardError(f"post-apply persistent mounts/options changed for {service}")
+        previous_env = previous.get("env_hashes", {})
+        actual_env = actual.get("env_hashes", {})
+        for key, value in previous_env.items():
+            if actual_env.get(key) != value:
+                raise GuardError(f"post-apply environment value changed for {service}: {key}")
+        additions = set(actual_env) - set(previous_env)
+        if additions - ALLOWED_ENV_ADDITIONS.get(service, set()):
+            raise GuardError(f"post-apply environment additions changed for {service}")
+        if not actual["state"].get("running"):
+            raise GuardError(f"post-apply service is not running: {service}")
+        if not (actual.get("health", {}).get("configured") or {}).get("configured"):
+            raise GuardError(f"post-apply healthcheck is not configured: {service}")
+        if actual["health"]["runtime"].get("status") != "healthy":
+            raise GuardError(f"post-apply service is not healthy: {service}")
+    if not current.get("job_quiescence", {}).get("quiescent"):
+        raise GuardError("post-apply queue snapshot is not quiescent")
+
+
+def require_clean_candidate_source(
+    path: Path, expected_commit: str, expected_tree: str | None = None
+) -> None:
+    if not path.exists():
+        raise GuardError(f"candidate source path does not exist: {path}")
+    actual = run(["git", "-C", str(path), "rev-parse", "HEAD"]).stdout.strip()
+    if actual != expected_commit:
+        raise GuardError(f"candidate source commit {actual} != {expected_commit}")
+    if expected_tree:
+        actual_tree = source_tree_hash(path)
+        if actual_tree != expected_tree:
+            raise GuardError(f"candidate source tree {actual_tree} != {expected_tree}")
+    status = run(["git", "-C", str(path), "status", "--porcelain", "--untracked-files=all"]).stdout
+    if status.strip():
+        raise GuardError("candidate source must be clean; refusing an untracked build")
+
+
+def verify_artifacts(root: Path, lock: dict[str, Any]) -> None:
+    expected = lock.get("artifacts") or {}
+    actual = artifact_hashes(root)
+    if set(expected) != set(actual):
+        raise GuardError(
+            "candidate artifact manifest does not match the guarded source files"
+        )
+    for name, digest in expected.items():
+        if actual.get(name) != digest:
+            raise GuardError(f"candidate artifact hash mismatch: {name}")
+
+
+@contextlib.contextmanager
+def advisory_lock(
+    *,
+    timeout_seconds: float = DEFAULT_LOCK_TIMEOUT_SECONDS,
+    deadline: OperationDeadline | None = None,
+) -> Iterator[None]:
+    """Hold a DB advisory lock across both targeted recreation batches."""
+    process = subprocess.Popen(
+        [
+            "podman",
+            "exec",
+            "-i",
+            "unstract-db",
+            "sh",
+            "-c",
+            'exec psql -XAtq -U "$POSTGRES_USER" -d "$POSTGRES_DB"',
+        ],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    assert process.stdin is not None
+    assert process.stdout is not None
+    timeout = deadline.remaining(timeout_seconds) if deadline else timeout_seconds
+    try:
+        process.stdin.write(ADVISORY_LOCK_SQL + "\n")
+        process.stdin.flush()
+    except (BrokenPipeError, OSError) as exc:
+        with contextlib.suppress(Exception):
+            process.kill()
+        raise GuardError("could not send the deployment lock query") from exc
+    ready, _, _ = select.select([process.stdout], [], [], timeout)
+    if not ready:
+        with contextlib.suppress(Exception):
+            process.kill()
+        with contextlib.suppress(Exception):
+            process.wait(timeout=5)
+        raise GuardError("timed out acquiring the Train deployment advisory lock")
+    result = process.stdout.readline().strip()
+    if result != "t":
+        with contextlib.suppress(Exception):
+            process.kill()
+        with contextlib.suppress(Exception):
+            process.wait(timeout=5)
+        raise GuardError("another Train health deployment already holds the advisory lock")
+    try:
+        yield
+    finally:
+        try:
+            process.stdin.write(ADVISORY_UNLOCK_SQL + "\n\\q\n")
+            process.stdin.flush()
+            process.wait(timeout=deadline.remaining(10) if deadline else 10)
+        except (OSError, subprocess.TimeoutExpired):
+            with contextlib.suppress(Exception):
+                process.kill()
+            with contextlib.suppress(Exception):
+                process.wait(timeout=5)
+
+
+@contextlib.contextmanager
+def local_operation_lock(
+    path: Path, *, deadline: OperationDeadline | None = None
+) -> Iterator[None]:
+    """Serialize this guard even while the DB container itself is replaced."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a+", encoding="utf-8") as handle:
+        while True:
+            try:
+                fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if deadline:
+                    delay = min(1.0, deadline.remaining())
+                else:
+                    delay = 1.0
+                time.sleep(delay)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+
+
+def capture_and_write(
+    project_dir: Path,
+    output: Path,
+    *,
+    operation_deadline: OperationDeadline | None = None,
+) -> dict[str, Any]:
+    snapshot = capture(project_dir, deadline=operation_deadline)
+    write_json(output, snapshot)
+    return snapshot
+
+
+def compose_args(compose_files: tuple[str, ...]) -> list[str]:
+    args = ["docker", "compose"]
+    for compose_file in compose_files:
+        args.extend(["-f", compose_file])
+    return args
+
+
+def write_image_override(lock: dict[str, Any], path: Path) -> None:
+    """Pin each target to the exact locked reference without touching source."""
+    lines = [
+        "# Generated by train_health_deployment_guard.py; do not edit.",
+        "services:",
+    ]
+    for service in TARGET_SERVICES:
+        reference = lock["images"][service]["reference"]
+        if any(character.isspace() for character in reference) or "\n" in reference:
+            raise GuardError(f"candidate image reference contains whitespace: {service}")
+        lines.extend([f"  {service}:", f"    image: {json.dumps(reference)}"])
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+@contextlib.contextmanager
+def candidate_image_override(lock: dict[str, Any]) -> Iterator[Path]:
+    handle = tempfile.NamedTemporaryFile(
+        mode="w", prefix="unstract-health-images-", suffix=".yaml", delete=False
+    )
+    path = Path(handle.name)
+    handle.close()
+    try:
+        write_image_override(lock, path)
+        yield path
+    finally:
+        with contextlib.suppress(FileNotFoundError):
+            path.unlink()
+
+
+def targeted_up(
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    services: tuple[str, ...],
+    *,
+    candidate_version: str,
+    probe_source: Path,
+    image_override: Path | None = None,
+    deadline: OperationDeadline | None = None,
+) -> None:
+    env = os.environ.copy()
+    env["VERSION"] = candidate_version
+    env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+    files = compose_files + ((str(image_override),) if image_override else ())
+    args = compose_args(files)
+    args.extend(
+        [
+            "up",
+            "-d",
+            "--no-deps",
+            "--force-recreate",
+            "--no-build",
+            "--pull",
+            "never",
+            *services,
+        ]
+    )
+    run(args, cwd=project_dir, env=env, deadline=deadline)
+
+
+def wait_healthy(
+    services: tuple[str, ...],
+    timeout_seconds: int = 240,
+    *,
+    operation_deadline: OperationDeadline | None = None,
+) -> None:
+    end = time.monotonic() + timeout_seconds
+    states: dict[str, Any] = {}
+    while time.monotonic() < end:
+        if operation_deadline:
+            operation_deadline.remaining()
+        current = service_map(
+            {"containers": inspect_project(deadline=operation_deadline)}
+        )
+        states = {
+            service: current.get(service, {}).get("health", {}).get("runtime", {}).get("status")
+            for service in services
+        }
+        if all(value == "healthy" for value in states.values()):
+            return
+        delay = min(5, max(0, end - time.monotonic()))
+        if operation_deadline:
+            delay = min(delay, operation_deadline.remaining())
+        if delay:
+            time.sleep(delay)
+    raise GuardError(f"targeted services did not become healthy: {states}")
+
+
+def wait_running(
+    services: tuple[str, ...],
+    timeout_seconds: int = 240,
+    *,
+    operation_deadline: OperationDeadline | None = None,
+) -> None:
+    end = time.monotonic() + timeout_seconds
+    states: dict[str, Any] = {}
+    while time.monotonic() < end:
+        if operation_deadline:
+            operation_deadline.remaining()
+        current = service_map(
+            {"containers": inspect_project(deadline=operation_deadline)}
+        )
+        states = {
+            service: current.get(service, {}).get("state", {}).get("status")
+            for service in services
+        }
+        if all(value == "running" for value in states.values()):
+            return
+        delay = min(5, max(0, end - time.monotonic()))
+        if operation_deadline:
+            delay = min(delay, operation_deadline.remaining())
+        if delay:
+            time.sleep(delay)
+    raise GuardError(f"targeted services did not become running: {states}")
+
+
+def commit_backups(
+    snapshot: dict[str, Any],
+    backup_dir: Path,
+    *,
+    deadline: OperationDeadline | None = None,
+) -> dict[str, Any]:
+    backup_dir.mkdir(parents=True, exist_ok=True)
+    write_json(backup_dir / "baseline.json", snapshot)
+    tag_prefix = "localhost/unstract-health-backup-"
+    backup_images: dict[str, Any] = {
+        "schema": BACKUP_SCHEMA,
+        "created_at": utc_now(),
+        "services": {},
+    }
+    for service in TARGET_SERVICES:
+        container = service_map(snapshot)[service]
+        tag = tag_prefix + service.replace("_", "-") + ":" + snapshot["captured_at"].replace(":", "").replace("+", "-")
+        run(
+            ["podman", "commit", "--pause=false", container["id"], tag],
+            deadline=deadline,
+        )
+        image_rows = parse_json_output(
+            run(["podman", "image", "inspect", tag], deadline=deadline),
+            f"backup image {service}",
+        )
+        if not image_rows or not image_rows[0].get("Id"):
+            raise GuardError(f"backup image has no immutable ID for {service}")
+        image = image_rows[0]
+        digest = image_digest(image)
+        backup_images["services"][service] = {
+            "reference": tag,
+            "id": image["Id"],
+            "digest": digest,
+            "old_container_id": container["id"],
+            "old_image": container.get("image"),
+            "old_health": container.get("health"),
+            "old_env_hashes": container.get("env_hashes"),
+            "old_mounts": container.get("mounts"),
+            "old_options": container.get("options"),
+            "old_networks": container.get("networks"),
+            "old_network_details": container.get("network_details"),
+            "old_name": container.get("name"),
+        }
+    write_json(backup_dir / "backup-images.json", backup_images)
+    return backup_images
+
+
+def backup_service_record(backup_images: dict[str, Any], service: str) -> dict[str, Any]:
+    records = backup_images.get("services")
+    if isinstance(records, dict) and isinstance(records.get(service), dict):
+        return records[service]
+    # Read manifests produced by the first preparation revision so rollback
+    # remains possible if the coordinator already has one on the private host.
+    reference = backup_images.get(service)
+    if isinstance(reference, str):
+        return {"reference": reference}
+    raise GuardError(f"backup manifest has no service record for {service}")
+
+
+def rollback_override(
+    backup_images: dict[str, Any], path: Path, services: tuple[str, ...] = TARGET_SERVICES
+) -> None:
+    lines = [
+        "# Generated by train_health_deployment_guard.py; do not edit.",
+        "services:",
+    ]
+    for service in services:
+        record = backup_service_record(backup_images, service)
+        reference = record.get("reference")
+        if not reference:
+            raise GuardError(f"backup image reference missing for {service}")
+        lines.extend(
+            [
+                f"  {service}:",
+                f"    image: {json.dumps(reference)}",
+                '    healthcheck: {test: ["NONE"]}',
+            ]
+        )
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def command_capture(args: argparse.Namespace) -> int:
+    deadline = OperationDeadline(args.operation_timeout)
+    capture_and_write(
+        Path(args.project_dir), Path(args.output), operation_deadline=deadline
+    )
+    return 0
+
+
+def prepare(
+    args: argparse.Namespace,
+    image_override: Path,
+    *,
+    operation_deadline: OperationDeadline,
+) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
+    baseline = load_baseline(Path(args.baseline))
+    lock = load_lock(Path(args.candidate_lock))
+    require_clean_candidate_source(
+        Path(args.candidate_source), lock["source_commit"], lock.get("source_tree")
+    )
+    verify_artifacts(Path(args.candidate_source), lock)
+    candidate_images = candidate_image_snapshot(lock, deadline=operation_deadline)
+    if not candidate_images:
+        raise GuardError("no candidate images were verified")
+    current = capture(Path(args.project_dir), deadline=operation_deadline)
+    compare_baseline_current(baseline, current, allow_new_probe=False)
+    compare_untargeted_runtime(baseline, current)
+    compare_source_and_quiescence(baseline, current)
+    config = compose_config(
+        Path(args.project_dir),
+        tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
+        candidate_version=lock["candidate_version"],
+        probe_source=Path(args.probe_source),
+        image_override=image_override,
+        deadline=operation_deadline,
+    )
+    check_candidate_config(config, baseline, lock)
+    return baseline, lock, current
+
+
+def record_replacements(
+    baseline: dict[str, Any],
+    current: dict[str, Any],
+    lock: dict[str, Any],
+    services: tuple[str, ...],
+    *,
+    strict: bool = True,
+) -> dict[str, Any]:
+    old, new = service_map(baseline), service_map(current)
+    records: dict[str, Any] = {}
+    unresolved: list[str] = []
+    for service in services:
+        previous = old.get(service)
+        actual = new.get(service)
+        if previous is None or actual is None:
+            if strict:
+                raise GuardError(f"cannot identify replacement container for {service}")
+            unresolved.append(service)
+            continue
+        if actual.get("id") == previous.get("id"):
+            continue
+        expected = lock["images"][service]
+        if actual.get("image", {}).get("id") != expected.get("id"):
+            if strict:
+                raise GuardError(
+                    f"replacement image cannot be identified safely for {service}"
+                )
+            unresolved.append(service)
+            continue
+        if actual.get("image", {}).get("digest") != expected.get("digest"):
+            if strict:
+                raise GuardError(
+                    f"replacement image digest cannot be identified safely for {service}"
+                )
+            unresolved.append(service)
+            continue
+        records[service] = {
+            "old_container_id": previous.get("id"),
+            "replacement_container_id": actual.get("id"),
+            "candidate_image_id": expected.get("id"),
+            "candidate_image_digest": expected.get("digest"),
+            "observed_at": current.get("captured_at"),
+        }
+    return {
+        "schema": REPLACEMENT_SCHEMA,
+        "services": records,
+        "unresolved": sorted(set(unresolved)),
+    }
+
+
+def write_replacement_manifest(
+    backup_dir: Path, manifest: dict[str, Any], *, name: str = "replacements.json"
+) -> None:
+    write_json(backup_dir / name, manifest)
+
+
+def verify_replacement_ids(
+    current: dict[str, Any], replacement_manifest: dict[str, Any]
+) -> None:
+    current_services = service_map(current)
+    for service, record in (replacement_manifest.get("services") or {}).items():
+        actual = current_services.get(service)
+        if actual is None or actual.get("id") != record.get("replacement_container_id"):
+            raise GuardError(
+                f"replacement container identity changed before rollback: {service}"
+            )
+
+
+def rollback_compose_files(args: argparse.Namespace) -> tuple[str, ...]:
+    configured = tuple(
+        getattr(args, "rollback_compose_file", None)
+        or getattr(args, "compose_file", None)
+        or DEFAULT_COMPOSE_FILES
+    )
+    # The old runtime did not have the new health overlay.  Keep the original
+    # owner files and append the generated image/health override last.
+    return tuple(
+        path
+        for path in configured
+        if "compose.train.worker-healthchecks.yaml" not in path
+        and "compose.train.healthchecks.yaml" not in path
+    )
+
+
+def verify_rollback_result(
+    baseline: dict[str, Any],
+    final: dict[str, Any],
+    backup_images: dict[str, Any],
+    replacement_manifest: dict[str, Any],
+    *,
+    operation_deadline: OperationDeadline,
+) -> None:
+    old, new = service_map(baseline), service_map(final)
+    for service in replacement_manifest.get("services", {}):
+        actual = new.get(service)
+        previous = old.get(service)
+        record = backup_service_record(backup_images, service)
+        if actual is None or previous is None:
+            raise GuardError(f"rollback service is missing: {service}")
+        if actual.get("image", {}).get("id") != record.get("id"):
+            raise GuardError(f"rollback backup image ID mismatch for {service}")
+        if record.get("digest") and actual.get("image", {}).get("digest") != record.get("digest"):
+            raise GuardError(f"rollback backup image digest mismatch for {service}")
+        compare_preserved_runtime(previous, actual, service)
+        old_health = previous.get("health") or {}
+        new_health = actual.get("health") or {}
+        if old_health.get("configured") != new_health.get("configured"):
+            raise GuardError(f"rollback health configuration changed for {service}")
+        if (old_health.get("runtime") or {}).get("status") != (
+            new_health.get("runtime") or {}
+        ).get("status"):
+            raise GuardError(f"rollback health state was not restored for {service}")
+    compare_untargeted_runtime(baseline, final)
+    compare_source_and_quiescence(baseline, final)
+
+
+def compensating_rollback(
+    args: argparse.Namespace,
+    baseline: dict[str, Any],
+    backup_images: dict[str, Any],
+    replacement_manifest: dict[str, Any],
+    backup_dir: Path,
+    *,
+    operation_deadline: OperationDeadline,
+) -> None:
+    services = tuple((replacement_manifest.get("services") or {}).keys())
+    if not services:
+        raise GuardError("no exact replacement IDs were recorded for compensating rollback")
+    current = capture(Path(args.project_dir), deadline=operation_deadline)
+    compare_source_and_quiescence(baseline, current)
+    verify_replacement_ids(current, replacement_manifest)
+    override = backup_dir / "compensating-rollback.override.yaml"
+    rollback_override(backup_images, override, services)
+    rollback_files = rollback_compose_files(args) + (str(override),)
+    with advisory_lock(deadline=operation_deadline):
+        # Recheck identity and quiescence after acquiring the DB lock.  The
+        # process may be disconnected when db itself is recreated; the local
+        # operation lock remains held for that bounded transaction.
+        locked = capture(Path(args.project_dir), deadline=operation_deadline)
+        compare_source_and_quiescence(baseline, locked)
+        verify_replacement_ids(locked, replacement_manifest)
+        targeted_up(
+            Path(args.project_dir),
+            rollback_files,
+            services,
+            candidate_version="rollback-unused",
+            probe_source=Path(args.probe_source),
+            deadline=operation_deadline,
+        )
+        wait_running(services, operation_deadline=operation_deadline)
+    final = capture(Path(args.project_dir), deadline=operation_deadline)
+    verify_rollback_result(
+        baseline,
+        final,
+        backup_images,
+        replacement_manifest,
+        operation_deadline=operation_deadline,
+    )
+    write_json(backup_dir / "post-compensating-rollback.json", final)
+    write_json(backup_dir / "compensating-rollback.json", replacement_manifest)
+
+
+def apply_batch(
+    args: argparse.Namespace,
+    baseline: dict[str, Any],
+    lock: dict[str, Any],
+    backup_dir: Path,
+    image_override: Path,
+    services: tuple[str, ...],
+    untouched_services: tuple[str, ...],
+    applied_services: tuple[str, ...],
+    *,
+    operation_deadline: OperationDeadline,
+) -> dict[str, Any]:
+    compose_files = tuple(args.compose_file or DEFAULT_COMPOSE_FILES)
+    with advisory_lock(deadline=operation_deadline):
+        fresh = capture(Path(args.project_dir), deadline=operation_deadline)
+        compare_untargeted_runtime(baseline, fresh)
+        compare_source_and_quiescence(baseline, fresh)
+        verify_untouched_targets(baseline, fresh, untouched_services)
+        if applied_services:
+            compare_post_apply(baseline, fresh, lock, applied_services)
+        candidate_image_snapshot(lock, deadline=operation_deadline)
+        config = compose_config(
+            Path(args.project_dir),
+            compose_files,
+            candidate_version=lock["candidate_version"],
+            probe_source=Path(args.probe_source),
+            image_override=image_override,
+            deadline=operation_deadline,
+        )
+        check_candidate_config(config, baseline, lock)
+        targeted_up(
+            Path(args.project_dir),
+            compose_files,
+            services,
+            candidate_version=lock["candidate_version"],
+            probe_source=Path(args.probe_source),
+            image_override=image_override,
+            deadline=operation_deadline,
+        )
+        observed = capture(Path(args.project_dir), deadline=operation_deadline)
+        replacements = record_replacements(baseline, observed, lock, services)
+        write_replacement_manifest(backup_dir, replacements, name=f"replacements-{services[0]}.json")
+        wait_healthy(services, operation_deadline=operation_deadline)
+        final = capture(Path(args.project_dir), deadline=operation_deadline)
+        compare_post_apply(baseline, final, lock, services)
+        compare_untargeted_runtime(baseline, final)
+        compare_source_and_quiescence(baseline, final)
+        return replacements
+
+
+def command_preflight(args: argparse.Namespace) -> int:
+    deadline = OperationDeadline(args.operation_timeout)
+    lock = load_lock(Path(args.candidate_lock))
+    with candidate_image_override(lock) as image_override:
+        prepare(args, image_override, operation_deadline=deadline)
+    print(
+        "preflight: candidate source, image lock, Compose identity, runtime, "
+        "data, network, environment, queue, and active-job state verified"
+    )
+    return 0
+
+
+def command_apply(args: argparse.Namespace) -> int:
+    if args.confirm != CONFIRM_TOKEN:
+        raise GuardError(f"apply requires --confirm {CONFIRM_TOKEN}")
+    operation_deadline = OperationDeadline(args.operation_timeout)
+    backup_dir = Path(args.backup_dir)
+    attempted: list[str] = []
+    applied: list[str] = []
+    backup_images: dict[str, Any] | None = None
+    replacement_manifest: dict[str, Any] = {
+        "schema": REPLACEMENT_SCHEMA,
+        "services": {},
+        "unresolved": [],
+    }
+    with local_operation_lock(backup_dir / ".guard.lock", deadline=operation_deadline):
+        lock_hint = load_lock(Path(args.candidate_lock))
+        with candidate_image_override(lock_hint) as image_override:
+            try:
+                baseline, lock, _ = prepare(
+                    args, image_override, operation_deadline=operation_deadline
+                )
+                with advisory_lock(deadline=operation_deadline):
+                    fresh = capture(Path(args.project_dir), deadline=operation_deadline)
+                    compare_baseline_current(baseline, fresh, allow_new_probe=False)
+                    compare_untargeted_runtime(baseline, fresh)
+                    compare_source_and_quiescence(baseline, fresh)
+                    candidate_image_snapshot(lock, deadline=operation_deadline)
+                    backup_images = commit_backups(
+                        fresh, backup_dir, deadline=operation_deadline
+                    )
+                rollback_override(backup_images, backup_dir / "rollback.override.yaml")
+                write_json(backup_dir / "candidate-images.json", lock["images"])
+
+                attempted.extend(WORKER_SERVICES)
+                worker_replacements = apply_batch(
+                    args,
+                    baseline,
+                    lock,
+                    backup_dir,
+                    image_override,
+                    WORKER_SERVICES,
+                    CORE_SERVICES,
+                    (),
+                    operation_deadline=operation_deadline,
+                )
+                replacement_manifest["services"].update(
+                    worker_replacements.get("services", {})
+                )
+                applied.extend(WORKER_SERVICES)
+                write_replacement_manifest(backup_dir, replacement_manifest)
+
+                attempted.extend(CORE_SERVICES)
+                core_replacements = apply_batch(
+                    args,
+                    baseline,
+                    lock,
+                    backup_dir,
+                    image_override,
+                    CORE_SERVICES,
+                    (),
+                    tuple(applied),
+                    operation_deadline=operation_deadline,
+                )
+                replacement_manifest["services"].update(
+                    core_replacements.get("services", {})
+                )
+                applied.extend(CORE_SERVICES)
+                write_replacement_manifest(backup_dir, replacement_manifest)
+                final = capture(Path(args.project_dir), deadline=operation_deadline)
+                compare_post_apply(baseline, final, lock, TARGET_SERVICES)
+                compare_untargeted_runtime(baseline, final)
+                compare_source_and_quiescence(baseline, final)
+                write_json(backup_dir / "post-apply.json", final)
+            except Exception as exc:
+                if backup_images is not None and attempted:
+                    try:
+                        failed_state = capture(
+                            Path(args.project_dir), deadline=operation_deadline
+                        )
+                        discovered = record_replacements(
+                            baseline,
+                            failed_state,
+                            lock,
+                            tuple(attempted),
+                            strict=False,
+                        )
+                        replacement_manifest["services"].update(
+                            discovered.get("services", {})
+                        )
+                        replacement_manifest["unresolved"] = sorted(
+                            set(replacement_manifest.get("unresolved", []))
+                            | set(discovered.get("unresolved", []))
+                        )
+                        write_replacement_manifest(
+                            backup_dir, replacement_manifest, name="failed-replacements.json"
+                        )
+                        if replacement_manifest["services"]:
+                            compensating_rollback(
+                                args,
+                                baseline,
+                                backup_images,
+                                replacement_manifest,
+                                backup_dir,
+                                operation_deadline=operation_deadline,
+                            )
+                    except Exception as rollback_error:
+                        raise GuardError(
+                            "guarded apply failed and compensating rollback failed; "
+                            f"manual recovery is required: {type(rollback_error).__name__}"
+                        ) from exc
+                raise
+    print(f"apply: verified {len(TARGET_SERVICES)} targeted services; backup={backup_dir}")
+    return 0
+
+
+def command_rollback(args: argparse.Namespace) -> int:
+    if args.confirm != CONFIRM_TOKEN:
+        raise GuardError(f"rollback requires --confirm {CONFIRM_TOKEN}")
+    operation_deadline = OperationDeadline(args.operation_timeout)
+    backup_dir = Path(args.backup_dir)
+    try:
+        backup_images = json.loads(
+            (backup_dir / "backup-images.json").read_text(encoding="utf-8")
+        )
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read backup image manifest: {exc}") from exc
+    if backup_images.get("schema") != BACKUP_SCHEMA:
+        raise GuardError("backup image manifest has an unsupported schema")
+    if set(backup_images.get("services") or {}) != set(TARGET_SERVICES):
+        raise GuardError("backup image manifest does not cover the exact target set")
+    replacement_path = backup_dir / "replacements.json"
+    if not replacement_path.exists():
+        raise GuardError("rollback requires the exact replacement ID manifest")
+    try:
+        replacement_manifest = json.loads(replacement_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read replacement manifest: {exc}") from exc
+    if replacement_manifest.get("schema") != REPLACEMENT_SCHEMA:
+        raise GuardError("replacement manifest has an unsupported schema")
+    baseline = load_baseline(backup_dir / "baseline.json")
+    with local_operation_lock(backup_dir / ".guard.lock", deadline=operation_deadline):
+        compensating_rollback(
+            args,
+            baseline,
+            backup_images,
+            replacement_manifest,
+            backup_dir,
+            operation_deadline=operation_deadline,
+        )
+        final = json.loads(
+            (backup_dir / "post-compensating-rollback.json").read_text(encoding="utf-8")
+        )
+        write_json(backup_dir / "post-rollback.json", final)
+    print(
+        "rollback: verified "
+        f"{len(replacement_manifest.get('services', {}))} exact replacement services; "
+        f"backup={backup_dir}"
+    )
+    return 0
+
+
+def add_common(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
+    parser.add_argument("--compose-file", action="append", default=None)
+    parser.add_argument("--baseline", required=True)
+    parser.add_argument("--candidate-source", required=True)
+    parser.add_argument("--candidate-lock", required=True)
+    parser.add_argument("--probe-source", required=True)
+    parser.add_argument(
+        "--operation-timeout",
+        type=float,
+        default=DEFAULT_APPLY_TIMEOUT_SECONDS,
+        help="total monotonic deadline for the guarded operation",
+    )
+
+
+def parser() -> argparse.ArgumentParser:
+    root = argparse.ArgumentParser(description=__doc__)
+    sub = root.add_subparsers(dest="command", required=True)
+    lock_parser = sub.add_parser(
+        "lock", help="lock a clean source commit and already-built candidate images"
+    )
+    lock_parser.add_argument("--candidate-source", required=True)
+    lock_parser.add_argument("--candidate-version", required=True)
+    lock_parser.add_argument("--image", action="append", required=True)
+    lock_parser.add_argument("--output", required=True)
+    capture_parser = sub.add_parser("capture", help="read-only sanitized runtime snapshot")
+    capture_parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
+    capture_parser.add_argument("--output", required=True)
+    capture_parser.add_argument(
+        "--operation-timeout",
+        type=float,
+        default=DEFAULT_COMMAND_TIMEOUT_SECONDS,
+    )
+    preflight_parser = sub.add_parser("preflight", help="read-only candidate and drift checks")
+    add_common(preflight_parser)
+    apply_parser = sub.add_parser("apply", help="explicit targeted recreation")
+    add_common(apply_parser)
+    apply_parser.add_argument("--backup-dir", required=True)
+    apply_parser.add_argument("--confirm", required=True)
+    rollback_parser = sub.add_parser("rollback", help="explicit targeted compensating rollback")
+    rollback_parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
+    rollback_parser.add_argument("--rollback-compose-file", action="append", default=None)
+    rollback_parser.add_argument("--probe-source", required=True)
+    rollback_parser.add_argument("--backup-dir", required=True)
+    rollback_parser.add_argument("--confirm", required=True)
+    rollback_parser.add_argument(
+        "--operation-timeout",
+        type=float,
+        default=DEFAULT_ROLLBACK_TIMEOUT_SECONDS,
+    )
+    return root
+
+
+def main() -> int:
+    args = parser().parse_args()
+    try:
+        if args.command == "capture":
+            return command_capture(args)
+        if args.command == "lock":
+            return command_lock(args)
+        if args.command == "preflight":
+            return command_preflight(args)
+        if args.command == "apply":
+            return command_apply(args)
+        if args.command == "rollback":
+            return command_rollback(args)
+        raise GuardError(f"unknown command {args.command}")
+    except GuardError as exc:
+        print(f"guard: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
new file mode 100644
index 0000000000..1d9e980d1b
--- /dev/null
+++ b/docs/train-unstract-health-deployment.md
@@ -0,0 +1,205 @@
+# Guarded Train health deployment
+
+The two Train overlays are deployable without checking out over the dirty Train
+source directory:
+
+- `docker/compose.train.worker-healthchecks.yaml` adds the runner and twelve
+  worker healthchecks and the two health-port environment additions needed by
+  the log consumers.
+- `docker/compose.train.healthchecks.yaml` adds read-only probe mounts and
+  bounded checks for the eleven core services. Set
+  `UNSTRACT_HEALTHCHECK_SOURCE` when the probe is staged outside the live
+  checkout; the default remains the repository-relative path for local use.
+- `docker/docker-compose-dev-essentials.yaml` uses the bounded
+  `postgres-readiness.sh` and `http-readiness.sh` probes for its pgvector,
+  Milvus MinIO, and Milvus services while preserving the local SSL entrypoint
+  and volume configuration.
+
+`docker/scripts/train_health_deployment_guard.py` is the transaction guard. Its
+`capture`, `lock`, and `preflight` commands are read-only. Only `apply` and
+`rollback` mutate the host, and both require `--confirm APPLY_UNSTRACT_HEALTH`.
+Every command has a finite subprocess timeout and every operation has a finite
+monotonic deadline. The guard never builds, pulls, deletes source, resets a
+checkout, or runs project-wide `up`/`down` commands.
+
+The target set is fixed at 24 services: runner plus twelve workers, followed
+by db, Redis, MinIO, reverse proxy, Qdrant, RabbitMQ, Weaviate, x2text,
+platform, backend, and frontend. The completed one-shot `minio-bootstrap`
+container is intentionally excluded.
+
+## Candidate preparation
+
+Build the six changed images from a clean detached worktree at the exact
+integrated source commit. Run this on an external builder or disposable build
+host; do not build from the dirty Train checkout:
+
+```sh
+CANDIDATE_COMMIT="$(git rev-parse HEAD)"
+CANDIDATE_SHORT="${CANDIDATE_COMMIT:0:8}"
+CANDIDATE_VERSION="goal09-${CANDIDATE_SHORT}"
+git worktree add --detach /var/tmp/unstract-goal09 "$CANDIDATE_COMMIT"
+cd /var/tmp/unstract-goal09
+VERSION="$CANDIDATE_VERSION" docker compose -f docker/docker-compose.build.yaml build --pull never \
+  backend frontend runner platform-service x2text-service worker-unified
+```
+
+The static database, broker, storage, proxy, and vector images are pinned by
+the candidate lock as well. Publish or import every image under an immutable
+reference without replacing an existing tag. After all 24 references are
+available to the rootless Train Podman context, generate the lock; the command
+inspects each image and records its full local ID and immutable digest:
+
+```sh
+python3 docker/scripts/train_health_deployment_guard.py lock \
+  --candidate-source /var/tmp/unstract-goal09 \
+  --candidate-version "$CANDIDATE_VERSION" \
+  --output /run/user/1000/unstract-goal09/candidate-lock.json \
+  --image runner=localhost/unstract/runner:"$CANDIDATE_VERSION" \
+  --image backend=localhost/unstract/backend:"$CANDIDATE_VERSION" \
+  --image frontend=localhost/unstract/frontend:"$CANDIDATE_VERSION" \
+  --image platform-service=localhost/unstract/platform-service:"$CANDIDATE_VERSION" \
+  --image worker-pg-orchestrator-api=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-orchestrator-general=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-fileproc=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-callback=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-scheduler=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-metrics=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-log-stream-consumer=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-executor=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-ide-callback=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-notification=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-pg-reaper=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image worker-log-history-scheduler-v2=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
+  --image db=docker.io/pgvector/pgvector:pg15 \
+  --image redis=docker.io/library/redis:7.2.3 \
+  --image minio=docker.io/minio/minio:latest \
+  --image reverse-proxy=docker.io/library/traefik:v3.6.2 \
+  --image qdrant=docker.io/qdrant/qdrant:v1.16.1 \
+  --image rabbitmq=docker.io/library/rabbitmq:4.1.0-management \
+  --image weaviate=docker.io/semitechnologies/weaviate:1.39.2 \
+  --image x2text-service=localhost/unstract/x2text-service:"$CANDIDATE_VERSION"
+```
+
+The final lock must include exactly one mapping for every target service and
+must contain no placeholder IDs or digests. The static image references should
+be the exact names and IDs already captured from the live stack, unless a
+deliberate static image change has separately been reviewed. The lock also
+contains the candidate commit's tree hash and hashes for the two overlays, the
+core and database probes, and the development essentials Compose file. The
+guard writes a temporary image override from this lock, so Compose cannot
+silently resolve a different registry or tag.
+
+Stage only the committed guard, overlays, and probe into a separate directory
+such as `/run/user/1000/unstract-goal09/source`; never copy over the live
+checkout and never use `rsync --delete`. The private Train Compose file and
+all existing `.env` files stay in their current location. The core overlay's
+`UNSTRACT_HEALTHCHECK_SOURCE` points at the staged probe.
+
+## Read-only preflight and bounded apply
+
+Capture a fresh baseline immediately before preflight, even if the earlier
+runtime snapshot is available. The capture hashes every dirty source file by
+length and SHA-256, and records active PostgreSQL claims/barriers and claimed
+queue rows in addition to RabbitMQ ready/unacknowledged messages. All queue
+checks are SELECT/list operations; they do not claim, consume, drain, or mutate
+work:
+
+```sh
+python3 docker/scripts/train_health_deployment_guard.py capture \
+  --project-dir /home/completetrain/etl.home.complete.tech \
+  --output /run/user/1000/unstract-goal09/baseline.json \
+  --operation-timeout 300
+```
+
+The capture hashes environment values without printing them, records full
+container IDs, image IDs/digests, writable-layer paths, mount source/target/
+options, networks, health metadata without health output, source dirty paths,
+and queue counts. It must show zero RabbitMQ messages, zero PostgreSQL queue
+rows, zero claimed or scheduled queue rows, zero active barriers, and zero
+orchestration claims. Terminal result and dedup rows are recorded separately
+and are not mistaken for active jobs.
+
+Use the live dirty Compose files plus the staged overlays. This preserves the
+local embedding includes, port changes, env files, named volumes, bind mounts,
+container names, and network ownership:
+
+```sh
+python3 docker/scripts/train_health_deployment_guard.py preflight \
+  --baseline /run/user/1000/unstract-goal09/baseline.json \
+  --project-dir /home/completetrain/etl.home.complete.tech \
+  --candidate-source /run/user/1000/unstract-goal09/source \
+  --candidate-lock /run/user/1000/unstract-goal09/candidate-lock.json \
+  --probe-source /run/user/1000/unstract-goal09/source/docker/healthchecks/unstract-services.sh \
+  --compose-file docker/docker-compose.yaml \
+  --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.worker-healthchecks.yaml \
+  --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.healthchecks.yaml \
+  --operation-timeout 1200
+```
+
+Preflight refuses a changed dirty-path list, container name, persistent mount
+source or access mode, network, candidate image ID/digest, source commit, or
+artifact hash. It also refuses missing images and any service outside the
+fixed target set. The mutation phase repeats the runtime identity and queue
+checks while holding the PostgreSQL advisory deployment lock. The lock
+serializes competing guarded deployments; the queue counts are rechecked
+immediately before each batch because independently running application
+producers cannot be stopped by an advisory lock.
+
+After a fresh quiescence check and automatic `podman commit` backup of each
+target container, `apply` recreates only the worker batch, waits for every
+worker healthcheck, verifies exact candidate image ID/digest, names, network
+attachments, container options, persistent mount sources/options, environment
+digests, and queue/active-job state, then reacquires the deployment lock and
+rechecks all of those preconditions before the eleven core services. It always
+uses `--no-deps --force-recreate --no-build --pull never`. No volumes or images
+are pruned and no source file is removed.
+
+The guard holds a local file lock for the entire transaction because replacing
+`db` necessarily disconnects a PostgreSQL advisory-lock session. It reacquires
+the DB advisory lock before each batch and immediately rechecks source hashes,
+untargeted container identities, queue quiescence, and candidate image IDs.
+
+```sh
+python3 docker/scripts/train_health_deployment_guard.py apply \
+  --confirm APPLY_UNSTRACT_HEALTH \
+  --backup-dir /run/user/1000/unstract-goal09/backup \
+  --baseline /run/user/1000/unstract-goal09/baseline.json \
+  --project-dir /home/completetrain/etl.home.complete.tech \
+  --candidate-source /run/user/1000/unstract-goal09/source \
+  --candidate-lock /run/user/1000/unstract-goal09/candidate-lock.json \
+  --probe-source /run/user/1000/unstract-goal09/source/docker/healthchecks/unstract-services.sh \
+  --compose-file docker/docker-compose.yaml \
+  --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.worker-healthchecks.yaml \
+  --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.healthchecks.yaml \
+  --operation-timeout 2400
+```
+
+## Compensating rollback
+
+The backup directory contains the sanitized baseline, committed backup images,
+the old runtime contract, and exact replacement container IDs. If either batch
+fails verification, the guard first captures the current IDs and image IDs. It
+automatically rolls back only containers whose replacement ID is known and
+whose image is the locked candidate; an unexpected or missing ID stops with a
+manual-recovery blocker instead of overwriting an external change. The
+compensating rollback recreates those services from the committed pre-change
+rootfs, disables the newly added healthchecks, and verifies the old image ID,
+health state, container options, network, mount/data, environment, source
+hashes, and queue state. Persistent named and bind-mounted data is never
+removed.
+
+```sh
+python3 docker/scripts/train_health_deployment_guard.py rollback \
+  --confirm APPLY_UNSTRACT_HEALTH \
+  --backup-dir /run/user/1000/unstract-goal09/backup \
+  --project-dir /home/completetrain/etl.home.complete.tech \
+  --probe-source /run/user/1000/unstract-goal09/source/docker/healthchecks/unstract-services.sh \
+  --rollback-compose-file docker/docker-compose.yaml \
+  --operation-timeout 1200
+```
+
+The current prepared state has no candidate image lock and has not executed
+these mutating phases. That is intentional: the live checkout contains dirty
+local Compose/embedding changes and the host remains unchanged until an
+independent image build, exact lock, fresh quiescence check, and coordinator
+authorization are available.
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index 73a5bc31f3..9c5f03cbe4 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -542,7 +542,7 @@ def test_train_overlay_mounts_read_only_probe_and_sets_core_checks() -> None:
     for service, probe_name in expected.items():
         config = overlay["services"][service]
         assert config["volumes"] == [
-            "./healthchecks/unstract-services.sh:/usr/local/bin/unstract-services.sh:ro"
+            "${UNSTRACT_HEALTHCHECK_SOURCE:-./healthchecks/unstract-services.sh}:/usr/local/bin/unstract-services.sh:ro"
         ]
         healthcheck = config["healthcheck"]
         assert healthcheck["test"] == [
@@ -554,3 +554,79 @@ def test_train_overlay_mounts_read_only_probe_and_sets_core_checks() -> None:
         assert healthcheck["timeout"] == "10s"
         assert healthcheck["retries"] == 3
         assert isinstance(healthcheck["start_period"], str)
+
+
+def test_train_worker_overlay_covers_runner_and_all_workers() -> None:
+    overlay_path = ROOT / "docker" / "compose.train.worker-healthchecks.yaml"
+    overlay = yaml.safe_load(overlay_path.read_text(encoding="utf-8"))
+    expected = {
+        "runner": 5002,
+        "worker-log-history-scheduler-v2": 8092,
+        "worker-pg-orchestrator-api": 8090,
+        "worker-pg-orchestrator-general": 8090,
+        "worker-pg-fileproc": 8090,
+        "worker-pg-callback": 8090,
+        "worker-pg-scheduler": 8090,
+        "worker-pg-metrics": 8090,
+        "worker-log-stream-consumer": 8091,
+        "worker-pg-executor": 8090,
+        "worker-pg-ide-callback": 8090,
+        "worker-pg-notification": 8090,
+        "worker-pg-reaper": 8086,
+    }
+
+    assert set(overlay["services"]) == set(expected)
+    for service, port in expected.items():
+        healthcheck = overlay["services"][service]["healthcheck"]
+        expected_path = "/health"
+        if service == "runner":
+            expected_path = "/v1/api/health"
+        assert healthcheck["test"][-1] == f"http://127.0.0.1:{port}{expected_path}"
+        assert healthcheck["interval"] == "30s"
+        assert healthcheck["timeout"] == "5s"
+        assert healthcheck["retries"] == 3
+        assert healthcheck["start_period"] == "30s"
+
+    assert overlay["services"]["worker-log-history-scheduler-v2"]["environment"] == {
+        "LOG_HISTORY_SCHEDULER_HEALTH_PORT": "8092",
+        "LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS": "${LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS:-120}",
+    }
+    assert overlay["services"]["worker-log-stream-consumer"]["environment"] == {
+        "LOG_STREAM_CONSUMER_HEALTH_PORT": "8091",
+        "LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS": "${LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS:-15}",
+    }
+
+
+def test_dev_essentials_uses_bounded_database_probes() -> None:
+    compose_path = ROOT / "docker" / "docker-compose-dev-essentials.yaml"
+    compose = yaml.safe_load(compose_path.read_text(encoding="utf-8"))
+    services = compose["services"]
+
+    postgres = services["postgres-vector"]
+    assert "./healthchecks/postgres-readiness.sh:/usr/local/bin/postgres-readiness.sh:ro" in postgres[
+        "volumes"
+    ]
+    assert postgres["healthcheck"] == {
+        "test": ["CMD", "/usr/local/bin/postgres-readiness.sh"],
+        "interval": "10s",
+        "timeout": "6s",
+        "retries": 5,
+    }
+    assert "./scripts/db-setup/postgres-vector-entrypoint.sh:/usr/local/bin/postgres-vector-entrypoint.sh:ro" in postgres[
+        "volumes"
+    ]
+    assert "postgres_vector_ssl:/var/lib/postgresql/ssl/" in postgres["volumes"]
+
+    for service, url in {
+        "milvus-minio": "http://127.0.0.1:9000/minio/health/ready",
+        "milvus": "http://127.0.0.1:9091/healthz",
+    }.items():
+        config = services[service]
+        assert "./healthchecks/http-readiness.sh:/usr/local/bin/http-readiness.sh:ro" in config[
+            "volumes"
+        ]
+        assert config["healthcheck"]["test"] == [
+            "CMD",
+            "/usr/local/bin/http-readiness.sh",
+            url,
+        ]

From 4b24f2e4998bcaba95a569cef5f57ab98d15a0a3 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:22:24 -0400
Subject: [PATCH 24/48] Bound health probe signal cleanup

---
 docker/healthchecks/unstract-services.sh | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index cbcfab1073..b48d591608 100755
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -62,6 +62,21 @@ rabbitmq_diagnostics_bin=${RABBITMQ_DIAGNOSTICS_BIN:-rabbitmq-diagnostics}
 pg_isready_bin=${PG_ISREADY_BIN:-pg_isready}
 psql_bin=${PSQL_BIN:-psql}
 
+# The timeout wrapper starts client commands in their own process group on the
+# supported GNU and BusyBox implementations.  A healthcheck can be signalled
+# while its shell is waiting for that wrapper; terminate both the group and
+# the wrapper PID so a client descendant cannot keep a FIFO open or delay the
+# shell's trap indefinitely.  The group form is allowed to fail for readers
+# that share the shell's process group, after which the direct PID is killed.
+terminate_process_group() {
+    terminate_process_pid=$1
+    [ -n "$terminate_process_pid" ] || return 0
+    kill -TERM -"$terminate_process_pid" >/dev/null 2>&1 || :
+    kill -TERM "$terminate_process_pid" >/dev/null 2>&1 || :
+    kill -KILL -"$terminate_process_pid" >/dev/null 2>&1 || :
+    kill -KILL "$terminate_process_pid" >/dev/null 2>&1 || :
+}
+
 # BusyBox wget has no max-filesize or max-redirect option. Stream through
 # bounded head processes, while capturing response headers so redirects can be
 # rejected even when the client follows them internally. The status file keeps
@@ -72,7 +87,7 @@ bounded_wget_cleanup() {
         "${bounded_wget_body_reader_pid-}" \
         "${bounded_wget_header_reader_pid-}"; do
         if [ -n "$bounded_wget_cleanup_pid" ]; then
-            kill "$bounded_wget_cleanup_pid" >/dev/null 2>&1 || :
+            terminate_process_group "$bounded_wget_cleanup_pid"
         fi
     done
     for bounded_wget_cleanup_pid in \
@@ -168,7 +183,7 @@ bounded_curl_cleanup() {
         "${bounded_curl_client_pid-}" \
         "${bounded_curl_body_reader_pid-}"; do
         if [ -n "$bounded_curl_cleanup_pid" ]; then
-            kill "$bounded_curl_cleanup_pid" >/dev/null 2>&1 || :
+            terminate_process_group "$bounded_curl_cleanup_pid"
         fi
     done
     for bounded_curl_cleanup_pid in \
@@ -246,7 +261,7 @@ bounded_exec_cleanup() {
         "${bounded_exec_client_pid-}" \
         "${bounded_exec_reader_pid-}"; do
         if [ -n "$bounded_exec_cleanup_pid" ]; then
-            kill "$bounded_exec_cleanup_pid" >/dev/null 2>&1 || :
+            terminate_process_group "$bounded_exec_cleanup_pid"
         fi
     done
     for bounded_exec_cleanup_pid in \

From 03404993c3329a221d7f426b0148d9431e6d5794 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 16:15:02 -0400
Subject: [PATCH 25/48] Close guarded health deployment review gaps

---
 .../scripts/train_health_deployment_guard.py  | 406 ++++++++++++++++--
 docs/train-unstract-health-deployment.md      |  37 +-
 2 files changed, 384 insertions(+), 59 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 538db3d6bc..999f72ba1e 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -48,6 +48,11 @@
 DEFAULT_ROLLBACK_TIMEOUT_SECONDS = 1200
 ADVISORY_LOCK_SQL = "SELECT pg_try_advisory_lock(hashtextextended('train-unstract-health-deploy', 0));"
 ADVISORY_UNLOCK_SQL = "SELECT pg_advisory_unlock(hashtextextended('train-unstract-health-deploy', 0));"
+EXPECTED_RUNTIME_ENDPOINT = "unix:///run/user/1000/podman/podman.sock"
+EXPECTED_RUNTIME_UID = 1000
+QUIESCENCE_REQUIRED_SAMPLES = 3
+QUIESCENCE_SAMPLE_INTERVAL_SECONDS = 2.0
+QUIESCENCE_MAX_WAIT_SECONDS = 10.0
 
 # Compose service names.  Keep this explicit so a typo or a newly added service
 # cannot silently turn a targeted deployment into a project-wide update.
@@ -101,6 +106,62 @@
 }
 
 
+def _probe_test(service: str) -> list[str]:
+    if service in CORE_SERVICES:
+        probe_name = {
+            "reverse-proxy": "proxy",
+            "qdrant": "vector-db",
+        }.get(service, service)
+        return ["CMD", "/usr/local/bin/unstract-services.sh", probe_name]
+    if service == "runner":
+        port, path = "5002", "/v1/api/health"
+    elif service == "worker-pg-reaper":
+        port, path = "8086", "/health"
+    elif service == "worker-log-history-scheduler-v2":
+        port, path = "8092", "/health"
+    elif service == "worker-log-stream-consumer":
+        port, path = "8091", "/health"
+    else:
+        port, path = "8090", "/health"
+    return [
+        "CMD",
+        "/usr/bin/curl",
+        "--fail",
+        "--silent",
+        "--show-error",
+        "--max-time",
+        "3",
+        f"http://127.0.0.1:{port}{path}",
+    ]
+
+
+def health_contract(service: str) -> dict[str, Any]:
+    if service in CORE_SERVICES:
+        start_period = {
+            "db": "30s",
+            "redis": "15s",
+            "minio": "60s",
+            "reverse-proxy": "60s",
+            "qdrant": "60s",
+            "rabbitmq": "60s",
+            "weaviate": "120s",
+            "x2text-service": "120s",
+            "platform-service": "120s",
+            "backend": "180s",
+            "frontend": "60s",
+        }[service]
+        timeout = "10s"
+    else:
+        start_period, timeout = "30s", "5s"
+    return {
+        "test": _probe_test(service),
+        "interval_ns": 30_000_000_000,
+        "timeout_ns": duration_ns(timeout),
+        "start_period_ns": duration_ns(start_period),
+        "retries": 3,
+    }
+
+
 class OperationDeadline:
     """Monotonic deadline shared by every command in one guarded operation."""
 
@@ -143,6 +204,54 @@ def sha256_file(path: Path) -> str:
         raise GuardError(f"cannot hash required artifact {path}: {exc}") from exc
 
 
+def duration_ns(value: Any) -> int | None:
+    """Normalize Compose duration strings and Podman nanosecond values."""
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return round(value)
+    text = str(value).strip().lower()
+    units = (
+        ("ns", 1),
+        ("us", 1_000),
+        ("µs", 1_000),
+        ("ms", 1_000_000),
+        ("s", 1_000_000_000),
+        ("m", 60_000_000_000),
+        ("h", 3_600_000_000_000),
+    )
+    for suffix, multiplier in units:
+        if text.endswith(suffix):
+            try:
+                return round(float(text[: -len(suffix)]) * multiplier)
+            except ValueError:
+                return None
+    return None
+
+
+def runtime_command_env(
+    command: list[str], env: dict[str, str] | None = None
+) -> dict[str, str] | None:
+    """Bind Docker-compatible Compose and direct Podman calls to one socket."""
+    if not command or command[0] not in {"docker", "podman"}:
+        return env
+    effective = os.environ.copy()
+    if env is not None:
+        effective.update(env)
+    for key in ("DOCKER_HOST", "CONTAINER_HOST"):
+        supplied = effective.get(key)
+        if supplied and supplied != EXPECTED_RUNTIME_ENDPOINT:
+            raise GuardError(
+                f"{key} must be {EXPECTED_RUNTIME_ENDPOINT} for the rootless Train runtime"
+            )
+        effective[key] = EXPECTED_RUNTIME_ENDPOINT
+    return effective
+
+
 def run(
     args: list[str],
     *,
@@ -154,6 +263,7 @@ def run(
     deadline: OperationDeadline | None = None,
 ) -> subprocess.CompletedProcess[str]:
     timeout = deadline.remaining(timeout_seconds) if deadline else timeout_seconds
+    env = runtime_command_env(args, env)
     try:
         result = subprocess.run(
             args,
@@ -216,12 +326,18 @@ def health_config(value: dict[str, Any] | None) -> dict[str, Any]:
     test = value.get("Test") or []
     return {
         "configured": True,
+        # Healthcheck command vectors contain no credentials and are retained
+        # so post-apply verification binds healthy status to the trusted probe.
+        "test": test,
         "test_sha256": sha256_bytes(json.dumps(test, separators=(",", ":")).encode()),
         "test_argv_count": len(test),
         "interval": value.get("Interval"),
         "timeout": value.get("Timeout"),
         "start_period": value.get("StartPeriod"),
         "retries": value.get("Retries"),
+        "interval_ns": duration_ns(value.get("Interval")),
+        "timeout_ns": duration_ns(value.get("Timeout")),
+        "start_period_ns": duration_ns(value.get("StartPeriod")),
     }
 
 
@@ -563,6 +679,93 @@ def queue_snapshot(deadline: OperationDeadline | None = None) -> dict[str, Any]:
     }
 
 
+def settled_queue_snapshot(deadline: OperationDeadline | None = None) -> dict[str, Any]:
+    """Require consecutive zero-work samples before any destructive change."""
+    started = time.monotonic()
+    end = started + QUIESCENCE_MAX_WAIT_SECONDS
+    consecutive = 0
+    observations: list[dict[str, Any]] = []
+    last: dict[str, Any] | None = None
+    while True:
+        last = queue_snapshot(deadline)
+        observations.append(
+            {
+                "observed_at": last.get("observed_at"),
+                "quiescent": last.get("quiescent"),
+                "rabbitmq_empty": (last.get("rabbitmq") or {}).get("empty"),
+                "postgres_counts": (last.get("postgres") or {}).get("counts"),
+            }
+        )
+        if last.get("quiescent"):
+            consecutive += 1
+            if consecutive >= QUIESCENCE_REQUIRED_SAMPLES:
+                last["stability"] = {
+                    "stable": True,
+                    "sample_count": len(observations),
+                    "stable_for_seconds": round(time.monotonic() - started, 3),
+                    "observations": observations,
+                }
+                return last
+        else:
+            consecutive = 0
+        remaining = end - time.monotonic()
+        if deadline:
+            remaining = min(remaining, deadline.remaining())
+        if remaining <= 0:
+            raise GuardError(
+                "queue and active-job state did not remain quiescent for the bounded stability interval"
+            )
+        time.sleep(min(QUIESCENCE_SAMPLE_INTERVAL_SECONDS, remaining))
+
+
+def runtime_context(*, deadline: OperationDeadline | None = None) -> dict[str, Any]:
+    """Prove direct Podman and Docker-compatible Compose share one rootless store."""
+    uid = os.getuid() if hasattr(os, "getuid") else None
+    if uid != EXPECTED_RUNTIME_UID:
+        raise GuardError(
+            f"guard must run as UID {EXPECTED_RUNTIME_UID}; observed {uid}"
+        )
+    podman_info = parse_json_output(
+        run(["podman", "info", "--format", "json"], deadline=deadline),
+        "Podman info",
+    )
+    host = podman_info.get("host") or {}
+    security = host.get("security") or {}
+    rootless = security.get("rootless")
+    if rootless is not True and str(rootless).lower() != "true":
+        raise GuardError("direct Podman runtime is not rootless")
+    store = podman_info.get("store") or {}
+    remote_socket = (host.get("remoteSocket") or {}).get("path")
+    if remote_socket and remote_socket not in EXPECTED_RUNTIME_ENDPOINT:
+        raise GuardError("direct Podman runtime reports a different API socket")
+    docker_info = parse_json_output(
+        run(["docker", "info", "--format", "{{json .}}"], deadline=deadline),
+        "Docker-compatible runtime info",
+    )
+    graph_root = store.get("graphRoot")
+    docker_root = docker_info.get("DockerRootDir")
+    if graph_root and docker_root and graph_root != docker_root:
+        raise GuardError(
+            "Docker Compose and direct Podman report different container stores"
+        )
+    return {
+        "endpoint": EXPECTED_RUNTIME_ENDPOINT,
+        "uid": uid,
+        "podman": {
+            "rootless": True,
+            "graph_root": graph_root,
+            "run_root": store.get("runRoot"),
+            "remote_socket": remote_socket,
+        },
+        "compose": {
+            "docker_host": EXPECTED_RUNTIME_ENDPOINT,
+            "server_version": docker_info.get("ServerVersion"),
+            "name": docker_info.get("Name"),
+            "docker_root_dir": docker_root,
+        },
+    }
+
+
 def capture(
     project_dir: Path, *, deadline: OperationDeadline | None = None
 ) -> dict[str, Any]:
@@ -575,8 +778,9 @@ def capture(
             "hostname": os.uname().nodename,
             "rootless_project": PROJECT,
         },
+        "runtime_context": runtime_context(deadline=deadline),
         "source": source_state(project_dir, deadline=deadline),
-        "job_quiescence": queue_snapshot(deadline),
+        "job_quiescence": settled_queue_snapshot(deadline),
         "containers": inspect_project(deadline=deadline),
     }
 
@@ -591,16 +795,21 @@ def service_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
     for container in snapshot.get("containers", []):
         service = container.get("compose", {}).get("com.docker.compose.service")
         if service:
+            if service in result:
+                raise GuardError(f"duplicate Compose service container in snapshot: {service}")
             result[service] = container
     return result
 
 
 def container_name_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
-    return {
-        container["name"]: container
-        for container in snapshot.get("containers", [])
-        if container.get("name")
-    }
+    result: dict[str, dict[str, Any]] = {}
+    for container in snapshot.get("containers", []):
+        name = container.get("name")
+        if name:
+            if name in result:
+                raise GuardError(f"duplicate container name in snapshot: {name}")
+            result[name] = container
+    return result
 
 
 def compare_preserved_runtime(
@@ -668,10 +877,13 @@ def compare_untargeted_runtime(
 def compare_source_and_quiescence(
     baseline: dict[str, Any], current: dict[str, Any]
 ) -> None:
+    if baseline.get("runtime_context") != current.get("runtime_context"):
+        raise GuardError("Compose and direct Podman runtime context changed")
     if baseline.get("source") != current.get("source"):
         raise GuardError("dirty live source state changed since baseline capture")
-    if not current.get("job_quiescence", {}).get("quiescent"):
-        raise GuardError("fresh queue or active-job quiescence check failed")
+    stability = current.get("job_quiescence", {}).get("stability") or {}
+    if not current.get("job_quiescence", {}).get("quiescent") or not stability.get("stable"):
+        raise GuardError("fresh queue or active-job settled-quiescence check failed")
 
 
 def verify_untouched_targets(
@@ -741,6 +953,11 @@ def load_baseline(path: Path) -> dict[str, Any]:
         raise GuardError("baseline source state is incomplete; capture a fresh baseline")
     if not isinstance(baseline.get("containers"), list):
         raise GuardError("baseline container snapshot is missing")
+    if not isinstance(baseline.get("runtime_context"), dict):
+        raise GuardError("baseline runtime context is missing; capture a fresh baseline")
+    stability = baseline.get("job_quiescence", {}).get("stability") or {}
+    if not stability.get("stable"):
+        raise GuardError("baseline quiescence is not settled; capture a fresh baseline")
     return baseline
 
 
@@ -842,10 +1059,52 @@ def compose_config(
     )
 
 
+def compose_environment(value: Any) -> dict[str, Any]:
+    if value is None:
+        return {}
+    if isinstance(value, list):
+        result: dict[str, Any] = {}
+        for item in value:
+            if not isinstance(item, str):
+                raise GuardError("Compose environment contains a non-string entry")
+            key, separator, item_value = item.partition("=")
+            result[key] = item_value if separator else None
+        return result
+    if isinstance(value, dict):
+        return {str(key): item for key, item in value.items()}
+    raise GuardError("Compose environment is not a mapping")
+
+
+def compose_value_hash(value: Any) -> dict[str, Any]:
+    if value is None:
+        raise GuardError("Compose environment contains a host-inherited value")
+    text = str(value)
+    return {"length": len(text), "sha256": sha256_bytes(text.encode())}
+
+
+def check_health_contract(
+    service: str, value: dict[str, Any], *, runtime: bool = False
+) -> None:
+    expected = health_contract(service)
+    test = value.get("test") if not runtime else value.get("test")
+    if test != expected["test"]:
+        raise GuardError(f"healthcheck command identity mismatch for {service}")
+    for field in ("interval_ns", "timeout_ns", "start_period_ns", "retries"):
+        actual = value.get(field)
+        if runtime and field.endswith("_ns") and actual is None:
+            # Older captures did not include normalized timing fields.  A
+            # fresh capture is required because timing identity is part of the
+            # deployment contract.
+            raise GuardError(f"healthcheck timing identity is missing for {service}")
+        if actual != expected[field]:
+            raise GuardError(f"healthcheck {field} mismatch for {service}")
+
+
 def check_candidate_config(
     config: dict[str, Any],
     baseline: dict[str, Any],
     lock: dict[str, Any],
+    baseline_config: dict[str, Any] | None = None,
 ) -> None:
     services = config.get("services") or {}
     missing = set(TARGET_SERVICES) - set(services)
@@ -874,9 +1133,38 @@ def check_candidate_config(
         if EXPECTED_NETWORK not in networks and "default" not in networks:
             raise GuardError(f"candidate service {service} leaves {EXPECTED_NETWORK}")
         healthcheck = candidate.get("healthcheck") or {}
+        unexpected_health_fields = set(healthcheck) - {
+            "test",
+            "interval",
+            "timeout",
+            "start_period",
+            "retries",
+        }
+        if unexpected_health_fields:
+            raise GuardError(
+                f"candidate healthcheck has unsupported fields for {service}: "
+                f"{sorted(unexpected_health_fields)}"
+            )
         health_test = healthcheck.get("test") or []
         if not health_test or health_test == ["NONE"]:
             raise GuardError(f"candidate healthcheck is missing for {service}")
+        check_health_contract(
+            service,
+            {
+                "test": health_test,
+                "interval_ns": duration_ns(healthcheck.get("interval")),
+                "timeout_ns": duration_ns(healthcheck.get("timeout")),
+                "start_period_ns": duration_ns(healthcheck.get("start_period")),
+                "retries": healthcheck.get("retries"),
+            },
+        )
+        probe_mounts = [
+            mount
+            for mount in candidate.get("volumes", [])
+            if mount.get("target") == PROBE_MOUNT_TARGET
+        ]
+        if len(probe_mounts) != 1 or probe_mounts[0].get("read_only") is not True:
+            raise GuardError(f"candidate trusted probe mount is missing or writable for {service}")
         old_mounts = {
             mount["destination"]: mount
             for mount in old[service].get("mounts", [])
@@ -902,45 +1190,43 @@ def check_candidate_config(
                     f"candidate changed {service} mount source for {destination}: "
                     f"{new_source} != {old_source}"
                 )
-            if old_mount.get("rw") is not None and new_mount.get("read_only") is not None:
-                if bool(old_mount["rw"]) == bool(new_mount["read_only"]):
-                    raise GuardError(f"candidate changed mount access for {service} {destination}")
+            candidate_read_only = new_mount.get("read_only", False)
+            if not isinstance(candidate_read_only, bool):
+                raise GuardError(
+                    f"candidate mount access is not a boolean for {service} {destination}"
+                )
+            if old_mount.get("rw") is not None:
+                expected_read_only = not bool(old_mount["rw"])
+                if candidate_read_only != expected_read_only:
+                    raise GuardError(
+                        f"candidate changed mount access for {service} {destination}"
+                    )
         if set(candidate_mounts) != set(old_mounts):
             raise GuardError(f"candidate changed data mounts for {service}")
 
-        candidate_environment = candidate.get("environment") or {}
-        if isinstance(candidate_environment, list):
-            candidate_environment = {
-                item.partition("=")[0]: item.partition("=")[2]
-                for item in candidate_environment
-                if isinstance(item, str)
-            }
-        if not isinstance(candidate_environment, dict):
-            raise GuardError(f"candidate environment is not a mapping for {service}")
-        old_environment = old[service].get("env_hashes") or {}
+        candidate_environment = compose_environment(candidate.get("environment"))
+        authored_services = (baseline_config or {}).get("services") or {}
+        if service not in authored_services:
+            raise GuardError(f"baseline Compose environment is missing for {service}")
+        old_authored_environment = compose_environment(
+            authored_services[service].get("environment")
+        )
         allowed_additions = ALLOWED_ENV_ADDITIONS.get(service, set())
+        for key, value in old_authored_environment.items():
+            if key not in candidate_environment:
+                raise GuardError(f"candidate removed authored environment for {service}: {key}")
+            if compose_value_hash(candidate_environment[key]) != compose_value_hash(value):
+                raise GuardError(f"candidate changed environment for {service}: {key}")
         for key, value in candidate_environment.items():
-            if value is None:
-                # Compose's null means "inherit from the host".  It is not a
-                # deterministic deployment contract and cannot be preflighted.
-                raise GuardError(f"candidate environment is host-inherited for {service}")
-            value_hash = {
-                "length": len(str(value)),
-                "sha256": sha256_bytes(str(value).encode()),
-            }
-            if key in old_environment and old_environment[key] != value_hash:
-                raise GuardError(f"candidate environment changed for {service}: {key}")
-            if key not in old_environment and key not in allowed_additions:
+            if key not in old_authored_environment and key not in allowed_additions:
                 raise GuardError(f"candidate added environment for {service}: {key}")
-        # Compose config does not include image-provided defaults such as PATH,
-        # while the runtime snapshot does.  The post-recreation inspect still
-        # compares the complete environment contract; only the two declared
-        # heartbeat additions are permitted.
 
 
 def compare_baseline_current(
     baseline: dict[str, Any], current: dict[str, Any], *, allow_new_probe: bool
 ) -> None:
+    if baseline.get("runtime_context") != current.get("runtime_context"):
+        raise GuardError("Compose and direct Podman runtime context changed")
     old = service_map(baseline)
     new = service_map(current)
     for service in TARGET_SERVICES:
@@ -973,8 +1259,9 @@ def compare_baseline_current(
             raise GuardError(f"runtime mounts/options drifted for {service}")
         if not previous.get("state", {}).get("running"):
             raise GuardError(f"baseline service {service} was not running")
-    if not current.get("job_quiescence", {}).get("quiescent"):
-        raise GuardError("fresh job quiescence check failed")
+    stability = current.get("job_quiescence", {}).get("stability") or {}
+    if not current.get("job_quiescence", {}).get("quiescent") or not stability.get("stable"):
+        raise GuardError("fresh job quiescence check did not remain settled")
 
 
 def compare_post_apply(
@@ -1021,12 +1308,15 @@ def compare_post_apply(
             raise GuardError(f"post-apply environment additions changed for {service}")
         if not actual["state"].get("running"):
             raise GuardError(f"post-apply service is not running: {service}")
-        if not (actual.get("health", {}).get("configured") or {}).get("configured"):
+        actual_health = actual.get("health", {}).get("configured") or {}
+        if not actual_health.get("configured"):
             raise GuardError(f"post-apply healthcheck is not configured: {service}")
+        check_health_contract(service, actual_health, runtime=True)
         if actual["health"]["runtime"].get("status") != "healthy":
             raise GuardError(f"post-apply service is not healthy: {service}")
-    if not current.get("job_quiescence", {}).get("quiescent"):
-        raise GuardError("post-apply queue snapshot is not quiescent")
+    stability = current.get("job_quiescence", {}).get("stability") or {}
+    if not current.get("job_quiescence", {}).get("quiescent") or not stability.get("stable"):
+        raise GuardError("post-apply queue snapshot is not settled")
 
 
 def require_clean_candidate_source(
@@ -1079,6 +1369,7 @@ def advisory_lock(
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         text=True,
+        env=runtime_command_env(["podman"], os.environ.copy()),
     )
     assert process.stdin is not None
     assert process.stdout is not None
@@ -1304,6 +1595,8 @@ def commit_backups(
             raise GuardError(f"backup image has no immutable ID for {service}")
         image = image_rows[0]
         digest = image_digest(image)
+        if not digest:
+            raise GuardError(f"backup image has no immutable digest for {service}")
         backup_images["services"][service] = {
             "reference": tag,
             "id": image["Id"],
@@ -1392,7 +1685,14 @@ def prepare(
         image_override=image_override,
         deadline=operation_deadline,
     )
-    check_candidate_config(config, baseline, lock)
+    authored_baseline = compose_config(
+        Path(args.project_dir),
+        tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
+        candidate_version=lock["candidate_version"],
+        probe_source=Path(args.probe_source),
+        deadline=operation_deadline,
+    )
+    check_candidate_config(config, baseline, lock, authored_baseline)
     return baseline, lock, current
 
 
@@ -1497,7 +1797,9 @@ def verify_rollback_result(
             raise GuardError(f"rollback service is missing: {service}")
         if actual.get("image", {}).get("id") != record.get("id"):
             raise GuardError(f"rollback backup image ID mismatch for {service}")
-        if record.get("digest") and actual.get("image", {}).get("digest") != record.get("digest"):
+        if not record.get("digest"):
+            raise GuardError(f"rollback backup image digest is missing for {service}")
+        if actual.get("image", {}).get("digest") != record.get("digest"):
             raise GuardError(f"rollback backup image digest mismatch for {service}")
         compare_preserved_runtime(previous, actual, service)
         old_health = previous.get("health") or {}
@@ -1587,7 +1889,19 @@ def apply_batch(
             image_override=image_override,
             deadline=operation_deadline,
         )
-        check_candidate_config(config, baseline, lock)
+        authored_baseline = compose_config(
+            Path(args.project_dir),
+            compose_files,
+            candidate_version=lock["candidate_version"],
+            probe_source=Path(args.probe_source),
+            deadline=operation_deadline,
+        )
+        check_candidate_config(config, baseline, lock, authored_baseline)
+        # Take the final settled sample after all preflight commands and
+        # immediately before the targeted Compose mutation.
+        final_quiescence = settled_queue_snapshot(operation_deadline)
+        if not final_quiescence.get("stability", {}).get("stable"):
+            raise GuardError("queue was not settled immediately before targeted recreation")
         targeted_up(
             Path(args.project_dir),
             compose_files,
@@ -1749,6 +2063,10 @@ def command_rollback(args: argparse.Namespace) -> int:
         raise GuardError("backup image manifest has an unsupported schema")
     if set(backup_images.get("services") or {}) != set(TARGET_SERVICES):
         raise GuardError("backup image manifest does not cover the exact target set")
+    for service in TARGET_SERVICES:
+        record = backup_service_record(backup_images, service)
+        if not record.get("id") or not record.get("digest"):
+            raise GuardError(f"backup image manifest lacks immutable identity for {service}")
     replacement_path = backup_dir / "replacements.json"
     if not replacement_path.exists():
         raise GuardError("rollback requires the exact replacement ID manifest")
diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
index 1d9e980d1b..69414ea625 100644
--- a/docs/train-unstract-health-deployment.md
+++ b/docs/train-unstract-health-deployment.md
@@ -29,9 +29,12 @@ container is intentionally excluded.
 
 ## Candidate preparation
 
-Build the six changed images from a clean detached worktree at the exact
-integrated source commit. Run this on an external builder or disposable build
-host; do not build from the dirty Train checkout:
+Build the two changed application images from a clean detached worktree at the
+exact integrated source commit. The runner source and shared worker source are
+the only image-bearing changes. Runner and the twelve worker services use those
+new image builds; the eleven core services use immutable image IDs/digests
+already captured from the live stack. Run this on an external builder or
+disposable build host; do not build from the dirty Train checkout:
 
 ```sh
 CANDIDATE_COMMIT="$(git rev-parse HEAD)"
@@ -40,7 +43,7 @@ CANDIDATE_VERSION="goal09-${CANDIDATE_SHORT}"
 git worktree add --detach /var/tmp/unstract-goal09 "$CANDIDATE_COMMIT"
 cd /var/tmp/unstract-goal09
 VERSION="$CANDIDATE_VERSION" docker compose -f docker/docker-compose.build.yaml build --pull never \
-  backend frontend runner platform-service x2text-service worker-unified
+  runner worker-unified
 ```
 
 The static database, broker, storage, proxy, and vector images are pinned by
@@ -55,9 +58,9 @@ python3 docker/scripts/train_health_deployment_guard.py lock \
   --candidate-version "$CANDIDATE_VERSION" \
   --output /run/user/1000/unstract-goal09/candidate-lock.json \
   --image runner=localhost/unstract/runner:"$CANDIDATE_VERSION" \
-  --image backend=localhost/unstract/backend:"$CANDIDATE_VERSION" \
-  --image frontend=localhost/unstract/frontend:"$CANDIDATE_VERSION" \
-  --image platform-service=localhost/unstract/platform-service:"$CANDIDATE_VERSION" \
+  --image backend=localhost/unstract/backend:all-active-prs-20260903-oauthfix-3a273af \
+  --image frontend=localhost/unstract/frontend:all-active-prs-20260903-oauthfix-a4da8ff \
+  --image platform-service=localhost/unstract/platform-service:all-active-prs-20260903-7b95921 \
   --image worker-pg-orchestrator-api=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
   --image worker-pg-orchestrator-general=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
   --image worker-pg-fileproc=localhost/unstract/worker-unified:"$CANDIDATE_VERSION" \
@@ -77,7 +80,7 @@ python3 docker/scripts/train_health_deployment_guard.py lock \
   --image qdrant=docker.io/qdrant/qdrant:v1.16.1 \
   --image rabbitmq=docker.io/library/rabbitmq:4.1.0-management \
   --image weaviate=docker.io/semitechnologies/weaviate:1.39.2 \
-  --image x2text-service=localhost/unstract/x2text-service:"$CANDIDATE_VERSION"
+  --image x2text-service=localhost/unstract/x2text-service:all-active-prs-20260903-7b95921
 ```
 
 The final lock must include exactly one mapping for every target service and
@@ -87,7 +90,10 @@ deliberate static image change has separately been reviewed. The lock also
 contains the candidate commit's tree hash and hashes for the two overlays, the
 core and database probes, and the development essentials Compose file. The
 guard writes a temporary image override from this lock, so Compose cannot
-silently resolve a different registry or tag.
+silently resolve a different registry or tag. Only `runner` and the twelve
+worker services point at the new build; backend, frontend, platform-service,
+x2text-service, and the seven core data services point at the captured static
+references.
 
 Stage only the committed guard, overlays, and probe into a separate directory
 such as `/run/user/1000/unstract-goal09/source`; never copy over the live
@@ -138,12 +144,13 @@ python3 docker/scripts/train_health_deployment_guard.py preflight \
 
 Preflight refuses a changed dirty-path list, container name, persistent mount
 source or access mode, network, candidate image ID/digest, source commit, or
-artifact hash. It also refuses missing images and any service outside the
-fixed target set. The mutation phase repeats the runtime identity and queue
-checks while holding the PostgreSQL advisory deployment lock. The lock
-serializes competing guarded deployments; the queue counts are rechecked
-immediately before each batch because independently running application
-producers cannot be stopped by an advisory lock.
+artifact hash. It also binds Compose and direct Podman to the rootless
+`/run/user/1000/podman/podman.sock` context, rejects duplicate service labels,
+and checks the exact trusted health command and timing for every target. It
+refuses missing images and any service outside the fixed target set. Each queue
+capture requires three consecutive zero-work samples at two-second intervals;
+the mutation phase takes a final settled sample immediately before each
+targeted Compose change while holding the PostgreSQL advisory deployment lock.
 
 After a fresh quiescence check and automatic `podman commit` backup of each
 target container, `apply` recreates only the worker batch, waits for every

From 002ad66464750b8e67f22a823c4d539140ffbea1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 16:42:58 -0400
Subject: [PATCH 26/48] Allow worker curl healthchecks without probe mounts

---
 .../scripts/train_health_deployment_guard.py  | 17 ++--
 .../test_train_health_deployment_guard.py     | 95 +++++++++++++++++++
 2 files changed, 105 insertions(+), 7 deletions(-)
 create mode 100644 tests/healthchecks/test_train_health_deployment_guard.py

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 999f72ba1e..85a9d4170f 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -1158,13 +1158,16 @@ def check_candidate_config(
                 "retries": healthcheck.get("retries"),
             },
         )
-        probe_mounts = [
-            mount
-            for mount in candidate.get("volumes", [])
-            if mount.get("target") == PROBE_MOUNT_TARGET
-        ]
-        if len(probe_mounts) != 1 or probe_mounts[0].get("read_only") is not True:
-            raise GuardError(f"candidate trusted probe mount is missing or writable for {service}")
+        if service in CORE_SERVICES:
+            probe_mounts = [
+                mount
+                for mount in candidate.get("volumes", [])
+                if mount.get("target") == PROBE_MOUNT_TARGET
+            ]
+            if len(probe_mounts) != 1 or probe_mounts[0].get("read_only") is not True:
+                raise GuardError(
+                    f"candidate trusted probe mount is missing or writable for {service}"
+                )
         old_mounts = {
             mount["destination"]: mount
             for mount in old[service].get("mounts", [])
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
new file mode 100644
index 0000000000..3d48776d5c
--- /dev/null
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import importlib.util
+from copy import deepcopy
+from pathlib import Path
+
+import pytest
+
+GUARD_PATH = Path(__file__).parents[2] / "docker/scripts/train_health_deployment_guard.py"
+SPEC = importlib.util.spec_from_file_location("train_health_deployment_guard", GUARD_PATH)
+assert SPEC and SPEC.loader
+guard = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(guard)
+
+
+def candidate_fixture() -> tuple[dict, dict, dict, dict]:
+    baseline = {"containers": []}
+    config = {
+        "networks": {"default": {"name": guard.EXPECTED_NETWORK}},
+        "services": {},
+    }
+    authored = {"services": {}}
+    lock = {"images": {}}
+    start_periods = {
+        "db": "30s",
+        "redis": "15s",
+        "minio": "60s",
+        "reverse-proxy": "60s",
+        "qdrant": "60s",
+        "rabbitmq": "60s",
+        "weaviate": "120s",
+        "x2text-service": "120s",
+        "platform-service": "120s",
+        "backend": "180s",
+        "frontend": "60s",
+    }
+    for service in guard.TARGET_SERVICES:
+        name = f"unstract-{service}"
+        baseline["containers"].append(
+            {
+                "compose": {"com.docker.compose.service": service},
+                "name": name,
+                "mounts": [],
+                "env_hashes": {},
+            }
+        )
+        healthcheck = {
+            "test": guard._probe_test(service),
+            "interval": "30s",
+            "timeout": "10s" if service in guard.CORE_SERVICES else "5s",
+            "start_period": start_periods.get(service, "30s"),
+            "retries": 3,
+        }
+        volumes = (
+            [
+                {
+                    "type": "bind",
+                    "source": "/staged/unstract-services.sh",
+                    "target": guard.PROBE_MOUNT_TARGET,
+                    "read_only": True,
+                }
+            ]
+            if service in guard.CORE_SERVICES
+            else []
+        )
+        service_config = {
+            "image": f"candidate/{service}",
+            "container_name": name,
+            "networks": {guard.EXPECTED_NETWORK: {}},
+            "volumes": volumes,
+            "environment": {},
+            "healthcheck": healthcheck,
+        }
+        config["services"][service] = service_config
+        authored["services"][service] = deepcopy(service_config)
+        lock["images"][service] = {
+            "reference": f"candidate/{service}",
+            "id": f"id-{service}",
+            "digest": f"sha256:{service}",
+        }
+    return config, baseline, lock, authored
+
+
+def test_worker_curl_checks_do_not_require_probe_mount() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+
+    guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_core_probe_checks_require_read_only_probe_mount() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    config["services"]["db"]["volumes"] = []
+
+    with pytest.raises(guard.GuardError, match="trusted probe mount"):
+        guard.check_candidate_config(config, baseline, lock, authored)

From e16e1cb66bb45e37d71c77e4d366154a24bb20ba Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 17:57:52 -0400
Subject: [PATCH 27/48] Track all guarded health deployment artifacts

---
 docker/compose.train.worker-healthchecks.yaml | 97 +++++++++++++++++++
 .../test_train_health_artifact_archive.py     | 49 ++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 docker/compose.train.worker-healthchecks.yaml
 create mode 100644 tests/healthchecks/test_train_health_artifact_archive.py

diff --git a/docker/compose.train.worker-healthchecks.yaml b/docker/compose.train.worker-healthchecks.yaml
new file mode 100644
index 0000000000..c53b440963
--- /dev/null
+++ b/docker/compose.train.worker-healthchecks.yaml
@@ -0,0 +1,97 @@
+# Worker and runner health coverage overlay for the dirty Train checkout.
+#
+# Apply this after docker/compose.train.yaml and before the core service
+# overlay.  It contains the same checks committed in docker-compose.yaml, so a
+# coordinator can stage the health configuration without checking out over
+# Train's local Compose and embedding changes.
+
+x-worker-healthcheck: &worker_healthcheck
+  interval: 30s
+  timeout: 5s
+  retries: 3
+  start_period: 30s
+
+services:
+  runner:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:5002/v1/api/health"]
+
+  worker-log-history-scheduler-v2:
+    environment:
+      LOG_HISTORY_SCHEDULER_HEALTH_PORT: "8092"
+      LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS: ${LOG_HISTORY_SCHEDULER_HEALTH_STALE_SECONDS:-120}
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8092/health"]
+
+  worker-pg-orchestrator-api:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-orchestrator-general:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-fileproc:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-callback:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-scheduler:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-metrics:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-log-stream-consumer:
+    environment:
+      LOG_STREAM_CONSUMER_HEALTH_PORT: "8091"
+      LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS: ${LOG_STREAM_CONSUMER_HEALTH_STALE_SECONDS:-15}
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8091/health"]
+
+  worker-pg-executor:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-ide-callback:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-notification:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8090/health"]
+
+  worker-pg-reaper:
+    healthcheck:
+      <<: *worker_healthcheck
+      test: ["CMD", "/usr/bin/curl", "--fail", "--silent", "--show-error",
+             "--max-time", "3", "http://127.0.0.1:8086/health"]
diff --git a/tests/healthchecks/test_train_health_artifact_archive.py b/tests/healthchecks/test_train_health_artifact_archive.py
new file mode 100644
index 0000000000..85946b3570
--- /dev/null
+++ b/tests/healthchecks/test_train_health_artifact_archive.py
@@ -0,0 +1,49 @@
+"""Ensure the guard's artifact inputs survive a clean Git checkout."""
+
+from __future__ import annotations
+
+import io
+import subprocess
+import tarfile
+from pathlib import Path
+
+ROOT = Path(__file__).parents[2]
+REQUIRED_ARTIFACTS = (
+    "docker/healthchecks/unstract-services.sh",
+    "docker/healthchecks/http-readiness.sh",
+    "docker/healthchecks/postgres-readiness.sh",
+    "docker/docker-compose-dev-essentials.yaml",
+    "docker/compose.train.healthchecks.yaml",
+    "docker/compose.train.worker-healthchecks.yaml",
+)
+
+
+def test_guard_artifacts_are_tracked_and_present_in_clean_git_archive(
+    tmp_path: Path,
+) -> None:
+    tracked = subprocess.run(
+        ["git", "ls-files", "--error-unmatch", "--", *REQUIRED_ARTIFACTS],
+        cwd=ROOT,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert tracked.returncode == 0, tracked.stderr
+
+    archive = subprocess.run(
+        ["git", "archive", "--format=tar", "HEAD", "--", *REQUIRED_ARTIFACTS],
+        cwd=ROOT,
+        check=False,
+        capture_output=True,
+    )
+    assert archive.returncode == 0, archive.stderr.decode(errors="replace")
+
+    with tarfile.open(fileobj=io.BytesIO(archive.stdout), mode="r:") as tar:
+        tar.extractall(tmp_path)
+
+    missing = [
+        path
+        for path in REQUIRED_ARTIFACTS
+        if not (tmp_path / path).is_file()
+    ]
+    assert not missing, f"clean Git archive omitted guarded artifacts: {missing}"

From 9f5f96d3c62b1ac5bead7e01192d9ae1c0ca2e85 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 18:18:37 -0400
Subject: [PATCH 28/48] Document Train Compose environment for guarded apply

---
 docs/train-unstract-health-deployment.md | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
index 69414ea625..c3f381b342 100644
--- a/docs/train-unstract-health-deployment.md
+++ b/docs/train-unstract-health-deployment.md
@@ -127,9 +127,13 @@ and are not mistaken for active jobs.
 
 Use the live dirty Compose files plus the staged overlays. This preserves the
 local embedding includes, port changes, env files, named volumes, bind mounts,
-container names, and network ownership:
+container names, and network ownership. The live `docker/.env` derives
+`TOOL_REGISTRY_CONFIG_SRC_PATH` from `PWD`, so preserve the Train Compose
+working-directory value while the guard still uses the project root for
+Compose:
 
 ```sh
+export PWD=/home/completetrain/etl.home.complete.tech/docker
 python3 docker/scripts/train_health_deployment_guard.py preflight \
   --baseline /run/user/1000/unstract-goal09/baseline.json \
   --project-dir /home/completetrain/etl.home.complete.tech \
@@ -137,6 +141,7 @@ python3 docker/scripts/train_health_deployment_guard.py preflight \
   --candidate-lock /run/user/1000/unstract-goal09/candidate-lock.json \
   --probe-source /run/user/1000/unstract-goal09/source/docker/healthchecks/unstract-services.sh \
   --compose-file docker/docker-compose.yaml \
+  --compose-file docker/compose.train.yaml \
   --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.worker-healthchecks.yaml \
   --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.healthchecks.yaml \
   --operation-timeout 1200
@@ -167,6 +172,7 @@ the DB advisory lock before each batch and immediately rechecks source hashes,
 untargeted container identities, queue quiescence, and candidate image IDs.
 
 ```sh
+export PWD=/home/completetrain/etl.home.complete.tech/docker
 python3 docker/scripts/train_health_deployment_guard.py apply \
   --confirm APPLY_UNSTRACT_HEALTH \
   --backup-dir /run/user/1000/unstract-goal09/backup \
@@ -176,6 +182,7 @@ python3 docker/scripts/train_health_deployment_guard.py apply \
   --candidate-lock /run/user/1000/unstract-goal09/candidate-lock.json \
   --probe-source /run/user/1000/unstract-goal09/source/docker/healthchecks/unstract-services.sh \
   --compose-file docker/docker-compose.yaml \
+  --compose-file docker/compose.train.yaml \
   --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.worker-healthchecks.yaml \
   --compose-file /run/user/1000/unstract-goal09/source/docker/compose.train.healthchecks.yaml \
   --operation-timeout 2400

From e639c7d6149399e0ff083eae40af850cdc7bd408 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 18:30:47 -0400
Subject: [PATCH 29/48] Normalize guarded Compose identities

---
 .../scripts/train_health_deployment_guard.py  |  90 ++++++---
 .../test_train_health_deployment_guard.py     | 171 ++++++++++++++++++
 2 files changed, 240 insertions(+), 21 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 85a9d4170f..b197958645 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -22,7 +22,9 @@
 import fcntl
 import hashlib
 import json
+import math
 import os
+import re
 import select
 import shlex
 import subprocess
@@ -30,6 +32,7 @@
 import tempfile
 import time
 from collections.abc import Iterator
+from decimal import Decimal, InvalidOperation
 from pathlib import Path
 from typing import Any
 
@@ -90,6 +93,18 @@
 SOURCE_STATE_SCHEMA = "unstract-source-state/v2"
 BACKUP_SCHEMA = "unstract-health-backup/v2"
 REPLACEMENT_SCHEMA = "unstract-health-replacements/v1"
+DURATION_TOKEN = re.compile(
+    r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
+)
+DURATION_UNITS_NS = {
+    "ns": 1,
+    "us": 1_000,
+    "µs": 1_000,
+    "ms": 1_000_000,
+    "s": 1_000_000_000,
+    "m": 60_000_000_000,
+    "h": 3_600_000_000_000,
+}
 
 # The two log consumers need these values to expose their source-level
 # heartbeat endpoints.  Every other environment value must survive a
@@ -205,32 +220,37 @@ def sha256_file(path: Path) -> str:
 
 
 def duration_ns(value: Any) -> int | None:
-    """Normalize Compose duration strings and Podman nanosecond values."""
+    """Normalize full Compose durations and Podman nanosecond values."""
     if value is None:
         return None
     if isinstance(value, bool):
         return None
     if isinstance(value, int):
-        return value
+        return value if value >= 0 else None
     if isinstance(value, float):
+        if not math.isfinite(value) or value < 0:
+            return None
         return round(value)
     text = str(value).strip().lower()
-    units = (
-        ("ns", 1),
-        ("us", 1_000),
-        ("µs", 1_000),
-        ("ms", 1_000_000),
-        ("s", 1_000_000_000),
-        ("m", 60_000_000_000),
-        ("h", 3_600_000_000_000),
-    )
-    for suffix, multiplier in units:
-        if text.endswith(suffix):
-            try:
-                return round(float(text[: -len(suffix)]) * multiplier)
-            except ValueError:
-                return None
-    return None
+    if text == "0":
+        return 0
+    position = 0
+    total = Decimal(0)
+    while position < len(text):
+        match = DURATION_TOKEN.match(text, position)
+        if match is None:
+            return None
+        try:
+            number = Decimal(match.group("number"))
+        except InvalidOperation:
+            return None
+        if not number.is_finite() or number < 0:
+            return None
+        total += number * DURATION_UNITS_NS[match.group("unit")]
+        if not total.is_finite():
+            return None
+        position = match.end()
+    return int(total.to_integral_value()) if position else None
 
 
 def runtime_command_env(
@@ -360,16 +380,38 @@ def health_runtime(value: dict[str, Any] | None) -> dict[str, Any]:
 
 
 def normalize_mount(mount: dict[str, Any]) -> dict[str, Any]:
+    source = mount.get("Source")
+    if mount.get("Type") != "volume":
+        source = normalize_bind_mount_source(source)
     return {
         "type": mount.get("Type"),
         "name": mount.get("Name"),
-        "source": mount.get("Source"),
+        "source": source,
         "destination": mount.get("Destination"),
         "rw": mount.get("RW"),
         "options": sorted(mount.get("Options") or []),
     }
 
 
+def normalize_bind_mount_source(value: Any) -> Any:
+    """Compare bind sources by absolute lexical identity without resolving symlinks."""
+    if not isinstance(value, str) or not value:
+        return value
+    return os.path.normpath(os.path.abspath(value))
+
+
+def compose_mount_source(config: dict[str, Any], mount: dict[str, Any]) -> Any:
+    """Resolve a Compose volume alias to its project-scoped name."""
+    source = mount.get("source")
+    if mount.get("type") != "volume" or not isinstance(source, str):
+        return source
+    volumes = config.get("volumes") or {}
+    definition = volumes.get(source)
+    if isinstance(definition, dict):
+        return definition.get("name") or source
+    return source
+
+
 def normalize_networks(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
     """Keep stable network identity while omitting replacement-specific IPs."""
     result: dict[str, dict[str, Any]] = {}
@@ -1187,8 +1229,14 @@ def check_candidate_config(
                 if old_mount.get("type") == "volume"
                 else old_mount.get("source")
             )
-            new_source = new_mount.get("source")
-            if old_source and new_source and old_source != new_source:
+            new_source = compose_mount_source(config, new_mount)
+            if old_mount.get("type") == "volume" or new_mount.get("type") == "volume":
+                sources_match = old_source == new_source
+            else:
+                sources_match = normalize_bind_mount_source(
+                    old_source
+                ) == normalize_bind_mount_source(new_source)
+            if old_source and new_source and not sources_match:
                 raise GuardError(
                     f"candidate changed {service} mount source for {destination}: "
                     f"{new_source} != {old_source}"
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 3d48776d5c..fa4cf073c8 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -87,9 +87,180 @@ def test_worker_curl_checks_do_not_require_probe_mount() -> None:
     guard.check_candidate_config(config, baseline, lock, authored)
 
 
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("1m0s", 60_000_000_000),
+        ("2m0s", 120_000_000_000),
+        ("3m0s", 180_000_000_000),
+        ("1.5s", 1_500_000_000),
+        ("250ms", 250_000_000),
+        ("1h2m3.25s", 3_723_250_000_000),
+        (30_000_000_000, 30_000_000_000),
+    ],
+)
+def test_duration_parser_accepts_full_compose_values(value, expected) -> None:
+    assert guard.duration_ns(value) == expected
+
+
+@pytest.mark.parametrize(
+    "value",
+    ["", "1mtrailing", "1e3s", "-1s", "nan", "inf", -1, float("nan")],
+)
+def test_duration_parser_rejects_invalid_or_nonfinite_values(value) -> None:
+    assert guard.duration_ns(value) is None
+
+
 def test_core_probe_checks_require_read_only_probe_mount() -> None:
     config, baseline, lock, authored = candidate_fixture()
     config["services"]["db"]["volumes"] = []
 
     with pytest.raises(guard.GuardError, match="trusted probe mount"):
         guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_bind_mount_source_accepts_lexical_parent_segment() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    source = "/home/completetrain/etl.home.complete.tech/unstract/tool-registry"
+    db_container = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"] == "db"
+    )
+    db_container["mounts"] = [
+        {
+            "type": "bind",
+            "source": source,
+            "destination": "/data/tool_registry_config",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    trusted_probe = {
+        "type": "bind",
+        "source": "/staged/unstract-services.sh",
+        "target": guard.PROBE_MOUNT_TARGET,
+        "read_only": True,
+    }
+    config["services"]["db"]["volumes"] = [
+        trusted_probe,
+        {
+            "type": "bind",
+            "source": "/home/completetrain/etl.home.complete.tech/docker/../unstract/tool-registry",
+            "target": "/data/tool_registry_config",
+            "read_only": False,
+        }
+    ]
+    authored["services"]["db"]["volumes"] = config["services"]["db"]["volumes"]
+
+    guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_bind_mount_source_rejects_distinct_path() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    db_container = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"] == "db"
+    )
+    db_container["mounts"] = [
+        {
+            "type": "bind",
+            "source": "/home/completetrain/etl.home.complete.tech/unstract/tool-registry",
+            "destination": "/data/tool_registry_config",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    trusted_probe = {
+        "type": "bind",
+        "source": "/staged/unstract-services.sh",
+        "target": guard.PROBE_MOUNT_TARGET,
+        "read_only": True,
+    }
+    config["services"]["db"]["volumes"] = [
+        trusted_probe,
+        {
+            "type": "bind",
+            "source": "/home/completetrain/etl.home.complete.tech/other/tool-registry",
+            "target": "/data/tool_registry_config",
+            "read_only": False,
+        }
+    ]
+    authored["services"]["db"]["volumes"] = config["services"]["db"]["volumes"]
+
+    with pytest.raises(guard.GuardError, match="mount source"):
+        guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_named_volume_alias_resolves_to_live_name() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    volume_name = "unstract-etl-home-complete-tech_prompt_studio_data"
+    executor = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"]
+        == "worker-pg-executor"
+    )
+    executor["mounts"] = [
+        {
+            "type": "volume",
+            "name": volume_name,
+            "source": f"/var/lib/containers/storage/volumes/{volume_name}/_data",
+            "destination": "/app/prompt-studio-data",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    config["volumes"] = {"prompt_studio_data": {"name": volume_name}}
+    config["services"]["worker-pg-executor"]["volumes"] = [
+        {
+            "type": "volume",
+            "source": "prompt_studio_data",
+            "target": "/app/prompt-studio-data",
+            "read_only": False,
+        }
+    ]
+    authored["services"]["worker-pg-executor"]["volumes"] = config["services"][
+        "worker-pg-executor"
+    ]["volumes"]
+
+    guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_named_volume_alias_rejects_wrong_live_name() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    volume_name = "unstract-etl-home-complete-tech_prompt_studio_data"
+    executor = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"]
+        == "worker-pg-executor"
+    )
+    executor["mounts"] = [
+        {
+            "type": "volume",
+            "name": volume_name,
+            "source": f"/var/lib/containers/storage/volumes/{volume_name}/_data",
+            "destination": "/app/prompt-studio-data",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    config["volumes"] = {
+        "prompt_studio_data": {"name": "other_prompt_studio_data"}
+    }
+    config["services"]["worker-pg-executor"]["volumes"] = [
+        {
+            "type": "volume",
+            "source": "prompt_studio_data",
+            "target": "/app/prompt-studio-data",
+            "read_only": False,
+        }
+    ]
+    authored["services"]["worker-pg-executor"]["volumes"] = config["services"][
+        "worker-pg-executor"
+    ]["volumes"]
+
+    with pytest.raises(guard.GuardError, match="mount source"):
+        guard.check_candidate_config(config, baseline, lock, authored)

From bf728a740f4dd208a878358aba3bc98ebf8fbfd1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 18:45:02 -0400
Subject: [PATCH 30/48] Harden guarded bind mount identity

---
 .../scripts/train_health_deployment_guard.py  | 41 ++++++++--
 .../test_train_health_deployment_guard.py     | 80 +++++++++++++++++++
 2 files changed, 116 insertions(+), 5 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index b197958645..dc4b880261 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -394,10 +394,39 @@ def normalize_mount(mount: dict[str, Any]) -> dict[str, Any]:
 
 
 def normalize_bind_mount_source(value: Any) -> Any:
-    """Compare bind sources by absolute lexical identity without resolving symlinks."""
+    """Return a bind source's filesystem-aware absolute identity.
+
+    Resolve symlinks before comparing paths so a parent segment such as
+    ``docker/../data`` cannot hide a symlink that redirects the bind source.
+    Keep relative-path handling separate from ``abspath`` because ``abspath``
+    normalizes ``..`` before symlink resolution.
+    """
     if not isinstance(value, str) or not value:
         return value
-    return os.path.normpath(os.path.abspath(value))
+    if not os.path.isabs(value):
+        value = os.path.join(os.getcwd(), value)
+    return os.path.realpath(value)
+
+
+def bind_mount_sources_match(left: Any, right: Any) -> bool:
+    """Compare bind sources while failing closed on uncertain filesystem state."""
+    left_path = normalize_bind_mount_source(left)
+    right_path = normalize_bind_mount_source(right)
+    if not isinstance(left_path, str) or not isinstance(right_path, str):
+        return False
+    try:
+        left_exists = os.path.exists(left_path)
+        right_exists = os.path.exists(right_path)
+    except OSError:
+        return False
+    if left_exists != right_exists:
+        return False
+    if left_exists:
+        try:
+            return os.path.samefile(left_path, right_path)
+        except OSError:
+            return False
+    return left_path == right_path
 
 
 def compose_mount_source(config: dict[str, Any], mount: dict[str, Any]) -> Any:
@@ -1224,6 +1253,10 @@ def check_candidate_config(
             new_mount = candidate_mounts.get(destination)
             if not new_mount:
                 raise GuardError(f"candidate removed {service} mount {destination}")
+            if old_mount.get("type") != new_mount.get("type"):
+                raise GuardError(
+                    f"candidate changed {service} mount type for {destination}"
+                )
             old_source = (
                 old_mount.get("name") or old_mount.get("source")
                 if old_mount.get("type") == "volume"
@@ -1233,9 +1266,7 @@ def check_candidate_config(
             if old_mount.get("type") == "volume" or new_mount.get("type") == "volume":
                 sources_match = old_source == new_source
             else:
-                sources_match = normalize_bind_mount_source(
-                    old_source
-                ) == normalize_bind_mount_source(new_source)
+                sources_match = bind_mount_sources_match(old_source, new_source)
             if old_source and new_source and not sources_match:
                 raise GuardError(
                     f"candidate changed {service} mount source for {destination}: "
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index fa4cf073c8..61cab689fd 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -193,6 +193,86 @@ def test_bind_mount_source_rejects_distinct_path() -> None:
         guard.check_candidate_config(config, baseline, lock, authored)
 
 
+def test_bind_mount_source_rejects_symlink_parent_to_distinct_physical_path(
+    tmp_path: Path,
+) -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    live_source = tmp_path / "data"
+    redirected_source = tmp_path / "other" / "data"
+    live_source.mkdir()
+    redirected_source.mkdir(parents=True)
+    symlink_parent = tmp_path / "docker"
+    symlink_parent.symlink_to(tmp_path / "other" / "nested", target_is_directory=True)
+    db_container = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"] == "db"
+    )
+    db_container["mounts"] = [
+        {
+            "type": "bind",
+            "source": str(live_source),
+            "destination": "/data/tool_registry_config",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    trusted_probe = {
+        "type": "bind",
+        "source": "/staged/unstract-services.sh",
+        "target": guard.PROBE_MOUNT_TARGET,
+        "read_only": True,
+    }
+    config["services"]["db"]["volumes"] = [
+        trusted_probe,
+        {
+            "type": "bind",
+            "source": str(symlink_parent / ".." / "data"),
+            "target": "/data/tool_registry_config",
+            "read_only": False,
+        },
+    ]
+    authored["services"]["db"]["volumes"] = config["services"]["db"]["volumes"]
+
+    with pytest.raises(guard.GuardError, match="mount source"):
+        guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_mount_type_change_rejected_even_when_source_matches() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    volume_name = "unstract-etl-home-complete-tech_prompt_studio_data"
+    executor = next(
+        container
+        for container in baseline["containers"]
+        if container["compose"]["com.docker.compose.service"]
+        == "worker-pg-executor"
+    )
+    executor["mounts"] = [
+        {
+            "type": "volume",
+            "name": volume_name,
+            "source": f"/var/lib/containers/storage/volumes/{volume_name}/_data",
+            "destination": "/app/prompt-studio-data",
+            "rw": True,
+            "options": [],
+        }
+    ]
+    config["services"]["worker-pg-executor"]["volumes"] = [
+        {
+            "type": "bind",
+            "source": volume_name,
+            "target": "/app/prompt-studio-data",
+            "read_only": False,
+        }
+    ]
+    authored["services"]["worker-pg-executor"]["volumes"] = config["services"][
+        "worker-pg-executor"
+    ]["volumes"]
+
+    with pytest.raises(guard.GuardError, match="mount type"):
+        guard.check_candidate_config(config, baseline, lock, authored)
+
+
 def test_named_volume_alias_resolves_to_live_name() -> None:
     config, baseline, lock, authored = candidate_fixture()
     volume_name = "unstract-etl-home-complete-tech_prompt_studio_data"

From f25e48cb18702e0b5ca9e9ad83306873c6cefb8c Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 19:11:24 -0400
Subject: [PATCH 31/48] Normalize recreated runtime identity

---
 .../scripts/train_health_deployment_guard.py  | 63 +++++++++++++++++--
 .../test_train_health_deployment_guard.py     | 52 +++++++++++++++
 2 files changed, 109 insertions(+), 6 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index dc4b880261..e8c2e10840 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -93,6 +93,7 @@
 SOURCE_STATE_SCHEMA = "unstract-source-state/v2"
 BACKUP_SCHEMA = "unstract-health-backup/v2"
 REPLACEMENT_SCHEMA = "unstract-health-replacements/v1"
+FAILURE_SCHEMA = "unstract-health-failure/v1"
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
 )
@@ -200,6 +201,21 @@ class GuardError(RuntimeError):
     """A precondition or postcondition failed."""
 
 
+def exception_reason(error: BaseException, *, limit: int = 240) -> str:
+    """Return bounded, single-line failure context without secret-looking values."""
+    text = " ".join(str(error).split())
+    text = re.sub(
+        r"(?i)(password|passwd|secret|token|authorization|api[_-]?key)(\s*[=:]\s*)\S+",
+        r"\1\2<redacted>",
+        text,
+    )
+    if not text:
+        text = "<no detail>"
+    if len(text) > limit:
+        text = text[: limit - 3] + "..."
+    return f"{type(error).__name__}: {text}"
+
+
 def utc_now() -> str:
     return dt.datetime.now(dt.UTC).isoformat()
 
@@ -315,12 +331,19 @@ def image_digest(image: dict[str, Any]) -> str | None:
     return image.get("Digest") or next(iter(image.get("RepoDigests") or []), None)
 
 
-def env_hashes(values: list[str] | None) -> dict[str, dict[str, Any]]:
+def env_hashes(
+    values: list[str] | None, *, container_id: str | None = None
+) -> dict[str, dict[str, Any]]:
     result: dict[str, dict[str, Any]] = {}
     for item in values or []:
         key, separator, value = item.partition("=")
         if not separator:
             value = ""
+        # Podman injects HOSTNAME from the container ID. Recreated containers
+        # therefore receive a new value even when the application environment
+        # is unchanged. Keep an explicit custom HOSTNAME in the contract.
+        if key == "HOSTNAME" and container_id and value == container_id[:12]:
+            continue
         result[key] = {"length": len(value), "sha256": sha256_bytes(value.encode())}
     return dict(sorted(result.items()))
 
@@ -344,6 +367,12 @@ def health_config(value: dict[str, Any] | None) -> dict[str, Any]:
     if not value:
         return {"configured": False}
     test = value.get("Test") or []
+    # Compose's ``healthcheck: {test: ["NONE"]}`` disables a healthcheck.
+    # Podman reports that override as a Healthcheck object while an unchanged
+    # container reports no object at all; normalize both to the same state so
+    # rollback verification compares effective configuration.
+    if test == ["NONE"]:
+        return {"configured": False}
     return {
         "configured": True,
         # Healthcheck command vectors contain no credentials and are retained
@@ -441,12 +470,18 @@ def compose_mount_source(config: dict[str, Any], mount: dict[str, Any]) -> Any:
     return source
 
 
-def normalize_networks(value: dict[str, Any]) -> dict[str, dict[str, Any]]:
+def normalize_networks(
+    value: dict[str, Any], *, container_id: str | None = None
+) -> dict[str, dict[str, Any]]:
     """Keep stable network identity while omitting replacement-specific IPs."""
     result: dict[str, dict[str, Any]] = {}
+    generated_alias = container_id[:12] if container_id else None
     for name, network in sorted(value.items()):
+        aliases = network.get("Aliases") or []
+        if generated_alias:
+            aliases = [alias for alias in aliases if alias != generated_alias]
         result[name] = {
-            "aliases": sorted(network.get("Aliases") or []),
+            "aliases": sorted(aliases),
             "network_mode": network.get("NetworkID") or None,
             "driver_opts": network.get("DriverOpts") or {},
         }
@@ -536,7 +571,7 @@ def inspect_project(
                     "configured": health_config(config.get("Healthcheck")),
                     "runtime": health_runtime(state.get("Health")),
                 },
-                "env_hashes": env_hashes(config.get("Env")),
+                "env_hashes": env_hashes(config.get("Env"), container_id=item.get("Id")),
                 "mounts": [normalize_mount(mount) for mount in item.get("Mounts") or []],
                 "options": runtime_options(item, config),
                 "graphdriver": {
@@ -545,7 +580,7 @@ def inspect_project(
                     "work_dir": (item.get("GraphDriver") or {}).get("Data", {}).get("WorkDir"),
                 },
                 "networks": sorted(networks),
-                "network_details": normalize_networks(networks),
+                "network_details": normalize_networks(networks, container_id=item.get("Id")),
                 "user": config.get("User"),
                 "working_dir": config.get("WorkingDir"),
                 "rootless_runtime": item.get("OCIRuntime"),
@@ -2089,11 +2124,20 @@ def command_apply(args: argparse.Namespace) -> int:
                 compare_source_and_quiescence(baseline, final)
                 write_json(backup_dir / "post-apply.json", final)
             except Exception as exc:
+                failure_record = {
+                    "schema": FAILURE_SCHEMA,
+                    "original_error": exception_reason(exc),
+                }
+                try:
+                    write_json(backup_dir / "apply-failure.json", failure_record)
+                except OSError:
+                    pass
                 if backup_images is not None and attempted:
                     try:
                         failed_state = capture(
                             Path(args.project_dir), deadline=operation_deadline
                         )
+                        write_json(backup_dir / "failed-state.json", failed_state)
                         discovered = record_replacements(
                             baseline,
                             failed_state,
@@ -2121,9 +2165,16 @@ def command_apply(args: argparse.Namespace) -> int:
                                 operation_deadline=operation_deadline,
                             )
                     except Exception as rollback_error:
+                        failure_record["rollback_error"] = exception_reason(rollback_error)
+                        try:
+                            write_json(backup_dir / "apply-failure.json", failure_record)
+                        except OSError:
+                            pass
                         raise GuardError(
                             "guarded apply failed and compensating rollback failed; "
-                            f"manual recovery is required: {type(rollback_error).__name__}"
+                            "manual recovery is required; "
+                            f"original failure: {failure_record['original_error']}; "
+                            f"recovery failure: {failure_record['rollback_error']}"
                         ) from exc
                 raise
     print(f"apply: verified {len(TARGET_SERVICES)} targeted services; backup={backup_dir}")
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 61cab689fd..c4e6e2ea5f 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -111,6 +111,58 @@ def test_duration_parser_rejects_invalid_or_nonfinite_values(value) -> None:
     assert guard.duration_ns(value) is None
 
 
+def test_healthcheck_none_is_equivalent_to_no_healthcheck() -> None:
+    assert guard.health_config(None) == {"configured": False}
+    assert guard.health_config({"Test": ["NONE"]}) == {"configured": False}
+    assert guard.health_config({"Test": ["CMD", "true"]})["configured"] is True
+
+
+def test_generated_hostname_is_excluded_but_custom_hostname_is_retained() -> None:
+    container_id = "abcdef0123456789"
+    generated = guard.env_hashes(
+        ["HOSTNAME=abcdef012345", "APP_MODE=prod"], container_id=container_id
+    )
+    custom = guard.env_hashes(
+        ["HOSTNAME=worker-custom", "APP_MODE=prod"], container_id=container_id
+    )
+
+    assert "HOSTNAME" not in generated
+    assert "HOSTNAME" in custom
+
+
+def test_generated_network_alias_is_excluded_but_explicit_alias_is_retained() -> None:
+    container_id = "abcdef0123456789"
+    network = {
+        "unstract-network": {
+            "Aliases": [
+                "abcdef012345",
+                "abcdef012345-explicit",
+                "unstract-runner",
+            ],
+            "NetworkID": "network-id",
+            "DriverOpts": {},
+        }
+    }
+
+    normalized = guard.normalize_networks(network, container_id=container_id)
+
+    assert normalized["unstract-network"]["aliases"] == [
+        "abcdef012345-explicit",
+        "unstract-runner",
+    ]
+
+
+def test_exception_reason_is_bounded_and_redacts_secret_assignments() -> None:
+    reason = guard.exception_reason(
+        RuntimeError("token=private-value password: another-private-value " + "x" * 500)
+    )
+
+    assert "private-value" not in reason
+    assert "another-private-value" not in reason
+    assert "<redacted>" in reason
+    assert len(reason) < 260
+
+
 def test_core_probe_checks_require_read_only_probe_mount() -> None:
     config, baseline, lock, authored = candidate_fixture()
     config["services"]["db"]["volumes"] = []

From dcaba1312795efad3be0c1c362a2463cafd82ba5 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 20:16:49 -0400
Subject: [PATCH 32/48] Preserve runtime environment across guarded recreation

---
 .../scripts/train_health_deployment_guard.py  | 384 +++++++++++++++++-
 .../test_train_health_deployment_guard.py     | 105 +++++
 2 files changed, 476 insertions(+), 13 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index e8c2e10840..9e5afe8ef4 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -27,6 +27,7 @@
 import re
 import select
 import shlex
+import stat
 import subprocess
 import sys
 import tempfile
@@ -94,6 +95,9 @@
 BACKUP_SCHEMA = "unstract-health-backup/v2"
 REPLACEMENT_SCHEMA = "unstract-health-replacements/v1"
 FAILURE_SCHEMA = "unstract-health-failure/v1"
+RUNTIME_ENVIRONMENT_SCHEMA = "unstract-health-runtime-environment/v1"
+RUNTIME_ENVIRONMENT_FILENAME = "runtime-environment.override.yaml"
+RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
 )
@@ -348,6 +352,25 @@ def env_hashes(
     return dict(sorted(result.items()))
 
 
+def environment_values(values: list[str] | None) -> dict[str, str]:
+    """Parse an inspect environment vector without exposing its values."""
+    result: dict[str, str] = {}
+    for item in values or []:
+        if not isinstance(item, str):
+            continue
+        key, separator, value = item.partition("=")
+        result[key] = value if separator else ""
+    return result
+
+
+def environment_hashes(
+    values: dict[str, str], *, container_id: str | None = None
+) -> dict[str, dict[str, Any]]:
+    return env_hashes(
+        [f"{key}={value}" for key, value in values.items()], container_id=container_id
+    )
+
+
 def selected_labels(labels: dict[str, str]) -> dict[str, str]:
     keys = (
         "com.docker.compose.project",
@@ -589,6 +612,37 @@ def inspect_project(
     return containers
 
 
+def inspect_runtime_environment(
+    project: str = PROJECT, *, deadline: OperationDeadline | None = None
+) -> dict[str, dict[str, Any]]:
+    """Read raw environment values privately for reviewed Compose preservation."""
+    ids_result = run(
+        ["podman", "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}"],
+        deadline=deadline,
+    )
+    ids = ids_result.stdout.split()
+    if not ids:
+        return {}
+    raw = parse_json_output(
+        run(["podman", "inspect", *ids], deadline=deadline),
+        "podman inspect runtime environment",
+    )
+    result: dict[str, dict[str, Any]] = {}
+    for item in raw:
+        config = item.get("Config") or {}
+        labels = config.get("Labels") or {}
+        service = labels.get("com.docker.compose.service")
+        if not service:
+            continue
+        if service in result:
+            raise GuardError(f"duplicate Compose service environment: {service}")
+        result[service] = {
+            "id": item.get("Id"),
+            "values": environment_values(config.get("Env")),
+        }
+    return result
+
+
 def source_state(
     project_dir: Path, *, deadline: OperationDeadline | None = None
 ) -> dict[str, Any]:
@@ -1132,6 +1186,7 @@ def candidate_image_snapshot(
             "reference": expected["reference"],
             "id": row.get("Id") or row.get("ID"),
             "digest": image_digest(row),
+            "environment": environment_values((row.get("Config") or {}).get("Env")),
         }
         if actual["id"] != expected["id"] or actual["digest"] != expected["digest"]:
             raise GuardError(
@@ -1150,12 +1205,17 @@ def compose_config(
     candidate_version: str,
     probe_source: Path,
     image_override: Path | None = None,
+    environment_override: Path | None = None,
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     env = os.environ.copy()
     env["VERSION"] = candidate_version
     env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
-    files = compose_files + ((str(image_override),) if image_override else ())
+    files = compose_files
+    if image_override:
+        files += (str(image_override),)
+    if environment_override:
+        files += (str(environment_override),)
     args = ["docker", "compose"]
     for compose_file in files:
         args.extend(["-f", compose_file])
@@ -1181,6 +1241,90 @@ def compose_environment(value: Any) -> dict[str, Any]:
     raise GuardError("Compose environment is not a mapping")
 
 
+def candidate_environment_values(
+    config: dict[str, Any], service: str, image_values: dict[str, str]
+) -> dict[str, str]:
+    """Merge candidate image defaults with the effective Compose service env."""
+    service_config = (config.get("services") or {}).get(service) or {}
+    values = dict(image_values)
+    for key, value in compose_environment(service_config.get("environment")).items():
+        if value is None:
+            raise GuardError(f"candidate environment inherits host value for {service}: {key}")
+        values[str(key)] = str(value)
+    # Docker/Podman exposes an explicit Compose hostname through HOSTNAME.
+    # When no hostname is authored, the runtime-generated container ID is
+    # normalized out of the baseline and is deliberately omitted here.
+    hostname = service_config.get("hostname")
+    if hostname is not None:
+        values["HOSTNAME"] = str(hostname)
+    elif "HOSTNAME" not in image_values:
+        values.pop("HOSTNAME", None)
+    return values
+
+
+def plan_runtime_environment_override(
+    baseline: dict[str, Any],
+    runtime_environment: dict[str, dict[str, Any]],
+    candidate_images: dict[str, dict[str, Any]],
+    config: dict[str, Any],
+    *,
+    services: tuple[str, ...] = TARGET_SERVICES,
+) -> tuple[dict[str, dict[str, str]], dict[str, set[str]]]:
+    """Plan private Compose values that restore the baseline env contract."""
+    baseline_services = service_map(baseline)
+    overrides: dict[str, dict[str, str]] = {}
+    reviewed_keys: dict[str, set[str]] = {}
+    for service in services:
+        previous = baseline_services.get(service)
+        observed = runtime_environment.get(service)
+        candidate = candidate_images.get(service)
+        if previous is None or observed is None or candidate is None:
+            raise GuardError(f"runtime environment plan is incomplete for {service}")
+        observed_id = observed.get("id")
+        observed_values = observed.get("values") or {}
+        observed_hashes = environment_hashes(observed_values, container_id=observed_id)
+        if observed_hashes != (previous.get("env_hashes") or {}):
+            raise GuardError(f"fresh runtime environment changed for {service}")
+        candidate_values = candidate_environment_values(
+            config, service, candidate.get("environment") or {}
+        )
+        baseline_hashes = {
+            key: value
+            for key, value in (previous.get("env_hashes") or {}).items()
+            if key not in RUNTIME_GENERATED_ENV_KEYS
+        }
+        candidate_hashes = {
+            key: value
+            for key, value in environment_hashes(candidate_values).items()
+            if key not in RUNTIME_GENERATED_ENV_KEYS
+        }
+        service_overrides: dict[str, str] = {}
+        for key, expected_hash in baseline_hashes.items():
+            if candidate_hashes.get(key) == expected_hash:
+                continue
+            if key == "HOSTNAME":
+                raise GuardError(f"candidate hostname changed for {service}")
+            if key not in observed_values:
+                raise GuardError(f"baseline environment value is unavailable for {service}: {key}")
+            service_overrides[key] = observed_values[key]
+        for key in candidate_hashes:
+            if key in baseline_hashes or key in ALLOWED_ENV_ADDITIONS.get(service, set()):
+                continue
+            raise GuardError(f"candidate added environment for {service}: {key}")
+        effective_values = dict(candidate_values)
+        effective_values.update(service_overrides)
+        effective_hashes = {
+            key: value
+            for key, value in environment_hashes(effective_values).items()
+            if key not in RUNTIME_GENERATED_ENV_KEYS
+        }
+        if any(effective_hashes.get(key) != value for key, value in baseline_hashes.items()):
+            raise GuardError(f"candidate environment cannot preserve {service}")
+        overrides[service] = service_overrides
+        reviewed_keys[service] = set(service_overrides)
+    return overrides, reviewed_keys
+
+
 def compose_value_hash(value: Any) -> dict[str, Any]:
     if value is None:
         raise GuardError("Compose environment contains a host-inherited value")
@@ -1211,6 +1355,7 @@ def check_candidate_config(
     baseline: dict[str, Any],
     lock: dict[str, Any],
     baseline_config: dict[str, Any] | None = None,
+    reviewed_environment_keys: dict[str, set[str]] | None = None,
 ) -> None:
     services = config.get("services") or {}
     missing = set(TARGET_SERVICES) - set(services)
@@ -1328,12 +1473,18 @@ def check_candidate_config(
         old_authored_environment = compose_environment(
             authored_services[service].get("environment")
         )
-        allowed_additions = ALLOWED_ENV_ADDITIONS.get(service, set())
+        reviewed_keys = (reviewed_environment_keys or {}).get(service, set())
+        allowed_additions = ALLOWED_ENV_ADDITIONS.get(service, set()) | reviewed_keys
         for key, value in old_authored_environment.items():
             if key not in candidate_environment:
-                raise GuardError(f"candidate removed authored environment for {service}: {key}")
+                if key not in reviewed_keys:
+                    raise GuardError(
+                        f"candidate removed authored environment for {service}: {key}"
+                    )
+                continue
             if compose_value_hash(candidate_environment[key]) != compose_value_hash(value):
-                raise GuardError(f"candidate changed environment for {service}: {key}")
+                if key not in reviewed_keys:
+                    raise GuardError(f"candidate changed environment for {service}: {key}")
         for key, value in candidate_environment.items():
             if key not in old_authored_environment and key not in allowed_additions:
                 raise GuardError(f"candidate added environment for {service}: {key}")
@@ -1582,6 +1733,65 @@ def write_image_override(lock: dict[str, Any], path: Path) -> None:
     path.write_text("\n".join(lines) + "\n", encoding="utf-8")
 
 
+def write_runtime_environment_override(
+    overrides: dict[str, dict[str, str]], path: Path, *, replace: bool = False
+) -> None:
+    """Write reviewed baseline values to a private mode-600 Compose override."""
+    lines = [
+        "# Generated by train_health_deployment_guard.py; do not edit.",
+        "services:",
+    ]
+    written_services = 0
+    for service in TARGET_SERVICES:
+        values = overrides.get(service) or {}
+        if not values:
+            continue
+        written_services += 1
+        lines.extend([f"  {service}:", "    environment:"])
+        for key in sorted(values, key=lambda item: str(item)):
+            value = values[key]
+            if not isinstance(key, str) or not isinstance(value, str):
+                raise GuardError(f"runtime environment entry is not a string for {service}")
+            if not key or any(character in key for character in "\r\n=\x00"):
+                raise GuardError(f"runtime environment key is invalid for {service}")
+            if "\x00" in value:
+                raise GuardError(f"runtime environment value is invalid for {service}: {key}")
+            # Compose interpolates ``$VAR`` in YAML values even when they are
+            # quoted.  ``$$`` is Compose's escaped literal dollar sign.
+            lines.append(f"      {json.dumps(key)}: {json.dumps(value.replace('$', '$$'))}")
+    if not written_services:
+        lines = [
+            "# Generated by train_health_deployment_guard.py; do not edit.",
+            "services: {}",
+        ]
+    path.parent.mkdir(parents=True, exist_ok=True)
+    flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if replace else os.O_EXCL)
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(path, flags, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write("\n".join(lines) + "\n")
+    except FileExistsError as exc:
+        raise GuardError(f"runtime environment override already exists: {path}") from exc
+    os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+
+
+def validate_private_override(path: Path, *, expected_sha256: str | None = None) -> None:
+    try:
+        metadata = path.lstat()
+    except OSError as exc:
+        raise GuardError(f"cannot read private runtime environment override: {path}") from exc
+    if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600:
+        raise GuardError(f"runtime environment override is not a private regular file: {path}")
+    if hasattr(os, "getuid") and metadata.st_uid != os.getuid():
+        raise GuardError(f"runtime environment override has the wrong owner: {path}")
+    if expected_sha256 is not None:
+        actual_sha256 = sha256_file(path)
+        if actual_sha256 != expected_sha256:
+            raise GuardError(f"runtime environment override changed: {path}")
+
+
 @contextlib.contextmanager
 def candidate_image_override(lock: dict[str, Any]) -> Iterator[Path]:
     handle = tempfile.NamedTemporaryFile(
@@ -1605,12 +1815,21 @@ def targeted_up(
     candidate_version: str,
     probe_source: Path,
     image_override: Path | None = None,
+    environment_override: Path | None = None,
+    environment_override_sha256: str | None = None,
     deadline: OperationDeadline | None = None,
 ) -> None:
     env = os.environ.copy()
     env["VERSION"] = candidate_version
     env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
-    files = compose_files + ((str(image_override),) if image_override else ())
+    files = compose_files
+    if image_override:
+        files += (str(image_override),)
+    if environment_override:
+        validate_private_override(
+            environment_override, expected_sha256=environment_override_sha256
+        )
+        files += (str(environment_override),)
     args = compose_args(files)
     args.extend(
         [
@@ -1687,14 +1906,30 @@ def commit_backups(
     snapshot: dict[str, Any],
     backup_dir: Path,
     *,
+    runtime_environment_override: Path,
+    runtime_environment_sha256: str,
+    reviewed_environment_keys: dict[str, set[str]],
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     backup_dir.mkdir(parents=True, exist_ok=True)
+    validate_private_override(
+        runtime_environment_override, expected_sha256=runtime_environment_sha256
+    )
     write_json(backup_dir / "baseline.json", snapshot)
     tag_prefix = "localhost/unstract-health-backup-"
     backup_images: dict[str, Any] = {
         "schema": BACKUP_SCHEMA,
         "created_at": utc_now(),
+        "runtime_environment": {
+            "schema": RUNTIME_ENVIRONMENT_SCHEMA,
+            "file": runtime_environment_override.name,
+            "sha256": runtime_environment_sha256,
+            "reviewed_environment_keys": {
+                service: sorted(keys)
+                for service, keys in reviewed_environment_keys.items()
+                if keys
+            },
+        },
         "services": {},
     }
     for service in TARGET_SERVICES:
@@ -1744,6 +1979,24 @@ def backup_service_record(backup_images: dict[str, Any], service: str) -> dict[s
     raise GuardError(f"backup manifest has no service record for {service}")
 
 
+def runtime_environment_backup(
+    backup_images: dict[str, Any], backup_dir: Path
+) -> tuple[Path, str]:
+    metadata = backup_images.get("runtime_environment")
+    if not isinstance(metadata, dict):
+        raise GuardError("backup manifest has no runtime environment metadata")
+    if metadata.get("schema") != RUNTIME_ENVIRONMENT_SCHEMA:
+        raise GuardError("backup runtime environment has an unsupported schema")
+    if metadata.get("file") != RUNTIME_ENVIRONMENT_FILENAME:
+        raise GuardError("backup runtime environment file is not the guarded override")
+    digest = metadata.get("sha256")
+    if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+        raise GuardError("backup runtime environment digest is invalid")
+    path = backup_dir / RUNTIME_ENVIRONMENT_FILENAME
+    validate_private_override(path, expected_sha256=digest)
+    return path, digest
+
+
 def rollback_override(
     backup_images: dict[str, Any], path: Path, services: tuple[str, ...] = TARGET_SERVICES
 ) -> None:
@@ -1779,8 +2032,12 @@ def prepare(
     args: argparse.Namespace,
     image_override: Path,
     *,
+    runtime_environment_override: Path,
+    replace_runtime_environment_override: bool = False,
     operation_deadline: OperationDeadline,
-) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
+) -> tuple[
+    dict[str, Any], dict[str, Any], dict[str, Any], dict[str, set[str]], str
+]:
     baseline = load_baseline(Path(args.baseline))
     lock = load_lock(Path(args.candidate_lock))
     require_clean_candidate_source(
@@ -1794,12 +2051,36 @@ def prepare(
     compare_baseline_current(baseline, current, allow_new_probe=False)
     compare_untargeted_runtime(baseline, current)
     compare_source_and_quiescence(baseline, current)
+    runtime_environment = inspect_runtime_environment(
+        deadline=operation_deadline
+    )
+    config = compose_config(
+        Path(args.project_dir),
+        tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
+        candidate_version=lock["candidate_version"],
+        probe_source=Path(args.probe_source),
+        image_override=image_override,
+        deadline=operation_deadline,
+    )
+    overrides, reviewed_keys = plan_runtime_environment_override(
+        baseline,
+        runtime_environment,
+        candidate_images,
+        config,
+    )
+    write_runtime_environment_override(
+        overrides,
+        runtime_environment_override,
+        replace=replace_runtime_environment_override,
+    )
+    validate_private_override(runtime_environment_override)
     config = compose_config(
         Path(args.project_dir),
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
         probe_source=Path(args.probe_source),
         image_override=image_override,
+        environment_override=runtime_environment_override,
         deadline=operation_deadline,
     )
     authored_baseline = compose_config(
@@ -1809,8 +2090,18 @@ def prepare(
         probe_source=Path(args.probe_source),
         deadline=operation_deadline,
     )
-    check_candidate_config(config, baseline, lock, authored_baseline)
-    return baseline, lock, current
+    check_candidate_config(
+        config,
+        baseline,
+        lock,
+        authored_baseline,
+        reviewed_environment_keys=reviewed_keys,
+    )
+    runtime_environment_sha256 = sha256_file(runtime_environment_override)
+    validate_private_override(
+        runtime_environment_override, expected_sha256=runtime_environment_sha256
+    )
+    return baseline, lock, current, reviewed_keys, runtime_environment_sha256
 
 
 def record_replacements(
@@ -1938,6 +2229,8 @@ def compensating_rollback(
     replacement_manifest: dict[str, Any],
     backup_dir: Path,
     *,
+    runtime_environment_override: Path,
+    runtime_environment_sha256: str,
     operation_deadline: OperationDeadline,
 ) -> None:
     services = tuple((replacement_manifest.get("services") or {}).keys())
@@ -1948,6 +2241,9 @@ def compensating_rollback(
     verify_replacement_ids(current, replacement_manifest)
     override = backup_dir / "compensating-rollback.override.yaml"
     rollback_override(backup_images, override, services)
+    validate_private_override(
+        runtime_environment_override, expected_sha256=runtime_environment_sha256
+    )
     rollback_files = rollback_compose_files(args) + (str(override),)
     with advisory_lock(deadline=operation_deadline):
         # Recheck identity and quiescence after acquiring the DB lock.  The
@@ -1962,6 +2258,8 @@ def compensating_rollback(
             services,
             candidate_version="rollback-unused",
             probe_source=Path(args.probe_source),
+            environment_override=runtime_environment_override,
+            environment_override_sha256=runtime_environment_sha256,
             deadline=operation_deadline,
         )
         wait_running(services, operation_deadline=operation_deadline)
@@ -1987,9 +2285,15 @@ def apply_batch(
     untouched_services: tuple[str, ...],
     applied_services: tuple[str, ...],
     *,
+    runtime_environment_override: Path,
+    runtime_environment_sha256: str,
+    reviewed_environment_keys: dict[str, set[str]],
     operation_deadline: OperationDeadline,
 ) -> dict[str, Any]:
     compose_files = tuple(args.compose_file or DEFAULT_COMPOSE_FILES)
+    validate_private_override(
+        runtime_environment_override, expected_sha256=runtime_environment_sha256
+    )
     with advisory_lock(deadline=operation_deadline):
         fresh = capture(Path(args.project_dir), deadline=operation_deadline)
         compare_untargeted_runtime(baseline, fresh)
@@ -2004,6 +2308,7 @@ def apply_batch(
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
             image_override=image_override,
+            environment_override=runtime_environment_override,
             deadline=operation_deadline,
         )
         authored_baseline = compose_config(
@@ -2013,7 +2318,13 @@ def apply_batch(
             probe_source=Path(args.probe_source),
             deadline=operation_deadline,
         )
-        check_candidate_config(config, baseline, lock, authored_baseline)
+        check_candidate_config(
+            config,
+            baseline,
+            lock,
+            authored_baseline,
+            reviewed_environment_keys=reviewed_environment_keys,
+        )
         # Take the final settled sample after all preflight commands and
         # immediately before the targeted Compose mutation.
         final_quiescence = settled_queue_snapshot(operation_deadline)
@@ -2026,6 +2337,8 @@ def apply_batch(
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
             image_override=image_override,
+            environment_override=runtime_environment_override,
+            environment_override_sha256=runtime_environment_sha256,
             deadline=operation_deadline,
         )
         observed = capture(Path(args.project_dir), deadline=operation_deadline)
@@ -2042,8 +2355,23 @@ def apply_batch(
 def command_preflight(args: argparse.Namespace) -> int:
     deadline = OperationDeadline(args.operation_timeout)
     lock = load_lock(Path(args.candidate_lock))
+    descriptor, temporary_name = tempfile.mkstemp(
+        prefix="unstract-health-environment-", suffix=".yaml"
+    )
+    os.close(descriptor)
+    runtime_environment_override = Path(temporary_name)
     with candidate_image_override(lock) as image_override:
-        prepare(args, image_override, operation_deadline=deadline)
+        try:
+            prepare(
+                args,
+                image_override,
+                runtime_environment_override=runtime_environment_override,
+                replace_runtime_environment_override=True,
+                operation_deadline=deadline,
+            )
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                runtime_environment_override.unlink()
     print(
         "preflight: candidate source, image lock, Compose identity, runtime, "
         "data, network, environment, queue, and active-job state verified"
@@ -2059,6 +2387,9 @@ def command_apply(args: argparse.Namespace) -> int:
     attempted: list[str] = []
     applied: list[str] = []
     backup_images: dict[str, Any] | None = None
+    runtime_environment_override = backup_dir / RUNTIME_ENVIRONMENT_FILENAME
+    runtime_environment_sha256 = ""
+    reviewed_environment_keys: dict[str, set[str]] = {}
     replacement_manifest: dict[str, Any] = {
         "schema": REPLACEMENT_SCHEMA,
         "services": {},
@@ -2068,8 +2399,17 @@ def command_apply(args: argparse.Namespace) -> int:
         lock_hint = load_lock(Path(args.candidate_lock))
         with candidate_image_override(lock_hint) as image_override:
             try:
-                baseline, lock, _ = prepare(
-                    args, image_override, operation_deadline=operation_deadline
+                (
+                    baseline,
+                    lock,
+                    _,
+                    reviewed_environment_keys,
+                    runtime_environment_sha256,
+                ) = prepare(
+                    args,
+                    image_override,
+                    runtime_environment_override=runtime_environment_override,
+                    operation_deadline=operation_deadline,
                 )
                 with advisory_lock(deadline=operation_deadline):
                     fresh = capture(Path(args.project_dir), deadline=operation_deadline)
@@ -2078,7 +2418,12 @@ def command_apply(args: argparse.Namespace) -> int:
                     compare_source_and_quiescence(baseline, fresh)
                     candidate_image_snapshot(lock, deadline=operation_deadline)
                     backup_images = commit_backups(
-                        fresh, backup_dir, deadline=operation_deadline
+                        fresh,
+                        backup_dir,
+                        runtime_environment_override=runtime_environment_override,
+                        runtime_environment_sha256=runtime_environment_sha256,
+                        reviewed_environment_keys=reviewed_environment_keys,
+                        deadline=operation_deadline,
                     )
                 rollback_override(backup_images, backup_dir / "rollback.override.yaml")
                 write_json(backup_dir / "candidate-images.json", lock["images"])
@@ -2093,6 +2438,9 @@ def command_apply(args: argparse.Namespace) -> int:
                     WORKER_SERVICES,
                     CORE_SERVICES,
                     (),
+                    runtime_environment_override=runtime_environment_override,
+                    runtime_environment_sha256=runtime_environment_sha256,
+                    reviewed_environment_keys=reviewed_environment_keys,
                     operation_deadline=operation_deadline,
                 )
                 replacement_manifest["services"].update(
@@ -2111,6 +2459,9 @@ def command_apply(args: argparse.Namespace) -> int:
                     CORE_SERVICES,
                     (),
                     tuple(applied),
+                    runtime_environment_override=runtime_environment_override,
+                    runtime_environment_sha256=runtime_environment_sha256,
+                    reviewed_environment_keys=reviewed_environment_keys,
                     operation_deadline=operation_deadline,
                 )
                 replacement_manifest["services"].update(
@@ -2162,6 +2513,8 @@ def command_apply(args: argparse.Namespace) -> int:
                                 backup_images,
                                 replacement_manifest,
                                 backup_dir,
+                                runtime_environment_override=runtime_environment_override,
+                                runtime_environment_sha256=runtime_environment_sha256,
                                 operation_deadline=operation_deadline,
                             )
                     except Exception as rollback_error:
@@ -2210,6 +2563,9 @@ def command_rollback(args: argparse.Namespace) -> int:
     if replacement_manifest.get("schema") != REPLACEMENT_SCHEMA:
         raise GuardError("replacement manifest has an unsupported schema")
     baseline = load_baseline(backup_dir / "baseline.json")
+    runtime_environment_override, runtime_environment_sha256 = runtime_environment_backup(
+        backup_images, backup_dir
+    )
     with local_operation_lock(backup_dir / ".guard.lock", deadline=operation_deadline):
         compensating_rollback(
             args,
@@ -2217,6 +2573,8 @@ def command_rollback(args: argparse.Namespace) -> int:
             backup_images,
             replacement_manifest,
             backup_dir,
+            runtime_environment_override=runtime_environment_override,
+            runtime_environment_sha256=runtime_environment_sha256,
             operation_deadline=operation_deadline,
         )
         final = json.loads(
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index c4e6e2ea5f..8944f73f43 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -163,6 +163,111 @@ def test_exception_reason_is_bounded_and_redacts_secret_assignments() -> None:
     assert len(reason) < 260
 
 
+def test_runtime_environment_plan_preserves_image_drift_and_missing_baseline_keys() -> None:
+    baseline_values = {
+        "qdrant": {"PATH": "/usr/local/bin", "QDRANT_DB": "baseline-db"},
+        "runner": {"PATH": "/usr/local/bin", "UNSTRACT_APPS_VERSION": "old"},
+    }
+    baseline = {
+        "containers": [
+            {
+                "compose": {"com.docker.compose.service": service},
+                "name": f"unstract-{service}",
+                "env_hashes": guard.environment_hashes(values),
+            }
+            for service, values in baseline_values.items()
+        ]
+    }
+    runtime_environment = {
+        service: {"id": f"{service}-id", "values": values}
+        for service, values in baseline_values.items()
+    }
+    candidate_images = {
+        "qdrant": {"environment": {"PATH": "/usr/local/bin"}},
+        "runner": {
+            "environment": {"PATH": "/usr/local/bin", "UNSTRACT_APPS_VERSION": "new"}
+        },
+    }
+    config = {"services": {"qdrant": {}, "runner": {}}}
+
+    overrides, reviewed = guard.plan_runtime_environment_override(
+        baseline,
+        runtime_environment,
+        candidate_images,
+        config,
+        services=("qdrant", "runner"),
+    )
+
+    assert overrides == {
+        "qdrant": {"QDRANT_DB": "baseline-db"},
+        "runner": {"UNSTRACT_APPS_VERSION": "old"},
+    }
+    assert reviewed == {
+        "qdrant": {"QDRANT_DB"},
+        "runner": {"UNSTRACT_APPS_VERSION"},
+    }
+
+
+def test_runtime_environment_plan_rejects_unreviewed_image_default() -> None:
+    baseline = {
+        "containers": [
+            {
+                "compose": {"com.docker.compose.service": "runner"},
+                "name": "unstract-runner",
+                "env_hashes": guard.environment_hashes({"PATH": "/usr/local/bin"}),
+            }
+        ]
+    }
+    runtime_environment = {
+        "runner": {"id": "runner-id", "values": {"PATH": "/usr/local/bin"}}
+    }
+    candidate_images = {
+        "runner": {
+            "environment": {"PATH": "/usr/local/bin", "UNREVIEWED_DEFAULT": "changed"}
+        }
+    }
+
+    with pytest.raises(guard.GuardError, match="added environment"):
+        guard.plan_runtime_environment_override(
+            baseline,
+            runtime_environment,
+            candidate_images,
+            {"services": {"runner": {}}},
+            services=("runner",),
+        )
+
+
+def test_reviewed_environment_override_is_allowed_by_compose_identity_guard() -> None:
+    config, baseline, lock, authored = candidate_fixture()
+    config["services"]["db"]["environment"]["BASELINE_ONLY"] = "preserved"
+
+    guard.check_candidate_config(
+        config,
+        baseline,
+        lock,
+        authored,
+        reviewed_environment_keys={"db": {"BASELINE_ONLY"}},
+    )
+
+
+def test_runtime_environment_override_is_private(tmp_path: Path) -> None:
+    path = tmp_path / "runtime-environment.override.yaml"
+
+    guard.write_runtime_environment_override(
+        {"qdrant": {"QDRANT_DB": "baseline-$DB-${DB_NAME}"}}, path
+    )
+    guard.validate_private_override(path)
+
+    assert path.stat().st_mode & 0o777 == 0o600
+    assert '"baseline-$$DB-$${DB_NAME}"' in path.read_text(encoding="utf-8")
+
+    digest = guard.sha256_file(path)
+    guard.validate_private_override(path, expected_sha256=digest)
+    path.write_text(path.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8")
+    with pytest.raises(guard.GuardError, match="override changed"):
+        guard.validate_private_override(path, expected_sha256=digest)
+
+
 def test_core_probe_checks_require_read_only_probe_mount() -> None:
     config, baseline, lock, authored = candidate_fixture()
     config["services"]["db"]["volumes"] = []

From ac4688e9f2930a8eabea8064a041f0102236a603 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 21:19:20 -0400
Subject: [PATCH 33/48] Persist guarded Compose deployment inputs

---
 .../scripts/train_health_deployment_guard.py  | 741 +++++++++++++-----
 docs/train-unstract-health-deployment.md      |   8 +-
 .../test_train_health_deployment_guard.py     | 143 ++++
 3 files changed, 704 insertions(+), 188 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 9e5afe8ef4..019698d86e 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -30,7 +30,6 @@
 import stat
 import subprocess
 import sys
-import tempfile
 import time
 from collections.abc import Iterator
 from decimal import Decimal, InvalidOperation
@@ -97,6 +96,10 @@
 FAILURE_SCHEMA = "unstract-health-failure/v1"
 RUNTIME_ENVIRONMENT_SCHEMA = "unstract-health-runtime-environment/v1"
 RUNTIME_ENVIRONMENT_FILENAME = "runtime-environment.override.yaml"
+CANDIDATE_IMAGE_SCHEMA = "unstract-health-candidate-image/v1"
+CANDIDATE_IMAGE_FILENAME = "candidate-image.override.yaml"
+COMPOSE_SETTINGS_SCHEMA = "unstract-health-compose-settings/v1"
+COMPOSE_SETTINGS_FILENAME = "compose-settings.env"
 RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
@@ -1205,18 +1208,46 @@ def compose_config(
     candidate_version: str,
     probe_source: Path,
     image_override: Path | None = None,
+    image_override_sha256: str | None = None,
     environment_override: Path | None = None,
+    environment_override_sha256: str | None = None,
+    settings_file: Path | None = None,
+    settings_file_sha256: str | None = None,
+    probe_source_sha256: str | None = None,
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     env = os.environ.copy()
-    env["VERSION"] = candidate_version
-    env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+    if settings_file:
+        settings = validate_compose_settings(
+            settings_file,
+            candidate_version=candidate_version,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            expected_sha256=settings_file_sha256,
+        )
+        env["VERSION"] = settings["VERSION"]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+        probe_source_sha256 = settings["UNSTRACT_HEALTHCHECK_SOURCE_SHA256"]
+    else:
+        env["VERSION"] = candidate_version
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+        validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
     files = compose_files
     if image_override:
+        validate_private_file(
+            image_override,
+            expected_sha256=image_override_sha256,
+            description="candidate image override",
+        )
         files += (str(image_override),)
     if environment_override:
+        validate_private_override(
+            environment_override, expected_sha256=environment_override_sha256
+        )
         files += (str(environment_override),)
     args = ["docker", "compose"]
+    if settings_file:
+        args.extend(["--env-file", str(settings_file)])
     for compose_file in files:
         args.extend(["-f", compose_file])
     args.extend(["config", "--format", "json"])
@@ -1262,6 +1293,106 @@ def candidate_environment_values(
     return values
 
 
+def validate_probe_source(path: Path, *, expected_sha256: str | None = None) -> str:
+    """Require the staged probe to remain the reviewed source artifact."""
+    try:
+        metadata = path.lstat()
+    except OSError as exc:
+        raise GuardError(f"cannot read health probe source: {path}") from exc
+    if not stat.S_ISREG(metadata.st_mode):
+        raise GuardError(f"health probe source is not a regular file: {path}")
+    actual_sha256 = sha256_file(path)
+    if expected_sha256 is not None and actual_sha256 != expected_sha256:
+        raise GuardError(f"health probe source changed: {path}")
+    return actual_sha256
+
+
+def compose_settings_values(
+    candidate_version: str,
+    probe_source: Path,
+    *,
+    probe_source_sha256: str | None = None,
+) -> dict[str, str]:
+    """Return the non-secret Compose interpolation values for a deployment."""
+    if not isinstance(candidate_version, str) or not candidate_version:
+        raise GuardError("candidate version must be a non-empty string")
+    if any(character in candidate_version for character in "\r\n=\x00"):
+        raise GuardError("candidate version contains an invalid character")
+    source = str(probe_source)
+    if any(character in source for character in "\r\n\x00"):
+        raise GuardError("health probe source contains an invalid character")
+    if probe_source_sha256 is None:
+        probe_source_sha256 = validate_probe_source(probe_source)
+    elif not re.fullmatch(r"[0-9a-f]{64}", probe_source_sha256):
+        raise GuardError("health probe source digest is invalid")
+    return {
+        "VERSION": candidate_version,
+        "UNSTRACT_HEALTHCHECK_SOURCE": source,
+        "UNSTRACT_HEALTHCHECK_SOURCE_SHA256": probe_source_sha256,
+    }
+
+
+def parse_compose_settings(path: Path) -> dict[str, str]:
+    """Read the small private interpolation file without exposing its values."""
+    values: dict[str, str] = {}
+    source_digest: str | None = None
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise GuardError(f"cannot read private Compose settings: {path}") from exc
+    for line in lines:
+        if not line or line.startswith("#"):
+            if line.startswith("# probe-source-sha256="):
+                if source_digest is not None:
+                    raise GuardError(
+                        f"private Compose settings contain duplicate probe digests: {path}"
+                    )
+                source_digest = line.split("=", 1)[1]
+            continue
+        key, separator, value = line.partition("=")
+        if not separator or key not in {
+            "VERSION",
+            "UNSTRACT_HEALTHCHECK_SOURCE",
+        }:
+            raise GuardError(f"private Compose settings contain an unsupported entry: {path}")
+        if key in values:
+            raise GuardError(f"private Compose settings contain a duplicate entry: {path}")
+        values[key] = value
+    if set(values) != {"VERSION", "UNSTRACT_HEALTHCHECK_SOURCE"}:
+        raise GuardError(f"private Compose settings are incomplete: {path}")
+    if source_digest is None:
+        raise GuardError(f"private Compose settings lack the probe source digest: {path}")
+    if not re.fullmatch(r"[0-9a-f]{64}", source_digest):
+        raise GuardError(f"private Compose settings have an invalid probe source digest: {path}")
+    values["UNSTRACT_HEALTHCHECK_SOURCE_SHA256"] = source_digest
+    return values
+
+
+def validate_compose_settings(
+    path: Path,
+    *,
+    candidate_version: str,
+    probe_source: Path,
+    probe_source_sha256: str | None = None,
+    expected_sha256: str | None = None,
+) -> dict[str, str]:
+    validate_private_file(path, expected_sha256=expected_sha256, description="Compose settings")
+    values = parse_compose_settings(path)
+    expected = compose_settings_values(
+        candidate_version,
+        probe_source,
+        probe_source_sha256=probe_source_sha256
+        or values["UNSTRACT_HEALTHCHECK_SOURCE_SHA256"],
+    )
+    if values != expected:
+        raise GuardError(f"durable Compose settings drifted: {path}")
+    validate_probe_source(
+        Path(values["UNSTRACT_HEALTHCHECK_SOURCE"]),
+        expected_sha256=values["UNSTRACT_HEALTHCHECK_SOURCE_SHA256"],
+    )
+    return values
+
+
 def plan_runtime_environment_override(
     baseline: dict[str, Any],
     runtime_environment: dict[str, dict[str, Any]],
@@ -1711,14 +1842,67 @@ def capture_and_write(
     return snapshot
 
 
-def compose_args(compose_files: tuple[str, ...]) -> list[str]:
+def compose_args(
+    compose_files: tuple[str, ...], *, settings_file: Path | None = None
+) -> list[str]:
     args = ["docker", "compose"]
+    if settings_file:
+        args.extend(["--env-file", str(settings_file)])
     for compose_file in compose_files:
         args.extend(["-f", compose_file])
     return args
 
 
-def write_image_override(lock: dict[str, Any], path: Path) -> None:
+def write_private_text(
+    path: Path,
+    text: str,
+    *,
+    replace: bool,
+    description: str,
+    reuse_if_identical: bool = False,
+) -> None:
+    """Write one private state file without creating a transient reference."""
+    if "\x00" in text:
+        raise GuardError(f"{description} contains a NUL byte")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    mode = stat.S_IRUSR | stat.S_IWUSR
+    if path.exists() and not replace and reuse_if_identical:
+        validate_private_file(path, description=description)
+        if sha256_file(path) != sha256_bytes(text.encode("utf-8")):
+            raise GuardError(f"{description} already exists with different contents: {path}")
+        return
+    if replace:
+        staging = path.with_name(f".{path.name}.{os.getpid()}.new")
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        try:
+            descriptor = os.open(staging, flags, mode)
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                handle.write(text)
+            os.chmod(staging, mode)
+            os.replace(staging, path)
+        except FileExistsError as exc:
+            raise GuardError(f"private state staging file already exists: {staging}") from exc
+        finally:
+            with contextlib.suppress(FileNotFoundError):
+                staging.unlink()
+    else:
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        try:
+            descriptor = os.open(path, flags, mode)
+            with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+                handle.write(text)
+        except FileExistsError as exc:
+            raise GuardError(f"{description} already exists: {path}") from exc
+    os.chmod(path, mode)
+
+
+def write_image_override(
+    lock: dict[str, Any], path: Path, *, replace: bool = False
+) -> None:
     """Pin each target to the exact locked reference without touching source."""
     lines = [
         "# Generated by train_health_deployment_guard.py; do not edit.",
@@ -1729,8 +1913,44 @@ def write_image_override(lock: dict[str, Any], path: Path) -> None:
         if any(character.isspace() for character in reference) or "\n" in reference:
             raise GuardError(f"candidate image reference contains whitespace: {service}")
         lines.extend([f"  {service}:", f"    image: {json.dumps(reference)}"])
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    write_private_text(
+        path,
+        "\n".join(lines) + "\n",
+        replace=replace,
+        description="candidate image override",
+        reuse_if_identical=True,
+    )
+
+
+def write_compose_settings(
+    candidate_version: str,
+    probe_source: Path,
+    path: Path,
+    *,
+    probe_source_sha256: str | None = None,
+    replace: bool = False,
+) -> None:
+    values = compose_settings_values(
+        candidate_version,
+        probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
+    text = "\n".join(
+        [
+            "# Generated by train_health_deployment_guard.py; do not edit.",
+            f"VERSION={values['VERSION']}",
+            f"UNSTRACT_HEALTHCHECK_SOURCE={values['UNSTRACT_HEALTHCHECK_SOURCE']}",
+            f"# probe-source-sha256={values['UNSTRACT_HEALTHCHECK_SOURCE_SHA256']}",
+            "",
+        ]
+    )
+    write_private_text(
+        path,
+        text,
+        replace=replace,
+        description="Compose settings",
+        reuse_if_identical=True,
+    )
 
 
 def write_runtime_environment_override(
@@ -1764,47 +1984,60 @@ def write_runtime_environment_override(
             "# Generated by train_health_deployment_guard.py; do not edit.",
             "services: {}",
         ]
-    path.parent.mkdir(parents=True, exist_ok=True)
-    flags = os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if replace else os.O_EXCL)
-    if hasattr(os, "O_NOFOLLOW"):
-        flags |= os.O_NOFOLLOW
-    try:
-        descriptor = os.open(path, flags, 0o600)
-        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
-            handle.write("\n".join(lines) + "\n")
-    except FileExistsError as exc:
-        raise GuardError(f"runtime environment override already exists: {path}") from exc
-    os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
+    write_private_text(
+        path,
+        "\n".join(lines) + "\n",
+        replace=replace,
+        description="runtime environment override",
+        reuse_if_identical=True,
+    )
 
 
-def validate_private_override(path: Path, *, expected_sha256: str | None = None) -> None:
+def validate_private_file(
+    path: Path,
+    *,
+    expected_sha256: str | None = None,
+    description: str,
+) -> None:
     try:
         metadata = path.lstat()
     except OSError as exc:
-        raise GuardError(f"cannot read private runtime environment override: {path}") from exc
+        raise GuardError(f"cannot read private {description}: {path}") from exc
     if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600:
-        raise GuardError(f"runtime environment override is not a private regular file: {path}")
+        raise GuardError(f"{description} is not a private regular file: {path}")
     if hasattr(os, "getuid") and metadata.st_uid != os.getuid():
-        raise GuardError(f"runtime environment override has the wrong owner: {path}")
+        raise GuardError(f"{description} has the wrong owner: {path}")
     if expected_sha256 is not None:
         actual_sha256 = sha256_file(path)
         if actual_sha256 != expected_sha256:
-            raise GuardError(f"runtime environment override changed: {path}")
+            raise GuardError(f"{description} changed: {path}")
 
 
-@contextlib.contextmanager
-def candidate_image_override(lock: dict[str, Any]) -> Iterator[Path]:
-    handle = tempfile.NamedTemporaryFile(
-        mode="w", prefix="unstract-health-images-", suffix=".yaml", delete=False
+def validate_private_override(path: Path, *, expected_sha256: str | None = None) -> None:
+    validate_private_file(
+        path,
+        expected_sha256=expected_sha256,
+        description="runtime environment override",
     )
-    path = Path(handle.name)
-    handle.close()
-    try:
-        write_image_override(lock, path)
-        yield path
-    finally:
-        with contextlib.suppress(FileNotFoundError):
-            path.unlink()
+
+
+def validate_candidate_image_override(
+    path: Path, *, expected_sha256: str | None = None
+) -> None:
+    validate_private_file(
+        path,
+        expected_sha256=expected_sha256,
+        description="candidate image override",
+    )
+
+
+def candidate_image_override(
+    lock: dict[str, Any], path: Path, *, replace: bool = False
+) -> Path:
+    """Materialize a durable private image override for every Compose replay."""
+    write_image_override(lock, path, replace=replace)
+    validate_candidate_image_override(path)
+    return path
 
 
 def targeted_up(
@@ -1815,22 +2048,41 @@ def targeted_up(
     candidate_version: str,
     probe_source: Path,
     image_override: Path | None = None,
+    image_override_sha256: str | None = None,
     environment_override: Path | None = None,
     environment_override_sha256: str | None = None,
+    settings_file: Path | None = None,
+    settings_file_sha256: str | None = None,
+    probe_source_sha256: str | None = None,
     deadline: OperationDeadline | None = None,
 ) -> None:
     env = os.environ.copy()
-    env["VERSION"] = candidate_version
-    env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+    if settings_file:
+        settings = validate_compose_settings(
+            settings_file,
+            candidate_version=candidate_version,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            expected_sha256=settings_file_sha256,
+        )
+        env["VERSION"] = settings["VERSION"]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+    else:
+        env["VERSION"] = candidate_version
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = str(probe_source)
+        validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
     files = compose_files
     if image_override:
+        validate_candidate_image_override(
+            image_override, expected_sha256=image_override_sha256
+        )
         files += (str(image_override),)
     if environment_override:
         validate_private_override(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = compose_args(files)
+    args = compose_args(files, settings_file=settings_file)
     args.extend(
         [
             "up",
@@ -1909,12 +2161,26 @@ def commit_backups(
     runtime_environment_override: Path,
     runtime_environment_sha256: str,
     reviewed_environment_keys: dict[str, set[str]],
+    image_override: Path,
+    image_override_sha256: str,
+    settings_file: Path,
+    settings_file_sha256: str,
+    probe_source: Path,
+    probe_source_sha256: str,
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     backup_dir.mkdir(parents=True, exist_ok=True)
     validate_private_override(
         runtime_environment_override, expected_sha256=runtime_environment_sha256
     )
+    validate_candidate_image_override(image_override, expected_sha256=image_override_sha256)
+    validate_compose_settings(
+        settings_file,
+        candidate_version=parse_compose_settings(settings_file)["VERSION"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        expected_sha256=settings_file_sha256,
+    )
     write_json(backup_dir / "baseline.json", snapshot)
     tag_prefix = "localhost/unstract-health-backup-"
     backup_images: dict[str, Any] = {
@@ -1930,6 +2196,19 @@ def commit_backups(
                 if keys
             },
         },
+        "candidate_image_override": {
+            "schema": CANDIDATE_IMAGE_SCHEMA,
+            "file": image_override.name,
+            "sha256": image_override_sha256,
+        },
+        "compose_settings": {
+            "schema": COMPOSE_SETTINGS_SCHEMA,
+            "file": settings_file.name,
+            "sha256": settings_file_sha256,
+            "probe_source": str(probe_source),
+            "probe_source_sha256": probe_source_sha256,
+            "candidate_version": parse_compose_settings(settings_file)["VERSION"],
+        },
         "services": {},
     }
     for service in TARGET_SERVICES:
@@ -2016,8 +2295,12 @@ def rollback_override(
                 '    healthcheck: {test: ["NONE"]}',
             ]
         )
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    write_private_text(
+        path,
+        "\n".join(lines) + "\n",
+        replace=True,
+        description="rollback image override",
+    )
 
 
 def command_capture(args: argparse.Namespace) -> int:
@@ -2033,7 +2316,9 @@ def prepare(
     image_override: Path,
     *,
     runtime_environment_override: Path,
+    settings_file: Path,
     replace_runtime_environment_override: bool = False,
+    replace_settings_file: bool = False,
     operation_deadline: OperationDeadline,
 ) -> tuple[
     dict[str, Any], dict[str, Any], dict[str, Any], dict[str, set[str]], str
@@ -2044,6 +2329,29 @@ def prepare(
         Path(args.candidate_source), lock["source_commit"], lock.get("source_tree")
     )
     verify_artifacts(Path(args.candidate_source), lock)
+    probe_source = Path(args.probe_source)
+    probe_source_sha256 = (lock.get("artifacts") or {}).get(
+        "docker/healthchecks/unstract-services.sh"
+    )
+    if not isinstance(probe_source_sha256, str):
+        raise GuardError("candidate lock lacks the guarded probe source digest")
+    validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
+    write_compose_settings(
+        lock["candidate_version"],
+        probe_source,
+        settings_file,
+        probe_source_sha256=probe_source_sha256,
+        replace=replace_settings_file,
+    )
+    validate_compose_settings(
+        settings_file,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
+    validate_candidate_image_override(image_override)
+    image_override_sha256 = sha256_file(image_override)
+    settings_file_sha256 = sha256_file(settings_file)
     candidate_images = candidate_image_snapshot(lock, deadline=operation_deadline)
     if not candidate_images:
         raise GuardError("no candidate images were verified")
@@ -2058,8 +2366,12 @@ def prepare(
         Path(args.project_dir),
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
-        probe_source=Path(args.probe_source),
+        probe_source=probe_source,
         image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
         deadline=operation_deadline,
     )
     overrides, reviewed_keys = plan_runtime_environment_override(
@@ -2078,16 +2390,23 @@ def prepare(
         Path(args.project_dir),
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
-        probe_source=Path(args.probe_source),
+        probe_source=probe_source,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
         image_override=image_override,
         environment_override=runtime_environment_override,
+        environment_override_sha256=sha256_file(runtime_environment_override),
         deadline=operation_deadline,
     )
     authored_baseline = compose_config(
         Path(args.project_dir),
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
-        probe_source=Path(args.probe_source),
+        probe_source=probe_source,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
         deadline=operation_deadline,
     )
     check_candidate_config(
@@ -2101,6 +2420,16 @@ def prepare(
     validate_private_override(
         runtime_environment_override, expected_sha256=runtime_environment_sha256
     )
+    validate_candidate_image_override(
+        image_override, expected_sha256=image_override_sha256
+    )
+    validate_compose_settings(
+        settings_file,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        expected_sha256=settings_file_sha256,
+    )
     return baseline, lock, current, reviewed_keys, runtime_environment_sha256
 
 
@@ -2285,6 +2614,10 @@ def apply_batch(
     untouched_services: tuple[str, ...],
     applied_services: tuple[str, ...],
     *,
+    settings_file: Path,
+    settings_file_sha256: str,
+    image_override_sha256: str,
+    probe_source_sha256: str,
     runtime_environment_override: Path,
     runtime_environment_sha256: str,
     reviewed_environment_keys: dict[str, set[str]],
@@ -2308,7 +2641,12 @@ def apply_batch(
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
             image_override=image_override,
+            image_override_sha256=image_override_sha256,
             environment_override=runtime_environment_override,
+            environment_override_sha256=runtime_environment_sha256,
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            probe_source_sha256=probe_source_sha256,
             deadline=operation_deadline,
         )
         authored_baseline = compose_config(
@@ -2316,6 +2654,9 @@ def apply_batch(
             compose_files,
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            probe_source_sha256=probe_source_sha256,
             deadline=operation_deadline,
         )
         check_candidate_config(
@@ -2337,8 +2678,12 @@ def apply_batch(
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
             image_override=image_override,
+            image_override_sha256=image_override_sha256,
             environment_override=runtime_environment_override,
             environment_override_sha256=runtime_environment_sha256,
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            probe_source_sha256=probe_source_sha256,
             deadline=operation_deadline,
         )
         observed = capture(Path(args.project_dir), deadline=operation_deadline)
@@ -2355,26 +2700,25 @@ def apply_batch(
 def command_preflight(args: argparse.Namespace) -> int:
     deadline = OperationDeadline(args.operation_timeout)
     lock = load_lock(Path(args.candidate_lock))
-    descriptor, temporary_name = tempfile.mkstemp(
-        prefix="unstract-health-environment-", suffix=".yaml"
+    state_dir = Path(args.candidate_lock).resolve().parent
+    image_override = candidate_image_override(
+        lock, state_dir / CANDIDATE_IMAGE_FILENAME, replace=True
+    )
+    runtime_environment_override = state_dir / RUNTIME_ENVIRONMENT_FILENAME
+    settings_file = state_dir / COMPOSE_SETTINGS_FILENAME
+    prepare(
+        args,
+        image_override,
+        runtime_environment_override=runtime_environment_override,
+        settings_file=settings_file,
+        replace_runtime_environment_override=True,
+        replace_settings_file=True,
+        operation_deadline=deadline,
     )
-    os.close(descriptor)
-    runtime_environment_override = Path(temporary_name)
-    with candidate_image_override(lock) as image_override:
-        try:
-            prepare(
-                args,
-                image_override,
-                runtime_environment_override=runtime_environment_override,
-                replace_runtime_environment_override=True,
-                operation_deadline=deadline,
-            )
-        finally:
-            with contextlib.suppress(FileNotFoundError):
-                runtime_environment_override.unlink()
     print(
         "preflight: candidate source, image lock, Compose identity, runtime, "
-        "data, network, environment, queue, and active-job state verified"
+        "data, network, environment, queue, and active-job state verified; "
+        f"durable_artifacts={state_dir}"
     )
     return 0
 
@@ -2383,7 +2727,7 @@ def command_apply(args: argparse.Namespace) -> int:
     if args.confirm != CONFIRM_TOKEN:
         raise GuardError(f"apply requires --confirm {CONFIRM_TOKEN}")
     operation_deadline = OperationDeadline(args.operation_timeout)
-    backup_dir = Path(args.backup_dir)
+    backup_dir = Path(args.backup_dir).resolve()
     attempted: list[str] = []
     applied: list[str] = []
     backup_images: dict[str, Any] | None = None
@@ -2397,139 +2741,164 @@ def command_apply(args: argparse.Namespace) -> int:
     }
     with local_operation_lock(backup_dir / ".guard.lock", deadline=operation_deadline):
         lock_hint = load_lock(Path(args.candidate_lock))
-        with candidate_image_override(lock_hint) as image_override:
-            try:
-                (
-                    baseline,
-                    lock,
-                    _,
-                    reviewed_environment_keys,
-                    runtime_environment_sha256,
-                ) = prepare(
-                    args,
-                    image_override,
-                    runtime_environment_override=runtime_environment_override,
-                    operation_deadline=operation_deadline,
-                )
-                with advisory_lock(deadline=operation_deadline):
-                    fresh = capture(Path(args.project_dir), deadline=operation_deadline)
-                    compare_baseline_current(baseline, fresh, allow_new_probe=False)
-                    compare_untargeted_runtime(baseline, fresh)
-                    compare_source_and_quiescence(baseline, fresh)
-                    candidate_image_snapshot(lock, deadline=operation_deadline)
-                    backup_images = commit_backups(
-                        fresh,
-                        backup_dir,
-                        runtime_environment_override=runtime_environment_override,
-                        runtime_environment_sha256=runtime_environment_sha256,
-                        reviewed_environment_keys=reviewed_environment_keys,
-                        deadline=operation_deadline,
-                    )
-                rollback_override(backup_images, backup_dir / "rollback.override.yaml")
-                write_json(backup_dir / "candidate-images.json", lock["images"])
-
-                attempted.extend(WORKER_SERVICES)
-                worker_replacements = apply_batch(
-                    args,
-                    baseline,
-                    lock,
-                    backup_dir,
-                    image_override,
-                    WORKER_SERVICES,
-                    CORE_SERVICES,
-                    (),
-                    runtime_environment_override=runtime_environment_override,
-                    runtime_environment_sha256=runtime_environment_sha256,
-                    reviewed_environment_keys=reviewed_environment_keys,
-                    operation_deadline=operation_deadline,
-                )
-                replacement_manifest["services"].update(
-                    worker_replacements.get("services", {})
-                )
-                applied.extend(WORKER_SERVICES)
-                write_replacement_manifest(backup_dir, replacement_manifest)
-
-                attempted.extend(CORE_SERVICES)
-                core_replacements = apply_batch(
-                    args,
-                    baseline,
-                    lock,
+        image_override = candidate_image_override(
+            lock_hint, backup_dir / CANDIDATE_IMAGE_FILENAME
+        )
+        settings_file = backup_dir / COMPOSE_SETTINGS_FILENAME
+        try:
+            (
+                baseline,
+                lock,
+                _,
+                reviewed_environment_keys,
+                runtime_environment_sha256,
+            ) = prepare(
+                args,
+                image_override,
+                runtime_environment_override=runtime_environment_override,
+                settings_file=settings_file,
+                operation_deadline=operation_deadline,
+            )
+            image_override_sha256 = sha256_file(image_override)
+            settings_file_sha256 = sha256_file(settings_file)
+            probe_source_sha256 = (lock.get("artifacts") or {}).get(
+                "docker/healthchecks/unstract-services.sh"
+            )
+            if not isinstance(probe_source_sha256, str):
+                raise GuardError("candidate lock lacks the guarded probe source digest")
+            with advisory_lock(deadline=operation_deadline):
+                fresh = capture(Path(args.project_dir), deadline=operation_deadline)
+                compare_baseline_current(baseline, fresh, allow_new_probe=False)
+                compare_untargeted_runtime(baseline, fresh)
+                compare_source_and_quiescence(baseline, fresh)
+                candidate_image_snapshot(lock, deadline=operation_deadline)
+                backup_images = commit_backups(
+                    fresh,
                     backup_dir,
-                    image_override,
-                    CORE_SERVICES,
-                    (),
-                    tuple(applied),
                     runtime_environment_override=runtime_environment_override,
                     runtime_environment_sha256=runtime_environment_sha256,
                     reviewed_environment_keys=reviewed_environment_keys,
-                    operation_deadline=operation_deadline,
-                )
-                replacement_manifest["services"].update(
-                    core_replacements.get("services", {})
+                    image_override=image_override,
+                    image_override_sha256=image_override_sha256,
+                    settings_file=settings_file,
+                    settings_file_sha256=settings_file_sha256,
+                    probe_source=Path(args.probe_source),
+                    probe_source_sha256=probe_source_sha256,
+                    deadline=operation_deadline,
                 )
-                applied.extend(CORE_SERVICES)
-                write_replacement_manifest(backup_dir, replacement_manifest)
-                final = capture(Path(args.project_dir), deadline=operation_deadline)
-                compare_post_apply(baseline, final, lock, TARGET_SERVICES)
-                compare_untargeted_runtime(baseline, final)
-                compare_source_and_quiescence(baseline, final)
-                write_json(backup_dir / "post-apply.json", final)
-            except Exception as exc:
-                failure_record = {
-                    "schema": FAILURE_SCHEMA,
-                    "original_error": exception_reason(exc),
-                }
+            rollback_override(backup_images, backup_dir / "rollback.override.yaml")
+            write_json(backup_dir / "candidate-images.json", lock["images"])
+
+            attempted.extend(WORKER_SERVICES)
+            worker_replacements = apply_batch(
+                args,
+                baseline,
+                lock,
+                backup_dir,
+                image_override,
+                WORKER_SERVICES,
+                CORE_SERVICES,
+                (),
+                settings_file=settings_file,
+                settings_file_sha256=settings_file_sha256,
+                image_override_sha256=image_override_sha256,
+                probe_source_sha256=probe_source_sha256,
+                runtime_environment_override=runtime_environment_override,
+                runtime_environment_sha256=runtime_environment_sha256,
+                reviewed_environment_keys=reviewed_environment_keys,
+                operation_deadline=operation_deadline,
+            )
+            replacement_manifest["services"].update(
+                worker_replacements.get("services", {})
+            )
+            applied.extend(WORKER_SERVICES)
+            write_replacement_manifest(backup_dir, replacement_manifest)
+
+            attempted.extend(CORE_SERVICES)
+            core_replacements = apply_batch(
+                args,
+                baseline,
+                lock,
+                backup_dir,
+                image_override,
+                CORE_SERVICES,
+                (),
+                tuple(applied),
+                settings_file=settings_file,
+                settings_file_sha256=settings_file_sha256,
+                image_override_sha256=image_override_sha256,
+                probe_source_sha256=probe_source_sha256,
+                runtime_environment_override=runtime_environment_override,
+                runtime_environment_sha256=runtime_environment_sha256,
+                reviewed_environment_keys=reviewed_environment_keys,
+                operation_deadline=operation_deadline,
+            )
+            replacement_manifest["services"].update(
+                core_replacements.get("services", {})
+            )
+            applied.extend(CORE_SERVICES)
+            write_replacement_manifest(backup_dir, replacement_manifest)
+            final = capture(Path(args.project_dir), deadline=operation_deadline)
+            compare_post_apply(baseline, final, lock, TARGET_SERVICES)
+            compare_untargeted_runtime(baseline, final)
+            compare_source_and_quiescence(baseline, final)
+            write_json(backup_dir / "post-apply.json", final)
+        except Exception as exc:
+            failure_record = {
+                "schema": FAILURE_SCHEMA,
+                "original_error": exception_reason(exc),
+            }
+            try:
+                write_json(backup_dir / "apply-failure.json", failure_record)
+            except OSError:
+                pass
+            if backup_images is not None and attempted:
                 try:
-                    write_json(backup_dir / "apply-failure.json", failure_record)
-                except OSError:
-                    pass
-                if backup_images is not None and attempted:
-                    try:
-                        failed_state = capture(
-                            Path(args.project_dir), deadline=operation_deadline
-                        )
-                        write_json(backup_dir / "failed-state.json", failed_state)
-                        discovered = record_replacements(
+                    failed_state = capture(
+                        Path(args.project_dir), deadline=operation_deadline
+                    )
+                    write_json(backup_dir / "failed-state.json", failed_state)
+                    discovered = record_replacements(
+                        baseline,
+                        failed_state,
+                        lock,
+                        tuple(attempted),
+                        strict=False,
+                    )
+                    replacement_manifest["services"].update(
+                        discovered.get("services", {})
+                    )
+                    replacement_manifest["unresolved"] = sorted(
+                        set(replacement_manifest.get("unresolved", []))
+                        | set(discovered.get("unresolved", []))
+                    )
+                    write_replacement_manifest(
+                        backup_dir, replacement_manifest, name="failed-replacements.json"
+                    )
+                    if replacement_manifest["services"]:
+                        compensating_rollback(
+                            args,
                             baseline,
-                            failed_state,
-                            lock,
-                            tuple(attempted),
-                            strict=False,
-                        )
-                        replacement_manifest["services"].update(
-                            discovered.get("services", {})
+                            backup_images,
+                            replacement_manifest,
+                            backup_dir,
+                            runtime_environment_override=runtime_environment_override,
+                            runtime_environment_sha256=runtime_environment_sha256,
+                            operation_deadline=operation_deadline,
                         )
-                        replacement_manifest["unresolved"] = sorted(
-                            set(replacement_manifest.get("unresolved", []))
-                            | set(discovered.get("unresolved", []))
-                        )
-                        write_replacement_manifest(
-                            backup_dir, replacement_manifest, name="failed-replacements.json"
-                        )
-                        if replacement_manifest["services"]:
-                            compensating_rollback(
-                                args,
-                                baseline,
-                                backup_images,
-                                replacement_manifest,
-                                backup_dir,
-                                runtime_environment_override=runtime_environment_override,
-                                runtime_environment_sha256=runtime_environment_sha256,
-                                operation_deadline=operation_deadline,
-                            )
-                    except Exception as rollback_error:
-                        failure_record["rollback_error"] = exception_reason(rollback_error)
-                        try:
-                            write_json(backup_dir / "apply-failure.json", failure_record)
-                        except OSError:
-                            pass
-                        raise GuardError(
-                            "guarded apply failed and compensating rollback failed; "
-                            "manual recovery is required; "
-                            f"original failure: {failure_record['original_error']}; "
-                            f"recovery failure: {failure_record['rollback_error']}"
-                        ) from exc
-                raise
+                except Exception as rollback_error:
+                    failure_record["rollback_error"] = exception_reason(rollback_error)
+                    try:
+                        write_json(backup_dir / "apply-failure.json", failure_record)
+                    except OSError:
+                        pass
+                    raise GuardError(
+                        "guarded apply failed and compensating rollback failed; "
+                        "manual recovery is required; "
+                        f"original failure: {failure_record['original_error']}; "
+                        f"recovery failure: {failure_record['rollback_error']}"
+                    ) from exc
+            raise
     print(f"apply: verified {len(TARGET_SERVICES)} targeted services; backup={backup_dir}")
     return 0
 
diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
index c3f381b342..ed8930cf3e 100644
--- a/docs/train-unstract-health-deployment.md
+++ b/docs/train-unstract-health-deployment.md
@@ -89,8 +89,12 @@ be the exact names and IDs already captured from the live stack, unless a
 deliberate static image change has separately been reviewed. The lock also
 contains the candidate commit's tree hash and hashes for the two overlays, the
 core and database probes, and the development essentials Compose file. The
-guard writes a temporary image override from this lock, so Compose cannot
-silently resolve a different registry or tag. Only `runner` and the twelve
+guard writes a private mode-600 image override and Compose settings file from
+this lock, so every health, environment, and image replay uses the same
+immutable references, `VERSION`, and staged probe path. These artifacts remain
+in the preflight state directory and apply backup directory for later startup
+or recovery; the guard validates their SHA-256 values before each Compose
+invocation. Only `runner` and the twelve
 worker services point at the new build; backend, frontend, platform-service,
 x2text-service, and the seven core data services point at the captured static
 references.
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 8944f73f43..8128352d2b 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import importlib.util
+import json
+import subprocess
 from copy import deepcopy
 from pathlib import Path
 
@@ -268,6 +270,147 @@ def test_runtime_environment_override_is_private(tmp_path: Path) -> None:
         guard.validate_private_override(path, expected_sha256=digest)
 
 
+def test_durable_compose_inputs_are_private_and_reusable(tmp_path: Path) -> None:
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nprintf probe\n", encoding="utf-8")
+    probe_sha256 = guard.sha256_file(probe)
+    lock = {
+        "images": {
+            service: {"reference": f"candidate/{service}"}
+            for service in guard.TARGET_SERVICES
+        }
+    }
+    image_override = tmp_path / guard.CANDIDATE_IMAGE_FILENAME
+    settings = tmp_path / guard.COMPOSE_SETTINGS_FILENAME
+
+    guard.candidate_image_override(lock, image_override)
+    guard.candidate_image_override(lock, image_override)
+    guard.write_compose_settings(
+        "goal09-test",
+        probe,
+        settings,
+        probe_source_sha256=probe_sha256,
+    )
+    guard.write_compose_settings(
+        "goal09-test",
+        probe,
+        settings,
+        probe_source_sha256=probe_sha256,
+    )
+
+    assert image_override.stat().st_mode & 0o777 == 0o600
+    assert settings.stat().st_mode & 0o777 == 0o600
+    guard.validate_compose_settings(
+        settings,
+        candidate_version="goal09-test",
+        probe_source=probe,
+        probe_source_sha256=probe_sha256,
+    )
+
+    probe.write_text("#!/bin/sh\nprintf changed\n", encoding="utf-8")
+    with pytest.raises(guard.GuardError, match="health probe source changed"):
+        guard.validate_compose_settings(
+            settings,
+            candidate_version="goal09-test",
+            probe_source=probe,
+            probe_source_sha256=probe_sha256,
+        )
+
+
+def test_compose_replay_consumes_durable_settings_and_overrides(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nprintf probe\n", encoding="utf-8")
+    probe_sha256 = guard.sha256_file(probe)
+    lock = {
+        "images": {
+            service: {"reference": f"candidate/{service}"}
+            for service in guard.TARGET_SERVICES
+        }
+    }
+    image_override = guard.candidate_image_override(
+        lock, tmp_path / guard.CANDIDATE_IMAGE_FILENAME
+    )
+    settings = tmp_path / guard.COMPOSE_SETTINGS_FILENAME
+    guard.write_compose_settings(
+        "goal09-test",
+        probe,
+        settings,
+        probe_source_sha256=probe_sha256,
+    )
+    settings_sha256 = guard.sha256_file(settings)
+    environment_override = tmp_path / guard.RUNTIME_ENVIRONMENT_FILENAME
+    guard.write_runtime_environment_override(
+        {"runner": {"APP_MODE": "test"}}, environment_override
+    )
+    calls: list[tuple[list[str], dict[str, str] | None]] = []
+
+    def fake_run(
+        args: list[str],
+        *,
+        cwd: Path | None = None,
+        env: dict[str, str] | None = None,
+        **_: object,
+    ) -> subprocess.CompletedProcess[str]:
+        calls.append((args, env))
+        if "config" in args:
+            return subprocess.CompletedProcess(args, 0, json.dumps({"services": {}}), "")
+        return subprocess.CompletedProcess(args, 0, "", "")
+
+    monkeypatch.setattr(guard, "run", fake_run)
+    guard.compose_config(
+        tmp_path,
+        ("compose.yaml",),
+        candidate_version="goal09-test",
+        probe_source=probe,
+        image_override=image_override,
+        image_override_sha256=guard.sha256_file(image_override),
+        environment_override=environment_override,
+        environment_override_sha256=guard.sha256_file(environment_override),
+        settings_file=settings,
+        settings_file_sha256=guard.sha256_file(settings),
+        probe_source_sha256=probe_sha256,
+    )
+    guard.targeted_up(
+        tmp_path,
+        ("compose.yaml",),
+        ("runner",),
+        candidate_version="goal09-test",
+        probe_source=probe,
+        image_override=image_override,
+        image_override_sha256=guard.sha256_file(image_override),
+        environment_override=environment_override,
+        environment_override_sha256=guard.sha256_file(environment_override),
+        settings_file=settings,
+        settings_file_sha256=guard.sha256_file(settings),
+        probe_source_sha256=probe_sha256,
+    )
+
+    assert len(calls) == 2
+    config_args, config_env = calls[0]
+    assert config_args[:3] == ["docker", "compose", "--env-file"]
+    assert str(settings) in config_args
+    assert str(image_override) in config_args
+    assert str(environment_override) in config_args
+    assert config_env and config_env["VERSION"] == "goal09-test"
+    assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(probe)
+    assert calls[1][0][-1] == "runner"
+
+    settings.write_text(settings.read_text(encoding="utf-8").replace("goal09-test", "tampered"), encoding="utf-8")
+    with pytest.raises(guard.GuardError, match="Compose settings changed"):
+        guard.targeted_up(
+            tmp_path,
+            ("compose.yaml",),
+            ("runner",),
+            candidate_version="goal09-test",
+            probe_source=probe,
+            settings_file=settings,
+            settings_file_sha256=settings_sha256,
+            probe_source_sha256=probe_sha256,
+        )
+
+
 def test_core_probe_checks_require_read_only_probe_mount() -> None:
     config, baseline, lock, authored = candidate_fixture()
     config["services"]["db"]["volumes"] = []

From 4ee386a08e1d4db4f11b9cb867b45b8e4cbb393b Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 21:50:48 -0400
Subject: [PATCH 34/48] Preserve live Compose inputs for guarded replay

---
 .../scripts/train_health_deployment_guard.py  | 479 ++++++++++++++++--
 .../test_train_health_deployment_guard.py     | 128 ++++-
 2 files changed, 569 insertions(+), 38 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 019698d86e..0a556b4c36 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -1,13 +1,15 @@
 #!/usr/bin/env python3
 """Guarded, targeted deployment helper for the Train Unstract health checks.
 
-The default operation is read-only.  ``capture`` records a sanitized runtime
-snapshot and ``preflight`` refuses to continue when the dirty live checkout,
-container identity, mounts, networks, or environment hashes drift.  The
-mutating ``apply`` and ``rollback`` phases require an explicit confirmation
-token and an external candidate image lock.  They recreate only the 24
-health-covered workloads in two bounded batches; they never build, pull,
-delete source files, reset the checkout, or run project-wide Compose commands.
+``capture`` records a sanitized runtime snapshot. ``preflight`` refuses to
+continue when the dirty live checkout, ignored Compose inputs, container
+identity, mounts, networks, or environment hashes drift, and refreshes the
+private durable replay files beside the candidate lock. The mutating ``start``,
+``apply``, and ``rollback`` phases require explicit confirmation tokens and an
+external candidate image lock. ``start`` is the authoritative whole-project
+startup path; ``apply`` and ``rollback`` recreate only the 24 health-covered
+workloads in two bounded batches. They never build, pull, delete source files,
+reset the checkout, or run project-wide destructive Compose commands.
 
 The script deliberately keeps secret values out of all output.  Environment
 values are represented by length and SHA-256 digest only, and health log
@@ -100,6 +102,9 @@
 CANDIDATE_IMAGE_FILENAME = "candidate-image.override.yaml"
 COMPOSE_SETTINGS_SCHEMA = "unstract-health-compose-settings/v1"
 COMPOSE_SETTINGS_FILENAME = "compose-settings.env"
+START_CONFIRM_TOKEN = "START_UNSTRACT_HEALTH"
+LIVE_COMPOSE_TRAIN = "docker/compose.train.yaml"
+LIVE_ENV_RELATIVE = "docker/.env"
 RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
@@ -646,8 +651,48 @@ def inspect_runtime_environment(
     return result
 
 
+def source_file_fingerprint(path: Path) -> dict[str, Any]:
+    """Hash one ignored Compose input without retaining its contents."""
+    try:
+        metadata = path.lstat()
+    except FileNotFoundError:
+        return {"missing": True}
+    except OSError as exc:
+        raise GuardError(f"cannot stat live Compose input {path}: {exc}") from exc
+    if stat.S_ISLNK(metadata.st_mode):
+        target = os.readlink(path)
+        resolved = path.resolve()
+        target_metadata = resolved.lstat()
+        if not stat.S_ISREG(target_metadata.st_mode):
+            raise GuardError(f"live Compose input does not resolve to a regular file: {path}")
+        return {
+            "symlink": target,
+            "target_bytes": target_metadata.st_size,
+            "target_sha256": sha256_file(resolved),
+        }
+    if not stat.S_ISREG(metadata.st_mode):
+        return {"special": True, "mode": stat.S_IFMT(metadata.st_mode)}
+    return {"bytes": metadata.st_size, "sha256": sha256_file(path)}
+
+
+def live_compose_inputs(
+    project_dir: Path, *, live_env_file: Path | None = None
+) -> dict[str, dict[str, Any]]:
+    env_path = live_env_file or (project_dir / LIVE_ENV_RELATIVE)
+    return {
+        LIVE_COMPOSE_TRAIN: source_file_fingerprint(project_dir / LIVE_COMPOSE_TRAIN),
+        "live_env_file": {
+            "path": str(env_path.resolve()),
+            **source_file_fingerprint(env_path),
+        },
+    }
+
+
 def source_state(
-    project_dir: Path, *, deadline: OperationDeadline | None = None
+    project_dir: Path,
+    *,
+    live_env_file: Path | None = None,
+    deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     head = run(
         ["git", "-C", str(project_dir), "rev-parse", "HEAD"], deadline=deadline
@@ -698,6 +743,7 @@ def source_state(
         "status_paths": paths,
         "status_hashes": status_hashes,
         "tracked_dirty_or_untracked_count": len(paths),
+        "live_inputs": live_compose_inputs(project_dir, live_env_file=live_env_file),
     }
 
 
@@ -930,7 +976,10 @@ def runtime_context(*, deadline: OperationDeadline | None = None) -> dict[str, A
 
 
 def capture(
-    project_dir: Path, *, deadline: OperationDeadline | None = None
+    project_dir: Path,
+    *,
+    live_env_file: Path | None = None,
+    deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     uid = os.getuid() if hasattr(os, "getuid") else None
     return {
@@ -942,7 +991,9 @@ def capture(
             "rootless_project": PROJECT,
         },
         "runtime_context": runtime_context(deadline=deadline),
-        "source": source_state(project_dir, deadline=deadline),
+        "source": source_state(
+            project_dir, live_env_file=live_env_file, deadline=deadline
+        ),
         "job_quiescence": settled_queue_snapshot(deadline),
         "containers": inspect_project(deadline=deadline),
     }
@@ -1114,6 +1165,10 @@ def load_baseline(path: Path) -> dict[str, Any]:
         raise GuardError("baseline has an unsupported schema; capture a fresh baseline")
     if baseline.get("source", {}).get("schema") != SOURCE_STATE_SCHEMA:
         raise GuardError("baseline source state is incomplete; capture a fresh baseline")
+    if not isinstance(baseline.get("source", {}).get("live_inputs"), dict):
+        raise GuardError(
+            "baseline live Compose input hashes are missing; capture a fresh baseline"
+        )
     if not isinstance(baseline.get("containers"), list):
         raise GuardError("baseline container snapshot is missing")
     if not isinstance(baseline.get("runtime_context"), dict):
@@ -1207,6 +1262,10 @@ def compose_config(
     *,
     candidate_version: str,
     probe_source: Path,
+    live_env_file: Path | None = None,
+    expected_source: dict[str, Any] | None = None,
+    candidate_source: Path | None = None,
+    candidate_lock: dict[str, Any] | None = None,
     image_override: Path | None = None,
     image_override_sha256: str | None = None,
     environment_override: Path | None = None,
@@ -1216,6 +1275,12 @@ def compose_config(
     probe_source_sha256: str | None = None,
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
+    verify_candidate_source_state(candidate_source, candidate_lock)
+    verify_live_compose_inputs(
+        project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+    )
     env = os.environ.copy()
     if settings_file:
         settings = validate_compose_settings(
@@ -1246,8 +1311,8 @@ def compose_config(
         )
         files += (str(environment_override),)
     args = ["docker", "compose"]
-    if settings_file:
-        args.extend(["--env-file", str(settings_file)])
+    if live_env_file:
+        args.extend(["--env-file", str(live_env_file)])
     for compose_file in files:
         args.extend(["-f", compose_file])
     args.extend(["config", "--format", "json"])
@@ -1747,6 +1812,36 @@ def verify_artifacts(root: Path, lock: dict[str, Any]) -> None:
             raise GuardError(f"candidate artifact hash mismatch: {name}")
 
 
+def verify_candidate_source_state(
+    candidate_source: Path | None, lock: dict[str, Any] | None
+) -> None:
+    if candidate_source is None or lock is None:
+        return
+    require_clean_candidate_source(
+        candidate_source, lock["source_commit"], lock.get("source_tree")
+    )
+    verify_artifacts(candidate_source, lock)
+
+
+def verify_live_compose_inputs(
+    project_dir: Path,
+    *,
+    live_env_file: Path | None,
+    expected_source: dict[str, Any] | None,
+) -> None:
+    """Recheck ignored live inputs before each Compose invocation."""
+    actual = live_compose_inputs(project_dir, live_env_file=live_env_file)
+    if expected_source is None:
+        return
+    expected = expected_source.get("live_inputs")
+    if not isinstance(expected, dict):
+        raise GuardError(
+            "baseline source state lacks live Compose input hashes; capture a fresh baseline"
+        )
+    if expected != actual:
+        raise GuardError("ignored live Compose input drifted since baseline capture")
+
+
 @contextlib.contextmanager
 def advisory_lock(
     *,
@@ -1835,19 +1930,24 @@ def capture_and_write(
     project_dir: Path,
     output: Path,
     *,
+    live_env_file: Path | None = None,
     operation_deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
-    snapshot = capture(project_dir, deadline=operation_deadline)
+    snapshot = capture(
+        project_dir,
+        live_env_file=live_env_file,
+        deadline=operation_deadline,
+    )
     write_json(output, snapshot)
     return snapshot
 
 
 def compose_args(
-    compose_files: tuple[str, ...], *, settings_file: Path | None = None
+    compose_files: tuple[str, ...], *, live_env_file: Path | None = None
 ) -> list[str]:
     args = ["docker", "compose"]
-    if settings_file:
-        args.extend(["--env-file", str(settings_file)])
+    if live_env_file:
+        args.extend(["--env-file", str(live_env_file)])
     for compose_file in compose_files:
         args.extend(["-f", compose_file])
     return args
@@ -1993,6 +2093,41 @@ def write_runtime_environment_override(
     )
 
 
+def reviewed_environment_keys_from_override(path: Path) -> dict[str, set[str]]:
+    """Recover reviewed key names from durable YAML without reading values aloud."""
+    validate_private_override(path)
+    result: dict[str, set[str]] = {}
+    current_service: str | None = None
+    in_environment = False
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise GuardError(f"cannot read runtime environment override: {path}") from exc
+    for line in lines:
+        if line.startswith("  ") and not line.startswith("    ") and line.endswith(":"):
+            current_service = line[2:-1]
+            in_environment = False
+            continue
+        if current_service and line == "    environment:":
+            in_environment = True
+            result.setdefault(current_service, set())
+            continue
+        if current_service and in_environment and line.startswith("      "):
+            key_text, separator, _ = line[6:].partition(":")
+            if not separator:
+                raise GuardError(f"runtime environment override has invalid YAML: {path}")
+            try:
+                key = json.loads(key_text.strip())
+            except json.JSONDecodeError as exc:
+                raise GuardError(
+                    f"runtime environment override has an invalid key: {path}"
+                ) from exc
+            if not isinstance(key, str):
+                raise GuardError(f"runtime environment override key is not a string: {path}")
+            result[current_service].add(key)
+    return result
+
+
 def validate_private_file(
     path: Path,
     *,
@@ -2047,6 +2182,10 @@ def targeted_up(
     *,
     candidate_version: str,
     probe_source: Path,
+    live_env_file: Path | None = None,
+    expected_source: dict[str, Any] | None = None,
+    candidate_source: Path | None = None,
+    candidate_lock: dict[str, Any] | None = None,
     image_override: Path | None = None,
     image_override_sha256: str | None = None,
     environment_override: Path | None = None,
@@ -2056,6 +2195,12 @@ def targeted_up(
     probe_source_sha256: str | None = None,
     deadline: OperationDeadline | None = None,
 ) -> None:
+    verify_candidate_source_state(candidate_source, candidate_lock)
+    verify_live_compose_inputs(
+        project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+    )
     env = os.environ.copy()
     if settings_file:
         settings = validate_compose_settings(
@@ -2082,7 +2227,7 @@ def targeted_up(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = compose_args(files, settings_file=settings_file)
+    args = compose_args(files, live_env_file=live_env_file)
     args.extend(
         [
             "up",
@@ -2098,6 +2243,54 @@ def targeted_up(
     run(args, cwd=project_dir, env=env, deadline=deadline)
 
 
+def compose_start(
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    *,
+    candidate_version: str,
+    probe_source: Path,
+    live_env_file: Path,
+    expected_source: dict[str, Any] | None,
+    candidate_source: Path,
+    candidate_lock: dict[str, Any],
+    image_override: Path,
+    image_override_sha256: str,
+    environment_override: Path,
+    environment_override_sha256: str,
+    settings_file: Path,
+    settings_file_sha256: str,
+    probe_source_sha256: str,
+    deadline: OperationDeadline | None = None,
+) -> None:
+    """Start the whole stack through the durable guarded Compose inputs."""
+    verify_candidate_source_state(candidate_source, candidate_lock)
+    verify_live_compose_inputs(
+        project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+    )
+    env = os.environ.copy()
+    settings = validate_compose_settings(
+        settings_file,
+        candidate_version=candidate_version,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        expected_sha256=settings_file_sha256,
+    )
+    env["VERSION"] = settings["VERSION"]
+    env["UNSTRACT_HEALTHCHECK_SOURCE"] = settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+    validate_candidate_image_override(
+        image_override, expected_sha256=image_override_sha256
+    )
+    validate_private_override(
+        environment_override, expected_sha256=environment_override_sha256
+    )
+    files = compose_files + (str(image_override), str(environment_override))
+    args = compose_args(files, live_env_file=live_env_file)
+    args.extend(["up", "-d", "--no-build", "--pull", "never"])
+    run(args, cwd=project_dir, env=env, deadline=deadline)
+
+
 def wait_healthy(
     services: tuple[str, ...],
     timeout_seconds: int = 240,
@@ -2305,12 +2498,22 @@ def rollback_override(
 
 def command_capture(args: argparse.Namespace) -> int:
     deadline = OperationDeadline(args.operation_timeout)
+    project_dir = Path(args.project_dir)
+    live_env_file = resolve_live_env_file(args, project_dir)
     capture_and_write(
-        Path(args.project_dir), Path(args.output), operation_deadline=deadline
+        project_dir,
+        Path(args.output),
+        live_env_file=live_env_file,
+        operation_deadline=deadline,
     )
     return 0
 
 
+def resolve_live_env_file(args: argparse.Namespace, project_dir: Path) -> Path:
+    configured = getattr(args, "live_env_file", None)
+    return Path(configured).resolve() if configured else (project_dir / LIVE_ENV_RELATIVE).resolve()
+
+
 def prepare(
     args: argparse.Namespace,
     image_override: Path,
@@ -2329,6 +2532,9 @@ def prepare(
         Path(args.candidate_source), lock["source_commit"], lock.get("source_tree")
     )
     verify_artifacts(Path(args.candidate_source), lock)
+    project_dir = Path(args.project_dir)
+    candidate_source = Path(args.candidate_source)
+    live_env_file = resolve_live_env_file(args, project_dir)
     probe_source = Path(args.probe_source)
     probe_source_sha256 = (lock.get("artifacts") or {}).get(
         "docker/healthchecks/unstract-services.sh"
@@ -2355,7 +2561,11 @@ def prepare(
     candidate_images = candidate_image_snapshot(lock, deadline=operation_deadline)
     if not candidate_images:
         raise GuardError("no candidate images were verified")
-    current = capture(Path(args.project_dir), deadline=operation_deadline)
+    current = capture(
+        project_dir,
+        live_env_file=live_env_file,
+        deadline=operation_deadline,
+    )
     compare_baseline_current(baseline, current, allow_new_probe=False)
     compare_untargeted_runtime(baseline, current)
     compare_source_and_quiescence(baseline, current)
@@ -2363,10 +2573,14 @@ def prepare(
         deadline=operation_deadline
     )
     config = compose_config(
-        Path(args.project_dir),
+        project_dir,
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
         probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         settings_file=settings_file,
@@ -2387,10 +2601,14 @@ def prepare(
     )
     validate_private_override(runtime_environment_override)
     config = compose_config(
-        Path(args.project_dir),
+        project_dir,
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
         probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
         settings_file=settings_file,
         settings_file_sha256=settings_file_sha256,
         probe_source_sha256=probe_source_sha256,
@@ -2400,10 +2618,14 @@ def prepare(
         deadline=operation_deadline,
     )
     authored_baseline = compose_config(
-        Path(args.project_dir),
+        project_dir,
         tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
         candidate_version=lock["candidate_version"],
         probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
         settings_file=settings_file,
         settings_file_sha256=settings_file_sha256,
         probe_source_sha256=probe_source_sha256,
@@ -2565,7 +2787,13 @@ def compensating_rollback(
     services = tuple((replacement_manifest.get("services") or {}).keys())
     if not services:
         raise GuardError("no exact replacement IDs were recorded for compensating rollback")
-    current = capture(Path(args.project_dir), deadline=operation_deadline)
+    project_dir = Path(args.project_dir)
+    live_env_file = resolve_live_env_file(args, project_dir)
+    current = capture(
+        project_dir,
+        live_env_file=live_env_file,
+        deadline=operation_deadline,
+    )
     compare_source_and_quiescence(baseline, current)
     verify_replacement_ids(current, replacement_manifest)
     override = backup_dir / "compensating-rollback.override.yaml"
@@ -2578,21 +2806,31 @@ def compensating_rollback(
         # Recheck identity and quiescence after acquiring the DB lock.  The
         # process may be disconnected when db itself is recreated; the local
         # operation lock remains held for that bounded transaction.
-        locked = capture(Path(args.project_dir), deadline=operation_deadline)
+        locked = capture(
+            project_dir,
+            live_env_file=live_env_file,
+            deadline=operation_deadline,
+        )
         compare_source_and_quiescence(baseline, locked)
         verify_replacement_ids(locked, replacement_manifest)
         targeted_up(
-            Path(args.project_dir),
+            project_dir,
             rollback_files,
             services,
             candidate_version="rollback-unused",
             probe_source=Path(args.probe_source),
+            live_env_file=live_env_file,
+            expected_source=baseline["source"],
             environment_override=runtime_environment_override,
             environment_override_sha256=runtime_environment_sha256,
             deadline=operation_deadline,
         )
         wait_running(services, operation_deadline=operation_deadline)
-    final = capture(Path(args.project_dir), deadline=operation_deadline)
+    final = capture(
+        project_dir,
+        live_env_file=live_env_file,
+        deadline=operation_deadline,
+    )
     verify_rollback_result(
         baseline,
         final,
@@ -2624,11 +2862,18 @@ def apply_batch(
     operation_deadline: OperationDeadline,
 ) -> dict[str, Any]:
     compose_files = tuple(args.compose_file or DEFAULT_COMPOSE_FILES)
+    project_dir = Path(args.project_dir)
+    candidate_source = Path(args.candidate_source)
+    live_env_file = resolve_live_env_file(args, project_dir)
     validate_private_override(
         runtime_environment_override, expected_sha256=runtime_environment_sha256
     )
     with advisory_lock(deadline=operation_deadline):
-        fresh = capture(Path(args.project_dir), deadline=operation_deadline)
+        fresh = capture(
+            project_dir,
+            live_env_file=live_env_file,
+            deadline=operation_deadline,
+        )
         compare_untargeted_runtime(baseline, fresh)
         compare_source_and_quiescence(baseline, fresh)
         verify_untouched_targets(baseline, fresh, untouched_services)
@@ -2636,10 +2881,14 @@ def apply_batch(
             compare_post_apply(baseline, fresh, lock, applied_services)
         candidate_image_snapshot(lock, deadline=operation_deadline)
         config = compose_config(
-            Path(args.project_dir),
+            project_dir,
             compose_files,
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
+            live_env_file=live_env_file,
+            expected_source=baseline["source"],
+            candidate_source=candidate_source,
+            candidate_lock=lock,
             image_override=image_override,
             image_override_sha256=image_override_sha256,
             environment_override=runtime_environment_override,
@@ -2650,10 +2899,14 @@ def apply_batch(
             deadline=operation_deadline,
         )
         authored_baseline = compose_config(
-            Path(args.project_dir),
+            project_dir,
             compose_files,
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
+            live_env_file=live_env_file,
+            expected_source=baseline["source"],
+            candidate_source=candidate_source,
+            candidate_lock=lock,
             settings_file=settings_file,
             settings_file_sha256=settings_file_sha256,
             probe_source_sha256=probe_source_sha256,
@@ -2672,11 +2925,15 @@ def apply_batch(
         if not final_quiescence.get("stability", {}).get("stable"):
             raise GuardError("queue was not settled immediately before targeted recreation")
         targeted_up(
-            Path(args.project_dir),
+            project_dir,
             compose_files,
             services,
             candidate_version=lock["candidate_version"],
             probe_source=Path(args.probe_source),
+            live_env_file=live_env_file,
+            expected_source=baseline["source"],
+            candidate_source=candidate_source,
+            candidate_lock=lock,
             image_override=image_override,
             image_override_sha256=image_override_sha256,
             environment_override=runtime_environment_override,
@@ -2686,11 +2943,19 @@ def apply_batch(
             probe_source_sha256=probe_source_sha256,
             deadline=operation_deadline,
         )
-        observed = capture(Path(args.project_dir), deadline=operation_deadline)
+        observed = capture(
+            project_dir,
+            live_env_file=live_env_file,
+            deadline=operation_deadline,
+        )
         replacements = record_replacements(baseline, observed, lock, services)
         write_replacement_manifest(backup_dir, replacements, name=f"replacements-{services[0]}.json")
         wait_healthy(services, operation_deadline=operation_deadline)
-        final = capture(Path(args.project_dir), deadline=operation_deadline)
+        final = capture(
+            project_dir,
+            live_env_file=live_env_file,
+            deadline=operation_deadline,
+        )
         compare_post_apply(baseline, final, lock, services)
         compare_untargeted_runtime(baseline, final)
         compare_source_and_quiescence(baseline, final)
@@ -2718,16 +2983,127 @@ def command_preflight(args: argparse.Namespace) -> int:
     print(
         "preflight: candidate source, image lock, Compose identity, runtime, "
         "data, network, environment, queue, and active-job state verified; "
-        f"durable_artifacts={state_dir}"
+        f"durable replay state refreshed at {state_dir}"
     )
     return 0
 
 
+def command_start(args: argparse.Namespace) -> int:
+    """Replay the durable state through the normal whole-project startup path."""
+    if args.confirm != START_CONFIRM_TOKEN:
+        raise GuardError(f"start requires --confirm {START_CONFIRM_TOKEN}")
+    operation_deadline = OperationDeadline(args.operation_timeout)
+    project_dir = Path(args.project_dir)
+    live_env_file = resolve_live_env_file(args, project_dir)
+    state_dir = Path(args.state_dir).resolve()
+    image_override = state_dir / CANDIDATE_IMAGE_FILENAME
+    settings_file = state_dir / COMPOSE_SETTINGS_FILENAME
+    runtime_environment_override = state_dir / RUNTIME_ENVIRONMENT_FILENAME
+    baseline = load_baseline(Path(args.baseline))
+    lock = load_lock(Path(args.candidate_lock))
+    candidate_source = Path(args.candidate_source)
+    probe_source = Path(args.probe_source)
+    require_clean_candidate_source(
+        candidate_source, lock["source_commit"], lock.get("source_tree")
+    )
+    verify_artifacts(candidate_source, lock)
+    probe_source_sha256 = (lock.get("artifacts") or {}).get(
+        "docker/healthchecks/unstract-services.sh"
+    )
+    if not isinstance(probe_source_sha256, str):
+        raise GuardError("candidate lock lacks the guarded probe source digest")
+    validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
+    validate_candidate_image_override(image_override)
+    validate_private_override(runtime_environment_override)
+    settings = validate_compose_settings(
+        settings_file,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
+    image_override_sha256 = sha256_file(image_override)
+    settings_file_sha256 = sha256_file(settings_file)
+    runtime_environment_sha256 = sha256_file(runtime_environment_override)
+    candidate_images = candidate_image_snapshot(lock, deadline=operation_deadline)
+    if not candidate_images:
+        raise GuardError("no candidate images were verified")
+    compose_files = tuple(args.compose_file or DEFAULT_COMPOSE_FILES)
+    config = compose_config(
+        project_dir,
+        compose_files,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=runtime_environment_override,
+        environment_override_sha256=runtime_environment_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
+        deadline=operation_deadline,
+    )
+    authored_baseline = compose_config(
+        project_dir,
+        compose_files,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
+        deadline=operation_deadline,
+    )
+    reviewed_keys = reviewed_environment_keys_from_override(runtime_environment_override)
+    check_candidate_config(
+        config,
+        baseline,
+        lock,
+        authored_baseline,
+        reviewed_environment_keys=reviewed_keys,
+    )
+    compose_start(
+        project_dir,
+        compose_files,
+        candidate_version=settings["VERSION"],
+        probe_source=probe_source,
+        live_env_file=live_env_file,
+        expected_source=baseline["source"],
+        candidate_source=candidate_source,
+        candidate_lock=lock,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=runtime_environment_override,
+        environment_override_sha256=runtime_environment_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        probe_source_sha256=probe_source_sha256,
+        deadline=operation_deadline,
+    )
+    wait_running(TARGET_SERVICES, operation_deadline=operation_deadline)
+    post = capture(
+        project_dir,
+        live_env_file=live_env_file,
+        deadline=operation_deadline,
+    )
+    write_json(state_dir / "post-start.json", post)
+    print(f"start: verified durable Compose replay; state={state_dir}")
+    return 0
+
+
 def command_apply(args: argparse.Namespace) -> int:
     if args.confirm != CONFIRM_TOKEN:
         raise GuardError(f"apply requires --confirm {CONFIRM_TOKEN}")
     operation_deadline = OperationDeadline(args.operation_timeout)
     backup_dir = Path(args.backup_dir).resolve()
+    project_dir = Path(args.project_dir)
+    live_env_file = resolve_live_env_file(args, project_dir)
     attempted: list[str] = []
     applied: list[str] = []
     backup_images: dict[str, Any] | None = None
@@ -2767,7 +3143,11 @@ def command_apply(args: argparse.Namespace) -> int:
             if not isinstance(probe_source_sha256, str):
                 raise GuardError("candidate lock lacks the guarded probe source digest")
             with advisory_lock(deadline=operation_deadline):
-                fresh = capture(Path(args.project_dir), deadline=operation_deadline)
+                fresh = capture(
+                    project_dir,
+                    live_env_file=live_env_file,
+                    deadline=operation_deadline,
+                )
                 compare_baseline_current(baseline, fresh, allow_new_probe=False)
                 compare_untargeted_runtime(baseline, fresh)
                 compare_source_and_quiescence(baseline, fresh)
@@ -2838,7 +3218,11 @@ def command_apply(args: argparse.Namespace) -> int:
             )
             applied.extend(CORE_SERVICES)
             write_replacement_manifest(backup_dir, replacement_manifest)
-            final = capture(Path(args.project_dir), deadline=operation_deadline)
+            final = capture(
+                project_dir,
+                live_env_file=live_env_file,
+                deadline=operation_deadline,
+            )
             compare_post_apply(baseline, final, lock, TARGET_SERVICES)
             compare_untargeted_runtime(baseline, final)
             compare_source_and_quiescence(baseline, final)
@@ -2855,7 +3239,9 @@ def command_apply(args: argparse.Namespace) -> int:
             if backup_images is not None and attempted:
                 try:
                     failed_state = capture(
-                        Path(args.project_dir), deadline=operation_deadline
+                        project_dir,
+                        live_env_file=live_env_file,
+                        deadline=operation_deadline,
                     )
                     write_json(backup_dir / "failed-state.json", failed_state)
                     discovered = record_replacements(
@@ -2961,6 +3347,11 @@ def command_rollback(args: argparse.Namespace) -> int:
 def add_common(parser: argparse.ArgumentParser) -> None:
     parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
     parser.add_argument("--compose-file", action="append", default=None)
+    parser.add_argument(
+        "--live-env-file",
+        default=None,
+        help="the existing private Compose .env; its values are read but never printed or copied",
+    )
     parser.add_argument("--baseline", required=True)
     parser.add_argument("--candidate-source", required=True)
     parser.add_argument("--candidate-lock", required=True)
@@ -2985,14 +3376,25 @@ def parser() -> argparse.ArgumentParser:
     lock_parser.add_argument("--output", required=True)
     capture_parser = sub.add_parser("capture", help="read-only sanitized runtime snapshot")
     capture_parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
+    capture_parser.add_argument("--live-env-file", default=None)
     capture_parser.add_argument("--output", required=True)
     capture_parser.add_argument(
         "--operation-timeout",
         type=float,
         default=DEFAULT_COMMAND_TIMEOUT_SECONDS,
     )
-    preflight_parser = sub.add_parser("preflight", help="read-only candidate and drift checks")
+    preflight_parser = sub.add_parser(
+        "preflight",
+        help="candidate and drift checks; refreshes private durable replay state",
+    )
     add_common(preflight_parser)
+    start_parser = sub.add_parser(
+        "start",
+        help="guarded whole-project startup using durable candidate/settings/environment state",
+    )
+    add_common(start_parser)
+    start_parser.add_argument("--state-dir", required=True)
+    start_parser.add_argument("--confirm", required=True)
     apply_parser = sub.add_parser("apply", help="explicit targeted recreation")
     add_common(apply_parser)
     apply_parser.add_argument("--backup-dir", required=True)
@@ -3000,6 +3402,7 @@ def parser() -> argparse.ArgumentParser:
     rollback_parser = sub.add_parser("rollback", help="explicit targeted compensating rollback")
     rollback_parser.add_argument("--project-dir", default=str(DEFAULT_PROJECT_DIR))
     rollback_parser.add_argument("--rollback-compose-file", action="append", default=None)
+    rollback_parser.add_argument("--live-env-file", default=None)
     rollback_parser.add_argument("--probe-source", required=True)
     rollback_parser.add_argument("--backup-dir", required=True)
     rollback_parser.add_argument("--confirm", required=True)
@@ -3020,6 +3423,8 @@ def main() -> int:
             return command_lock(args)
         if args.command == "preflight":
             return command_preflight(args)
+        if args.command == "start":
+            return command_start(args)
         if args.command == "apply":
             return command_apply(args)
         if args.command == "rollback":
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 8128352d2b..37749fe034 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -2,6 +2,7 @@
 
 import importlib.util
 import json
+import shutil
 import subprocess
 from copy import deepcopy
 from pathlib import Path
@@ -344,6 +345,12 @@ def test_compose_replay_consumes_durable_settings_and_overrides(
     guard.write_runtime_environment_override(
         {"runner": {"APP_MODE": "test"}}, environment_override
     )
+    live_env = tmp_path / "docker" / ".env"
+    live_env.parent.mkdir()
+    live_env.write_text(
+        "TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry\nCOMPOSE_PROJECT_NAME=test\n",
+        encoding="utf-8",
+    )
     calls: list[tuple[list[str], dict[str, str] | None]] = []
 
     def fake_run(
@@ -355,6 +362,10 @@ def fake_run(
     ) -> subprocess.CompletedProcess[str]:
         calls.append((args, env))
         if "config" in args:
+            env_file = Path(args[args.index("--env-file") + 1])
+            assert "TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry" in env_file.read_text(
+                encoding="utf-8"
+            )
             return subprocess.CompletedProcess(args, 0, json.dumps({"services": {}}), "")
         return subprocess.CompletedProcess(args, 0, "", "")
 
@@ -364,6 +375,7 @@ def fake_run(
         ("compose.yaml",),
         candidate_version="goal09-test",
         probe_source=probe,
+        live_env_file=live_env,
         image_override=image_override,
         image_override_sha256=guard.sha256_file(image_override),
         environment_override=environment_override,
@@ -378,6 +390,7 @@ def fake_run(
         ("runner",),
         candidate_version="goal09-test",
         probe_source=probe,
+        live_env_file=live_env,
         image_override=image_override,
         image_override_sha256=guard.sha256_file(image_override),
         environment_override=environment_override,
@@ -390,13 +403,34 @@ def fake_run(
     assert len(calls) == 2
     config_args, config_env = calls[0]
     assert config_args[:3] == ["docker", "compose", "--env-file"]
-    assert str(settings) in config_args
+    assert str(live_env) in config_args
+    assert str(settings) not in config_args
     assert str(image_override) in config_args
     assert str(environment_override) in config_args
     assert config_env and config_env["VERSION"] == "goal09-test"
     assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(probe)
     assert calls[1][0][-1] == "runner"
 
+    monkeypatch.setattr(guard, "verify_candidate_source_state", lambda *_: None)
+    guard.compose_start(
+        tmp_path,
+        ("compose.yaml",),
+        candidate_version="goal09-test",
+        probe_source=probe,
+        live_env_file=live_env,
+        expected_source=None,
+        candidate_source=tmp_path,
+        candidate_lock=lock,
+        image_override=image_override,
+        image_override_sha256=guard.sha256_file(image_override),
+        environment_override=environment_override,
+        environment_override_sha256=guard.sha256_file(environment_override),
+        settings_file=settings,
+        settings_file_sha256=guard.sha256_file(settings),
+        probe_source_sha256=probe_sha256,
+    )
+    assert calls[2][0][-5:] == ["up", "-d", "--no-build", "--pull", "never"]
+
     settings.write_text(settings.read_text(encoding="utf-8").replace("goal09-test", "tampered"), encoding="utf-8")
     with pytest.raises(guard.GuardError, match="Compose settings changed"):
         guard.targeted_up(
@@ -405,12 +439,104 @@ def fake_run(
             ("runner",),
             candidate_version="goal09-test",
             probe_source=probe,
+            live_env_file=live_env,
             settings_file=settings,
             settings_file_sha256=settings_sha256,
             probe_source_sha256=probe_sha256,
         )
 
 
+def test_compose_rejects_ignored_live_input_drift(tmp_path: Path) -> None:
+    project_dir = tmp_path
+    (project_dir / "docker").mkdir()
+    train_compose = project_dir / guard.LIVE_COMPOSE_TRAIN
+    live_env = project_dir / guard.LIVE_ENV_RELATIVE
+    train_compose.write_text("services: {}\n", encoding="utf-8")
+    live_env.write_text("TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry\n", encoding="utf-8")
+    expected = {"live_inputs": guard.live_compose_inputs(project_dir, live_env_file=live_env)}
+    live_env.write_text("TOOL_REGISTRY_CONFIG_SRC_PATH=/changed\n", encoding="utf-8")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+
+    with pytest.raises(guard.GuardError, match="ignored live Compose input drifted"):
+        guard.compose_config(
+            project_dir,
+            ("compose.yaml",),
+            candidate_version="goal09-test",
+            probe_source=probe,
+            live_env_file=live_env,
+            expected_source=expected,
+            probe_source_sha256=guard.sha256_file(probe),
+        )
+
+
+def test_compose_rechecks_candidate_artifacts_before_each_config(tmp_path: Path) -> None:
+    source = tmp_path / "candidate"
+    source.mkdir()
+    source_root = GUARD_PATH.parents[2]
+    for relative in (
+        "docker/healthchecks/unstract-services.sh",
+        "docker/healthchecks/http-readiness.sh",
+        "docker/healthchecks/postgres-readiness.sh",
+        "docker/docker-compose-dev-essentials.yaml",
+        "docker/compose.train.healthchecks.yaml",
+        "docker/compose.train.worker-healthchecks.yaml",
+    ):
+        destination = source / relative
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(source_root / relative, destination)
+    subprocess.run(["git", "init", "-q"], cwd=source, check=True)
+    subprocess.run(["git", "config", "user.email", "test@example.invalid"], cwd=source, check=True)
+    subprocess.run(["git", "config", "user.name", "test"], cwd=source, check=True)
+    subprocess.run(["git", "add", "."], cwd=source, check=True)
+    subprocess.run(["git", "commit", "-qm", "candidate"], cwd=source, check=True)
+    lock = {
+        "source_commit": subprocess.check_output(
+            ["git", "rev-parse", "HEAD"], cwd=source, text=True
+        ).strip(),
+        "source_tree": subprocess.check_output(
+            ["git", "rev-parse", "HEAD^{tree}"], cwd=source, text=True
+        ).strip(),
+        "artifacts": guard.artifact_hashes(source),
+    }
+    probe = source / "docker/healthchecks/unstract-services.sh"
+
+    real_run = guard.run
+
+    def fake_run(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+        if args and args[0] == "git":
+            return real_run(args, **kwargs)
+        return subprocess.CompletedProcess(args, 0, json.dumps({"services": {}}), "")
+
+    monkeypatch = pytest.MonkeyPatch()
+    monkeypatch.setattr(guard, "run", fake_run)
+    try:
+        guard.compose_config(
+            tmp_path,
+            ("compose.yaml",),
+            candidate_version="goal09-test",
+            probe_source=probe,
+            candidate_source=source,
+            candidate_lock=lock,
+            probe_source_sha256=guard.sha256_file(probe),
+        )
+        (source / "docker/compose.train.healthchecks.yaml").write_text(
+            "services: {}\n", encoding="utf-8"
+        )
+        with pytest.raises(guard.GuardError, match="candidate source must be clean"):
+            guard.compose_config(
+                tmp_path,
+                ("compose.yaml",),
+                candidate_version="goal09-test",
+                probe_source=probe,
+                candidate_source=source,
+                candidate_lock=lock,
+                probe_source_sha256=guard.sha256_file(probe),
+            )
+    finally:
+        monkeypatch.undo()
+
+
 def test_core_probe_checks_require_read_only_probe_mount() -> None:
     config, baseline, lock, authored = candidate_fixture()
     config["services"]["db"]["volumes"] = []

From 512ef27df78ea1cc4a665e761aaaf88f2e6b5b9b Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 23:00:17 -0400
Subject: [PATCH 35/48] Add guarded durable Unstract startup replay

---
 docker/healthchecks/unstract-services.sh      |  63 +++-
 .../scripts/train_health_deployment_guard.py  | 293 +++++++++++++++++-
 docker/scripts/unstract_durable_replay.py     | 112 +++++++
 .../60-unstract-durable-replay.conf           |   6 +
 .../systemd/unstract-durable-replay.service   |  22 ++
 docs/train-unstract-health-deployment.md      | 115 ++++++-
 .../test_train_health_deployment_guard.py     |  67 ++++
 .../test_unstract_durable_replay.py           |  92 ++++++
 8 files changed, 747 insertions(+), 23 deletions(-)
 create mode 100644 docker/scripts/unstract_durable_replay.py
 create mode 100644 docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf
 create mode 100644 docker/systemd/unstract-durable-replay.service
 create mode 100644 tests/healthchecks/test_unstract_durable_replay.py

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index b48d591608..c6b88e6e48 100755
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -77,6 +77,28 @@ terminate_process_group() {
     kill -KILL "$terminate_process_pid" >/dev/null 2>&1 || :
 }
 
+# A trapped signal can remain pending while a POSIX shell is blocked in a
+# foreground `wait`. Poll the child instead, so the shell gets a chance to run
+# its cleanup trap between short sleeps. The final wait only runs after the
+# child has exited and is therefore bounded even when a healthcheck is
+# terminated while its client owns one of the capture FIFOs.
+wait_for_child() {
+    wait_for_child_pid=$1
+    wait_for_child_ticks=0
+    wait_for_child_limit=$((timeout_seconds * 20 + 40))
+    while kill -0 "$wait_for_child_pid" >/dev/null 2>&1; do
+        wait_for_child_ticks=$((wait_for_child_ticks + 1))
+        if [ "$wait_for_child_ticks" -ge "$wait_for_child_limit" ]; then
+            terminate_process_group "$wait_for_child_pid"
+            return 124
+        fi
+        # GNU and BusyBox sleep both support sub-second intervals; keeping the
+        # interval short bounds signal latency without a busy loop.
+        sleep 0.05
+    done
+    wait "$wait_for_child_pid"
+}
+
 # BusyBox wget has no max-filesize or max-redirect option. Stream through
 # bounded head processes, while capturing response headers so redirects can be
 # rejected even when the client follows them internally. The status file keeps
@@ -115,6 +137,12 @@ bounded_wget_finish() {
     bounded_wget_cleanup
 }
 
+bounded_wget_abort() {
+    trap - HUP INT TERM EXIT
+    bounded_wget_cleanup
+    exit 143
+}
+
 bounded_wget() {
     bounded_wget_url=$1
     bounded_wget_limit=$2
@@ -127,7 +155,8 @@ bounded_wget() {
     bounded_wget_client_pid=
     bounded_wget_body_reader_pid=
     bounded_wget_header_reader_pid=
-    trap bounded_wget_cleanup HUP INT TERM EXIT
+    trap bounded_wget_abort HUP INT TERM
+    trap bounded_wget_cleanup EXIT
 
     bounded_wget_headers=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-headers.XXXXXX" 2>/dev/null) || return 1
     bounded_wget_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || return 1
@@ -144,7 +173,7 @@ bounded_wget() {
     "$timeout_bin" "$timeout_seconds" "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" \
         >"$bounded_wget_body_fifo" 2>"$bounded_wget_header_fifo" &
     bounded_wget_client_pid=$!
-    if wait "$bounded_wget_client_pid"; then
+    if wait_for_child "$bounded_wget_client_pid"; then
         bounded_wget_status=0
     else
         bounded_wget_status=$?
@@ -208,6 +237,12 @@ bounded_curl_finish() {
     bounded_curl_cleanup
 }
 
+bounded_curl_abort() {
+    trap - HUP INT TERM EXIT
+    bounded_curl_cleanup
+    exit 143
+}
+
 bounded_curl() {
     bounded_curl_url=$1
     bounded_curl_limit=$2
@@ -217,7 +252,8 @@ bounded_curl() {
     bounded_curl_body_fifo=
     bounded_curl_client_pid=
     bounded_curl_body_reader_pid=
-    trap bounded_curl_cleanup HUP INT TERM EXIT
+    trap bounded_curl_abort HUP INT TERM
+    trap bounded_curl_cleanup EXIT
     bounded_curl_body_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null) || return 1
     bounded_curl_status_file=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-status.XXXXXX" 2>/dev/null) || return 1
     bounded_curl_body_fifo=$("$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body-fifo.XXXXXX" 2>/dev/null) || return 1
@@ -228,7 +264,7 @@ bounded_curl() {
     "$timeout_bin" "$timeout_seconds" "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
         --max-time "$timeout_seconds" "$bounded_curl_url" >"$bounded_curl_body_fifo" &
     bounded_curl_client_pid=$!
-    if wait "$bounded_curl_client_pid"; then
+    if wait_for_child "$bounded_curl_client_pid"; then
         bounded_curl_status=0
     else
         bounded_curl_status=$?
@@ -286,6 +322,12 @@ bounded_exec_finish() {
     bounded_exec_cleanup
 }
 
+bounded_exec_abort() {
+    trap - HUP INT TERM EXIT
+    bounded_exec_cleanup
+    exit 143
+}
+
 bounded_exec() {
     bounded_exec_limit=$1
     shift
@@ -295,7 +337,8 @@ bounded_exec() {
     bounded_exec_body_fifo=
     bounded_exec_client_pid=
     bounded_exec_reader_pid=
-    trap bounded_exec_cleanup HUP INT TERM EXIT
+    trap bounded_exec_abort HUP INT TERM
+    trap bounded_exec_cleanup EXIT
 
     bounded_exec_body_file=$(
         "$mktemp_bin" "${TMPDIR:-/tmp}/unstract-health-body.XXXXXX" 2>/dev/null
@@ -313,7 +356,7 @@ bounded_exec() {
     bounded_exec_reader_pid=$!
     "$timeout_bin" "$timeout_seconds" "$@" >"$bounded_exec_body_fifo" 2>/dev/null &
     bounded_exec_client_pid=$!
-    if wait "$bounded_exec_client_pid"; then
+    if wait_for_child "$bounded_exec_client_pid"; then
         bounded_exec_status=0
     else
         bounded_exec_status=$?
@@ -342,7 +385,7 @@ bounded_exec() {
 }
 
 probe_weaviate() {
-    bounded_wget "$weaviate_url" 65536 2>/dev/null || fail
+    bounded_wget "$weaviate_url" 65536 || fail
     body=$("$head_bin" -c 65536 "$bounded_wget_result_file") || {
         bounded_wget_finish
         fail
@@ -395,7 +438,7 @@ probe_redis() {
 }
 
 probe_proxy() {
-    bounded_wget "$proxy_url" 65536 2>/dev/null || fail
+    bounded_wget "$proxy_url" 65536 || fail
     body=$("$head_bin" -c 65536 "$bounded_wget_result_file") || {
         bounded_wget_finish
         fail
@@ -416,7 +459,7 @@ probe_minio() {
     # MinIO's unauthenticated readiness endpoint reports cluster readiness and
     # avoids a mutating S3 operation or a dependency on an mc alias file.
     bounded_curl "${MINIO_READY_URL:-http://127.0.0.1:9000/minio/health/ready}" 1024 \
-        >/dev/null 2>&1 || fail
+        || fail
     bounded_curl_finish
 }
 
@@ -493,7 +536,7 @@ with opener.open(request, timeout=float(sys.argv[2])) as response:
 }
 
 probe_frontend() {
-    bounded_curl "$frontend_url" 65536 2>/dev/null || fail
+    bounded_curl "$frontend_url" 65536 || fail
     body=$("$head_bin" -c 65536 "$bounded_curl_result_file") || {
         bounded_curl_finish
         fail
diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 0a556b4c36..491155e3ad 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -105,6 +105,8 @@
 START_CONFIRM_TOKEN = "START_UNSTRACT_HEALTH"
 LIVE_COMPOSE_TRAIN = "docker/compose.train.yaml"
 LIVE_ENV_RELATIVE = "docker/.env"
+REPLAY_MANIFEST_SCHEMA = "unstract-durable-replay/v1"
+REPLAY_MANIFEST_FILENAME = "durable-replay-manifest.json"
 RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
@@ -1004,6 +1006,16 @@ def write_json(path: Path, value: Any) -> None:
     path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
 
 
+def write_private_json(path: Path, value: Any, *, description: str) -> None:
+    """Write sanitized replay metadata with the same private contract as overrides."""
+    write_private_text(
+        path,
+        json.dumps(value, indent=2, sort_keys=True) + "\n",
+        replace=True,
+        description=description,
+    )
+
+
 def service_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
     result: dict[str, dict[str, Any]] = {}
     for container in snapshot.get("containers", []):
@@ -1316,6 +1328,27 @@ def compose_config(
     for compose_file in files:
         args.extend(["-f", compose_file])
     args.extend(["config", "--format", "json"])
+    final_settings = verify_compose_inputs_before_run(
+        project_dir=project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+        candidate_source=candidate_source,
+        candidate_lock=candidate_lock,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=environment_override,
+        environment_override_sha256=environment_override_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        candidate_version=candidate_version,
+    )
+    if final_settings:
+        env["VERSION"] = final_settings["VERSION"]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+            "UNSTRACT_HEALTHCHECK_SOURCE"
+        ]
     return parse_json_output(
         run(args, cwd=project_dir, env=env, deadline=deadline), "Compose config"
     )
@@ -1458,6 +1491,133 @@ def validate_compose_settings(
     return values
 
 
+def write_replay_manifest(
+    path: Path,
+    *,
+    lock_path: Path,
+    lock: dict[str, Any],
+    image_override: Path,
+    settings_file: Path,
+    runtime_environment_override: Path,
+    probe_source: Path,
+    probe_source_sha256: str,
+) -> None:
+    """Bind every private replay input to a durable, sanitized hash record."""
+    validate_candidate_image_override(image_override)
+    settings = validate_compose_settings(
+        settings_file,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
+    validate_private_override(runtime_environment_override)
+    write_private_json(
+        path,
+        {
+            "schema": REPLAY_MANIFEST_SCHEMA,
+            "candidate_version": lock["candidate_version"],
+            "source_commit": lock["source_commit"],
+            "source_tree": lock["source_tree"],
+            "candidate_lock_sha256": sha256_file(lock_path),
+            "probe_source_sha256": probe_source_sha256,
+            "files": {
+                CANDIDATE_IMAGE_FILENAME: sha256_file(image_override),
+                COMPOSE_SETTINGS_FILENAME: sha256_file(settings_file),
+                RUNTIME_ENVIRONMENT_FILENAME: sha256_file(runtime_environment_override),
+            },
+            "runtime_environment_keys": {
+                service: sorted(keys)
+                for service, keys in reviewed_environment_keys_from_override(
+                    runtime_environment_override
+                ).items()
+                if keys
+            },
+            "settings": {
+                "VERSION": settings["VERSION"],
+                "UNSTRACT_HEALTHCHECK_SOURCE_SHA256": settings[
+                    "UNSTRACT_HEALTHCHECK_SOURCE_SHA256"
+                ],
+            },
+        },
+        description="durable replay manifest",
+    )
+
+
+def load_replay_manifest(
+    path: Path,
+    *,
+    lock_path: Path,
+    lock: dict[str, Any],
+    image_override: Path,
+    settings_file: Path,
+    runtime_environment_override: Path,
+    probe_source: Path,
+    probe_source_sha256: str,
+) -> dict[str, Any]:
+    """Verify the persisted private replay files before normal startup."""
+    validate_private_file(path, description="durable replay manifest")
+    try:
+        manifest = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read durable replay manifest {path}: {exc}") from exc
+    if manifest.get("schema") != REPLAY_MANIFEST_SCHEMA:
+        raise GuardError("durable replay manifest has an unsupported schema")
+    if manifest.get("candidate_version") != lock.get("candidate_version"):
+        raise GuardError("durable replay candidate version changed")
+    if manifest.get("source_commit") != lock.get("source_commit"):
+        raise GuardError("durable replay source commit changed")
+    if manifest.get("source_tree") != lock.get("source_tree"):
+        raise GuardError("durable replay source tree changed")
+    expected_lock_sha256 = manifest.get("candidate_lock_sha256")
+    if not isinstance(expected_lock_sha256, str):
+        raise GuardError("durable replay manifest lacks candidate lock digest")
+    if sha256_file(lock_path) != expected_lock_sha256:
+        raise GuardError("candidate lock changed since durable replay was prepared")
+    if manifest.get("probe_source_sha256") != probe_source_sha256:
+        raise GuardError("durable replay probe digest changed")
+    files = manifest.get("files")
+    if not isinstance(files, dict):
+        raise GuardError("durable replay manifest lacks private file digests")
+    image_sha256 = files.get(CANDIDATE_IMAGE_FILENAME)
+    settings_sha256 = files.get(COMPOSE_SETTINGS_FILENAME)
+    runtime_sha256 = files.get(RUNTIME_ENVIRONMENT_FILENAME)
+    if not all(isinstance(value, str) for value in (image_sha256, settings_sha256, runtime_sha256)):
+        raise GuardError("durable replay manifest has incomplete private file digests")
+    validate_candidate_image_override(image_override, expected_sha256=image_sha256)
+    settings = validate_compose_settings(
+        settings_file,
+        candidate_version=lock["candidate_version"],
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        expected_sha256=settings_sha256,
+    )
+    if manifest.get("settings") != {
+        "VERSION": settings["VERSION"],
+        "UNSTRACT_HEALTHCHECK_SOURCE_SHA256": settings[
+            "UNSTRACT_HEALTHCHECK_SOURCE_SHA256"
+        ],
+    }:
+        raise GuardError("durable Compose settings metadata changed")
+    validate_private_override(
+        runtime_environment_override, expected_sha256=runtime_sha256
+    )
+    expected_keys = manifest.get("runtime_environment_keys") or {}
+    if expected_keys != {
+        service: sorted(keys)
+        for service, keys in reviewed_environment_keys_from_override(
+            runtime_environment_override
+        ).items()
+        if keys
+    }:
+        raise GuardError("runtime environment contract changed since durable replay was prepared")
+    return {
+        "image_override_sha256": image_sha256,
+        "settings_sha256": settings_sha256,
+        "runtime_environment_sha256": runtime_sha256,
+        "settings": settings,
+    }
+
+
 def plan_runtime_environment_override(
     baseline: dict[str, Any],
     runtime_environment: dict[str, dict[str, Any]],
@@ -1842,6 +2002,52 @@ def verify_live_compose_inputs(
         raise GuardError("ignored live Compose input drifted since baseline capture")
 
 
+def verify_compose_inputs_before_run(
+    *,
+    project_dir: Path,
+    live_env_file: Path | None,
+    expected_source: dict[str, Any] | None,
+    candidate_source: Path | None,
+    candidate_lock: dict[str, Any] | None,
+    probe_source: Path,
+    probe_source_sha256: str | None,
+    image_override: Path | None,
+    image_override_sha256: str | None,
+    environment_override: Path | None,
+    environment_override_sha256: str | None,
+    settings_file: Path | None,
+    settings_file_sha256: str | None,
+    candidate_version: str,
+) -> dict[str, str] | None:
+    """Recheck every mutable Compose input immediately before subprocess start."""
+    verify_candidate_source_state(candidate_source, candidate_lock)
+    verify_live_compose_inputs(
+        project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+    )
+    if settings_file:
+        settings = validate_compose_settings(
+            settings_file,
+            candidate_version=candidate_version,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            expected_sha256=settings_file_sha256,
+        )
+    else:
+        validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
+        settings = None
+    if image_override:
+        validate_candidate_image_override(
+            image_override, expected_sha256=image_override_sha256
+        )
+    if environment_override:
+        validate_private_override(
+            environment_override, expected_sha256=environment_override_sha256
+        )
+    return settings
+
+
 @contextlib.contextmanager
 def advisory_lock(
     *,
@@ -2240,6 +2446,27 @@ def targeted_up(
             *services,
         ]
     )
+    final_settings = verify_compose_inputs_before_run(
+        project_dir=project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+        candidate_source=candidate_source,
+        candidate_lock=candidate_lock,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=environment_override,
+        environment_override_sha256=environment_override_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        candidate_version=candidate_version,
+    )
+    if final_settings:
+        env["VERSION"] = final_settings["VERSION"]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+            "UNSTRACT_HEALTHCHECK_SOURCE"
+        ]
     run(args, cwd=project_dir, env=env, deadline=deadline)
 
 
@@ -2288,6 +2515,27 @@ def compose_start(
     files = compose_files + (str(image_override), str(environment_override))
     args = compose_args(files, live_env_file=live_env_file)
     args.extend(["up", "-d", "--no-build", "--pull", "never"])
+    final_settings = verify_compose_inputs_before_run(
+        project_dir=project_dir,
+        live_env_file=live_env_file,
+        expected_source=expected_source,
+        candidate_source=candidate_source,
+        candidate_lock=candidate_lock,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=environment_override,
+        environment_override_sha256=environment_override_sha256,
+        settings_file=settings_file,
+        settings_file_sha256=settings_file_sha256,
+        candidate_version=candidate_version,
+    )
+    if final_settings:
+        env["VERSION"] = final_settings["VERSION"]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+            "UNSTRACT_HEALTHCHECK_SOURCE"
+        ]
     run(args, cwd=project_dir, env=env, deadline=deadline)
 
 
@@ -2980,6 +3228,23 @@ def command_preflight(args: argparse.Namespace) -> int:
         replace_settings_file=True,
         operation_deadline=deadline,
     )
+    lock = load_lock(Path(args.candidate_lock))
+    probe_source = Path(args.probe_source)
+    probe_source_sha256 = (lock.get("artifacts") or {}).get(
+        "docker/healthchecks/unstract-services.sh"
+    )
+    if not isinstance(probe_source_sha256, str):
+        raise GuardError("candidate lock lacks the guarded probe source digest")
+    write_replay_manifest(
+        state_dir / REPLAY_MANIFEST_FILENAME,
+        lock_path=Path(args.candidate_lock),
+        lock=lock,
+        image_override=image_override,
+        settings_file=settings_file,
+        runtime_environment_override=runtime_environment_override,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
     print(
         "preflight: candidate source, image lock, Compose identity, runtime, "
         "data, network, environment, queue, and active-job state verified; "
@@ -3014,16 +3279,26 @@ def command_start(args: argparse.Namespace) -> int:
         raise GuardError("candidate lock lacks the guarded probe source digest")
     validate_probe_source(probe_source, expected_sha256=probe_source_sha256)
     validate_candidate_image_override(image_override)
-    validate_private_override(runtime_environment_override)
+    replay_manifest = load_replay_manifest(
+        state_dir / REPLAY_MANIFEST_FILENAME,
+        lock_path=Path(args.candidate_lock),
+        lock=lock,
+        image_override=image_override,
+        settings_file=settings_file,
+        runtime_environment_override=runtime_environment_override,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+    )
     settings = validate_compose_settings(
         settings_file,
         candidate_version=lock["candidate_version"],
         probe_source=probe_source,
         probe_source_sha256=probe_source_sha256,
+        expected_sha256=replay_manifest["settings_sha256"],
     )
-    image_override_sha256 = sha256_file(image_override)
-    settings_file_sha256 = sha256_file(settings_file)
-    runtime_environment_sha256 = sha256_file(runtime_environment_override)
+    image_override_sha256 = replay_manifest["image_override_sha256"]
+    settings_file_sha256 = replay_manifest["settings_sha256"]
+    runtime_environment_sha256 = replay_manifest["runtime_environment_sha256"]
     candidate_images = candidate_image_snapshot(lock, deadline=operation_deadline)
     if not candidate_images:
         raise GuardError("no candidate images were verified")
@@ -3166,6 +3441,16 @@ def command_apply(args: argparse.Namespace) -> int:
                     probe_source_sha256=probe_source_sha256,
                     deadline=operation_deadline,
                 )
+            write_replay_manifest(
+                backup_dir / REPLAY_MANIFEST_FILENAME,
+                lock_path=Path(args.candidate_lock),
+                lock=lock,
+                image_override=image_override,
+                settings_file=settings_file,
+                runtime_environment_override=runtime_environment_override,
+                probe_source=Path(args.probe_source),
+                probe_source_sha256=probe_source_sha256,
+            )
             rollback_override(backup_images, backup_dir / "rollback.override.yaml")
             write_json(backup_dir / "candidate-images.json", lock["images"])
 
diff --git a/docker/scripts/unstract_durable_replay.py b/docker/scripts/unstract_durable_replay.py
new file mode 100644
index 0000000000..069bda26b1
--- /dev/null
+++ b/docker/scripts/unstract_durable_replay.py
@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""Run the durable Unstract replay owned by the Train user systemd manager.
+
+The systemd unit supplies only operator-owned, non-secret paths through its
+EnvironmentFile.  This launcher validates the path contract and constructs
+the guarded ``start`` command without a shell, so a malformed state file
+fails closed before Compose is reached.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+CONFIRM_TOKEN = "START_UNSTRACT_HEALTH"
+PYTHON = "/usr/bin/python3"
+REQUIRED_PATHS = (
+    "UNSTRACT_GUARD",
+    "UNSTRACT_STATE_DIR",
+    "UNSTRACT_PROJECT_DIR",
+    "UNSTRACT_LIVE_ENV_FILE",
+    "UNSTRACT_BASELINE",
+    "UNSTRACT_CANDIDATE_SOURCE",
+    "UNSTRACT_CANDIDATE_LOCK",
+    "UNSTRACT_PROBE_SOURCE",
+    "UNSTRACT_COMPOSE_BASE",
+    "UNSTRACT_COMPOSE_TRAIN",
+    "UNSTRACT_COMPOSE_WORKER_HEALTHCHECKS",
+    "UNSTRACT_COMPOSE_CORE_HEALTHCHECKS",
+    "UNSTRACT_COMPOSE_ENV_DIR",
+)
+
+
+def _required(environ: dict[str, str], name: str) -> str:
+    value = environ.get(name, "")
+    if not value:
+        raise ValueError(f"{name} is required")
+    return value
+
+
+def _absolute(environ: dict[str, str], name: str) -> str:
+    value = _required(environ, name)
+    if not Path(value).is_absolute():
+        raise ValueError(f"{name} must be an absolute path")
+    return value
+
+
+def build_start_args(environ: dict[str, str]) -> list[str]:
+    """Build the exact guarded startup command from systemd's environment."""
+    for name in REQUIRED_PATHS:
+        _absolute(environ, name)
+
+    confirm = environ.get("UNSTRACT_START_CONFIRM", "")
+    if confirm != CONFIRM_TOKEN:
+        raise ValueError("UNSTRACT_START_CONFIRM does not authorize durable replay")
+
+    timeout = environ.get("UNSTRACT_OPERATION_TIMEOUT", "2400")
+    if not timeout.isdigit() or int(timeout) <= 0:
+        raise ValueError("UNSTRACT_OPERATION_TIMEOUT must be a positive integer")
+
+    args = [
+        PYTHON,
+        _absolute(environ, "UNSTRACT_GUARD"),
+        "start",
+        "--state-dir",
+        _absolute(environ, "UNSTRACT_STATE_DIR"),
+        "--project-dir",
+        _absolute(environ, "UNSTRACT_PROJECT_DIR"),
+        "--live-env-file",
+        _absolute(environ, "UNSTRACT_LIVE_ENV_FILE"),
+        "--baseline",
+        _absolute(environ, "UNSTRACT_BASELINE"),
+        "--candidate-source",
+        _absolute(environ, "UNSTRACT_CANDIDATE_SOURCE"),
+        "--candidate-lock",
+        _absolute(environ, "UNSTRACT_CANDIDATE_LOCK"),
+        "--probe-source",
+        _absolute(environ, "UNSTRACT_PROBE_SOURCE"),
+    ]
+    for name in (
+        "UNSTRACT_COMPOSE_BASE",
+        "UNSTRACT_COMPOSE_TRAIN",
+        "UNSTRACT_COMPOSE_WORKER_HEALTHCHECKS",
+        "UNSTRACT_COMPOSE_CORE_HEALTHCHECKS",
+    ):
+        args.extend(("--compose-file", _absolute(environ, name)))
+    args.extend(("--operation-timeout", timeout, "--confirm", CONFIRM_TOKEN))
+    return args
+
+
+def compose_environment(environ: dict[str, str]) -> dict[str, str]:
+    """Return the child environment with the live Compose interpolation root."""
+    result = dict(environ)
+    result["PWD"] = _absolute(environ, "UNSTRACT_COMPOSE_ENV_DIR")
+    return result
+
+
+def main() -> int:
+    environ = dict(os.environ)
+    try:
+        args = build_start_args(environ)
+    except ValueError as exc:
+        print(f"unstract durable replay: {exc}", file=sys.stderr)
+        return 2
+    os.environ.update(compose_environment(environ))
+    os.execv(args[0], args)
+    return 127
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf b/docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf
new file mode 100644
index 0000000000..393a0a57f2
--- /dev/null
+++ b/docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf
@@ -0,0 +1,6 @@
+[Unit]
+# podman-restart.service is the existing generic owner for the Unstract
+# containers' unless-stopped policy. Require the guarded Compose replay first
+# so a manual or boot-triggered generic restart cannot bypass durable state.
+Requires=unstract-durable-replay.service
+After=unstract-durable-replay.service
diff --git a/docker/systemd/unstract-durable-replay.service b/docker/systemd/unstract-durable-replay.service
new file mode 100644
index 0000000000..2c443304a0
--- /dev/null
+++ b/docker/systemd/unstract-durable-replay.service
@@ -0,0 +1,22 @@
+[Unit]
+Description=Guarded durable Unstract Compose replay
+Documentation=file:%h/.local/libexec/unstract-durable-replay.py
+Wants=network-online.target podman.socket
+Requires=train-rootless-boot-recovery.service
+After=network-online.target podman.socket train-rootless-boot-recovery.service
+Before=podman-restart.service
+ConditionPathExists=%h/.config/unstract/durable-replay.env
+ConditionPathExists=%h/.local/libexec/unstract-durable-replay.py
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=%h/.config/unstract/durable-replay.env
+ExecStart=/usr/bin/python3 %h/.local/libexec/unstract-durable-replay.py
+# Compose can wait on database and health transitions; do not let the user
+# manager kill the guarded replay while it is within its own deadline.
+TimeoutStartSec=2700
+KillMode=process
+
+[Install]
+WantedBy=default.target
diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
index ed8930cf3e..c644d0dc65 100644
--- a/docs/train-unstract-health-deployment.md
+++ b/docs/train-unstract-health-deployment.md
@@ -15,12 +15,15 @@ source directory:
   Milvus MinIO, and Milvus services while preserving the local SSL entrypoint
   and volume configuration.
 
-`docker/scripts/train_health_deployment_guard.py` is the transaction guard. Its
-`capture`, `lock`, and `preflight` commands are read-only. Only `apply` and
-`rollback` mutate the host, and both require `--confirm APPLY_UNSTRACT_HEALTH`.
-Every command has a finite subprocess timeout and every operation has a finite
-monotonic deadline. The guard never builds, pulls, deletes source, resets a
-checkout, or runs project-wide `up`/`down` commands.
+`docker/scripts/train_health_deployment_guard.py` is the transaction guard.
+`capture` and `lock` are read-only. `preflight` verifies the candidate and
+refreshes private mode-600 replay files beside the lock, so it is an intentional
+state write. `start`, `apply`, and `rollback` can mutate the running Compose
+project; `start` requires `--confirm START_UNSTRACT_HEALTH`, while `apply` and
+`rollback` require `--confirm APPLY_UNSTRACT_HEALTH`. Every command has a
+finite subprocess timeout and every operation has a finite monotonic deadline.
+The guard never builds, pulls, deletes source, resets a checkout, or runs
+project-wide destructive `down` commands.
 
 The target set is fixed at 24 services: runner plus twelve workers, followed
 by db, Redis, MinIO, reverse proxy, Qdrant, RabbitMQ, Weaviate, x2text,
@@ -89,9 +92,12 @@ be the exact names and IDs already captured from the live stack, unless a
 deliberate static image change has separately been reviewed. The lock also
 contains the candidate commit's tree hash and hashes for the two overlays, the
 core and database probes, and the development essentials Compose file. The
-guard writes a private mode-600 image override and Compose settings file from
+guard writes private mode-600 image, settings, runtime-environment, and
+replay-manifest files from
 this lock, so every health, environment, and image replay uses the same
-immutable references, `VERSION`, and staged probe path. These artifacts remain
+immutable references, `VERSION`, and staged probe path. The replay manifest
+binds those three private files, the candidate lock, source commit/tree, probe
+digest, and reviewed environment-key set. These artifacts remain
 in the preflight state directory and apply backup directory for later startup
 or recovery; the guard validates their SHA-256 values before each Compose
 invocation. Only `runner` and the twelve
@@ -105,7 +111,98 @@ checkout and never use `rsync --delete`. The private Train Compose file and
 all existing `.env` files stay in their current location. The core overlay's
 `UNSTRACT_HEALTHCHECK_SOURCE` points at the staged probe.
 
-## Read-only preflight and bounded apply
+## Existing Train startup owner
+
+The Train user manager currently has two relevant owners. The enabled
+`train-rootless-boot-recovery.service` runs the host's bounded helper, whose
+contract is to validate and start only the exact rows in its private manifest;
+that helper deliberately does not run Compose. The enabled generic
+`podman-restart.service` starts existing containers selected by restart policy.
+That generic path is how Unstract's existing `unless-stopped` containers can
+return after a user-manager restart, but it does not read the candidate lock,
+the staged overlays, or the durable replay files. A deployment that installs
+only the guard therefore still loses its candidate state on ordinary recovery.
+
+This repository now carries the small owner integration:
+
+- `docker/systemd/unstract-durable-replay.service` runs the guarded whole-stack
+  `start` path after the bounded rootless recovery and before generic restart.
+- `docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf`
+  makes the ordering a requirement even when an operator starts
+  `podman-restart.service` manually.
+- `docker/scripts/unstract_durable_replay.py` validates the mode-600 systemd
+  environment contract and invokes the guard without a shell.
+
+The unit is deliberately separate from the application checkout. Install the
+launcher, unit, and drop-in only after `apply` has produced a verified backup
+directory. The environment file below contains paths and the explicit startup
+confirmation token; it contains no credentials or copied `.env` values:
+
+```sh
+REMOTE_ROOT=/home/completetrain/train-health-coverage-20260908/unstract-build-03404993-2042
+REMOTE_SOURCE="$REMOTE_ROOT/guard-source-4ee386a0"
+REMOTE_LOCK="$REMOTE_ROOT/candidate-lock-4ee386a0.json"
+REMOTE_BACKUP="$REMOTE_ROOT/deployment-backup-4ee386a0"
+LIVE_ROOT=/home/completetrain/etl.home.complete.tech
+LIVE_ENV="$LIVE_ROOT/docker/.env"
+REMOTE_PROBE="$REMOTE_SOURCE/docker/healthchecks/unstract-services.sh"
+
+install -d -m700 "$HOME/.config/unstract" "$HOME/.local/libexec"
+umask 077
+cat >"$HOME/.config/unstract/durable-replay.env.new" <<EOF
+UNSTRACT_GUARD=$REMOTE_SOURCE/docker/scripts/train_health_deployment_guard.py
+UNSTRACT_STATE_DIR=$REMOTE_BACKUP
+UNSTRACT_PROJECT_DIR=$LIVE_ROOT
+UNSTRACT_LIVE_ENV_FILE=$LIVE_ENV
+UNSTRACT_BASELINE=$REMOTE_BACKUP/baseline.json
+UNSTRACT_CANDIDATE_SOURCE=$REMOTE_SOURCE
+UNSTRACT_CANDIDATE_LOCK=$REMOTE_LOCK
+UNSTRACT_PROBE_SOURCE=$REMOTE_PROBE
+UNSTRACT_COMPOSE_BASE=$LIVE_ROOT/docker/docker-compose.yaml
+UNSTRACT_COMPOSE_TRAIN=$LIVE_ROOT/docker/compose.train.yaml
+UNSTRACT_COMPOSE_WORKER_HEALTHCHECKS=$REMOTE_SOURCE/docker/compose.train.worker-healthchecks.yaml
+UNSTRACT_COMPOSE_CORE_HEALTHCHECKS=$REMOTE_SOURCE/docker/compose.train.healthchecks.yaml
+UNSTRACT_COMPOSE_ENV_DIR=$LIVE_ROOT/docker
+UNSTRACT_OPERATION_TIMEOUT=2400
+UNSTRACT_START_CONFIRM=START_UNSTRACT_HEALTH
+EOF
+chmod 600 "$HOME/.config/unstract/durable-replay.env.new"
+mv -f "$HOME/.config/unstract/durable-replay.env.new" "$HOME/.config/unstract/durable-replay.env"
+
+install -m755 "$REMOTE_SOURCE/docker/scripts/unstract_durable_replay.py" \
+  "$HOME/.local/libexec/unstract-durable-replay.py.new"
+mv -f "$HOME/.local/libexec/unstract-durable-replay.py.new" \
+  "$HOME/.local/libexec/unstract-durable-replay.py"
+install -m644 "$REMOTE_SOURCE/docker/systemd/unstract-durable-replay.service" \
+  "$HOME/.config/systemd/user/unstract-durable-replay.service.new"
+mv -f "$HOME/.config/systemd/user/unstract-durable-replay.service.new" \
+  "$HOME/.config/systemd/user/unstract-durable-replay.service"
+install -d -m755 "$HOME/.config/systemd/user/podman-restart.service.d"
+install -m644 "$REMOTE_SOURCE/docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf" \
+  "$HOME/.config/systemd/user/podman-restart.service.d/60-unstract-durable-replay.conf.new"
+mv -f "$HOME/.config/systemd/user/podman-restart.service.d/60-unstract-durable-replay.conf.new" \
+  "$HOME/.config/systemd/user/podman-restart.service.d/60-unstract-durable-replay.conf"
+systemctl --user daemon-reload
+systemd-analyze --user verify unstract-durable-replay.service podman-restart.service
+systemctl --user enable unstract-durable-replay.service
+```
+
+The launcher sets `PWD` to `UNSTRACT_COMPOSE_ENV_DIR` before `exec`, so the
+existing live `.env` can resolve its `${PWD}` path interpolation even when the
+user manager starts the unit from a different working directory. Manual
+`capture`, `preflight`, and `apply` commands must likewise export
+`PWD=$LIVE_ROOT/docker` as shown below.
+
+The install must preserve fresh backups of any prior env, launcher, unit, and
+drop-in before replacement. If the owner integration must be reverted, stop
+the durable unit, disable it, restore those exact backups, remove the drop-in
+only if it was absent before, and run `systemctl --user daemon-reload`; do not
+fall back to an unguarded `podman-restart.service` while the candidate state is
+uncertain. A failed durable unit intentionally blocks the generic restart path
+until the state or source hashes are repaired. Starting the unit is a normal
+Compose startup mutation and is a separate approved action after installation.
+
+## Preflight and bounded apply
 
 Capture a fresh baseline immediately before preflight, even if the earlier
 runtime snapshot is available. The capture hashes every dirty source file by
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 37749fe034..b996eae589 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -318,6 +318,73 @@ def test_durable_compose_inputs_are_private_and_reusable(tmp_path: Path) -> None
         )
 
 
+def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) -> None:
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    probe_sha256 = guard.sha256_file(probe)
+    lock = {
+        "schema": "unstract-health-candidate/v1",
+        "candidate_version": "goal09-test",
+        "source_commit": "a" * 40,
+        "source_tree": "b" * 40,
+        "images": {
+            service: {"reference": f"candidate/{service}"}
+            for service in guard.TARGET_SERVICES
+        },
+    }
+    lock_path = tmp_path / "candidate-lock.json"
+    lock_path.write_text(json.dumps(lock), encoding="utf-8")
+    image_override = guard.candidate_image_override(
+        lock, tmp_path / guard.CANDIDATE_IMAGE_FILENAME
+    )
+    settings = tmp_path / guard.COMPOSE_SETTINGS_FILENAME
+    guard.write_compose_settings(
+        lock["candidate_version"],
+        probe,
+        settings,
+        probe_source_sha256=probe_sha256,
+    )
+    runtime = tmp_path / guard.RUNTIME_ENVIRONMENT_FILENAME
+    guard.write_runtime_environment_override({}, runtime)
+    manifest = tmp_path / guard.REPLAY_MANIFEST_FILENAME
+
+    guard.write_replay_manifest(
+        manifest,
+        lock_path=lock_path,
+        lock=lock,
+        image_override=image_override,
+        settings_file=settings,
+        runtime_environment_override=runtime,
+        probe_source=probe,
+        probe_source_sha256=probe_sha256,
+    )
+    loaded = guard.load_replay_manifest(
+        manifest,
+        lock_path=lock_path,
+        lock=lock,
+        image_override=image_override,
+        settings_file=settings,
+        runtime_environment_override=runtime,
+        probe_source=probe,
+        probe_source_sha256=probe_sha256,
+    )
+    assert loaded["runtime_environment_sha256"] == guard.sha256_file(runtime)
+    assert manifest.stat().st_mode & 0o777 == 0o600
+
+    runtime.write_text(runtime.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8")
+    with pytest.raises(guard.GuardError, match="runtime environment override changed"):
+        guard.load_replay_manifest(
+            manifest,
+            lock_path=lock_path,
+            lock=lock,
+            image_override=image_override,
+            settings_file=settings,
+            runtime_environment_override=runtime,
+            probe_source=probe,
+            probe_source_sha256=probe_sha256,
+        )
+
+
 def test_compose_replay_consumes_durable_settings_and_overrides(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
diff --git a/tests/healthchecks/test_unstract_durable_replay.py b/tests/healthchecks/test_unstract_durable_replay.py
new file mode 100644
index 0000000000..f3e942d9f1
--- /dev/null
+++ b/tests/healthchecks/test_unstract_durable_replay.py
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).parents[2]
+LAUNCHER = ROOT / "docker/scripts/unstract_durable_replay.py"
+UNIT = ROOT / "docker/systemd/unstract-durable-replay.service"
+DROP_IN = ROOT / "docker/systemd/podman-restart.service.d/60-unstract-durable-replay.conf"
+SPEC = importlib.util.spec_from_file_location("unstract_durable_replay", LAUNCHER)
+assert SPEC and SPEC.loader
+launcher = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(launcher)
+
+
+def launcher_environment(tmp_path: Path) -> dict[str, str]:
+    paths = {
+        "UNSTRACT_GUARD": tmp_path / "source/docker/scripts/train_health_deployment_guard.py",
+        "UNSTRACT_STATE_DIR": tmp_path / "state",
+        "UNSTRACT_PROJECT_DIR": tmp_path / "project",
+        "UNSTRACT_LIVE_ENV_FILE": tmp_path / "project/docker/.env",
+        "UNSTRACT_BASELINE": tmp_path / "state/baseline.json",
+        "UNSTRACT_CANDIDATE_SOURCE": tmp_path / "source",
+        "UNSTRACT_CANDIDATE_LOCK": tmp_path / "state/candidate-lock.json",
+        "UNSTRACT_PROBE_SOURCE": tmp_path / "source/docker/healthchecks/unstract-services.sh",
+        "UNSTRACT_COMPOSE_BASE": tmp_path / "project/docker/docker-compose.yaml",
+        "UNSTRACT_COMPOSE_TRAIN": tmp_path / "project/docker/compose.train.yaml",
+        "UNSTRACT_COMPOSE_WORKER_HEALTHCHECKS": tmp_path / "source/docker/compose.train.worker-healthchecks.yaml",
+        "UNSTRACT_COMPOSE_CORE_HEALTHCHECKS": tmp_path / "source/docker/compose.train.healthchecks.yaml",
+        "UNSTRACT_COMPOSE_ENV_DIR": tmp_path / "project/docker",
+    }
+    return {name: str(path) for name, path in paths.items()} | {
+        "UNSTRACT_START_CONFIRM": launcher.CONFIRM_TOKEN,
+        "UNSTRACT_OPERATION_TIMEOUT": "2400",
+    }
+
+
+def test_launcher_builds_guarded_start_with_all_persistent_inputs(tmp_path: Path) -> None:
+    environment = launcher_environment(tmp_path)
+    args = launcher.build_start_args(environment)
+
+    assert args[:3] == [
+        launcher.PYTHON,
+        str(tmp_path / "source/docker/scripts/train_health_deployment_guard.py"),
+        "start",
+    ]
+    assert args[-4:] == [
+        "--operation-timeout",
+        "2400",
+        "--confirm",
+        launcher.CONFIRM_TOKEN,
+    ]
+    assert args.count("--compose-file") == 4
+    for value in environment.values():
+        if value.startswith("/"):
+            if value != environment["UNSTRACT_COMPOSE_ENV_DIR"]:
+                assert value in args
+    assert launcher.compose_environment(environment)["PWD"] == environment[
+        "UNSTRACT_COMPOSE_ENV_DIR"
+    ]
+
+
+@pytest.mark.parametrize(
+    ("name", "value", "message"),
+    [
+        ("UNSTRACT_GUARD", "relative/guard.py", "absolute path"),
+        ("UNSTRACT_START_CONFIRM", "APPLY_UNSTRACT_HEALTH", "does not authorize"),
+        ("UNSTRACT_OPERATION_TIMEOUT", "0", "positive integer"),
+    ],
+)
+def test_launcher_fails_closed_on_invalid_environment(
+    tmp_path: Path, name: str, value: str, message: str
+) -> None:
+    environment = launcher_environment(tmp_path)
+    environment[name] = value
+
+    with pytest.raises(ValueError, match=message):
+        launcher.build_start_args(environment)
+
+
+def test_systemd_owner_orders_guard_before_generic_restart() -> None:
+    unit = UNIT.read_text(encoding="utf-8")
+    drop_in = DROP_IN.read_text(encoding="utf-8")
+
+    assert "Requires=train-rootless-boot-recovery.service" in unit
+    assert "Before=podman-restart.service" in unit
+    assert "ExecStart=/usr/bin/python3 %h/.local/libexec/unstract-durable-replay.py" in unit
+    assert "WantedBy=default.target" in unit
+    assert "Requires=unstract-durable-replay.service" in drop_in
+    assert "After=unstract-durable-replay.service" in drop_in

From af9fa4b43a7aad8bf6533a2d5b2d6edcdd769d10 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Tue, 8 Sep 2026 23:42:44 -0400
Subject: [PATCH 36/48] Bind durable Compose inputs at launch

---
 .../scripts/train_health_deployment_guard.py  | 294 ++++++++++++++----
 .../systemd/unstract-durable-replay.service   |   4 +
 .../test_train_health_deployment_guard.py     |  17 +-
 .../test_unstract_durable_replay.py           |   1 +
 4 files changed, 258 insertions(+), 58 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 491155e3ad..508358baa9 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -32,6 +32,7 @@
 import stat
 import subprocess
 import sys
+import tempfile
 import time
 from collections.abc import Iterator
 from decimal import Decimal, InvalidOperation
@@ -311,6 +312,7 @@ def run(
     input_text: str | None = None,
     timeout_seconds: float = DEFAULT_COMMAND_TIMEOUT_SECONDS,
     deadline: OperationDeadline | None = None,
+    pass_fds: tuple[int, ...] = (),
 ) -> subprocess.CompletedProcess[str]:
     timeout = deadline.remaining(timeout_seconds) if deadline else timeout_seconds
     env = runtime_command_env(args, env)
@@ -324,6 +326,7 @@ def run(
             capture_output=True,
             check=False,
             timeout=timeout,
+            pass_fds=pass_fds,
         )
     except subprocess.TimeoutExpired as exc:
         raise GuardError(f"command timed out: {shlex.join(args)}") from exc
@@ -334,6 +337,147 @@ def run(
     return result
 
 
+def _read_fd(fd: int) -> bytes:
+    """Read an open input and leave its offset at the beginning."""
+    try:
+        os.lseek(fd, 0, os.SEEK_SET)
+        chunks: list[bytes] = []
+        while True:
+            chunk = os.read(fd, 1024 * 1024)
+            if not chunk:
+                break
+            chunks.append(chunk)
+        os.lseek(fd, 0, os.SEEK_SET)
+    except OSError as exc:
+        raise GuardError(f"cannot snapshot Compose input: {exc}") from exc
+    return b"".join(chunks)
+
+
+def _immutable_snapshot_fd(data: bytes) -> tuple[int, tempfile.TemporaryFile[bytes] | None]:
+    """Create a sealed descriptor containing one Compose input snapshot."""
+    temporary: tempfile.TemporaryFile[bytes] | None = None
+    try:
+        fd = os.memfd_create(
+            "unstract-compose-input",
+            os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING,
+        )
+    except AttributeError:
+        # Linux Train hosts provide memfd_create. Keep the guard importable on
+        # other platforms for static checks, while retaining an unlinked
+        # descriptor as the best available private snapshot there.
+        temporary = tempfile.TemporaryFile(mode="w+b")
+        fd = temporary.fileno()
+    try:
+        view = memoryview(data)
+        while view:
+            written = os.write(fd, view)
+            if written <= 0:
+                raise OSError("snapshot write made no progress")
+            view = view[written:]
+        os.lseek(fd, 0, os.SEEK_SET)
+        if temporary is None:
+            fcntl.fcntl(
+                fd,
+                fcntl.F_ADD_SEALS,
+                fcntl.F_SEAL_WRITE
+                | fcntl.F_SEAL_GROW
+                | fcntl.F_SEAL_SHRINK
+                | fcntl.F_SEAL_SEAL,
+            )
+        else:
+            os.fchmod(fd, 0o400)
+    except BaseException:
+        if temporary is not None:
+            temporary.close()
+        else:
+            os.close(fd)
+        raise
+    return fd, temporary
+
+
+@contextlib.contextmanager
+def bound_private_compose_file(
+    path: Path,
+    *,
+    expected_sha256: str | None,
+    description: str,
+) -> Iterator[tuple[str, int]]:
+    """Bind a verified private Compose file to an immutable child-visible fd.
+
+    Hashing the pathname and then passing that pathname to Compose leaves a
+    same-user write window between verification and subprocess open. Read the
+    reviewed bytes first, seal a memfd snapshot, and pass the descriptor to
+    Compose through ``/proc/self/fd``. The original path can change after this
+    point without changing the bytes that the child parses.
+    """
+    flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        source_fd = os.open(path, flags)
+    except OSError as exc:
+        raise GuardError(f"cannot read private {description}: {path}") from exc
+    snapshot_fd: int | None = None
+    temporary: tempfile.TemporaryFile[bytes] | None = None
+    try:
+        metadata = os.fstat(source_fd)
+        if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600:
+            raise GuardError(f"{description} is not a private regular file: {path}")
+        if hasattr(os, "getuid") and metadata.st_uid != os.getuid():
+            raise GuardError(f"{description} has the wrong owner: {path}")
+        data = _read_fd(source_fd)
+        actual_sha256 = sha256_bytes(data)
+        if expected_sha256 is not None and actual_sha256 != expected_sha256:
+            raise GuardError(f"{description} changed: {path}")
+        snapshot_fd, temporary = _immutable_snapshot_fd(data)
+        yield f"/proc/self/fd/{snapshot_fd}", snapshot_fd
+    finally:
+        try:
+            os.close(source_fd)
+        except OSError:
+            pass
+        if temporary is not None:
+            temporary.close()
+        elif snapshot_fd is not None:
+            try:
+                os.close(snapshot_fd)
+            except OSError:
+                pass
+
+
+@contextlib.contextmanager
+def bound_private_compose_inputs(
+    *,
+    image_override: Path | None,
+    image_override_sha256: str | None,
+    environment_override: Path | None,
+    environment_override_sha256: str | None,
+) -> Iterator[tuple[dict[str, str], tuple[int, ...]]]:
+    """Snapshot each private override and return path replacements plus fds."""
+    replacements: dict[str, str] = {}
+    pass_fds: list[int] = []
+    with contextlib.ExitStack() as stack:
+        inputs = (
+            (image_override, image_override_sha256, "candidate image override"),
+            (
+                environment_override,
+                environment_override_sha256,
+                "runtime environment override",
+            ),
+        )
+        for path, expected_sha256, description in inputs:
+            if path is None:
+                continue
+            proc_path, fd = stack.enter_context(
+                bound_private_compose_file(
+                    path,
+                    expected_sha256=expected_sha256,
+                    description=description,
+                )
+            )
+            replacements[str(path)] = proc_path
+            pass_fds.append(fd)
+        yield replacements, tuple(pass_fds)
+
+
 def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
     try:
         return json.loads(result.stdout)
@@ -1328,30 +1472,42 @@ def compose_config(
     for compose_file in files:
         args.extend(["-f", compose_file])
     args.extend(["config", "--format", "json"])
-    final_settings = verify_compose_inputs_before_run(
-        project_dir=project_dir,
-        live_env_file=live_env_file,
-        expected_source=expected_source,
-        candidate_source=candidate_source,
-        candidate_lock=candidate_lock,
-        probe_source=probe_source,
-        probe_source_sha256=probe_source_sha256,
+    with bound_private_compose_inputs(
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
-        settings_file=settings_file,
-        settings_file_sha256=settings_file_sha256,
-        candidate_version=candidate_version,
-    )
-    if final_settings:
-        env["VERSION"] = final_settings["VERSION"]
-        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-            "UNSTRACT_HEALTHCHECK_SOURCE"
-        ]
-    return parse_json_output(
-        run(args, cwd=project_dir, env=env, deadline=deadline), "Compose config"
-    )
+    ) as (bound_paths, pass_fds):
+        bound_args = [bound_paths.get(argument, argument) for argument in args]
+        final_settings = verify_compose_inputs_before_run(
+            project_dir=project_dir,
+            live_env_file=live_env_file,
+            expected_source=expected_source,
+            candidate_source=candidate_source,
+            candidate_lock=candidate_lock,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            image_override=image_override,
+            image_override_sha256=image_override_sha256,
+            environment_override=environment_override,
+            environment_override_sha256=environment_override_sha256,
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            candidate_version=candidate_version,
+        )
+        if final_settings:
+            env["VERSION"] = final_settings["VERSION"]
+            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+                "UNSTRACT_HEALTHCHECK_SOURCE"
+            ]
+        result = run(
+            bound_args,
+            cwd=project_dir,
+            env=env,
+            deadline=deadline,
+            pass_fds=pass_fds,
+        )
+    return parse_json_output(result, "Compose config")
 
 
 def compose_environment(value: Any) -> dict[str, Any]:
@@ -2446,28 +2602,41 @@ def targeted_up(
             *services,
         ]
     )
-    final_settings = verify_compose_inputs_before_run(
-        project_dir=project_dir,
-        live_env_file=live_env_file,
-        expected_source=expected_source,
-        candidate_source=candidate_source,
-        candidate_lock=candidate_lock,
-        probe_source=probe_source,
-        probe_source_sha256=probe_source_sha256,
+    with bound_private_compose_inputs(
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
-        settings_file=settings_file,
-        settings_file_sha256=settings_file_sha256,
-        candidate_version=candidate_version,
-    )
-    if final_settings:
-        env["VERSION"] = final_settings["VERSION"]
-        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-            "UNSTRACT_HEALTHCHECK_SOURCE"
-        ]
-    run(args, cwd=project_dir, env=env, deadline=deadline)
+    ) as (bound_paths, pass_fds):
+        bound_args = [bound_paths.get(argument, argument) for argument in args]
+        final_settings = verify_compose_inputs_before_run(
+            project_dir=project_dir,
+            live_env_file=live_env_file,
+            expected_source=expected_source,
+            candidate_source=candidate_source,
+            candidate_lock=candidate_lock,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            image_override=image_override,
+            image_override_sha256=image_override_sha256,
+            environment_override=environment_override,
+            environment_override_sha256=environment_override_sha256,
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            candidate_version=candidate_version,
+        )
+        if final_settings:
+            env["VERSION"] = final_settings["VERSION"]
+            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+                "UNSTRACT_HEALTHCHECK_SOURCE"
+            ]
+        run(
+            bound_args,
+            cwd=project_dir,
+            env=env,
+            deadline=deadline,
+            pass_fds=pass_fds,
+        )
 
 
 def compose_start(
@@ -2515,28 +2684,41 @@ def compose_start(
     files = compose_files + (str(image_override), str(environment_override))
     args = compose_args(files, live_env_file=live_env_file)
     args.extend(["up", "-d", "--no-build", "--pull", "never"])
-    final_settings = verify_compose_inputs_before_run(
-        project_dir=project_dir,
-        live_env_file=live_env_file,
-        expected_source=expected_source,
-        candidate_source=candidate_source,
-        candidate_lock=candidate_lock,
-        probe_source=probe_source,
-        probe_source_sha256=probe_source_sha256,
+    with bound_private_compose_inputs(
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
-        settings_file=settings_file,
-        settings_file_sha256=settings_file_sha256,
-        candidate_version=candidate_version,
-    )
-    if final_settings:
-        env["VERSION"] = final_settings["VERSION"]
-        env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-            "UNSTRACT_HEALTHCHECK_SOURCE"
-        ]
-    run(args, cwd=project_dir, env=env, deadline=deadline)
+    ) as (bound_paths, pass_fds):
+        bound_args = [bound_paths.get(argument, argument) for argument in args]
+        final_settings = verify_compose_inputs_before_run(
+            project_dir=project_dir,
+            live_env_file=live_env_file,
+            expected_source=expected_source,
+            candidate_source=candidate_source,
+            candidate_lock=candidate_lock,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            image_override=image_override,
+            image_override_sha256=image_override_sha256,
+            environment_override=environment_override,
+            environment_override_sha256=environment_override_sha256,
+            settings_file=settings_file,
+            settings_file_sha256=settings_file_sha256,
+            candidate_version=candidate_version,
+        )
+        if final_settings:
+            env["VERSION"] = final_settings["VERSION"]
+            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
+                "UNSTRACT_HEALTHCHECK_SOURCE"
+            ]
+        run(
+            bound_args,
+            cwd=project_dir,
+            env=env,
+            deadline=deadline,
+            pass_fds=pass_fds,
+        )
 
 
 def wait_healthy(
diff --git a/docker/systemd/unstract-durable-replay.service b/docker/systemd/unstract-durable-replay.service
index 2c443304a0..feb6d1c1eb 100644
--- a/docker/systemd/unstract-durable-replay.service
+++ b/docker/systemd/unstract-durable-replay.service
@@ -5,6 +5,10 @@ Wants=network-online.target podman.socket
 Requires=train-rootless-boot-recovery.service
 After=network-online.target podman.socket train-rootless-boot-recovery.service
 Before=podman-restart.service
+# A restart of the generic owner must stop this settled oneshot first. Its
+# Requires edge in the podman-restart drop-in then schedules a fresh replay in
+# the same restart transaction instead of reusing RemainAfterExit state.
+PartOf=podman-restart.service
 ConditionPathExists=%h/.config/unstract/durable-replay.env
 ConditionPathExists=%h/.local/libexec/unstract-durable-replay.py
 
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index b996eae589..4b8b9ef4f1 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -429,6 +429,18 @@ def fake_run(
     ) -> subprocess.CompletedProcess[str]:
         calls.append((args, env))
         if "config" in args:
+            bound_files = [
+                Path(argument)
+                for argument in args
+                if argument.startswith("/proc/self/fd/")
+            ]
+            assert len(bound_files) == 2
+            original_image = image_override.read_bytes()
+            image_override.write_text("tampered after final verification\n", encoding="utf-8")
+            try:
+                assert any(path.read_bytes() == original_image for path in bound_files)
+            finally:
+                image_override.write_bytes(original_image)
             env_file = Path(args[args.index("--env-file") + 1])
             assert "TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry" in env_file.read_text(
                 encoding="utf-8"
@@ -472,8 +484,9 @@ def fake_run(
     assert config_args[:3] == ["docker", "compose", "--env-file"]
     assert str(live_env) in config_args
     assert str(settings) not in config_args
-    assert str(image_override) in config_args
-    assert str(environment_override) in config_args
+    assert str(image_override) not in config_args
+    assert str(environment_override) not in config_args
+    assert sum(argument.startswith("/proc/self/fd/") for argument in config_args) == 2
     assert config_env and config_env["VERSION"] == "goal09-test"
     assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(probe)
     assert calls[1][0][-1] == "runner"
diff --git a/tests/healthchecks/test_unstract_durable_replay.py b/tests/healthchecks/test_unstract_durable_replay.py
index f3e942d9f1..b10f30239d 100644
--- a/tests/healthchecks/test_unstract_durable_replay.py
+++ b/tests/healthchecks/test_unstract_durable_replay.py
@@ -87,6 +87,7 @@ def test_systemd_owner_orders_guard_before_generic_restart() -> None:
     assert "Requires=train-rootless-boot-recovery.service" in unit
     assert "Before=podman-restart.service" in unit
     assert "ExecStart=/usr/bin/python3 %h/.local/libexec/unstract-durable-replay.py" in unit
+    assert "PartOf=podman-restart.service" in unit
     assert "WantedBy=default.target" in unit
     assert "Requires=unstract-durable-replay.service" in drop_in
     assert "After=unstract-durable-replay.service" in drop_in

From ae814b34611f1e0682fddd616ff100aaa2296d7c Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 00:56:51 -0400
Subject: [PATCH 37/48] Harden durable Compose input binding

---
 .../scripts/train_health_deployment_guard.py  | 179 ++++++++++++++----
 .../systemd/unstract-durable-replay.service   |   2 -
 .../test_train_health_deployment_guard.py     |  50 +++--
 .../test_unstract_durable_replay.py           |   2 +
 4 files changed, 176 insertions(+), 57 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 508358baa9..ceab2f2f17 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -21,6 +21,7 @@
 import argparse
 import contextlib
 import datetime as dt
+import errno
 import fcntl
 import hashlib
 import json
@@ -395,33 +396,61 @@ def _immutable_snapshot_fd(data: bytes) -> tuple[int, tempfile.TemporaryFile[byt
     return fd, temporary
 
 
+def _compose_input_path(project_dir: Path, value: str | Path) -> Path:
+    """Resolve a Compose input the same way the explicit project directory does."""
+    path = Path(value)
+    if path.is_absolute():
+        return path
+    return project_dir.resolve() / path
+
+
+def _open_compose_input(path: Path) -> int:
+    """Open a Compose input without following a mutable pathname in place."""
+    flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        return os.open(path, flags)
+    except OSError as exc:
+        # A live .env or Compose file may be a symlink. Resolve it once and
+        # snapshot the target; the final source fingerprint check still runs
+        # before the child launch, while the child consumes the bound bytes.
+        if exc.errno != getattr(errno, "ELOOP", 40):
+            raise
+        try:
+            return os.open(path.resolve(strict=True), flags)
+        except OSError:
+            raise exc
+
+
 @contextlib.contextmanager
-def bound_private_compose_file(
+def bound_compose_file(
     path: Path,
     *,
     expected_sha256: str | None,
     description: str,
+    private: bool = False,
 ) -> Iterator[tuple[str, int]]:
-    """Bind a verified private Compose file to an immutable child-visible fd.
+    """Bind one Compose input to an immutable child-visible descriptor.
 
     Hashing the pathname and then passing that pathname to Compose leaves a
     same-user write window between verification and subprocess open. Read the
     reviewed bytes first, seal a memfd snapshot, and pass the descriptor to
-    Compose through ``/proc/self/fd``. The original path can change after this
-    point without changing the bytes that the child parses.
+    Compose through ``/proc/self/fd``. The original path can change after
+    this point without changing the bytes that the child parses. Private
+    replay files retain their stricter mode and ownership contract.
     """
-    flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
     try:
-        source_fd = os.open(path, flags)
+        source_fd = _open_compose_input(path)
     except OSError as exc:
         raise GuardError(f"cannot read private {description}: {path}") from exc
     snapshot_fd: int | None = None
     temporary: tempfile.TemporaryFile[bytes] | None = None
     try:
         metadata = os.fstat(source_fd)
-        if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600:
+        if not stat.S_ISREG(metadata.st_mode):
+            raise GuardError(f"{description} is not a regular file: {path}")
+        if private and stat.S_IMODE(metadata.st_mode) != 0o600:
             raise GuardError(f"{description} is not a private regular file: {path}")
-        if hasattr(os, "getuid") and metadata.st_uid != os.getuid():
+        if private and hasattr(os, "getuid") and metadata.st_uid != os.getuid():
             raise GuardError(f"{description} has the wrong owner: {path}")
         data = _read_fd(source_fd)
         actual_sha256 = sha256_bytes(data)
@@ -444,36 +473,82 @@ def bound_private_compose_file(
 
 
 @contextlib.contextmanager
-def bound_private_compose_inputs(
+def bound_compose_inputs(
     *,
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    live_env_file: Path | None,
+    probe_source: Path,
+    probe_source_sha256: str | None,
     image_override: Path | None,
     image_override_sha256: str | None,
     environment_override: Path | None,
     environment_override_sha256: str | None,
 ) -> Iterator[tuple[dict[str, str], tuple[int, ...]]]:
-    """Snapshot each private override and return path replacements plus fds."""
+    """Snapshot every file Compose and its interpolation environment consume."""
     replacements: dict[str, str] = {}
     pass_fds: list[int] = []
     with contextlib.ExitStack() as stack:
-        inputs = (
-            (image_override, image_override_sha256, "candidate image override"),
+        inputs: list[tuple[str, Path, str | None, str, bool]] = []
+        for compose_file in compose_files:
+            inputs.append(
+                (
+                    compose_file,
+                    _compose_input_path(project_dir, compose_file),
+                    None,
+                    "Compose file",
+                    False,
+                )
+            )
+        if live_env_file:
+            inputs.append(
+                (
+                    str(live_env_file),
+                    _compose_input_path(project_dir, live_env_file),
+                    None,
+                    "Compose environment file",
+                    False,
+                )
+            )
+        inputs.append(
             (
-                environment_override,
-                environment_override_sha256,
-                "runtime environment override",
-            ),
+                str(probe_source),
+                _compose_input_path(project_dir, probe_source),
+                probe_source_sha256,
+                "health probe source",
+                False,
+            )
         )
-        for path, expected_sha256, description in inputs:
-            if path is None:
-                continue
+        if image_override:
+            inputs.append(
+                (
+                    str(image_override),
+                    image_override,
+                    image_override_sha256,
+                    "candidate image override",
+                    True,
+                )
+            )
+        if environment_override:
+            inputs.append(
+                (
+                    str(environment_override),
+                    environment_override,
+                    environment_override_sha256,
+                    "runtime environment override",
+                    True,
+                )
+            )
+        for argument, path, expected_sha256, description, private in inputs:
             proc_path, fd = stack.enter_context(
-                bound_private_compose_file(
+                bound_compose_file(
                     path,
                     expected_sha256=expected_sha256,
                     description=description,
+                    private=private,
                 )
             )
-            replacements[str(path)] = proc_path
+            replacements[argument] = proc_path
             pass_fds.append(fd)
         yield replacements, tuple(pass_fds)
 
@@ -1466,13 +1541,14 @@ def compose_config(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = ["docker", "compose"]
-    if live_env_file:
-        args.extend(["--env-file", str(live_env_file)])
-    for compose_file in files:
-        args.extend(["-f", compose_file])
+    args = compose_args(project_dir, files, live_env_file=live_env_file)
     args.extend(["config", "--format", "json"])
-    with bound_private_compose_inputs(
+    with bound_compose_inputs(
+        project_dir=project_dir,
+        compose_files=files,
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
@@ -1497,9 +1573,11 @@ def compose_config(
         )
         if final_settings:
             env["VERSION"] = final_settings["VERSION"]
-            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-                "UNSTRACT_HEALTHCHECK_SOURCE"
-            ]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = bound_paths.get(
+            str(probe_source), final_settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+            if final_settings
+            else env["UNSTRACT_HEALTHCHECK_SOURCE"]
+        )
         result = run(
             bound_args,
             cwd=project_dir,
@@ -2305,9 +2383,12 @@ def capture_and_write(
 
 
 def compose_args(
-    compose_files: tuple[str, ...], *, live_env_file: Path | None = None
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    *,
+    live_env_file: Path | None = None,
 ) -> list[str]:
-    args = ["docker", "compose"]
+    args = ["docker", "compose", "--project-directory", str(project_dir.resolve())]
     if live_env_file:
         args.extend(["--env-file", str(live_env_file)])
     for compose_file in compose_files:
@@ -2589,7 +2670,7 @@ def targeted_up(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = compose_args(files, live_env_file=live_env_file)
+    args = compose_args(project_dir, files, live_env_file=live_env_file)
     args.extend(
         [
             "up",
@@ -2602,7 +2683,12 @@ def targeted_up(
             *services,
         ]
     )
-    with bound_private_compose_inputs(
+    with bound_compose_inputs(
+        project_dir=project_dir,
+        compose_files=files,
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
@@ -2627,9 +2713,11 @@ def targeted_up(
         )
         if final_settings:
             env["VERSION"] = final_settings["VERSION"]
-            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-                "UNSTRACT_HEALTHCHECK_SOURCE"
-            ]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = bound_paths.get(
+            str(probe_source), final_settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+            if final_settings
+            else env["UNSTRACT_HEALTHCHECK_SOURCE"]
+        )
         run(
             bound_args,
             cwd=project_dir,
@@ -2682,9 +2770,14 @@ def compose_start(
         environment_override, expected_sha256=environment_override_sha256
     )
     files = compose_files + (str(image_override), str(environment_override))
-    args = compose_args(files, live_env_file=live_env_file)
+    args = compose_args(project_dir, files, live_env_file=live_env_file)
     args.extend(["up", "-d", "--no-build", "--pull", "never"])
-    with bound_private_compose_inputs(
+    with bound_compose_inputs(
+        project_dir=project_dir,
+        compose_files=files,
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
         image_override=image_override,
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
@@ -2709,9 +2802,11 @@ def compose_start(
         )
         if final_settings:
             env["VERSION"] = final_settings["VERSION"]
-            env["UNSTRACT_HEALTHCHECK_SOURCE"] = final_settings[
-                "UNSTRACT_HEALTHCHECK_SOURCE"
-            ]
+        env["UNSTRACT_HEALTHCHECK_SOURCE"] = bound_paths.get(
+            str(probe_source), final_settings["UNSTRACT_HEALTHCHECK_SOURCE"]
+            if final_settings
+            else env["UNSTRACT_HEALTHCHECK_SOURCE"]
+        )
         run(
             bound_args,
             cwd=project_dir,
diff --git a/docker/systemd/unstract-durable-replay.service b/docker/systemd/unstract-durable-replay.service
index feb6d1c1eb..f66d9f6f36 100644
--- a/docker/systemd/unstract-durable-replay.service
+++ b/docker/systemd/unstract-durable-replay.service
@@ -9,8 +9,6 @@ Before=podman-restart.service
 # Requires edge in the podman-restart drop-in then schedules a fresh replay in
 # the same restart transaction instead of reusing RemainAfterExit state.
 PartOf=podman-restart.service
-ConditionPathExists=%h/.config/unstract/durable-replay.env
-ConditionPathExists=%h/.local/libexec/unstract-durable-replay.py
 
 [Service]
 Type=oneshot
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 4b8b9ef4f1..a15f89157f 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -418,6 +418,8 @@ def test_compose_replay_consumes_durable_settings_and_overrides(
         "TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry\nCOMPOSE_PROJECT_NAME=test\n",
         encoding="utf-8",
     )
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services: {}\n", encoding="utf-8")
     calls: list[tuple[list[str], dict[str, str] | None]] = []
 
     def fake_run(
@@ -425,26 +427,46 @@ def fake_run(
         *,
         cwd: Path | None = None,
         env: dict[str, str] | None = None,
-        **_: object,
+        **kwargs: object,
     ) -> subprocess.CompletedProcess[str]:
         calls.append((args, env))
         if "config" in args:
-            bound_files = [
-                Path(argument)
+            bound_files = {
+                argument
                 for argument in args
                 if argument.startswith("/proc/self/fd/")
-            ]
-            assert len(bound_files) == 2
-            original_image = image_override.read_bytes()
-            image_override.write_text("tampered after final verification\n", encoding="utf-8")
+            }
+            assert len(bound_files) == 4
+            assert str(live_env) not in args
+            assert str(compose) not in args
+            assert str(image_override) not in args
+            assert str(environment_override) not in args
+            assert env is not None
+            bound_probe = env["UNSTRACT_HEALTHCHECK_SOURCE"]
+            assert bound_probe.startswith("/proc/self/fd/")
+            bound_paths = {Path(value) for value in bound_files | {bound_probe}}
+            assert len(bound_paths) == 5
+            originals = {
+                compose: compose.read_bytes(),
+                live_env: live_env.read_bytes(),
+                probe: probe.read_bytes(),
+                image_override: image_override.read_bytes(),
+                environment_override: environment_override.read_bytes(),
+            }
+            for source in originals:
+                source.write_bytes(b"tampered after final verification\n")
             try:
-                assert any(path.read_bytes() == original_image for path in bound_files)
+                for source, original in originals.items():
+                    assert any(path.read_bytes() == original for path in bound_paths), source
             finally:
-                image_override.write_bytes(original_image)
+                for source, original in originals.items():
+                    source.write_bytes(original)
             env_file = Path(args[args.index("--env-file") + 1])
             assert "TOOL_REGISTRY_CONFIG_SRC_PATH=/srv/tool-registry" in env_file.read_text(
                 encoding="utf-8"
             )
+            assert Path(bound_probe).read_bytes() == originals[probe]
+            assert kwargs["pass_fds"]
             return subprocess.CompletedProcess(args, 0, json.dumps({"services": {}}), "")
         return subprocess.CompletedProcess(args, 0, "", "")
 
@@ -481,14 +503,15 @@ def fake_run(
 
     assert len(calls) == 2
     config_args, config_env = calls[0]
-    assert config_args[:3] == ["docker", "compose", "--env-file"]
-    assert str(live_env) in config_args
+    assert config_args[:3] == ["docker", "compose", "--project-directory"]
+    assert str(tmp_path.resolve()) in config_args
+    assert str(live_env) not in config_args
     assert str(settings) not in config_args
     assert str(image_override) not in config_args
     assert str(environment_override) not in config_args
-    assert sum(argument.startswith("/proc/self/fd/") for argument in config_args) == 2
+    assert sum(argument.startswith("/proc/self/fd/") for argument in config_args) == 4
     assert config_env and config_env["VERSION"] == "goal09-test"
-    assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(probe)
+    assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"].startswith("/proc/self/fd/")
     assert calls[1][0][-1] == "runner"
 
     monkeypatch.setattr(guard, "verify_candidate_source_state", lambda *_: None)
@@ -551,6 +574,7 @@ def test_compose_rejects_ignored_live_input_drift(tmp_path: Path) -> None:
 
 
 def test_compose_rechecks_candidate_artifacts_before_each_config(tmp_path: Path) -> None:
+    (tmp_path / "compose.yaml").write_text("services: {}\n", encoding="utf-8")
     source = tmp_path / "candidate"
     source.mkdir()
     source_root = GUARD_PATH.parents[2]
diff --git a/tests/healthchecks/test_unstract_durable_replay.py b/tests/healthchecks/test_unstract_durable_replay.py
index b10f30239d..68b8fafc9a 100644
--- a/tests/healthchecks/test_unstract_durable_replay.py
+++ b/tests/healthchecks/test_unstract_durable_replay.py
@@ -66,6 +66,7 @@ def test_launcher_builds_guarded_start_with_all_persistent_inputs(tmp_path: Path
     ("name", "value", "message"),
     [
         ("UNSTRACT_GUARD", "relative/guard.py", "absolute path"),
+        ("UNSTRACT_COMPOSE_BASE", "", "UNSTRACT_COMPOSE_BASE is required"),
         ("UNSTRACT_START_CONFIRM", "APPLY_UNSTRACT_HEALTH", "does not authorize"),
         ("UNSTRACT_OPERATION_TIMEOUT", "0", "positive integer"),
     ],
@@ -89,5 +90,6 @@ def test_systemd_owner_orders_guard_before_generic_restart() -> None:
     assert "ExecStart=/usr/bin/python3 %h/.local/libexec/unstract-durable-replay.py" in unit
     assert "PartOf=podman-restart.service" in unit
     assert "WantedBy=default.target" in unit
+    assert "ConditionPathExists" not in unit
     assert "Requires=unstract-durable-replay.service" in drop_in
     assert "After=unstract-durable-replay.service" in drop_in

From 5a75f1f4ce0115f276a88eb52b3719da58396ba5 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 02:16:22 -0400
Subject: [PATCH 38/48] Use retained daemon-visible Compose snapshots

---
 .../scripts/train_health_deployment_guard.py  | 861 ++++++++++++++----
 docs/train-unstract-health-deployment.md      |  26 +-
 .../test_train_health_deployment_guard.py     | 175 +++-
 3 files changed, 868 insertions(+), 194 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index ceab2f2f17..8d57531547 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -21,7 +21,6 @@
 import argparse
 import contextlib
 import datetime as dt
-import errno
 import fcntl
 import hashlib
 import json
@@ -38,7 +37,7 @@
 from collections.abc import Iterator
 from decimal import Decimal, InvalidOperation
 from pathlib import Path
-from typing import Any
+from typing import Any, NamedTuple
 
 PROJECT = "unstract-etl-home-complete-tech"
 DEFAULT_PROJECT_DIR = Path("/home/completetrain/etl.home.complete.tech")
@@ -107,8 +106,13 @@
 START_CONFIRM_TOKEN = "START_UNSTRACT_HEALTH"
 LIVE_COMPOSE_TRAIN = "docker/compose.train.yaml"
 LIVE_ENV_RELATIVE = "docker/.env"
-REPLAY_MANIFEST_SCHEMA = "unstract-durable-replay/v1"
+REPLAY_MANIFEST_SCHEMA = "unstract-durable-replay/v2"
 REPLAY_MANIFEST_FILENAME = "durable-replay-manifest.json"
+COMPOSE_SNAPSHOT_SCHEMA = "unstract-compose-snapshot/v1"
+COMPOSE_SNAPSHOT_MANIFEST_FILENAME = "compose-snapshot-manifest.json"
+COMPOSE_SNAPSHOT_DIR_PREFIX = "compose-snapshot-"
+COMPOSE_SNAPSHOT_PROBE_RELATIVE = Path("__helper__/unstract-services.sh")
+COMPOSE_SNAPSHOT_PRIVATE_RELATIVE = Path("__private__")
 RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
@@ -138,6 +142,22 @@
 }
 
 
+class ComposeSnapshot(NamedTuple):
+    """A daemon-visible immutable copy of every Compose launch input."""
+
+    root: Path
+    manifest_path: Path
+    project_dir: Path
+    paths: dict[str, str]
+
+    @property
+    def probe_path(self) -> Path:
+        return Path(self.paths["__probe_source__"])
+
+    def path_for(self, value: str | Path) -> str:
+        return self.paths.get(str(value), str(value))
+
+
 def _probe_test(service: str) -> list[str]:
     if service in CORE_SERVICES:
         probe_name = {
@@ -338,138 +358,539 @@ def run(
     return result
 
 
-def _read_fd(fd: int) -> bytes:
-    """Read an open input and leave its offset at the beginning."""
+def _compose_input_path(project_dir: Path, value: str | Path) -> Path:
+    """Resolve a Compose input the same way the explicit project directory does."""
+    path = Path(value)
+    if path.is_absolute():
+        return path
+    return project_dir.resolve() / path
+
+
+def _read_compose_input(path: Path, *, description: str) -> bytes:
+    """Read one Compose input while resolving a symlink only once."""
     try:
-        os.lseek(fd, 0, os.SEEK_SET)
-        chunks: list[bytes] = []
-        while True:
-            chunk = os.read(fd, 1024 * 1024)
-            if not chunk:
-                break
-            chunks.append(chunk)
-        os.lseek(fd, 0, os.SEEK_SET)
+        metadata = path.lstat()
+        resolved = path.resolve(strict=True) if stat.S_ISLNK(metadata.st_mode) else path
+        resolved_metadata = resolved.stat()
+        if not stat.S_ISREG(resolved_metadata.st_mode):
+            raise GuardError(f"{description} is not a regular file: {path}")
+        flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
+        try:
+            descriptor = os.open(resolved, flags)
+        except OSError:
+            # The fallback is only for platforms without O_NOFOLLOW support.
+            descriptor = os.open(resolved, os.O_RDONLY | getattr(os, "O_CLOEXEC", 0))
+        try:
+            chunks: list[bytes] = []
+            while True:
+                chunk = os.read(descriptor, 1024 * 1024)
+                if not chunk:
+                    break
+                chunks.append(chunk)
+            return b"".join(chunks)
+        finally:
+            os.close(descriptor)
+    except GuardError:
+        raise
     except OSError as exc:
-        raise GuardError(f"cannot snapshot Compose input: {exc}") from exc
-    return b"".join(chunks)
+        raise GuardError(f"cannot read {description}: {path}") from exc
 
 
-def _immutable_snapshot_fd(data: bytes) -> tuple[int, tempfile.TemporaryFile[bytes] | None]:
-    """Create a sealed descriptor containing one Compose input snapshot."""
-    temporary: tempfile.TemporaryFile[bytes] | None = None
+def _snapshot_source_relative(project_dir: Path, source: Path, *, fallback: str) -> Path:
+    """Choose a safe snapshot path while retaining project-relative layout."""
+    project = project_dir.resolve()
+    source = source.resolve()
     try:
-        fd = os.memfd_create(
-            "unstract-compose-input",
-            os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING,
-        )
-    except AttributeError:
-        # Linux Train hosts provide memfd_create. Keep the guard importable on
-        # other platforms for static checks, while retaining an unlinked
-        # descriptor as the best available private snapshot there.
-        temporary = tempfile.TemporaryFile(mode="w+b")
-        fd = temporary.fileno()
+        relative = source.relative_to(project)
+    except ValueError:
+        digest = sha256_bytes(str(source).encode("utf-8"))[:16]
+        return Path("__external__") / digest / fallback
+    if not relative.parts or any(part in {"", ".", ".."} for part in relative.parts):
+        raise GuardError(f"Compose input has an unsafe relative path: {source}")
+    return relative
+
+
+def _compose_include_paths(path: Path) -> list[Path]:
+    """Find literal Compose ``include`` entries without parsing interpolation."""
     try:
-        view = memoryview(data)
-        while view:
-            written = os.write(fd, view)
-            if written <= 0:
-                raise OSError("snapshot write made no progress")
-            view = view[written:]
-        os.lseek(fd, 0, os.SEEK_SET)
-        if temporary is None:
-            fcntl.fcntl(
-                fd,
-                fcntl.F_ADD_SEALS,
-                fcntl.F_SEAL_WRITE
-                | fcntl.F_SEAL_GROW
-                | fcntl.F_SEAL_SHRINK
-                | fcntl.F_SEAL_SEAL,
-            )
-        else:
-            os.fchmod(fd, 0o400)
-    except BaseException:
-        if temporary is not None:
-            temporary.close()
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise GuardError(f"cannot read Compose include source: {path}") from exc
+    result: list[Path] = []
+    for index, line in enumerate(lines):
+        stripped = line.strip()
+        if not stripped.startswith("include:"):
+            continue
+        indent = len(line) - len(line.lstrip())
+        tail = stripped[len("include:") :].strip()
+        values: list[str] = []
+        if tail.startswith("[") and tail.endswith("]"):
+            values.extend(part.strip().strip("'\"") for part in tail[1:-1].split(","))
+        elif tail and not tail.startswith("#"):
+            values.append(tail.strip("'\""))
+        for following in lines[index + 1 :]:
+            if following.strip() and len(following) - len(following.lstrip()) <= indent:
+                break
+            candidate = following.strip()
+            if candidate.startswith("-"):
+                value = candidate[1:].split("#", 1)[0].strip().strip("'\"")
+                if value:
+                    values.append(value)
+        for value in values:
+            if value and "${" not in value:
+                result.append((path.parent / value).resolve())
+    return result
+
+
+def _compose_relative_references(path: Path) -> list[tuple[str, bool]]:
+    """Find literal relative env-file and bind-source references.
+
+    Compose files in the guarded stack use the short list syntax for these
+    fields.  Keeping this scanner lexical avoids making the systemd launcher
+    depend on a YAML package while still allowing us to freeze the files the
+    provider opens itself.
+    """
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise GuardError(f"cannot read Compose reference source: {path}") from exc
+    result: list[tuple[str, bool]] = []
+    section: str | None = None
+    section_indent = -1
+    for line in lines:
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        indent = len(line) - len(line.lstrip())
+        if section is not None and indent <= section_indent and not stripped.startswith("-"):
+            section = None
+        if stripped.startswith("env_file:"):
+            section, section_indent = "env_file", indent
+            tail = stripped[len("env_file:") :].strip()
+            if tail and not tail.startswith("#"):
+                result.append((tail.strip("'\""), True))
+            continue
+        if stripped.startswith("volumes:"):
+            section, section_indent = "volumes", indent
+            continue
+        if section is None or not stripped.startswith("-"):
+            continue
+        value = stripped[1:].split("#", 1)[0].strip().strip("'\"")
+        if section == "env_file":
+            reference = value
+            private = True
         else:
-            os.close(fd)
+            # Long syntax starts with ``type:``/``source:`` and is handled by
+            # the provider directly; the repository's guarded files use the
+            # short ``source:target[:options]`` form.
+            if value.startswith(("type:", "source:")):
+                continue
+            reference = value.split(":", 1)[0]
+            private = False
+        if reference.startswith((".", "..")) and "${" not in reference:
+            result.append((reference, private))
+    return result
+
+
+def _write_snapshot_file(
+    root: Path,
+    relative: Path,
+    data: bytes,
+    *,
+    private: bool,
+) -> str:
+    """Write a snapshot file and return its content digest."""
+    destination = root / relative
+    try:
+        destination.relative_to(root)
+    except ValueError as exc:
+        raise GuardError(f"snapshot path escapes its root: {relative}") from exc
+    destination.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
+    mode = 0o400 if private else 0o444
+    flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0)
+    if hasattr(os, "O_NOFOLLOW"):
+        flags |= os.O_NOFOLLOW
+    try:
+        descriptor = os.open(destination, flags, mode)
+        with os.fdopen(descriptor, "wb") as handle:
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+    except OSError as exc:
+        raise GuardError(f"cannot write Compose snapshot input: {destination}") from exc
+    os.chmod(destination, mode)
+    return sha256_bytes(data)
+
+
+def _write_snapshot_directory_passthrough(
+    root: Path,
+    relative: Path,
+    source: Path,
+) -> str:
+    """Keep a runtime data directory at its original daemon-visible location."""
+    destination = root / relative
+    destination.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
+    try:
+        resolved = source.resolve(strict=True)
+        if not resolved.is_dir():
+            raise GuardError(f"Compose directory reference is not a directory: {source}")
+        destination.relative_to(root)
+        if destination.is_symlink():
+            if destination.resolve(strict=True) != resolved:
+                raise GuardError(f"Compose directory reference changed: {destination}")
+            return str(resolved)
+        if destination.exists():
+            raise GuardError(f"Compose directory reference collides with a snapshot file: {destination}")
+        os.symlink(resolved, destination, target_is_directory=True)
+    except GuardError:
         raise
-    return fd, temporary
+    except OSError as exc:
+        raise GuardError(f"cannot preserve Compose directory reference: {source}") from exc
+    return str(resolved)
 
 
-def _compose_input_path(project_dir: Path, value: str | Path) -> Path:
-    """Resolve a Compose input the same way the explicit project directory does."""
-    path = Path(value)
-    if path.is_absolute():
-        return path
-    return project_dir.resolve() / path
+def _freeze_snapshot_tree(root: Path) -> None:
+    """Make every snapshot directory traversable but non-user-writable."""
+    for directory in sorted(
+        (path for path in root.rglob("*") if path.is_dir() and not path.is_symlink()),
+        reverse=True,
+    ):
+        os.chmod(directory, 0o500 if directory.name == COMPOSE_SNAPSHOT_PRIVATE_RELATIVE.name else 0o555)
+    os.chmod(root, 0o555)
+
 
+def materialize_compose_snapshot(
+    *,
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    live_env_file: Path | None,
+    probe_source: Path,
+    probe_source_sha256: str | None,
+    image_override: Path | None,
+    image_override_sha256: str | None,
+    environment_override: Path | None,
+    environment_override_sha256: str | None,
+    destination: Path,
+) -> ComposeSnapshot:
+    """Create a durable daemon-visible snapshot preserving Compose includes."""
+    if destination.exists():
+        raise GuardError(f"Compose snapshot destination already exists: {destination}")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    destination.mkdir(mode=0o755)
+    paths: dict[str, str] = {}
+    entries: dict[str, dict[str, Any]] = {}
+    passthroughs: dict[str, str] = {}
+    copied: set[Path] = set()
+    private_sources = {
+        source.resolve()
+        for source in (image_override, environment_override)
+        if source is not None
+    }
 
-def _open_compose_input(path: Path) -> int:
-    """Open a Compose input without following a mutable pathname in place."""
-    flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0)
+    def copy_source(
+        source: Path,
+        relative: Path,
+        *,
+        key: str | None = None,
+        expected_sha256: str | None = None,
+        private: bool = False,
+        description: str = "Compose input",
+    ) -> None:
+        resolved = source.resolve(strict=True)
+        if resolved in copied and (destination / relative).is_file():
+            if key is not None:
+                paths[key] = str((destination / relative).resolve())
+            return
+        data = _read_compose_input(source, description=description)
+        actual = sha256_bytes(data)
+        if expected_sha256 is not None and actual != expected_sha256:
+            raise GuardError(f"{description} changed: {source}")
+        digest = _write_snapshot_file(destination, relative, data, private=private)
+        copied.add(resolved)
+        entries[str(relative)] = {
+            "sha256": digest,
+            "private": private,
+        }
+        if key is not None:
+            paths[key] = str((destination / relative).resolve())
+
+    compose_sources: list[Path] = []
+    pending_includes: list[Path] = []
+    for argument in compose_files:
+        source = _compose_input_path(project_dir, argument)
+        resolved = source.resolve()
+        if resolved in private_sources:
+            relative = COMPOSE_SNAPSHOT_PRIVATE_RELATIVE / source.name
+            copy_source(
+                source,
+                relative,
+                key=argument,
+                private=True,
+                expected_sha256=(
+                    image_override_sha256
+                    if image_override is not None and resolved == image_override.resolve()
+                    else environment_override_sha256
+                ),
+                description="private Compose override",
+            )
+        else:
+            relative = _snapshot_source_relative(
+                project_dir, source, fallback=Path(argument).name
+            )
+            copy_source(source, relative, key=argument, description="Compose file")
+        compose_sources.append(source)
+        pending_includes.extend(_compose_include_paths(source))
+    seen_includes: set[Path] = set()
+    while pending_includes:
+        source = pending_includes.pop(0)
+        source = source.resolve()
+        if source in seen_includes:
+            continue
+        seen_includes.add(source)
+        relative = _snapshot_source_relative(
+            project_dir, source, fallback=source.name
+        )
+        copy_source(source, relative, description="Compose include")
+        compose_sources.append(source)
+        pending_includes.extend(_compose_include_paths(source))
+
+    for compose_source in compose_sources:
+        for reference, private in _compose_relative_references(compose_source):
+            source = (compose_source.parent / reference).resolve()
+            relative = _snapshot_source_relative(
+                project_dir, source, fallback=source.name
+            )
+            if source.is_dir():
+                passthroughs[str(relative)] = _write_snapshot_directory_passthrough(
+                    destination, relative, source
+                )
+                continue
+            copy_source(
+                source,
+                relative,
+                private=private,
+                description="private Compose env file" if private else "Compose bind file",
+            )
+
+    if live_env_file is not None:
+        source = _compose_input_path(project_dir, live_env_file)
+        relative = _snapshot_source_relative(project_dir, source, fallback=source.name)
+        copy_source(
+            source,
+            relative,
+            key=str(live_env_file),
+            private=True,
+            description="Compose environment file",
+        )
+
+    copy_source(
+        probe_source,
+        COMPOSE_SNAPSHOT_PROBE_RELATIVE,
+        key=str(probe_source),
+        expected_sha256=probe_source_sha256,
+        description="health probe source",
+    )
+    paths["__probe_source__"] = paths[str(probe_source)]
+    for source, expected_sha256, description in (
+        (image_override, image_override_sha256, "candidate image override"),
+        (environment_override, environment_override_sha256, "runtime environment override"),
+    ):
+        if source is None:
+            continue
+        relative = COMPOSE_SNAPSHOT_PRIVATE_RELATIVE / source.name
+        copy_source(
+            source,
+            relative,
+            key=str(source),
+            expected_sha256=expected_sha256,
+            private=True,
+            description=description,
+        )
+
+    manifest_path = destination / COMPOSE_SNAPSHOT_MANIFEST_FILENAME
+    manifest = {
+        "schema": COMPOSE_SNAPSHOT_SCHEMA,
+        "root": str(destination.resolve()),
+        "project_dir": str(project_dir.resolve()),
+        "paths": {
+            key: str(Path(value).relative_to(destination.resolve()))
+            for key, value in paths.items()
+        },
+        "entries": entries,
+        "passthroughs": passthroughs,
+    }
     try:
-        return os.open(path, flags)
+        manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8")
+        os.chmod(manifest_path, 0o400)
     except OSError as exc:
-        # A live .env or Compose file may be a symlink. Resolve it once and
-        # snapshot the target; the final source fingerprint check still runs
-        # before the child launch, while the child consumes the bound bytes.
-        if exc.errno != getattr(errno, "ELOOP", 40):
-            raise
-        try:
-            return os.open(path.resolve(strict=True), flags)
-        except OSError:
-            raise exc
+        raise GuardError(f"cannot write Compose snapshot manifest: {manifest_path}") from exc
+    _freeze_snapshot_tree(destination)
+    return ComposeSnapshot(
+        root=destination.resolve(),
+        manifest_path=manifest_path.resolve(),
+        project_dir=project_dir.resolve(),
+        paths={key: str(destination / relative) for key, relative in manifest["paths"].items()},
+    )
 
 
-@contextlib.contextmanager
-def bound_compose_file(
-    path: Path,
-    *,
-    expected_sha256: str | None,
-    description: str,
-    private: bool = False,
-) -> Iterator[tuple[str, int]]:
-    """Bind one Compose input to an immutable child-visible descriptor.
-
-    Hashing the pathname and then passing that pathname to Compose leaves a
-    same-user write window between verification and subprocess open. Read the
-    reviewed bytes first, seal a memfd snapshot, and pass the descriptor to
-    Compose through ``/proc/self/fd``. The original path can change after
-    this point without changing the bytes that the child parses. Private
-    replay files retain their stricter mode and ownership contract.
-    """
+def _validate_snapshot_file(path: Path, expected_sha256: str, *, private: bool) -> None:
     try:
-        source_fd = _open_compose_input(path)
+        metadata = path.lstat()
     except OSError as exc:
-        raise GuardError(f"cannot read private {description}: {path}") from exc
-    snapshot_fd: int | None = None
-    temporary: tempfile.TemporaryFile[bytes] | None = None
+        raise GuardError(f"cannot read Compose snapshot file: {path}") from exc
+    if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) & 0o222:
+        raise GuardError(f"Compose snapshot file is writable or not regular: {path}")
+    if hasattr(os, "getuid") and os.getuid() == 0 and metadata.st_uid != 0:
+        raise GuardError(f"Compose snapshot file is not root-owned: {path}")
+    actual = sha256_file(path)
+    if actual != expected_sha256:
+        raise GuardError(f"Compose snapshot file changed: {path}")
+    if private and stat.S_IMODE(metadata.st_mode) != 0o400:
+        raise GuardError(f"private Compose snapshot file has an unsafe mode: {path}")
+
+
+def _validate_snapshot_parents(path: Path, root: Path) -> None:
+    """Ensure no snapshot directory can be replaced by the launching user."""
+    current = path.parent
+    while True:
+        try:
+            metadata = current.lstat()
+        except OSError as exc:
+            raise GuardError(f"cannot read Compose snapshot directory: {current}") from exc
+        if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) & 0o222:
+            raise GuardError(f"Compose snapshot directory is writable or not a directory: {current}")
+        if hasattr(os, "getuid") and os.getuid() == 0 and metadata.st_uid != 0:
+            raise GuardError(f"Compose snapshot directory is not root-owned: {current}")
+        if current == root:
+            return
+        if current.parent == current:
+            raise GuardError("Compose snapshot directory escaped its root")
+        current = current.parent
+
+
+def load_compose_snapshot(path: Path) -> ComposeSnapshot:
+    """Validate a retained snapshot immediately before a Compose launch."""
     try:
-        metadata = os.fstat(source_fd)
-        if not stat.S_ISREG(metadata.st_mode):
-            raise GuardError(f"{description} is not a regular file: {path}")
-        if private and stat.S_IMODE(metadata.st_mode) != 0o600:
-            raise GuardError(f"{description} is not a private regular file: {path}")
-        if private and hasattr(os, "getuid") and metadata.st_uid != os.getuid():
-            raise GuardError(f"{description} has the wrong owner: {path}")
-        data = _read_fd(source_fd)
-        actual_sha256 = sha256_bytes(data)
-        if expected_sha256 is not None and actual_sha256 != expected_sha256:
-            raise GuardError(f"{description} changed: {path}")
-        snapshot_fd, temporary = _immutable_snapshot_fd(data)
-        yield f"/proc/self/fd/{snapshot_fd}", snapshot_fd
-    finally:
+        metadata = path.lstat()
+        if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) & 0o222:
+            raise GuardError(f"Compose snapshot manifest is writable or not regular: {path}")
+        manifest = json.loads(path.read_text(encoding="utf-8"))
+    except GuardError:
+        raise
+    except (OSError, json.JSONDecodeError) as exc:
+        raise GuardError(f"cannot read Compose snapshot manifest: {path}") from exc
+    if manifest.get("schema") != COMPOSE_SNAPSHOT_SCHEMA:
+        raise GuardError("Compose snapshot manifest has an unsupported schema")
+    root = Path(manifest.get("root", "")).resolve()
+    if root != path.resolve().parent or not root.is_dir():
+        raise GuardError("Compose snapshot root does not match its manifest")
+    root_metadata = root.lstat()
+    if stat.S_IMODE(root_metadata.st_mode) & 0o222:
+        raise GuardError("Compose snapshot root is writable")
+    if hasattr(os, "getuid") and os.getuid() == 0 and root_metadata.st_uid != 0:
+        raise GuardError("Compose snapshot root is not root-owned")
+    entries = manifest.get("entries")
+    paths_manifest = manifest.get("paths")
+    passthroughs = manifest.get("passthroughs") or {}
+    if (
+        not isinstance(entries, dict)
+        or not isinstance(paths_manifest, dict)
+        or not isinstance(passthroughs, dict)
+    ):
+        raise GuardError("Compose snapshot manifest is incomplete")
+    paths: dict[str, str] = {}
+    for relative, entry in entries.items():
+        if not isinstance(relative, str) or not isinstance(entry, dict):
+            raise GuardError("Compose snapshot manifest contains an invalid entry")
+        candidate = (root / relative).resolve()
+        if candidate.parent == root and candidate.name == COMPOSE_SNAPSHOT_MANIFEST_FILENAME:
+            raise GuardError("Compose snapshot manifest lists itself as an input")
         try:
-            os.close(source_fd)
-        except OSError:
-            pass
-        if temporary is not None:
-            temporary.close()
-        elif snapshot_fd is not None:
-            try:
-                os.close(snapshot_fd)
-            except OSError:
-                pass
+            candidate.relative_to(root)
+        except ValueError as exc:
+            raise GuardError("Compose snapshot entry escapes its root") from exc
+        _validate_snapshot_parents(candidate, root)
+        digest = entry.get("sha256")
+        if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+            raise GuardError("Compose snapshot entry has an invalid digest")
+        _validate_snapshot_file(candidate, digest, private=bool(entry.get("private")))
+    for relative, target in passthroughs.items():
+        if not isinstance(relative, str) or not isinstance(target, str):
+            raise GuardError("Compose snapshot directory passthrough is invalid")
+        candidate = root / relative
+        try:
+            candidate.relative_to(root)
+            _validate_snapshot_parents(candidate, root)
+            metadata = candidate.lstat()
+            actual_target = candidate.resolve(strict=True)
+        except (OSError, ValueError) as exc:
+            raise GuardError("Compose snapshot directory passthrough is invalid") from exc
+        if not stat.S_ISLNK(metadata.st_mode) or not actual_target.is_dir():
+            raise GuardError("Compose snapshot directory passthrough is not a directory symlink")
+        if str(actual_target) != str(Path(target).resolve()):
+            raise GuardError("Compose snapshot directory passthrough target changed")
+    for key, relative in paths_manifest.items():
+        if not isinstance(key, str) or not isinstance(relative, str):
+            raise GuardError("Compose snapshot path mapping is invalid")
+        candidate = (root / relative).resolve()
+        try:
+            candidate.relative_to(root)
+        except ValueError as exc:
+            raise GuardError("Compose snapshot path escapes its root") from exc
+        if relative not in entries:
+            raise GuardError("Compose snapshot path mapping has no hashed entry")
+        paths[key] = str(candidate)
+    if "__probe_source__" not in paths:
+        raise GuardError("Compose snapshot lacks the immutable probe source")
+    project_dir_value = manifest.get("project_dir")
+    if not isinstance(project_dir_value, str) or not project_dir_value:
+        raise GuardError("Compose snapshot project directory is invalid")
+    project_dir = Path(project_dir_value).resolve()
+    if not Path(project_dir_value).is_absolute() or not project_dir.exists():
+        raise GuardError("Compose snapshot project directory is invalid")
+    return ComposeSnapshot(
+        root=root,
+        manifest_path=path.resolve(),
+        project_dir=project_dir,
+        paths=paths,
+    )
+
+
+@contextlib.contextmanager
+def _compose_snapshot_context(
+    *,
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    live_env_file: Path | None,
+    probe_source: Path,
+    probe_source_sha256: str | None,
+    image_override: Path | None,
+    image_override_sha256: str | None,
+    environment_override: Path | None,
+    environment_override_sha256: str | None,
+    snapshot: ComposeSnapshot | None,
+) -> Iterator[ComposeSnapshot]:
+    if snapshot is not None:
+        validated = load_compose_snapshot(snapshot.manifest_path)
+        if validated != snapshot:
+            raise GuardError("Compose snapshot changed after it was loaded")
+        if snapshot.project_dir != project_dir.resolve():
+            raise GuardError("Compose snapshot project directory changed")
+        yield snapshot
+        return
+    with tempfile.TemporaryDirectory(prefix=".unstract-compose-") as temporary:
+        yield materialize_compose_snapshot(
+            project_dir=project_dir,
+            compose_files=compose_files,
+            live_env_file=live_env_file,
+            probe_source=probe_source,
+            probe_source_sha256=probe_source_sha256,
+            image_override=image_override,
+            image_override_sha256=image_override_sha256,
+            environment_override=environment_override,
+            environment_override_sha256=environment_override_sha256,
+            destination=Path(temporary) / "tree",
+        )
 
 
 @contextlib.contextmanager
@@ -484,73 +905,37 @@ def bound_compose_inputs(
     image_override_sha256: str | None,
     environment_override: Path | None,
     environment_override_sha256: str | None,
+    snapshot: ComposeSnapshot | None = None,
 ) -> Iterator[tuple[dict[str, str], tuple[int, ...]]]:
-    """Snapshot every file Compose and its interpolation environment consume."""
-    replacements: dict[str, str] = {}
-    pass_fds: list[int] = []
-    with contextlib.ExitStack() as stack:
-        inputs: list[tuple[str, Path, str | None, str, bool]] = []
-        for compose_file in compose_files:
-            inputs.append(
-                (
-                    compose_file,
-                    _compose_input_path(project_dir, compose_file),
-                    None,
-                    "Compose file",
-                    False,
-                )
-            )
+    """Bind every Compose input to a daemon-visible immutable snapshot path."""
+    with _compose_snapshot_context(
+        project_dir=project_dir,
+        compose_files=compose_files,
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=environment_override,
+        environment_override_sha256=environment_override_sha256,
+        snapshot=snapshot,
+    ) as active:
+        required = list(compose_files)
         if live_env_file:
-            inputs.append(
-                (
-                    str(live_env_file),
-                    _compose_input_path(project_dir, live_env_file),
-                    None,
-                    "Compose environment file",
-                    False,
-                )
-            )
-        inputs.append(
-            (
-                str(probe_source),
-                _compose_input_path(project_dir, probe_source),
-                probe_source_sha256,
-                "health probe source",
-                False,
-            )
-        )
+            required.append(str(live_env_file))
+        required.append(str(probe_source))
         if image_override:
-            inputs.append(
-                (
-                    str(image_override),
-                    image_override,
-                    image_override_sha256,
-                    "candidate image override",
-                    True,
-                )
-            )
+            required.append(str(image_override))
         if environment_override:
-            inputs.append(
-                (
-                    str(environment_override),
-                    environment_override,
-                    environment_override_sha256,
-                    "runtime environment override",
-                    True,
-                )
-            )
-        for argument, path, expected_sha256, description, private in inputs:
-            proc_path, fd = stack.enter_context(
-                bound_compose_file(
-                    path,
-                    expected_sha256=expected_sha256,
-                    description=description,
-                    private=private,
-                )
+            required.append(str(environment_override))
+        missing = [argument for argument in required if argument not in active.paths]
+        if missing:
+            raise GuardError(
+                "Compose snapshot does not contain every launch input: "
+                + ", ".join(missing)
             )
-            replacements[argument] = proc_path
-            pass_fds.append(fd)
-        yield replacements, tuple(pass_fds)
+        replacements = {argument: active.path_for(argument) for argument in required}
+        yield replacements, ()
 
 
 def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
@@ -1504,6 +1889,7 @@ def compose_config(
     settings_file: Path | None = None,
     settings_file_sha256: str | None = None,
     probe_source_sha256: str | None = None,
+    snapshot: ComposeSnapshot | None = None,
     deadline: OperationDeadline | None = None,
 ) -> dict[str, Any]:
     verify_candidate_source_state(candidate_source, candidate_lock)
@@ -1553,6 +1939,7 @@ def compose_config(
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
+        snapshot=snapshot,
     ) as (bound_paths, pass_fds):
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
@@ -1735,6 +2122,7 @@ def write_replay_manifest(
     runtime_environment_override: Path,
     probe_source: Path,
     probe_source_sha256: str,
+    snapshot: ComposeSnapshot | None = None,
 ) -> None:
     """Bind every private replay input to a durable, sanitized hash record."""
     validate_candidate_image_override(image_override)
@@ -1745,6 +2133,9 @@ def write_replay_manifest(
         probe_source_sha256=probe_source_sha256,
     )
     validate_private_override(runtime_environment_override)
+    if snapshot is None:
+        raise GuardError("durable replay requires a retained Compose snapshot")
+    load_compose_snapshot(snapshot.manifest_path)
     write_private_json(
         path,
         {
@@ -1772,6 +2163,11 @@ def write_replay_manifest(
                     "UNSTRACT_HEALTHCHECK_SOURCE_SHA256"
                 ],
             },
+            "compose_snapshot": {
+                "schema": COMPOSE_SNAPSHOT_SCHEMA,
+                "manifest": str(snapshot.manifest_path),
+                "manifest_sha256": sha256_file(snapshot.manifest_path),
+            },
         },
         description="durable replay manifest",
     )
@@ -1787,6 +2183,7 @@ def load_replay_manifest(
     runtime_environment_override: Path,
     probe_source: Path,
     probe_source_sha256: str,
+    state_dir: Path | None = None,
 ) -> dict[str, Any]:
     """Verify the persisted private replay files before normal startup."""
     validate_private_file(path, description="durable replay manifest")
@@ -1844,11 +2241,30 @@ def load_replay_manifest(
         if keys
     }:
         raise GuardError("runtime environment contract changed since durable replay was prepared")
+    snapshot_metadata = manifest.get("compose_snapshot")
+    if not isinstance(snapshot_metadata, dict):
+        raise GuardError("durable replay manifest lacks the Compose snapshot")
+    if snapshot_metadata.get("schema") != COMPOSE_SNAPSHOT_SCHEMA:
+        raise GuardError("durable replay Compose snapshot has an unsupported schema")
+    snapshot_manifest_text = snapshot_metadata.get("manifest")
+    snapshot_manifest_sha256 = snapshot_metadata.get("manifest_sha256")
+    if not isinstance(snapshot_manifest_text, str) or not isinstance(snapshot_manifest_sha256, str):
+        raise GuardError("durable replay Compose snapshot metadata is incomplete")
+    snapshot_manifest = Path(snapshot_manifest_text).resolve()
+    if state_dir is not None:
+        try:
+            snapshot_manifest.relative_to(state_dir.resolve())
+        except ValueError as exc:
+            raise GuardError("durable replay Compose snapshot is outside state") from exc
+    if sha256_file(snapshot_manifest) != snapshot_manifest_sha256:
+        raise GuardError("durable replay Compose snapshot manifest changed")
+    snapshot = load_compose_snapshot(snapshot_manifest)
     return {
         "image_override_sha256": image_sha256,
         "settings_sha256": settings_sha256,
         "runtime_environment_sha256": runtime_sha256,
         "settings": settings,
+        "compose_snapshot": snapshot,
     }
 
 
@@ -2636,6 +3052,7 @@ def targeted_up(
     settings_file: Path | None = None,
     settings_file_sha256: str | None = None,
     probe_source_sha256: str | None = None,
+    snapshot: ComposeSnapshot | None = None,
     deadline: OperationDeadline | None = None,
 ) -> None:
     verify_candidate_source_state(candidate_source, candidate_lock)
@@ -2693,6 +3110,7 @@ def targeted_up(
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
+        snapshot=snapshot,
     ) as (bound_paths, pass_fds):
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
@@ -2744,6 +3162,7 @@ def compose_start(
     settings_file: Path,
     settings_file_sha256: str,
     probe_source_sha256: str,
+    snapshot: ComposeSnapshot | None = None,
     deadline: OperationDeadline | None = None,
 ) -> None:
     """Start the whole stack through the durable guarded Compose inputs."""
@@ -2782,6 +3201,7 @@ def compose_start(
         image_override_sha256=image_override_sha256,
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
+        snapshot=snapshot,
     ) as (bound_paths, pass_fds):
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
@@ -3039,6 +3459,39 @@ def resolve_live_env_file(args: argparse.Namespace, project_dir: Path) -> Path:
     return Path(configured).resolve() if configured else (project_dir / LIVE_ENV_RELATIVE).resolve()
 
 
+def compose_snapshot_destination(state_dir: Path) -> Path:
+    """Return a fresh retained snapshot directory below guarded state."""
+    stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%dT%H%M%S%fZ")
+    return state_dir / f"{COMPOSE_SNAPSHOT_DIR_PREFIX}{stamp}-{os.getpid()}"
+
+
+def create_compose_snapshot(
+    *,
+    state_dir: Path,
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    live_env_file: Path | None,
+    probe_source: Path,
+    probe_source_sha256: str | None,
+    image_override: Path | None,
+    image_override_sha256: str | None,
+    environment_override: Path | None,
+    environment_override_sha256: str | None,
+) -> ComposeSnapshot:
+    return materialize_compose_snapshot(
+        project_dir=project_dir,
+        compose_files=compose_files,
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=image_override_sha256,
+        environment_override=environment_override,
+        environment_override_sha256=environment_override_sha256,
+        destination=compose_snapshot_destination(state_dir),
+    )
+
+
 def prepare(
     args: argparse.Namespace,
     image_override: Path,
@@ -3327,6 +3780,18 @@ def compensating_rollback(
         runtime_environment_override, expected_sha256=runtime_environment_sha256
     )
     rollback_files = rollback_compose_files(args) + (str(override),)
+    rollback_snapshot = create_compose_snapshot(
+        state_dir=backup_dir,
+        project_dir=project_dir,
+        compose_files=rollback_files,
+        live_env_file=live_env_file,
+        probe_source=Path(args.probe_source),
+        probe_source_sha256=None,
+        image_override=override,
+        image_override_sha256=sha256_file(override),
+        environment_override=runtime_environment_override,
+        environment_override_sha256=runtime_environment_sha256,
+    )
     with advisory_lock(deadline=operation_deadline):
         # Recheck identity and quiescence after acquiring the DB lock.  The
         # process may be disconnected when db itself is recreated; the local
@@ -3346,8 +3811,11 @@ def compensating_rollback(
             probe_source=Path(args.probe_source),
             live_env_file=live_env_file,
             expected_source=baseline["source"],
+            image_override=override,
+            image_override_sha256=sha256_file(override),
             environment_override=runtime_environment_override,
             environment_override_sha256=runtime_environment_sha256,
+            snapshot=rollback_snapshot,
             deadline=operation_deadline,
         )
         wait_running(services, operation_deadline=operation_deadline)
@@ -3384,6 +3852,7 @@ def apply_batch(
     runtime_environment_override: Path,
     runtime_environment_sha256: str,
     reviewed_environment_keys: dict[str, set[str]],
+    snapshot: ComposeSnapshot,
     operation_deadline: OperationDeadline,
 ) -> dict[str, Any]:
     compose_files = tuple(args.compose_file or DEFAULT_COMPOSE_FILES)
@@ -3421,6 +3890,7 @@ def apply_batch(
             settings_file=settings_file,
             settings_file_sha256=settings_file_sha256,
             probe_source_sha256=probe_source_sha256,
+            snapshot=snapshot,
             deadline=operation_deadline,
         )
         authored_baseline = compose_config(
@@ -3435,6 +3905,7 @@ def apply_batch(
             settings_file=settings_file,
             settings_file_sha256=settings_file_sha256,
             probe_source_sha256=probe_source_sha256,
+            snapshot=snapshot,
             deadline=operation_deadline,
         )
         check_candidate_config(
@@ -3506,12 +3977,26 @@ def command_preflight(args: argparse.Namespace) -> int:
         operation_deadline=deadline,
     )
     lock = load_lock(Path(args.candidate_lock))
+    project_dir = Path(args.project_dir)
+    live_env_file = resolve_live_env_file(args, project_dir)
     probe_source = Path(args.probe_source)
     probe_source_sha256 = (lock.get("artifacts") or {}).get(
         "docker/healthchecks/unstract-services.sh"
     )
     if not isinstance(probe_source_sha256, str):
         raise GuardError("candidate lock lacks the guarded probe source digest")
+    snapshot = create_compose_snapshot(
+        state_dir=state_dir,
+        project_dir=project_dir,
+        compose_files=tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
+        live_env_file=live_env_file,
+        probe_source=probe_source,
+        probe_source_sha256=probe_source_sha256,
+        image_override=image_override,
+        image_override_sha256=sha256_file(image_override),
+        environment_override=runtime_environment_override,
+        environment_override_sha256=sha256_file(runtime_environment_override),
+    )
     write_replay_manifest(
         state_dir / REPLAY_MANIFEST_FILENAME,
         lock_path=Path(args.candidate_lock),
@@ -3521,6 +4006,7 @@ def command_preflight(args: argparse.Namespace) -> int:
         runtime_environment_override=runtime_environment_override,
         probe_source=probe_source,
         probe_source_sha256=probe_source_sha256,
+        snapshot=snapshot,
     )
     print(
         "preflight: candidate source, image lock, Compose identity, runtime, "
@@ -3565,7 +4051,9 @@ def command_start(args: argparse.Namespace) -> int:
         runtime_environment_override=runtime_environment_override,
         probe_source=probe_source,
         probe_source_sha256=probe_source_sha256,
+        state_dir=state_dir,
     )
+    snapshot = replay_manifest["compose_snapshot"]
     settings = validate_compose_settings(
         settings_file,
         candidate_version=lock["candidate_version"],
@@ -3596,6 +4084,7 @@ def command_start(args: argparse.Namespace) -> int:
         settings_file=settings_file,
         settings_file_sha256=settings_file_sha256,
         probe_source_sha256=probe_source_sha256,
+        snapshot=snapshot,
         deadline=operation_deadline,
     )
     authored_baseline = compose_config(
@@ -3610,6 +4099,7 @@ def command_start(args: argparse.Namespace) -> int:
         settings_file=settings_file,
         settings_file_sha256=settings_file_sha256,
         probe_source_sha256=probe_source_sha256,
+        snapshot=snapshot,
         deadline=operation_deadline,
     )
     reviewed_keys = reviewed_environment_keys_from_override(runtime_environment_override)
@@ -3636,6 +4126,7 @@ def command_start(args: argparse.Namespace) -> int:
         settings_file=settings_file,
         settings_file_sha256=settings_file_sha256,
         probe_source_sha256=probe_source_sha256,
+        snapshot=snapshot,
         deadline=operation_deadline,
     )
     wait_running(TARGET_SERVICES, operation_deadline=operation_deadline)
@@ -3661,6 +4152,7 @@ def command_apply(args: argparse.Namespace) -> int:
     backup_images: dict[str, Any] | None = None
     runtime_environment_override = backup_dir / RUNTIME_ENVIRONMENT_FILENAME
     runtime_environment_sha256 = ""
+    compose_snapshot: ComposeSnapshot | None = None
     reviewed_environment_keys: dict[str, set[str]] = {}
     replacement_manifest: dict[str, Any] = {
         "schema": REPLACEMENT_SCHEMA,
@@ -3694,6 +4186,18 @@ def command_apply(args: argparse.Namespace) -> int:
             )
             if not isinstance(probe_source_sha256, str):
                 raise GuardError("candidate lock lacks the guarded probe source digest")
+            compose_snapshot = create_compose_snapshot(
+                state_dir=backup_dir,
+                project_dir=project_dir,
+                compose_files=tuple(args.compose_file or DEFAULT_COMPOSE_FILES),
+                live_env_file=live_env_file,
+                probe_source=Path(args.probe_source),
+                probe_source_sha256=probe_source_sha256,
+                image_override=image_override,
+                image_override_sha256=image_override_sha256,
+                environment_override=runtime_environment_override,
+                environment_override_sha256=runtime_environment_sha256,
+            )
             with advisory_lock(deadline=operation_deadline):
                 fresh = capture(
                     project_dir,
@@ -3727,6 +4231,7 @@ def command_apply(args: argparse.Namespace) -> int:
                 runtime_environment_override=runtime_environment_override,
                 probe_source=Path(args.probe_source),
                 probe_source_sha256=probe_source_sha256,
+                snapshot=compose_snapshot,
             )
             rollback_override(backup_images, backup_dir / "rollback.override.yaml")
             write_json(backup_dir / "candidate-images.json", lock["images"])
@@ -3748,6 +4253,7 @@ def command_apply(args: argparse.Namespace) -> int:
                 runtime_environment_override=runtime_environment_override,
                 runtime_environment_sha256=runtime_environment_sha256,
                 reviewed_environment_keys=reviewed_environment_keys,
+                snapshot=compose_snapshot,
                 operation_deadline=operation_deadline,
             )
             replacement_manifest["services"].update(
@@ -3773,6 +4279,7 @@ def command_apply(args: argparse.Namespace) -> int:
                 runtime_environment_override=runtime_environment_override,
                 runtime_environment_sha256=runtime_environment_sha256,
                 reviewed_environment_keys=reviewed_environment_keys,
+                snapshot=compose_snapshot,
                 operation_deadline=operation_deadline,
             )
             replacement_manifest["services"].update(
diff --git a/docs/train-unstract-health-deployment.md b/docs/train-unstract-health-deployment.md
index c644d0dc65..ee1cfa9159 100644
--- a/docs/train-unstract-health-deployment.md
+++ b/docs/train-unstract-health-deployment.md
@@ -97,9 +97,14 @@ replay-manifest files from
 this lock, so every health, environment, and image replay uses the same
 immutable references, `VERSION`, and staged probe path. The replay manifest
 binds those three private files, the candidate lock, source commit/tree, probe
-digest, and reviewed environment-key set. These artifacts remain
-in the preflight state directory and apply backup directory for later startup
-or recovery; the guard validates their SHA-256 values before each Compose
+digest, reviewed environment-key set, and a retained Compose snapshot. That
+snapshot copies every Compose file and literal `include` target into a
+daemon-visible tree with the original relative layout, stores the live env
+file and private overrides in the same tree, and places the probe at a stable
+helper mount path. Snapshot files and directories are non-writable and every
+entry is hash-checked before Compose starts. These artifacts remain in the
+preflight state directory and apply backup directory for later startup or
+recovery; the guard validates their SHA-256 values before each Compose
 invocation. Only `runner` and the twelve
 worker services point at the new build; backend, frontend, platform-service,
 x2text-service, and the seven core data services point at the captured static
@@ -226,12 +231,15 @@ rows, zero claimed or scheduled queue rows, zero active barriers, and zero
 orchestration claims. Terminal result and dedup rows are recorded separately
 and are not mistaken for active jobs.
 
-Use the live dirty Compose files plus the staged overlays. This preserves the
-local embedding includes, port changes, env files, named volumes, bind mounts,
-container names, and network ownership. The live `docker/.env` derives
-`TOOL_REGISTRY_CONFIG_SRC_PATH` from `PWD`, so preserve the Train Compose
-working-directory value while the guard still uses the project root for
-Compose:
+Use the live dirty Compose files plus the staged overlays. Preflight freezes
+those files, their literal includes, referenced env and bind files, the live
+`docker/.env`, and the health probe into a retained daemon-visible snapshot.
+Runtime data directories remain at their original paths through validated
+snapshot passthroughs. This preserves the local embedding includes, port
+changes, env files, named volumes, bind mounts, container names, and network
+ownership. The live `docker/.env` derives `TOOL_REGISTRY_CONFIG_SRC_PATH` from
+`PWD`, so preserve the Train Compose working-directory value while the guard
+still passes the project root explicitly to Compose:
 
 ```sh
 export PWD=/home/completetrain/etl.home.complete.tech/docker
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index a15f89157f..67817a6e48 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -2,8 +2,10 @@
 
 import importlib.util
 import json
+import os
 import shutil
 import subprocess
+import sys
 from copy import deepcopy
 from pathlib import Path
 
@@ -346,6 +348,24 @@ def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) ->
     )
     runtime = tmp_path / guard.RUNTIME_ENVIRONMENT_FILENAME
     guard.write_runtime_environment_override({}, runtime)
+    compose = tmp_path / "docker" / "compose.yaml"
+    compose.parent.mkdir()
+    compose.write_text("include:\n  - included.yaml\nservices: {}\n", encoding="utf-8")
+    (compose.parent / "included.yaml").write_text("services: {}\n", encoding="utf-8")
+    live_env = tmp_path / "docker" / ".env"
+    live_env.write_text("COMPOSE_PROJECT_NAME=test\n", encoding="utf-8")
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=tmp_path,
+        compose_files=("docker/compose.yaml", str(image_override), str(runtime)),
+        live_env_file=live_env,
+        probe_source=probe,
+        probe_source_sha256=probe_sha256,
+        image_override=image_override,
+        image_override_sha256=guard.sha256_file(image_override),
+        environment_override=runtime,
+        environment_override_sha256=guard.sha256_file(runtime),
+    )
     manifest = tmp_path / guard.REPLAY_MANIFEST_FILENAME
 
     guard.write_replay_manifest(
@@ -357,6 +377,7 @@ def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) ->
         runtime_environment_override=runtime,
         probe_source=probe,
         probe_source_sha256=probe_sha256,
+        snapshot=snapshot,
     )
     loaded = guard.load_replay_manifest(
         manifest,
@@ -367,8 +388,10 @@ def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) ->
         runtime_environment_override=runtime,
         probe_source=probe,
         probe_source_sha256=probe_sha256,
+        state_dir=tmp_path / "state",
     )
     assert loaded["runtime_environment_sha256"] == guard.sha256_file(runtime)
+    assert loaded["compose_snapshot"].probe_path.read_bytes() == probe.read_bytes()
     assert manifest.stat().st_mode & 0o777 == 0o600
 
     runtime.write_text(runtime.read_text(encoding="utf-8") + "# changed\n", encoding="utf-8")
@@ -382,6 +405,7 @@ def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) ->
             runtime_environment_override=runtime,
             probe_source=probe,
             probe_source_sha256=probe_sha256,
+            state_dir=tmp_path / "state",
         )
 
 
@@ -419,7 +443,20 @@ def test_compose_replay_consumes_durable_settings_and_overrides(
         encoding="utf-8",
     )
     compose = tmp_path / "compose.yaml"
-    compose.write_text("services: {}\n", encoding="utf-8")
+    compose.write_text("include:\n  - included.yaml\nservices: {}\n", encoding="utf-8")
+    (tmp_path / "included.yaml").write_text("services: {}\n", encoding="utf-8")
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=tmp_path,
+        compose_files=("compose.yaml", str(image_override), str(environment_override)),
+        live_env_file=live_env,
+        probe_source=probe,
+        probe_source_sha256=probe_sha256,
+        image_override=image_override,
+        image_override_sha256=guard.sha256_file(image_override),
+        environment_override=environment_override,
+        environment_override_sha256=guard.sha256_file(environment_override),
+    )
     calls: list[tuple[list[str], dict[str, str] | None]] = []
 
     def fake_run(
@@ -432,18 +469,21 @@ def fake_run(
         calls.append((args, env))
         if "config" in args:
             bound_files = {
-                argument
-                for argument in args
-                if argument.startswith("/proc/self/fd/")
+                args[index + 1]
+                for index, argument in enumerate(args[:-1])
+                if argument == "-f"
             }
+            bound_files.add(args[args.index("--env-file") + 1])
             assert len(bound_files) == 4
+            assert all(path.startswith(str(snapshot.root)) for path in bound_files)
+            assert all(Path(path).is_file() for path in bound_files)
             assert str(live_env) not in args
             assert str(compose) not in args
             assert str(image_override) not in args
             assert str(environment_override) not in args
             assert env is not None
             bound_probe = env["UNSTRACT_HEALTHCHECK_SOURCE"]
-            assert bound_probe.startswith("/proc/self/fd/")
+            assert bound_probe == str(snapshot.probe_path)
             bound_paths = {Path(value) for value in bound_files | {bound_probe}}
             assert len(bound_paths) == 5
             originals = {
@@ -466,7 +506,7 @@ def fake_run(
                 encoding="utf-8"
             )
             assert Path(bound_probe).read_bytes() == originals[probe]
-            assert kwargs["pass_fds"]
+            assert kwargs["pass_fds"] == ()
             return subprocess.CompletedProcess(args, 0, json.dumps({"services": {}}), "")
         return subprocess.CompletedProcess(args, 0, "", "")
 
@@ -484,6 +524,7 @@ def fake_run(
         settings_file=settings,
         settings_file_sha256=guard.sha256_file(settings),
         probe_source_sha256=probe_sha256,
+        snapshot=snapshot,
     )
     guard.targeted_up(
         tmp_path,
@@ -499,6 +540,7 @@ def fake_run(
         settings_file=settings,
         settings_file_sha256=guard.sha256_file(settings),
         probe_source_sha256=probe_sha256,
+        snapshot=snapshot,
     )
 
     assert len(calls) == 2
@@ -509,9 +551,9 @@ def fake_run(
     assert str(settings) not in config_args
     assert str(image_override) not in config_args
     assert str(environment_override) not in config_args
-    assert sum(argument.startswith("/proc/self/fd/") for argument in config_args) == 4
+    assert sum(argument == "-f" for argument in config_args) == 3
     assert config_env and config_env["VERSION"] == "goal09-test"
-    assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"].startswith("/proc/self/fd/")
+    assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(snapshot.probe_path)
     assert calls[1][0][-1] == "runner"
 
     monkeypatch.setattr(guard, "verify_candidate_source_state", lambda *_: None)
@@ -531,6 +573,7 @@ def fake_run(
         settings_file=settings,
         settings_file_sha256=guard.sha256_file(settings),
         probe_source_sha256=probe_sha256,
+        snapshot=snapshot,
     )
     assert calls[2][0][-5:] == ["up", "-d", "--no-build", "--pull", "never"]
 
@@ -549,6 +592,122 @@ def fake_run(
         )
 
 
+def test_compose_snapshot_probe_is_visible_to_a_separate_process(tmp_path: Path) -> None:
+    project = tmp_path / "project"
+    (project / "docker").mkdir(parents=True)
+    compose = project / "docker" / "docker-compose.yaml"
+    compose.write_text("include:\n  - docker-compose-dev-essentials.yaml\nservices: {}\n", encoding="utf-8")
+    (project / "docker" / "docker-compose-dev-essentials.yaml").write_text(
+        "services: {}\n", encoding="utf-8"
+    )
+    env_file = project / "docker" / ".env"
+    env_file.write_text("COMPOSE_PROJECT_NAME=snapshot-test\n", encoding="utf-8")
+    probe = tmp_path / "source" / "unstract-services.sh"
+    probe.parent.mkdir()
+    probe.write_text("#!/bin/sh\nprintf immutable-probe\n", encoding="utf-8")
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=project,
+        compose_files=("docker/docker-compose.yaml",),
+        live_env_file=env_file,
+        probe_source=probe,
+        probe_source_sha256=guard.sha256_file(probe),
+        image_override=None,
+        image_override_sha256=None,
+        environment_override=None,
+        environment_override_sha256=None,
+    )
+
+    child = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            "import pathlib,sys; p=pathlib.Path(sys.argv[1]); "
+            "assert not str(p).startswith('/proc/self/fd/'); print(p.read_text())",
+            str(snapshot.probe_path),
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert child.stdout == "#!/bin/sh\nprintf immutable-probe\n\n"
+    assert (snapshot.root / "docker" / "docker-compose-dev-essentials.yaml").is_file()
+    assert guard.load_compose_snapshot(snapshot.manifest_path).probe_path == snapshot.probe_path
+
+
+def test_compose_snapshot_rejects_tampered_retained_input(tmp_path: Path) -> None:
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services: {}\n", encoding="utf-8")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=tmp_path,
+        compose_files=("compose.yaml",),
+        live_env_file=None,
+        probe_source=probe,
+        probe_source_sha256=guard.sha256_file(probe),
+        image_override=None,
+        image_override_sha256=None,
+        environment_override=None,
+        environment_override_sha256=None,
+    )
+    os.chmod(snapshot.probe_path, 0o600)
+    snapshot.probe_path.write_text("tampered\n", encoding="utf-8")
+    with pytest.raises(guard.GuardError, match="writable or not regular|changed"):
+        guard.load_compose_snapshot(snapshot.manifest_path)
+
+
+@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker Compose provider is unavailable")
+def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None:
+    project = tmp_path / "project"
+    (project / "docker").mkdir(parents=True)
+    (project / "docker" / "docker-compose.yaml").write_text(
+        "include:\n  - docker-compose-dev-essentials.yaml\nservices: {}\n",
+        encoding="utf-8",
+    )
+    (project / "docker" / "docker-compose-dev-essentials.yaml").write_text(
+        "services:\n  included:\n    image: busybox:latest\n", encoding="utf-8"
+    )
+    env_file = project / "docker" / ".env"
+    env_file.write_text("COMPOSE_PROJECT_NAME=snapshot-provider-test\n", encoding="utf-8")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=project,
+        compose_files=("docker/docker-compose.yaml",),
+        live_env_file=env_file,
+        probe_source=probe,
+        probe_source_sha256=guard.sha256_file(probe),
+        image_override=None,
+        image_override_sha256=None,
+        environment_override=None,
+        environment_override_sha256=None,
+    )
+    result = subprocess.run(
+        [
+            "docker",
+            "compose",
+            "--project-directory",
+            str(project),
+            "--env-file",
+            str(snapshot.path_for(str(env_file))),
+            "-f",
+            snapshot.path_for("docker/docker-compose.yaml"),
+            "config",
+            "--format",
+            "json",
+        ],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr
+    rendered = json.loads(result.stdout)
+    assert "included" in rendered["services"]
+
+
 def test_compose_rejects_ignored_live_input_drift(tmp_path: Path) -> None:
     project_dir = tmp_path
     (project_dir / "docker").mkdir()

From 5f95b6d338ca410bf0129e920385433975dc112e Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 03:13:03 -0400
Subject: [PATCH 39/48] Freeze Compose project paths in durable snapshots

---
 .../scripts/train_health_deployment_guard.py  | 96 +++++++++++++------
 .../test_train_health_deployment_guard.py     | 70 ++++++++++++--
 2 files changed, 130 insertions(+), 36 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 8d57531547..fd74609ca1 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -498,8 +498,15 @@ def _write_snapshot_file(
     data: bytes,
     *,
     private: bool,
+    executable: bool = False,
 ) -> str:
-    """Write a snapshot file and return its content digest."""
+    """Write a snapshot file and return its content digest.
+
+    Snapshot inputs are immutable after materialization, but executable
+    helpers still need their execute bits when Compose runs them in a
+    container.  Preserve only execute bits from the reviewed source while
+    stripping every write bit.
+    """
     destination = root / relative
     try:
         destination.relative_to(root)
@@ -507,6 +514,8 @@ def _write_snapshot_file(
         raise GuardError(f"snapshot path escapes its root: {relative}") from exc
     destination.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
     mode = 0o400 if private else 0o444
+    if executable and not private:
+        mode |= 0o111
     flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0)
     if hasattr(os, "O_NOFOLLOW"):
         flags |= os.O_NOFOLLOW
@@ -605,7 +614,14 @@ def copy_source(
         actual = sha256_bytes(data)
         if expected_sha256 is not None and actual != expected_sha256:
             raise GuardError(f"{description} changed: {source}")
-        digest = _write_snapshot_file(destination, relative, data, private=private)
+        source_mode = resolved.stat().st_mode
+        digest = _write_snapshot_file(
+            destination,
+            relative,
+            data,
+            private=private,
+            executable=bool(source_mode & 0o111),
+        )
         copied.add(resolved)
         entries[str(relative)] = {
             "sha256": digest,
@@ -774,6 +790,8 @@ def load_compose_snapshot(path: Path) -> ComposeSnapshot:
         metadata = path.lstat()
         if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) & 0o222:
             raise GuardError(f"Compose snapshot manifest is writable or not regular: {path}")
+        if hasattr(os, "getuid") and os.getuid() == 0 and metadata.st_uid != 0:
+            raise GuardError(f"Compose snapshot manifest is not root-owned: {path}")
         manifest = json.loads(path.read_text(encoding="utf-8"))
     except GuardError:
         raise
@@ -906,7 +924,7 @@ def bound_compose_inputs(
     environment_override: Path | None,
     environment_override_sha256: str | None,
     snapshot: ComposeSnapshot | None = None,
-) -> Iterator[tuple[dict[str, str], tuple[int, ...]]]:
+) -> Iterator[tuple[dict[str, str], tuple[int, ...], ComposeSnapshot]]:
     """Bind every Compose input to a daemon-visible immutable snapshot path."""
     with _compose_snapshot_context(
         project_dir=project_dir,
@@ -935,7 +953,7 @@ def bound_compose_inputs(
                 + ", ".join(missing)
             )
         replacements = {argument: active.path_for(argument) for argument in required}
-        yield replacements, ()
+        yield replacements, (), active
 
 
 def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
@@ -1927,8 +1945,6 @@ def compose_config(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = compose_args(project_dir, files, live_env_file=live_env_file)
-    args.extend(["config", "--format", "json"])
     with bound_compose_inputs(
         project_dir=project_dir,
         compose_files=files,
@@ -1940,7 +1956,14 @@ def compose_config(
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
         snapshot=snapshot,
-    ) as (bound_paths, pass_fds):
+    ) as (bound_paths, pass_fds, active_snapshot):
+        args = compose_args(
+            project_dir,
+            files,
+            live_env_file=live_env_file,
+            snapshot=active_snapshot,
+        )
+        args.extend(["config", "--format", "json"])
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
             project_dir=project_dir,
@@ -1967,7 +1990,7 @@ def compose_config(
         )
         result = run(
             bound_args,
-            cwd=project_dir,
+            cwd=active_snapshot.root,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
@@ -2803,8 +2826,15 @@ def compose_args(
     compose_files: tuple[str, ...],
     *,
     live_env_file: Path | None = None,
+    snapshot: ComposeSnapshot | None = None,
 ) -> list[str]:
-    args = ["docker", "compose", "--project-directory", str(project_dir.resolve())]
+    # Compose resolves includes, relative env files, and relative bind sources
+    # from this explicit project directory.  Once inputs are frozen, all of
+    # those paths must resolve inside the retained snapshot tree; pointing the
+    # provider at the live checkout would re-open mutable files after the
+    # guard's final verification.
+    effective_project_dir = snapshot.root if snapshot is not None else project_dir.resolve()
+    args = ["docker", "compose", "--project-directory", str(effective_project_dir)]
     if live_env_file:
         args.extend(["--env-file", str(live_env_file)])
     for compose_file in compose_files:
@@ -3087,19 +3117,6 @@ def targeted_up(
             environment_override, expected_sha256=environment_override_sha256
         )
         files += (str(environment_override),)
-    args = compose_args(project_dir, files, live_env_file=live_env_file)
-    args.extend(
-        [
-            "up",
-            "-d",
-            "--no-deps",
-            "--force-recreate",
-            "--no-build",
-            "--pull",
-            "never",
-            *services,
-        ]
-    )
     with bound_compose_inputs(
         project_dir=project_dir,
         compose_files=files,
@@ -3111,7 +3128,25 @@ def targeted_up(
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
         snapshot=snapshot,
-    ) as (bound_paths, pass_fds):
+    ) as (bound_paths, pass_fds, active_snapshot):
+        args = compose_args(
+            project_dir,
+            files,
+            live_env_file=live_env_file,
+            snapshot=active_snapshot,
+        )
+        args.extend(
+            [
+                "up",
+                "-d",
+                "--no-deps",
+                "--force-recreate",
+                "--no-build",
+                "--pull",
+                "never",
+                *services,
+            ]
+        )
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
             project_dir=project_dir,
@@ -3138,7 +3173,7 @@ def targeted_up(
         )
         run(
             bound_args,
-            cwd=project_dir,
+            cwd=active_snapshot.root,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
@@ -3189,8 +3224,6 @@ def compose_start(
         environment_override, expected_sha256=environment_override_sha256
     )
     files = compose_files + (str(image_override), str(environment_override))
-    args = compose_args(project_dir, files, live_env_file=live_env_file)
-    args.extend(["up", "-d", "--no-build", "--pull", "never"])
     with bound_compose_inputs(
         project_dir=project_dir,
         compose_files=files,
@@ -3202,7 +3235,14 @@ def compose_start(
         environment_override=environment_override,
         environment_override_sha256=environment_override_sha256,
         snapshot=snapshot,
-    ) as (bound_paths, pass_fds):
+    ) as (bound_paths, pass_fds, active_snapshot):
+        args = compose_args(
+            project_dir,
+            files,
+            live_env_file=live_env_file,
+            snapshot=active_snapshot,
+        )
+        args.extend(["up", "-d", "--no-build", "--pull", "never"])
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
             project_dir=project_dir,
@@ -3229,7 +3269,7 @@ def compose_start(
         )
         run(
             bound_args,
-            cwd=project_dir,
+            cwd=active_snapshot.root,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 67817a6e48..1b0ad0c3e4 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -3,6 +3,7 @@
 import importlib.util
 import json
 import os
+import shlex
 import shutil
 import subprocess
 import sys
@@ -10,6 +11,7 @@
 from pathlib import Path
 
 import pytest
+import yaml
 
 GUARD_PATH = Path(__file__).parents[2] / "docker/scripts/train_health_deployment_guard.py"
 SPEC = importlib.util.spec_from_file_location("train_health_deployment_guard", GUARD_PATH)
@@ -18,6 +20,24 @@
 SPEC.loader.exec_module(guard)
 
 
+def _real_compose_provider() -> list[str]:
+    """Return a locally installed Docker/Podman Compose provider for config-only tests."""
+    configured = os.environ.get("UNSTRACT_COMPOSE_PROVIDER")
+    if configured:
+        provider = shlex.split(configured)
+        if provider and shutil.which(provider[0]):
+            return provider
+        pytest.fail("UNSTRACT_COMPOSE_PROVIDER is not executable")
+    for provider in (
+        ("docker", "compose"),
+        ("podman", "compose"),
+        ("docker-compose",),
+    ):
+        if shutil.which(provider[0]):
+            return list(provider)
+    pytest.skip("Docker or Podman Compose provider is unavailable")
+
+
 def candidate_fixture() -> tuple[dict, dict, dict, dict]:
     baseline = {"containers": []}
     config = {
@@ -546,7 +566,7 @@ def fake_run(
     assert len(calls) == 2
     config_args, config_env = calls[0]
     assert config_args[:3] == ["docker", "compose", "--project-directory"]
-    assert str(tmp_path.resolve()) in config_args
+    assert str(snapshot.root) in config_args
     assert str(live_env) not in config_args
     assert str(settings) not in config_args
     assert str(image_override) not in config_args
@@ -605,6 +625,7 @@ def test_compose_snapshot_probe_is_visible_to_a_separate_process(tmp_path: Path)
     probe = tmp_path / "source" / "unstract-services.sh"
     probe.parent.mkdir()
     probe.write_text("#!/bin/sh\nprintf immutable-probe\n", encoding="utf-8")
+    os.chmod(probe, 0o755)
     snapshot = guard.create_compose_snapshot(
         state_dir=tmp_path / "state",
         project_dir=project,
@@ -631,6 +652,14 @@ def test_compose_snapshot_probe_is_visible_to_a_separate_process(tmp_path: Path)
         text=True,
     )
     assert child.stdout == "#!/bin/sh\nprintf immutable-probe\n\n"
+    executed = subprocess.run(
+        [str(snapshot.probe_path)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert executed.stdout == "immutable-probe"
+    assert snapshot.probe_path.stat().st_mode & 0o777 == 0o555
     assert (snapshot.root / "docker" / "docker-compose-dev-essentials.yaml").is_file()
     assert guard.load_compose_snapshot(snapshot.manifest_path).probe_path == snapshot.probe_path
 
@@ -658,7 +687,6 @@ def test_compose_snapshot_rejects_tampered_retained_input(tmp_path: Path) -> Non
         guard.load_compose_snapshot(snapshot.manifest_path)
 
 
-@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker Compose provider is unavailable")
 def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None:
     project = tmp_path / "project"
     (project / "docker").mkdir(parents=True)
@@ -669,6 +697,23 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
     (project / "docker" / "docker-compose-dev-essentials.yaml").write_text(
         "services:\n  included:\n    image: busybox:latest\n", encoding="utf-8"
     )
+    relative_env = project / "docker" / "relative.env"
+    relative_env.write_text("SNAPSHOT_MARKER=from-snapshot\n", encoding="utf-8")
+    relative_data = project / "docker" / "relative-data"
+    relative_data.mkdir()
+    (relative_data / "marker").write_text("runtime-data\n", encoding="utf-8")
+    (project / "docker" / "docker-compose.yaml").write_text(
+        "include:\n"
+        "  - docker-compose-dev-essentials.yaml\n"
+        "services:\n"
+        "  relative:\n"
+        "    image: busybox:latest\n"
+        "    env_file:\n"
+        "      - ./relative.env\n"
+        "    volumes:\n"
+        "      - ./relative-data:/data:ro\n",
+        encoding="utf-8",
+    )
     env_file = project / "docker" / ".env"
     env_file.write_text("COMPOSE_PROJECT_NAME=snapshot-provider-test\n", encoding="utf-8")
     probe = tmp_path / "probe.sh"
@@ -685,27 +730,36 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
         environment_override=None,
         environment_override_sha256=None,
     )
+    provider = _real_compose_provider()
     result = subprocess.run(
         [
-            "docker",
-            "compose",
+            *provider,
             "--project-directory",
-            str(project),
+            str(snapshot.root),
             "--env-file",
             str(snapshot.path_for(str(env_file))),
             "-f",
             snapshot.path_for("docker/docker-compose.yaml"),
             "config",
-            "--format",
-            "json",
+            *( ["--format", "json"] if provider[0] == "docker" else []),
         ],
         check=False,
         capture_output=True,
         text=True,
     )
     assert result.returncode == 0, result.stderr
-    rendered = json.loads(result.stdout)
+    rendered = (
+        json.loads(result.stdout)
+        if provider[0] == "docker"
+        else yaml.safe_load(result.stdout)
+    )
     assert "included" in rendered["services"]
+    assert (
+        rendered["services"]["relative"]["environment"]["SNAPSHOT_MARKER"]
+        == "from-snapshot"
+    )
+    rendered_source = Path(rendered["services"]["relative"]["volumes"][0]["source"])
+    assert rendered_source.resolve() == relative_data.resolve()
 
 
 def test_compose_rejects_ignored_live_input_drift(tmp_path: Path) -> None:

From f2ea30aa2f9b97763f2e71e88ea205cc437685ce Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 05:00:57 -0400
Subject: [PATCH 40/48] Fix Compose snapshot project directory

---
 .../scripts/train_health_deployment_guard.py  | 43 ++++++++++++++---
 .../test_train_health_deployment_guard.py     | 46 +++++++++++++++----
 2 files changed, 75 insertions(+), 14 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index fd74609ca1..289d1ddce8 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -366,6 +366,36 @@ def _compose_input_path(project_dir: Path, value: str | Path) -> Path:
     return project_dir.resolve() / path
 
 
+def _compose_project_directory(
+    project_dir: Path,
+    compose_files: tuple[str, ...],
+    *,
+    snapshot: ComposeSnapshot | None = None,
+) -> Path:
+    """Return Compose's path base for the first launch file.
+
+    Compose uses the first ``-f`` file's parent directory as its default
+    project directory. An explicit ``--project-directory`` overrides that
+    default for includes and merged-file relative paths, so a frozen snapshot
+    must preserve the first file's parent rather than promote the snapshot
+    root to the path base.
+    """
+    if not compose_files:
+        raise GuardError("Compose launch requires at least one Compose file")
+    first = str(compose_files[0])
+    if snapshot is None:
+        return _compose_input_path(project_dir, first).resolve().parent
+    frozen = snapshot.paths.get(first)
+    if frozen is None:
+        raise GuardError("Compose snapshot does not contain the base Compose file")
+    frozen_path = Path(frozen).resolve()
+    try:
+        frozen_path.relative_to(snapshot.root)
+    except ValueError as exc:
+        raise GuardError("Compose snapshot base file escapes its root") from exc
+    return frozen_path.parent
+
+
 def _read_compose_input(path: Path, *, description: str) -> bytes:
     """Read one Compose input while resolving a symlink only once."""
     try:
@@ -2828,12 +2858,13 @@ def compose_args(
     live_env_file: Path | None = None,
     snapshot: ComposeSnapshot | None = None,
 ) -> list[str]:
-    # Compose resolves includes, relative env files, and relative bind sources
-    # from this explicit project directory.  Once inputs are frozen, all of
-    # those paths must resolve inside the retained snapshot tree; pointing the
-    # provider at the live checkout would re-open mutable files after the
-    # guard's final verification.
-    effective_project_dir = snapshot.root if snapshot is not None else project_dir.resolve()
+    # Compose resolves the base file's includes and merged-file relative
+    # references from --project-directory. The snapshot retains the original
+    # layout, so selecting its frozen base file parent keeps those resolutions
+    # inside the snapshot without changing their meaning.
+    effective_project_dir = _compose_project_directory(
+        project_dir, compose_files, snapshot=snapshot
+    )
     args = ["docker", "compose", "--project-directory", str(effective_project_dir)]
     if live_env_file:
         args.extend(["--env-file", str(live_env_file)])
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 1b0ad0c3e4..0a39bfe142 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -661,7 +661,34 @@ def test_compose_snapshot_probe_is_visible_to_a_separate_process(tmp_path: Path)
     assert executed.stdout == "immutable-probe"
     assert snapshot.probe_path.stat().st_mode & 0o777 == 0o555
     assert (snapshot.root / "docker" / "docker-compose-dev-essentials.yaml").is_file()
-    assert guard.load_compose_snapshot(snapshot.manifest_path).probe_path == snapshot.probe_path
+    source_args = guard.compose_args(
+        project,
+        ("docker/docker-compose.yaml",),
+        live_env_file=env_file,
+    )
+    assert source_args[:4] == [
+        "docker",
+        "compose",
+        "--project-directory",
+        str(project / "docker"),
+    ]
+    snapshot_args = guard.compose_args(
+        project,
+        ("docker/docker-compose.yaml",),
+        live_env_file=env_file,
+        snapshot=snapshot,
+    )
+    assert snapshot_args[:4] == [
+        "docker",
+        "compose",
+        "--project-directory",
+        str(snapshot.root / "docker"),
+    ]
+    assert snapshot_args[-2:] == ["-f", "docker/docker-compose.yaml"]
+    assert (
+        guard.load_compose_snapshot(snapshot.manifest_path).probe_path
+        == snapshot.probe_path
+    )
 
 
 def test_compose_snapshot_rejects_tampered_retained_input(tmp_path: Path) -> None:
@@ -731,17 +758,20 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
         environment_override_sha256=None,
     )
     provider = _real_compose_provider()
+    launch_args = guard.compose_args(
+        project,
+        ("docker/docker-compose.yaml",),
+        live_env_file=env_file,
+        snapshot=snapshot,
+    )
+    bound_args = [snapshot.path_for(argument) for argument in launch_args[2:]]
+    assert bound_args[:2] == ["--project-directory", str(snapshot.root / "docker")]
     result = subprocess.run(
         [
             *provider,
-            "--project-directory",
-            str(snapshot.root),
-            "--env-file",
-            str(snapshot.path_for(str(env_file))),
-            "-f",
-            snapshot.path_for("docker/docker-compose.yaml"),
+            *bound_args,
             "config",
-            *( ["--format", "json"] if provider[0] == "docker" else []),
+            *(["--format", "json"] if provider[0] == "docker" else []),
         ],
         check=False,
         capture_output=True,

From 3ce6f3c97edbf77460fe38a4110edc3ead281b39 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 07:06:54 -0400
Subject: [PATCH 41/48] fix: enforce strict runtime hostname identity

---
 .../scripts/train_health_deployment_guard.py  |  99 +++++++++----
 .../test_train_health_deployment_guard.py     | 132 +++++++++++++++++-
 2 files changed, 198 insertions(+), 33 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 289d1ddce8..8bb1e26eaa 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -113,7 +113,6 @@
 COMPOSE_SNAPSHOT_DIR_PREFIX = "compose-snapshot-"
 COMPOSE_SNAPSHOT_PROBE_RELATIVE = Path("__helper__/unstract-services.sh")
 COMPOSE_SNAPSHOT_PRIVATE_RELATIVE = Path("__private__")
-RUNTIME_GENERATED_ENV_KEYS = frozenset({"HOME", "container"})
 DURATION_TOKEN = re.compile(
     r"(?P<number>(?:\d+(?:\.\d*)?|\.\d+))(?P<unit>ns|us|µs|ms|h|m|s)"
 )
@@ -998,17 +997,36 @@ def image_digest(image: dict[str, Any]) -> str | None:
 
 
 def env_hashes(
-    values: list[str] | None, *, container_id: str | None = None
+    values: list[str] | None,
+    *,
+    container_id: str | None = None,
+    config_hostname: Any = None,
 ) -> dict[str, dict[str, Any]]:
     result: dict[str, dict[str, Any]] = {}
+    seen: set[str] = set()
     for item in values or []:
+        if not isinstance(item, str):
+            raise GuardError("runtime environment contains a non-string entry")
         key, separator, value = item.partition("=")
         if not separator:
             value = ""
-        # Podman injects HOSTNAME from the container ID. Recreated containers
-        # therefore receive a new value even when the application environment
-        # is unchanged. Keep an explicit custom HOSTNAME in the contract.
-        if key == "HOSTNAME" and container_id and value == container_id[:12]:
+        if not key:
+            raise GuardError("runtime environment contains an empty key")
+        if key in seen:
+            raise GuardError(f"runtime environment contains a duplicate key: {key}")
+        seen.add(key)
+        # Podman injects HOSTNAME from the container ID only when both inspect
+        # fields have the current ID prefix.  Recreated containers then receive
+        # a new value even when the application environment is unchanged.  An
+        # authored image/Compose environment or fixed hostname remains part of
+        # the contract, including a HOSTNAME value that merely resembles an ID.
+        if (
+            key == "HOSTNAME"
+            and isinstance(container_id, str)
+            and len(container_id) >= 12
+            and value == container_id[:12]
+            and config_hostname == container_id[:12]
+        ):
             continue
         result[key] = {"length": len(value), "sha256": sha256_bytes(value.encode())}
     return dict(sorted(result.items()))
@@ -1019,17 +1037,26 @@ def environment_values(values: list[str] | None) -> dict[str, str]:
     result: dict[str, str] = {}
     for item in values or []:
         if not isinstance(item, str):
-            continue
+            raise GuardError("runtime environment contains a non-string entry")
         key, separator, value = item.partition("=")
+        if not key:
+            raise GuardError("runtime environment contains an empty key")
+        if key in result:
+            raise GuardError(f"runtime environment contains a duplicate key: {key}")
         result[key] = value if separator else ""
     return result
 
 
 def environment_hashes(
-    values: dict[str, str], *, container_id: str | None = None
+    values: dict[str, str],
+    *,
+    container_id: str | None = None,
+    config_hostname: Any = None,
 ) -> dict[str, dict[str, Any]]:
     return env_hashes(
-        [f"{key}={value}" for key, value in values.items()], container_id=container_id
+        [f"{key}={value}" for key, value in values.items()],
+        container_id=container_id,
+        config_hostname=config_hostname,
     )
 
 
@@ -1256,7 +1283,11 @@ def inspect_project(
                     "configured": health_config(config.get("Healthcheck")),
                     "runtime": health_runtime(state.get("Health")),
                 },
-                "env_hashes": env_hashes(config.get("Env"), container_id=item.get("Id")),
+                "env_hashes": env_hashes(
+                    config.get("Env"),
+                    container_id=item.get("Id"),
+                    config_hostname=config.get("Hostname"),
+                ),
                 "mounts": [normalize_mount(mount) for mount in item.get("Mounts") or []],
                 "options": runtime_options(item, config),
                 "graphdriver": {
@@ -1300,6 +1331,7 @@ def inspect_runtime_environment(
             raise GuardError(f"duplicate Compose service environment: {service}")
         result[service] = {
             "id": item.get("Id"),
+            "hostname": config.get("Hostname"),
             "values": environment_values(config.get("Env")),
         }
     return result
@@ -2037,10 +2069,21 @@ def compose_environment(value: Any) -> dict[str, Any]:
             if not isinstance(item, str):
                 raise GuardError("Compose environment contains a non-string entry")
             key, separator, item_value = item.partition("=")
+            if not key:
+                raise GuardError("Compose environment contains an empty key")
+            if key in result:
+                raise GuardError(f"Compose environment contains a duplicate key: {key}")
             result[key] = item_value if separator else None
         return result
     if isinstance(value, dict):
-        return {str(key): item for key, item in value.items()}
+        result: dict[str, Any] = {}
+        for key, item in value.items():
+            if not isinstance(key, str) or not key:
+                raise GuardError("Compose environment contains a non-string or empty key")
+            if key in result:
+                raise GuardError(f"Compose environment contains a duplicate key: {key}")
+            result[key] = item
+        return result
     raise GuardError("Compose environment is not a mapping")
 
 
@@ -2050,17 +2093,19 @@ def candidate_environment_values(
     """Merge candidate image defaults with the effective Compose service env."""
     service_config = (config.get("services") or {}).get(service) or {}
     values = dict(image_values)
-    for key, value in compose_environment(service_config.get("environment")).items():
+    compose_values = compose_environment(service_config.get("environment"))
+    for key, value in compose_values.items():
         if value is None:
             raise GuardError(f"candidate environment inherits host value for {service}: {key}")
         values[str(key)] = str(value)
     # Docker/Podman exposes an explicit Compose hostname through HOSTNAME.
-    # When no hostname is authored, the runtime-generated container ID is
-    # normalized out of the baseline and is deliberately omitted here.
+    # When neither `hostname:` nor an image/Compose HOSTNAME is authored, the
+    # runtime-generated container ID is normalized out of the baseline and is
+    # deliberately omitted here.
     hostname = service_config.get("hostname")
     if hostname is not None:
         values["HOSTNAME"] = str(hostname)
-    elif "HOSTNAME" not in image_values:
+    elif "HOSTNAME" not in image_values and "HOSTNAME" not in compose_values:
         values.pop("HOSTNAME", None)
     return values
 
@@ -2341,22 +2386,18 @@ def plan_runtime_environment_override(
             raise GuardError(f"runtime environment plan is incomplete for {service}")
         observed_id = observed.get("id")
         observed_values = observed.get("values") or {}
-        observed_hashes = environment_hashes(observed_values, container_id=observed_id)
+        observed_hashes = environment_hashes(
+            observed_values,
+            container_id=observed_id,
+            config_hostname=observed.get("hostname"),
+        )
         if observed_hashes != (previous.get("env_hashes") or {}):
             raise GuardError(f"fresh runtime environment changed for {service}")
         candidate_values = candidate_environment_values(
             config, service, candidate.get("environment") or {}
         )
-        baseline_hashes = {
-            key: value
-            for key, value in (previous.get("env_hashes") or {}).items()
-            if key not in RUNTIME_GENERATED_ENV_KEYS
-        }
-        candidate_hashes = {
-            key: value
-            for key, value in environment_hashes(candidate_values).items()
-            if key not in RUNTIME_GENERATED_ENV_KEYS
-        }
+        baseline_hashes = previous.get("env_hashes") or {}
+        candidate_hashes = environment_hashes(candidate_values)
         service_overrides: dict[str, str] = {}
         for key, expected_hash in baseline_hashes.items():
             if candidate_hashes.get(key) == expected_hash:
@@ -2372,11 +2413,7 @@ def plan_runtime_environment_override(
             raise GuardError(f"candidate added environment for {service}: {key}")
         effective_values = dict(candidate_values)
         effective_values.update(service_overrides)
-        effective_hashes = {
-            key: value
-            for key, value in environment_hashes(effective_values).items()
-            if key not in RUNTIME_GENERATED_ENV_KEYS
-        }
+        effective_hashes = environment_hashes(effective_values)
         if any(effective_hashes.get(key) != value for key, value in baseline_hashes.items()):
             raise GuardError(f"candidate environment cannot preserve {service}")
         overrides[service] = service_overrides
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 0a39bfe142..372a831dbb 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -145,16 +145,142 @@ def test_healthcheck_none_is_equivalent_to_no_healthcheck() -> None:
 def test_generated_hostname_is_excluded_but_custom_hostname_is_retained() -> None:
     container_id = "abcdef0123456789"
     generated = guard.env_hashes(
-        ["HOSTNAME=abcdef012345", "APP_MODE=prod"], container_id=container_id
+        ["HOSTNAME=abcdef012345", "APP_MODE=prod"],
+        container_id=container_id,
+        config_hostname="abcdef012345",
     )
     custom = guard.env_hashes(
-        ["HOSTNAME=worker-custom", "APP_MODE=prod"], container_id=container_id
+        ["HOSTNAME=worker-custom", "APP_MODE=prod"],
+        container_id=container_id,
+        config_hostname="abcdef012345",
     )
 
     assert "HOSTNAME" not in generated
     assert "HOSTNAME" in custom
 
 
+def test_hostname_normalization_requires_the_current_container_id() -> None:
+    old_id = "abcdef0123456789"
+    new_id = "fedcba9876543210"
+
+    old = guard.env_hashes(
+        ["HOSTNAME=abcdef012345"],
+        container_id=old_id,
+        config_hostname="abcdef012345",
+    )
+    new = guard.env_hashes(
+        ["HOSTNAME=fedcba987654"],
+        container_id=new_id,
+        config_hostname="fedcba987654",
+    )
+    fixed_old_value = guard.env_hashes(
+        ["HOSTNAME=abcdef012345"],
+        container_id=new_id,
+        config_hostname="fedcba987654",
+    )
+    mismatched_config_hostname = guard.env_hashes(
+        ["HOSTNAME=fedcba987654"],
+        container_id=new_id,
+        config_hostname="source-fixed",
+    )
+
+    assert old == new == {}
+    assert "HOSTNAME" in fixed_old_value
+    assert "HOSTNAME" in mismatched_config_hostname
+
+
+def test_runtime_environment_rejects_duplicate_keys() -> None:
+    with pytest.raises(guard.GuardError, match="duplicate key: APP_MODE"):
+        guard.env_hashes(["APP_MODE=first", "APP_MODE=second"])
+    with pytest.raises(guard.GuardError, match="duplicate key: HOSTNAME"):
+        guard.env_hashes(
+            ["HOSTNAME=abcdef012345", "HOSTNAME=abcdef012345"],
+            container_id="abcdef0123456789",
+            config_hostname="abcdef012345",
+        )
+    with pytest.raises(guard.GuardError, match="duplicate key: APP_MODE"):
+        guard.environment_values(["APP_MODE=first", "APP_MODE=second"])
+    with pytest.raises(guard.GuardError, match="duplicate key: APP_MODE"):
+        guard.compose_environment(["APP_MODE=first", "APP_MODE=second"])
+    with pytest.raises(guard.GuardError, match="non-string or empty key"):
+        guard.compose_environment({1: "first", "1": "second"})
+
+
+@pytest.mark.parametrize("key", ["HOME", "container"])
+def test_runtime_environment_plan_preserves_every_non_hostname_key(key: str) -> None:
+    baseline = {
+        "containers": [
+            {
+                "compose": {"com.docker.compose.service": "runner"},
+                "name": "unstract-runner",
+                "env_hashes": guard.environment_hashes({key: "baseline"}),
+            }
+        ]
+    }
+    runtime_environment = {"runner": {"id": "runner-id", "values": {key: "baseline"}}}
+    candidate_images = {"runner": {"environment": {key: "candidate"}}}
+
+    overrides, reviewed = guard.plan_runtime_environment_override(
+        baseline,
+        runtime_environment,
+        candidate_images,
+        {"services": {"runner": {}}},
+        services=("runner",),
+    )
+
+    assert overrides == {"runner": {key: "baseline"}}
+    assert reviewed == {"runner": {key}}
+
+
+def test_runtime_environment_plan_rejects_hostname_without_both_generated_fields() -> None:
+    container_id = "abcdef0123456789"
+    baseline = {
+        "containers": [
+            {
+                "compose": {"com.docker.compose.service": "runner"},
+                "name": "unstract-runner",
+                "env_hashes": guard.environment_hashes(
+                    {"HOSTNAME": "abcdef012345"},
+                    container_id=container_id,
+                    config_hostname="abcdef012345",
+                ),
+            }
+        ]
+    }
+    runtime_environment = {
+        "runner": {
+            "id": container_id,
+            "hostname": "source-fixed",
+            "values": {"HOSTNAME": "abcdef012345"},
+        }
+    }
+
+    with pytest.raises(guard.GuardError, match="fresh runtime environment changed"):
+        guard.plan_runtime_environment_override(
+            baseline,
+            runtime_environment,
+            {"runner": {"environment": {}}},
+            {"services": {"runner": {}}},
+            services=("runner",),
+        )
+
+
+def test_candidate_environment_keeps_explicit_hostname_override() -> None:
+    config = {"services": {"runner": {"environment": {"HOSTNAME": "source-fixed"}}}}
+
+    values = guard.candidate_environment_values(config, "runner", {})
+
+    assert values == {"HOSTNAME": "source-fixed"}
+
+    fixed_hostname = guard.candidate_environment_values(
+        {"services": {"runner": {"hostname": "fixed-source-hostname"}}},
+        "runner",
+        {},
+    )
+
+    assert fixed_hostname == {"HOSTNAME": "fixed-source-hostname"}
+
+
 def test_generated_network_alias_is_excluded_but_explicit_alias_is_retained() -> None:
     container_id = "abcdef0123456789"
     network = {
@@ -735,6 +861,7 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
         "services:\n"
         "  relative:\n"
         "    image: busybox:latest\n"
+        "    hostname: source-fixed-hostname\n"
         "    env_file:\n"
         "      - ./relative.env\n"
         "    volumes:\n"
@@ -784,6 +911,7 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
         else yaml.safe_load(result.stdout)
     )
     assert "included" in rendered["services"]
+    assert rendered["services"]["relative"]["hostname"] == "source-fixed-hostname"
     assert (
         rendered["services"]["relative"]["environment"]["SNAPSHOT_MARKER"]
         == "from-snapshot"

From b7e189732b448e183aa26fbd1ecc5d16097f4aa0 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 09:27:09 -0400
Subject: [PATCH 42/48] fix: normalize frozen Compose bind paths

---
 .../scripts/train_health_deployment_guard.py  | 194 ++++++++++-
 .../test_train_health_deployment_guard.py     | 301 +++++++++++++++++-
 2 files changed, 470 insertions(+), 25 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 8bb1e26eaa..3730755b86 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -108,7 +108,7 @@
 LIVE_ENV_RELATIVE = "docker/.env"
 REPLAY_MANIFEST_SCHEMA = "unstract-durable-replay/v2"
 REPLAY_MANIFEST_FILENAME = "durable-replay-manifest.json"
-COMPOSE_SNAPSHOT_SCHEMA = "unstract-compose-snapshot/v1"
+COMPOSE_SNAPSHOT_SCHEMA = "unstract-compose-snapshot/v2"
 COMPOSE_SNAPSHOT_MANIFEST_FILENAME = "compose-snapshot-manifest.json"
 COMPOSE_SNAPSHOT_DIR_PREFIX = "compose-snapshot-"
 COMPOSE_SNAPSHOT_PROBE_RELATIVE = Path("__helper__/unstract-services.sh")
@@ -148,6 +148,12 @@ class ComposeSnapshot(NamedTuple):
     manifest_path: Path
     project_dir: Path
     paths: dict[str, str]
+    # Maps a rendered snapshot-relative bind source back to the authoritative
+    # project-tree path it represented when the snapshot was frozen.  Compose
+    # providers are allowed to retain the snapshot spelling in ``config``
+    # output, but that temporary spelling must never become the mount identity
+    # compared with the running service.
+    bind_sources: dict[str, str]
 
     @property
     def probe_path(self) -> Path:
@@ -618,6 +624,7 @@ def materialize_compose_snapshot(
     paths: dict[str, str] = {}
     entries: dict[str, dict[str, Any]] = {}
     passthroughs: dict[str, str] = {}
+    bind_sources: dict[str, str] = {}
     copied: set[Path] = set()
     private_sources = {
         source.resolve()
@@ -659,6 +666,37 @@ def copy_source(
         if key is not None:
             paths[key] = str((destination / relative).resolve())
 
+    def register_bind_source(relative: Path, source: Path) -> None:
+        """Bind a rendered snapshot path to one exact project-tree source.
+
+        The snapshot may retain an immutable copy (or a protected symlink) for
+        Compose itself.  Its rendered ``config`` can still spell the temporary
+        snapshot path, so record only ordinary project-relative bind sources
+        that can be mapped back to the authoritative tree without inference.
+        """
+        if (
+            not relative.parts
+            or relative.is_absolute()
+            or any(part in {"", ".", ".."} for part in relative.parts)
+            or relative.parts[0].startswith("__")
+        ):
+            raise GuardError(f"Compose bind source has an unsafe snapshot path: {source}")
+        try:
+            authoritative = source.resolve(strict=True)
+            expected = (project_dir.resolve() / relative).resolve(strict=True)
+        except OSError as exc:
+            raise GuardError(f"cannot resolve Compose bind source: {source}") from exc
+        if authoritative != expected:
+            raise GuardError(
+                "Compose bind source does not map to the authoritative project tree: "
+                f"{source}"
+            )
+        key = str(relative)
+        prior = bind_sources.get(key)
+        if prior is not None and prior != str(authoritative):
+            raise GuardError(f"Compose bind source mapping changed: {source}")
+        bind_sources[key] = str(authoritative)
+
     compose_sources: list[Path] = []
     pending_includes: list[Path] = []
     for argument in compose_files:
@@ -709,13 +747,15 @@ def copy_source(
                 passthroughs[str(relative)] = _write_snapshot_directory_passthrough(
                     destination, relative, source
                 )
-                continue
-            copy_source(
-                source,
-                relative,
-                private=private,
-                description="private Compose env file" if private else "Compose bind file",
-            )
+            else:
+                copy_source(
+                    source,
+                    relative,
+                    private=private,
+                    description="private Compose env file" if private else "Compose bind file",
+                )
+            if not private:
+                register_bind_source(relative, source)
 
     if live_env_file is not None:
         source = _compose_input_path(project_dir, live_env_file)
@@ -763,6 +803,7 @@ def copy_source(
         },
         "entries": entries,
         "passthroughs": passthroughs,
+        "bind_sources": bind_sources,
     }
     try:
         manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n", encoding="utf-8")
@@ -775,6 +816,7 @@ def copy_source(
         manifest_path=manifest_path.resolve(),
         project_dir=project_dir.resolve(),
         paths={key: str(destination / relative) for key, relative in manifest["paths"].items()},
+        bind_sources=bind_sources,
     )
 
 
@@ -839,10 +881,12 @@ def load_compose_snapshot(path: Path) -> ComposeSnapshot:
     entries = manifest.get("entries")
     paths_manifest = manifest.get("paths")
     passthroughs = manifest.get("passthroughs") or {}
+    bind_sources = manifest.get("bind_sources")
     if (
         not isinstance(entries, dict)
         or not isinstance(paths_manifest, dict)
         or not isinstance(passthroughs, dict)
+        or not isinstance(bind_sources, dict)
     ):
         raise GuardError("Compose snapshot manifest is incomplete")
     paths: dict[str, str] = {}
@@ -895,11 +939,46 @@ def load_compose_snapshot(path: Path) -> ComposeSnapshot:
     project_dir = Path(project_dir_value).resolve()
     if not Path(project_dir_value).is_absolute() or not project_dir.exists():
         raise GuardError("Compose snapshot project directory is invalid")
+    validated_bind_sources: dict[str, str] = {}
+    for relative, target in bind_sources.items():
+        if not isinstance(relative, str) or not isinstance(target, str):
+            raise GuardError("Compose snapshot bind source mapping is invalid")
+        relative_path = Path(relative)
+        if (
+            not relative_path.parts
+            or relative_path.is_absolute()
+            or any(part in {"", ".", ".."} for part in relative_path.parts)
+            or relative_path.parts[0].startswith("__")
+            or relative not in entries and relative not in passthroughs
+        ):
+            raise GuardError("Compose snapshot bind source mapping is unsafe")
+        if relative in entries and bool(entries[relative].get("private")):
+            raise GuardError("Compose snapshot bind source mapping references private input")
+        try:
+            authoritative = Path(target)
+            if not authoritative.is_absolute():
+                raise ValueError("bind source is not absolute")
+            expected = (project_dir / relative_path).resolve(strict=True)
+            actual = authoritative.resolve(strict=True)
+        except (OSError, ValueError) as exc:
+            raise GuardError("Compose snapshot bind source mapping is invalid") from exc
+        if actual != expected:
+            raise GuardError("Compose snapshot bind source no longer matches the project tree")
+        if relative in passthroughs:
+            try:
+                if (root / relative_path).resolve(strict=True) != actual:
+                    raise GuardError(
+                        "Compose snapshot bind source passthrough target changed"
+                    )
+            except OSError as exc:
+                raise GuardError("Compose snapshot bind source passthrough is invalid") from exc
+        validated_bind_sources[relative] = str(actual)
     return ComposeSnapshot(
         root=root,
         manifest_path=path.resolve(),
         project_dir=project_dir,
         paths=paths,
+        bind_sources=validated_bind_sources,
     )
 
 
@@ -985,6 +1064,100 @@ def bound_compose_inputs(
         yield replacements, (), active
 
 
+def _snapshot_bind_source_relative(value: str, snapshot: ComposeSnapshot) -> str | None:
+    """Return a lexical snapshot-relative bind source, if one was rendered.
+
+    Do not resolve the candidate before deciding whether it belongs to the
+    snapshot: directory passthroughs intentionally resolve to their original
+    host paths.  A provider spelling a source inside the snapshot must match a
+    manifest-authorized bind source exactly; an unrecognized or escaping
+    spelling is a hard error rather than a fallback to the temporary path.
+    """
+    if not os.path.isabs(value):
+        return None
+    root = os.path.abspath(str(snapshot.root))
+    candidate = os.path.abspath(value)
+    claims_snapshot_root = value == root or value.startswith(root + os.sep)
+    try:
+        inside_snapshot = os.path.commonpath((root, candidate)) == root
+        resolves_inside_snapshot = (
+            os.path.commonpath((root, os.path.realpath(value))) == root
+        )
+    except ValueError:
+        return None
+    if claims_snapshot_root and not inside_snapshot:
+        raise GuardError("Compose rendered bind source escapes its snapshot")
+    if not inside_snapshot:
+        if resolves_inside_snapshot:
+            raise GuardError("Compose rendered bind source aliases its snapshot")
+        return None
+    relative = Path(os.path.relpath(candidate, root))
+    if (
+        not relative.parts
+        or relative == Path(".")
+        or relative.is_absolute()
+        or any(part in {"", ".", ".."} for part in relative.parts)
+    ):
+        raise GuardError("Compose rendered bind source is not a snapshot input")
+    return str(relative)
+
+
+def normalize_snapshot_bind_sources(
+    config: dict[str, Any], snapshot: ComposeSnapshot
+) -> dict[str, Any]:
+    """Map provider-rendered snapshot bind paths to authoritative host paths.
+
+    Compose still reads all includes, env files, and overrides from the frozen
+    snapshot.  This changes only rendered bind identities used for guard
+    comparison, so a temporary snapshot path cannot be mistaken for a changed
+    persistent mount after its temporary directory is removed.
+    """
+    services = config.get("services")
+    if not isinstance(services, dict):
+        return config
+    for service, definition in services.items():
+        if not isinstance(definition, dict):
+            continue
+        mounts = definition.get("volumes")
+        if mounts is None:
+            continue
+        if not isinstance(mounts, list):
+            raise GuardError(f"Compose rendered volumes are invalid for {service}")
+        for mount in mounts:
+            if not isinstance(mount, dict):
+                raise GuardError(f"Compose rendered mount is invalid for {service}")
+            if mount.get("type") != "bind":
+                continue
+            source = mount.get("source")
+            if mount.get("target") == PROBE_MOUNT_TARGET:
+                expected_probe = os.path.abspath(str(snapshot.probe_path))
+                if (
+                    not isinstance(source, str)
+                    or os.path.abspath(source) != expected_probe
+                ):
+                    raise GuardError(
+                        "Compose rendered trusted probe mount does not use "
+                        "the frozen probe source"
+                    )
+                # The immutable probe deliberately lives inside the snapshot.
+                # It is content-hashed separately and excluded from persistent
+                # mount identity comparison, so do not treat it as project data.
+                continue
+            if not isinstance(source, str) or not source:
+                continue
+            relative = _snapshot_bind_source_relative(source, snapshot)
+            if relative is None:
+                continue
+            authoritative = snapshot.bind_sources.get(relative)
+            if authoritative is None:
+                raise GuardError(
+                    "Compose rendered bind source is not authorized by its snapshot: "
+                    f"{source}"
+                )
+            mount["source"] = authoritative
+    return config
+
+
 def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
     try:
         return json.loads(result.stdout)
@@ -2057,7 +2230,10 @@ def compose_config(
             deadline=deadline,
             pass_fds=pass_fds,
         )
-    return parse_json_output(result, "Compose config")
+        rendered = parse_json_output(result, "Compose config")
+        if not isinstance(rendered, dict):
+            raise GuardError("Compose config did not return an object")
+        return normalize_snapshot_bind_sources(rendered, active_snapshot)
 
 
 def compose_environment(value: Any) -> dict[str, Any]:
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 372a831dbb..e44c6299c5 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -840,7 +840,9 @@ def test_compose_snapshot_rejects_tampered_retained_input(tmp_path: Path) -> Non
         guard.load_compose_snapshot(snapshot.manifest_path)
 
 
-def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None:
+def test_real_compose_provider_preserves_snapshot_runner_paths_and_env(
+    tmp_path: Path,
+) -> None:
     project = tmp_path / "project"
     (project / "docker").mkdir(parents=True)
     (project / "docker" / "docker-compose.yaml").write_text(
@@ -850,26 +852,38 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
     (project / "docker" / "docker-compose-dev-essentials.yaml").write_text(
         "services:\n  included:\n    image: busybox:latest\n", encoding="utf-8"
     )
-    relative_env = project / "docker" / "relative.env"
-    relative_env.write_text("SNAPSHOT_MARKER=from-snapshot\n", encoding="utf-8")
-    relative_data = project / "docker" / "relative-data"
-    relative_data.mkdir()
-    (relative_data / "marker").write_text("runtime-data\n", encoding="utf-8")
+    workflow_data = project / "docker" / "workflow_data"
+    workflow_data.mkdir()
+    (workflow_data / "marker").write_text("runtime-data\n", encoding="utf-8")
+    tool_registry = project / "tool-registry"
+    tool_registry.mkdir()
+    (tool_registry / "marker").write_text("registry-data\n", encoding="utf-8")
+    runner_env = project / "runner" / ".env"
+    runner_env.parent.mkdir()
+    runner_env.write_text("RUNNER_ENV_FILE=from-runner-env-file\n", encoding="utf-8")
     (project / "docker" / "docker-compose.yaml").write_text(
         "include:\n"
         "  - docker-compose-dev-essentials.yaml\n"
         "services:\n"
-        "  relative:\n"
+        "  runner:\n"
         "    image: busybox:latest\n"
         "    hostname: source-fixed-hostname\n"
         "    env_file:\n"
-        "      - ./relative.env\n"
+        "      - ../runner/.env\n"
+        "    environment:\n"
+        "      RUNNER_SOURCE_ENV: ${RUNNER_SOURCE_ENV}\n"
         "    volumes:\n"
-        "      - ./relative-data:/data:ro\n",
+        "      - ./workflow_data:/data\n"
+        "      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config\n",
         encoding="utf-8",
     )
     env_file = project / "docker" / ".env"
-    env_file.write_text("COMPOSE_PROJECT_NAME=snapshot-provider-test\n", encoding="utf-8")
+    env_file.write_text(
+        "COMPOSE_PROJECT_NAME=snapshot-provider-test\n"
+        "RUNNER_SOURCE_ENV=from-source-env\n"
+        f"TOOL_REGISTRY_CONFIG_SRC_PATH={tool_registry}\n",
+        encoding="utf-8",
+    )
     probe = tmp_path / "probe.sh"
     probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
     snapshot = guard.create_compose_snapshot(
@@ -911,13 +925,268 @@ def test_real_compose_provider_resolves_snapshot_include(tmp_path: Path) -> None
         else yaml.safe_load(result.stdout)
     )
     assert "included" in rendered["services"]
-    assert rendered["services"]["relative"]["hostname"] == "source-fixed-hostname"
-    assert (
-        rendered["services"]["relative"]["environment"]["SNAPSHOT_MARKER"]
-        == "from-snapshot"
+    runner = rendered["services"]["runner"]
+    assert runner["hostname"] == "source-fixed-hostname"
+    assert runner["environment"]["RUNNER_SOURCE_ENV"] == "from-source-env"
+    assert runner["environment"]["RUNNER_ENV_FILE"] == "from-runner-env-file"
+    normalized = guard.normalize_snapshot_bind_sources(rendered, snapshot)
+    mounts = {mount["target"]: mount for mount in normalized["services"]["runner"]["volumes"]}
+    assert Path(mounts["/data"]["source"]).resolve() == workflow_data.resolve()
+    assert Path(mounts["/data/tool_registry_config"]["source"]).resolve() == tool_registry.resolve()
+
+
+def test_compose_config_maps_temporary_runner_data_bind_to_project_tree(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    project = tmp_path / "project"
+    docker_dir = project / "docker"
+    docker_dir.mkdir(parents=True)
+    workflow_data = docker_dir / "workflow_data"
+    workflow_data.mkdir()
+    tool_registry = project / "tool-registry"
+    tool_registry.mkdir()
+    compose = docker_dir / "docker-compose.yaml"
+    compose.write_text(
+        "services:\n"
+        "  runner:\n"
+        "    image: busybox:latest\n"
+        "    volumes:\n"
+        "      - ./workflow_data:/data\n"
+        "      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config\n"
+        "  db:\n"
+        "    image: busybox:latest\n"
+        "    volumes:\n"
+        "      - ${UNSTRACT_HEALTHCHECK_SOURCE}:"
+        "/usr/local/bin/unstract-services.sh:ro\n",
+        encoding="utf-8",
+    )
+    (docker_dir / "compose.train.yaml").write_text("services: {}\n", encoding="utf-8")
+    live_env = docker_dir / ".env"
+    live_env.write_text(
+        "TOOL_REGISTRY_CONFIG_SRC_PATH=" + str(tool_registry) + "\n",
+        encoding="utf-8",
+    )
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    temporary_roots: list[Path] = []
+
+    def fake_run(
+        args: list[str], *, env: dict[str, str] | None = None, **_: object
+    ) -> subprocess.CompletedProcess[str]:
+        assert env is not None
+        project_directory = Path(args[args.index("--project-directory") + 1])
+        temporary_roots.append(project_directory.parent)
+        bound_env = Path(args[args.index("--env-file") + 1])
+        assert "TOOL_REGISTRY_CONFIG_SRC_PATH=" + str(tool_registry) in bound_env.read_text(
+            encoding="utf-8"
+        )
+        assert env["VERSION"] == "goal09-test"
+        assert env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(
+            project_directory.parent / "__helper__" / "unstract-services.sh"
+        )
+        return subprocess.CompletedProcess(
+            args,
+            0,
+            json.dumps(
+                {
+                    "services": {
+                        "runner": {
+                            "volumes": [
+                                {
+                                    "type": "bind",
+                                    "source": str(project_directory / "workflow_data"),
+                                    "target": "/data",
+                                },
+                                {
+                                    "type": "bind",
+                                    "source": str(tool_registry),
+                                    "target": "/data/tool_registry_config",
+                                },
+                            ]
+                        },
+                        "db": {
+                            "volumes": [
+                                {
+                                    "type": "bind",
+                                    "source": env["UNSTRACT_HEALTHCHECK_SOURCE"],
+                                    "target": guard.PROBE_MOUNT_TARGET,
+                                    "read_only": True,
+                                }
+                            ]
+                        }
+                    }
+                }
+            ),
+            "",
+        )
+
+    monkeypatch.setattr(guard, "run", fake_run)
+    config = guard.compose_config(
+        project,
+        ("docker/docker-compose.yaml",),
+        candidate_version="goal09-test",
+        probe_source=probe,
+        live_env_file=live_env,
+        probe_source_sha256=guard.sha256_file(probe),
+    )
+
+    mounts = {mount["target"]: mount for mount in config["services"]["runner"]["volumes"]}
+    assert mounts["/data"]["source"] == str(workflow_data.resolve())
+    assert mounts["/data/tool_registry_config"]["source"] == str(tool_registry.resolve())
+    assert temporary_roots
+    probe_mount = config["services"]["db"]["volumes"][0]
+    assert probe_mount["source"] == str(
+        temporary_roots[0] / "__helper__" / "unstract-services.sh"
+    )
+    assert not temporary_roots[0].exists()
+
+
+def test_compose_config_rejects_wrong_snapshot_probe_bind(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    project = tmp_path / "project"
+    docker_dir = project / "docker"
+    docker_dir.mkdir(parents=True)
+    compose = docker_dir / "docker-compose.yaml"
+    compose.write_text(
+        "services:\n"
+        "  db:\n"
+        "    image: busybox:latest\n"
+        "    volumes:\n"
+        "      - ${UNSTRACT_HEALTHCHECK_SOURCE}:"
+        "/usr/local/bin/unstract-services.sh:ro\n",
+        encoding="utf-8",
+    )
+    (docker_dir / "compose.train.yaml").write_text("services: {}\n", encoding="utf-8")
+    live_env = docker_dir / ".env"
+    live_env.write_text("COMPOSE_PROJECT_NAME=test\n", encoding="utf-8")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+
+    def fake_run(args: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+        project_directory = Path(args[args.index("--project-directory") + 1])
+        return subprocess.CompletedProcess(
+            args,
+            0,
+            json.dumps(
+                {
+                    "services": {
+                        "db": {
+                            "volumes": [
+                                {
+                                    "type": "bind",
+                                    "source": str(project_directory / "untracked"),
+                                    "target": guard.PROBE_MOUNT_TARGET,
+                                    "read_only": True,
+                                }
+                            ]
+                        }
+                    }
+                }
+            ),
+            "",
+        )
+
+    monkeypatch.setattr(guard, "run", fake_run)
+    with pytest.raises(
+        guard.GuardError, match="trusted probe mount does not use the frozen probe source"
+    ):
+        guard.compose_config(
+            project,
+            ("docker/docker-compose.yaml",),
+            candidate_version="goal09-test",
+            probe_source=probe,
+            live_env_file=live_env,
+            probe_source_sha256=guard.sha256_file(probe),
+        )
+
+
+def test_compose_config_rejects_unrecognized_snapshot_bind_source(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    project = tmp_path / "project"
+    docker_dir = project / "docker"
+    docker_dir.mkdir(parents=True)
+    (docker_dir / "workflow_data").mkdir()
+    compose = docker_dir / "docker-compose.yaml"
+    compose.write_text(
+        "services:\n  runner:\n    image: busybox:latest\n    volumes:\n      - ./workflow_data:/data\n",
+        encoding="utf-8",
+    )
+    (docker_dir / "compose.train.yaml").write_text("services: {}\n", encoding="utf-8")
+    live_env = docker_dir / ".env"
+    live_env.write_text("COMPOSE_PROJECT_NAME=test\n", encoding="utf-8")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+
+    def fake_run(args: list[str], **_: object) -> subprocess.CompletedProcess[str]:
+        project_directory = Path(args[args.index("--project-directory") + 1])
+        return subprocess.CompletedProcess(
+            args,
+            0,
+            json.dumps(
+                {
+                    "services": {
+                        "runner": {
+                            "volumes": [
+                                {
+                                    "type": "bind",
+                                    "source": str(project_directory / "untracked"),
+                                    "target": "/data",
+                                }
+                            ]
+                        }
+                    }
+                }
+            ),
+            "",
+        )
+
+    monkeypatch.setattr(guard, "run", fake_run)
+    with pytest.raises(guard.GuardError, match="not authorized by its snapshot"):
+        guard.compose_config(
+            project,
+            ("docker/docker-compose.yaml",),
+            candidate_version="goal09-test",
+            probe_source=probe,
+            live_env_file=live_env,
+            probe_source_sha256=guard.sha256_file(probe),
+        )
+
+
+def test_compose_snapshot_reloads_authoritative_bind_source_mapping(
+    tmp_path: Path,
+) -> None:
+    project = tmp_path / "project"
+    docker_dir = project / "docker"
+    docker_dir.mkdir(parents=True)
+    workflow_data = docker_dir / "workflow_data"
+    workflow_data.mkdir()
+    compose = docker_dir / "docker-compose.yaml"
+    compose.write_text(
+        "services:\n  runner:\n    image: busybox:latest\n    volumes:\n      - ./workflow_data:/data\n",
+        encoding="utf-8",
+    )
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state",
+        project_dir=project,
+        compose_files=("docker/docker-compose.yaml",),
+        live_env_file=None,
+        probe_source=probe,
+        probe_source_sha256=guard.sha256_file(probe),
+        image_override=None,
+        image_override_sha256=None,
+        environment_override=None,
+        environment_override_sha256=None,
     )
-    rendered_source = Path(rendered["services"]["relative"]["volumes"][0]["source"])
-    assert rendered_source.resolve() == relative_data.resolve()
+
+    loaded = guard.load_compose_snapshot(snapshot.manifest_path)
+    assert loaded.bind_sources == {
+        "docker/workflow_data": str(workflow_data.resolve())
+    }
 
 
 def test_compose_rejects_ignored_live_input_drift(tmp_path: Path) -> None:

From e8ff6d4720792b8f3d206c63dc8975edc035c470 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 15:35:28 -0400
Subject: [PATCH 43/48] Guard post-recreation quiescence before next batch

---
 .../scripts/train_health_deployment_guard.py  | 167 +++++++++++++--
 .../test_train_health_deployment_guard.py     | 197 ++++++++++++++++++
 2 files changed, 351 insertions(+), 13 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index 3730755b86..fb499f59fc 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -59,6 +59,11 @@
 QUIESCENCE_REQUIRED_SAMPLES = 3
 QUIESCENCE_SAMPLE_INTERVAL_SECONDS = 2.0
 QUIESCENCE_MAX_WAIT_SECONDS = 10.0
+# A recreation may need time to settle before the existing strict zero-work
+# predicate is met again.  The longer post-recreation window does not permit
+# work to be consumed, cleared, or ignored: every following mutation still
+# requires three consecutive all-zero samples.
+POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS = 120.0
 
 # Compose service names.  Keep this explicit so a typo or a newly added service
 # cannot silently turn a targeted deployment into a project-wide update.
@@ -242,6 +247,39 @@ class GuardError(RuntimeError):
     """A precondition or postcondition failed."""
 
 
+class QuiescenceTimeout(GuardError):
+    """A bounded zero-work wait that retains only sanitized observations."""
+
+    def __init__(
+        self,
+        *,
+        phase: str,
+        max_wait_seconds: float,
+        observations: list[dict[str, Any]],
+        reason: str = "bounded settled-quiescence interval elapsed",
+    ) -> None:
+        self.phase = phase
+        self.max_wait_seconds = max_wait_seconds
+        self.observations = observations
+        self.reason = reason
+        super().__init__(
+            "queue and active-job state did not remain quiescent for the bounded "
+            f"stability interval (phase={phase})"
+        )
+
+    def evidence(self) -> dict[str, Any]:
+        """Return failure evidence without command output or environment values."""
+        return {
+            "schema": "unstract-health-quiescence-timeout/v1",
+            "reason": self.reason,
+            "phase": self.phase,
+            "max_wait_seconds": self.max_wait_seconds,
+            "required_consecutive_samples": QUIESCENCE_REQUIRED_SAMPLES,
+            "sample_interval_seconds": QUIESCENCE_SAMPLE_INTERVAL_SECONDS,
+            "observations": self.observations,
+        }
+
+
 def exception_reason(error: BaseException, *, limit: int = 240) -> str:
     """Return bounded, single-line failure context without secret-looking values."""
     text = " ".join(str(error).split())
@@ -1747,15 +1785,32 @@ def queue_snapshot(deadline: OperationDeadline | None = None) -> dict[str, Any]:
     }
 
 
-def settled_queue_snapshot(deadline: OperationDeadline | None = None) -> dict[str, Any]:
+def settled_queue_snapshot(
+    deadline: OperationDeadline | None = None,
+    *,
+    max_wait_seconds: float = QUIESCENCE_MAX_WAIT_SECONDS,
+    phase: str = "guarded operation",
+) -> dict[str, Any]:
     """Require consecutive zero-work samples before any destructive change."""
+    if not math.isfinite(max_wait_seconds) or max_wait_seconds <= 0:
+        raise GuardError("quiescence wait must be a positive finite duration")
     started = time.monotonic()
-    end = started + QUIESCENCE_MAX_WAIT_SECONDS
+    end = started + max_wait_seconds
     consecutive = 0
     observations: list[dict[str, Any]] = []
     last: dict[str, Any] | None = None
     while True:
-        last = queue_snapshot(deadline)
+        try:
+            last = queue_snapshot(deadline)
+        except GuardError as exc:
+            if deadline is not None and time.monotonic() >= deadline.ends_at:
+                raise QuiescenceTimeout(
+                    phase=phase,
+                    max_wait_seconds=max_wait_seconds,
+                    observations=observations,
+                    reason="guarded operation deadline elapsed before settled quiescence",
+                ) from exc
+            raise
         observations.append(
             {
                 "observed_at": last.get("observed_at"),
@@ -1778,10 +1833,20 @@ def settled_queue_snapshot(deadline: OperationDeadline | None = None) -> dict[st
             consecutive = 0
         remaining = end - time.monotonic()
         if deadline:
-            remaining = min(remaining, deadline.remaining())
+            try:
+                remaining = min(remaining, deadline.remaining())
+            except GuardError as exc:
+                raise QuiescenceTimeout(
+                    phase=phase,
+                    max_wait_seconds=max_wait_seconds,
+                    observations=observations,
+                    reason="guarded operation deadline elapsed before settled quiescence",
+                ) from exc
         if remaining <= 0:
-            raise GuardError(
-                "queue and active-job state did not remain quiescent for the bounded stability interval"
+            raise QuiescenceTimeout(
+                phase=phase,
+                max_wait_seconds=max_wait_seconds,
+                observations=observations,
             )
         time.sleep(min(QUIESCENCE_SAMPLE_INTERVAL_SECONDS, remaining))
 
@@ -1839,8 +1904,31 @@ def capture(
     *,
     live_env_file: Path | None = None,
     deadline: OperationDeadline | None = None,
+    require_settled_quiescence: bool = True,
+    quiescence_max_wait_seconds: float = QUIESCENCE_MAX_WAIT_SECONDS,
+    quiescence_phase: str = "capture",
 ) -> dict[str, Any]:
+    """Capture sanitized state, optionally retaining one raw transition sample.
+
+    A raw sample is only for recording exact replacement IDs and timeout context
+    after a recreation.  It is never accepted as a precondition for another
+    mutation; callers that could mutate must use the default settled capture.
+    """
     uid = os.getuid() if hasattr(os, "getuid") else None
+    current_runtime_context = runtime_context(deadline=deadline)
+    current_source = source_state(
+        project_dir, live_env_file=live_env_file, deadline=deadline
+    )
+    job_quiescence = (
+        settled_queue_snapshot(
+            deadline,
+            max_wait_seconds=quiescence_max_wait_seconds,
+            phase=quiescence_phase,
+        )
+        if require_settled_quiescence
+        else queue_snapshot(deadline)
+    )
+    containers = inspect_project(deadline=deadline)
     return {
         "schema": "unstract-deployment-prep/v2",
         "captured_at": utc_now(),
@@ -1849,12 +1937,10 @@ def capture(
             "hostname": os.uname().nodename,
             "rootless_project": PROJECT,
         },
-        "runtime_context": runtime_context(deadline=deadline),
-        "source": source_state(
-            project_dir, live_env_file=live_env_file, deadline=deadline
-        ),
-        "job_quiescence": settled_queue_snapshot(deadline),
-        "containers": inspect_project(deadline=deadline),
+        "runtime_context": current_runtime_context,
+        "source": current_source,
+        "job_quiescence": job_quiescence,
+        "containers": containers,
     }
 
 
@@ -1873,6 +1959,34 @@ def write_private_json(path: Path, value: Any, *, description: str) -> None:
     )
 
 
+def write_post_recreation_quiescence(
+    backup_dir: Path,
+    services: tuple[str, ...],
+    snapshot: dict[str, Any],
+    *,
+    stage: str,
+) -> None:
+    """Persist a small, sanitized observation around one recreated batch."""
+    if not services:
+        raise GuardError("post-recreation quiescence evidence needs at least one service")
+    if stage not in {"observed", "settled"}:
+        raise GuardError(f"unsupported post-recreation quiescence stage: {stage}")
+    job_quiescence = snapshot.get("job_quiescence")
+    if not isinstance(job_quiescence, dict):
+        raise GuardError("post-recreation snapshot lacks job quiescence")
+    write_private_json(
+        backup_dir / f"post-recreation-{stage}-{services[0]}.json",
+        {
+            "schema": "unstract-health-post-recreation-quiescence/v1",
+            "stage": stage,
+            "services": list(services),
+            "captured_at": snapshot.get("captured_at"),
+            "job_quiescence": job_quiescence,
+        },
+        description=f"post-recreation {stage} quiescence evidence",
+    )
+
+
 def service_map(snapshot: dict[str, Any]) -> dict[str, dict[str, Any]]:
     result: dict[str, dict[str, Any]] = {}
     for container in snapshot.get("containers", []):
@@ -4055,6 +4169,8 @@ def compensating_rollback(
         project_dir,
         live_env_file=live_env_file,
         deadline=operation_deadline,
+        quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
+        quiescence_phase=f"pre-compensating-rollback:{services[0]}",
     )
     compare_source_and_quiescence(baseline, current)
     verify_replacement_ids(current, replacement_manifest)
@@ -4084,6 +4200,8 @@ def compensating_rollback(
             project_dir,
             live_env_file=live_env_file,
             deadline=operation_deadline,
+            quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
+            quiescence_phase=f"locked-compensating-rollback:{services[0]}",
         )
         compare_source_and_quiescence(baseline, locked)
         verify_replacement_ids(locked, replacement_manifest)
@@ -4107,6 +4225,8 @@ def compensating_rollback(
         project_dir,
         live_env_file=live_env_file,
         deadline=operation_deadline,
+        quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
+        quiescence_phase=f"post-compensating-rollback:{services[0]}",
     )
     verify_rollback_result(
         baseline,
@@ -4201,7 +4321,10 @@ def apply_batch(
         )
         # Take the final settled sample after all preflight commands and
         # immediately before the targeted Compose mutation.
-        final_quiescence = settled_queue_snapshot(operation_deadline)
+        final_quiescence = settled_queue_snapshot(
+            operation_deadline,
+            phase=f"pre-recreation:{services[0]}",
+        )
         if not final_quiescence.get("stability", {}).get("stable"):
             raise GuardError("queue was not settled immediately before targeted recreation")
         targeted_up(
@@ -4227,6 +4350,10 @@ def apply_batch(
             project_dir,
             live_env_file=live_env_file,
             deadline=operation_deadline,
+            require_settled_quiescence=False,
+        )
+        write_post_recreation_quiescence(
+            backup_dir, services, observed, stage="observed"
         )
         replacements = record_replacements(baseline, observed, lock, services)
         write_replacement_manifest(backup_dir, replacements, name=f"replacements-{services[0]}.json")
@@ -4235,6 +4362,11 @@ def apply_batch(
             project_dir,
             live_env_file=live_env_file,
             deadline=operation_deadline,
+            quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
+            quiescence_phase=f"post-recreation:{services[0]}",
+        )
+        write_post_recreation_quiescence(
+            backup_dir, services, final, stage="settled"
         )
         compare_post_apply(baseline, final, lock, services)
         compare_untargeted_runtime(baseline, final)
@@ -4575,6 +4707,8 @@ def command_apply(args: argparse.Namespace) -> int:
                 project_dir,
                 live_env_file=live_env_file,
                 deadline=operation_deadline,
+                quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
+                quiescence_phase="post-apply",
             )
             compare_post_apply(baseline, final, lock, TARGET_SERVICES)
             compare_untargeted_runtime(baseline, final)
@@ -4585,6 +4719,8 @@ def command_apply(args: argparse.Namespace) -> int:
                 "schema": FAILURE_SCHEMA,
                 "original_error": exception_reason(exc),
             }
+            if isinstance(exc, QuiescenceTimeout):
+                failure_record["quiescence_timeout"] = exc.evidence()
             try:
                 write_json(backup_dir / "apply-failure.json", failure_record)
             except OSError:
@@ -4595,6 +4731,7 @@ def command_apply(args: argparse.Namespace) -> int:
                         project_dir,
                         live_env_file=live_env_file,
                         deadline=operation_deadline,
+                        require_settled_quiescence=False,
                     )
                     write_json(backup_dir / "failed-state.json", failed_state)
                     discovered = record_replacements(
@@ -4627,6 +4764,10 @@ def command_apply(args: argparse.Namespace) -> int:
                         )
                 except Exception as rollback_error:
                     failure_record["rollback_error"] = exception_reason(rollback_error)
+                    if isinstance(rollback_error, QuiescenceTimeout):
+                        failure_record["rollback_quiescence_timeout"] = (
+                            rollback_error.evidence()
+                        )
                     try:
                         write_json(backup_dir / "apply-failure.json", failure_record)
                     except OSError:
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index e44c6299c5..e107656640 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import importlib.util
 import json
 import os
@@ -9,6 +10,7 @@
 import sys
 from copy import deepcopy
 from pathlib import Path
+from types import SimpleNamespace
 
 import pytest
 import yaml
@@ -1514,3 +1516,198 @@ def test_named_volume_alias_rejects_wrong_live_name() -> None:
 
     with pytest.raises(guard.GuardError, match="mount source"):
         guard.check_candidate_config(config, baseline, lock, authored)
+
+
+def test_settled_queue_timeout_retains_sanitized_counts_and_phase(monkeypatch) -> None:
+    def non_quiescent_snapshot(_deadline):
+        return {
+            "observed_at": "2026-09-09T17:14:00+00:00",
+            "quiescent": False,
+            "rabbitmq": {"empty": True},
+            "postgres": {
+                "counts": {
+                    "pg_queue_message": 1,
+                    "pg_queue_claimed": 0,
+                    "pg_active_barriers": 0,
+                    "pg_orchestration_claims": 0,
+                }
+            },
+        }
+
+    monkeypatch.setattr(guard, "queue_snapshot", non_quiescent_snapshot)
+
+    with pytest.raises(guard.QuiescenceTimeout) as raised:
+        guard.settled_queue_snapshot(
+            max_wait_seconds=0.001,
+            phase="post-recreation:runner",
+        )
+
+    evidence = raised.value.evidence()
+    assert evidence["reason"] == "bounded settled-quiescence interval elapsed"
+    assert evidence["phase"] == "post-recreation:runner"
+    assert evidence["observations"]
+    assert evidence["observations"][0]["postgres_counts"]["pg_queue_message"] == 1
+
+
+def test_settled_queue_deadline_timeout_retains_counts_and_reason(monkeypatch) -> None:
+    def non_quiescent_snapshot(_deadline):
+        return {
+            "observed_at": "2026-09-09T17:14:00+00:00",
+            "quiescent": False,
+            "rabbitmq": {"empty": True},
+            "postgres": {"counts": {"pg_queue_message": 2}},
+        }
+
+    class ExhaustedDeadline:
+        def remaining(self) -> float:
+            raise guard.GuardError("guarded operation exceeded its total deadline")
+
+    monkeypatch.setattr(guard, "queue_snapshot", non_quiescent_snapshot)
+
+    with pytest.raises(guard.QuiescenceTimeout) as raised:
+        guard.settled_queue_snapshot(
+            ExhaustedDeadline(),
+            max_wait_seconds=120,
+            phase="post-recreation:runner",
+        )
+
+    evidence = raised.value.evidence()
+    assert evidence["reason"] == "guarded operation deadline elapsed before settled quiescence"
+    assert evidence["observations"][-1]["postgres_counts"]["pg_queue_message"] == 2
+
+
+def test_raw_transition_capture_never_substitutes_for_a_settled_capture(monkeypatch) -> None:
+    raw_job_state = {"quiescent": False, "postgres": {"counts": {"pg_queue_message": 1}}}
+    monkeypatch.setattr(guard, "runtime_context", lambda **_kwargs: {"runtime": "ok"})
+    monkeypatch.setattr(guard, "source_state", lambda *_args, **_kwargs: {"source": "ok"})
+    monkeypatch.setattr(guard, "queue_snapshot", lambda _deadline: raw_job_state)
+    monkeypatch.setattr(guard, "inspect_project", lambda **_kwargs: [])
+
+    def settled_must_not_run(*_args, **_kwargs):
+        raise AssertionError("raw transition capture must not claim settled quiescence")
+
+    monkeypatch.setattr(guard, "settled_queue_snapshot", settled_must_not_run)
+    snapshot = guard.capture(
+        Path("/project"),
+        require_settled_quiescence=False,
+    )
+
+    assert snapshot["job_quiescence"] is raw_job_state
+    assert snapshot["job_quiescence"]["quiescent"] is False
+
+
+def test_post_recreation_evidence_is_private_and_contains_only_job_state(tmp_path: Path) -> None:
+    snapshot = {
+        "captured_at": "2026-09-09T17:14:00+00:00",
+        "job_quiescence": {
+            "quiescent": False,
+            "postgres": {"counts": {"pg_queue_message": 1}},
+        },
+    }
+
+    guard.write_post_recreation_quiescence(
+        tmp_path,
+        ("runner",),
+        snapshot,
+        stage="observed",
+    )
+
+    path = tmp_path / "post-recreation-observed-runner.json"
+    evidence = json.loads(path.read_text(encoding="utf-8"))
+    assert path.stat().st_mode & 0o777 == 0o600
+    assert evidence == {
+        "captured_at": "2026-09-09T17:14:00+00:00",
+        "job_quiescence": snapshot["job_quiescence"],
+        "schema": "unstract-health-post-recreation-quiescence/v1",
+        "services": ["runner"],
+        "stage": "observed",
+    }
+
+
+def test_apply_batch_records_raw_transition_then_requires_strict_settlement(monkeypatch) -> None:
+    events: list[str] = []
+    capture_kwargs: list[dict] = []
+    capture_results = [{}, {}, {}]
+    args = SimpleNamespace(
+        compose_file=None,
+        project_dir="/project",
+        candidate_source="/candidate",
+        probe_source="/probe",
+    )
+    baseline = {"source": {}}
+    lock = {"candidate_version": "candidate"}
+
+    def fake_capture(*_args, **kwargs):
+        capture_kwargs.append(kwargs)
+        events.append(
+            "raw-capture"
+            if kwargs.get("require_settled_quiescence") is False
+            else "settled-capture"
+        )
+        return capture_results.pop(0)
+
+    monkeypatch.setattr(guard, "resolve_live_env_file", lambda *_args: Path("/env"))
+    monkeypatch.setattr(guard, "validate_private_override", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(guard, "advisory_lock", lambda **_kwargs: contextlib.nullcontext())
+    monkeypatch.setattr(guard, "capture", fake_capture)
+    monkeypatch.setattr(guard, "compare_untargeted_runtime", lambda *_args: None)
+    monkeypatch.setattr(guard, "compare_source_and_quiescence", lambda *_args: None)
+    monkeypatch.setattr(guard, "verify_untouched_targets", lambda *_args: None)
+    monkeypatch.setattr(guard, "candidate_image_snapshot", lambda *_args, **_kwargs: {})
+    monkeypatch.setattr(guard, "compose_config", lambda *_args, **_kwargs: {})
+    monkeypatch.setattr(guard, "check_candidate_config", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(
+        guard,
+        "settled_queue_snapshot",
+        lambda *_args, **_kwargs: {"stability": {"stable": True}},
+    )
+    monkeypatch.setattr(guard, "targeted_up", lambda *_args, **_kwargs: events.append("targeted-up"))
+    monkeypatch.setattr(
+        guard,
+        "write_post_recreation_quiescence",
+        lambda *_args, stage, **_kwargs: events.append(f"evidence-{stage}"),
+    )
+    monkeypatch.setattr(
+        guard,
+        "record_replacements",
+        lambda *_args, **_kwargs: {"services": {"runner": {}}},
+    )
+    monkeypatch.setattr(guard, "write_replacement_manifest", lambda *_args, **_kwargs: None)
+    monkeypatch.setattr(guard, "wait_healthy", lambda *_args, **_kwargs: events.append("healthy"))
+    monkeypatch.setattr(guard, "compare_post_apply", lambda *_args: None)
+
+    result = guard.apply_batch(
+        args,
+        baseline,
+        lock,
+        Path("/backup"),
+        Path("/image-override"),
+        ("runner",),
+        (),
+        (),
+        settings_file=Path("/settings"),
+        settings_file_sha256="settings-sha",
+        image_override_sha256="image-sha",
+        probe_source_sha256="probe-sha",
+        runtime_environment_override=Path("/runtime-override"),
+        runtime_environment_sha256="runtime-sha",
+        reviewed_environment_keys={},
+        snapshot=SimpleNamespace(),
+        operation_deadline=guard.OperationDeadline(60),
+    )
+
+    assert result == {"services": {"runner": {}}}
+    assert events == [
+        "settled-capture",
+        "targeted-up",
+        "raw-capture",
+        "evidence-observed",
+        "healthy",
+        "settled-capture",
+        "evidence-settled",
+    ]
+    assert capture_kwargs[1]["require_settled_quiescence"] is False
+    assert capture_kwargs[2]["quiescence_max_wait_seconds"] == (
+        guard.POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS
+    )
+    assert capture_kwargs[2]["quiescence_phase"] == "post-recreation:runner"

From a4870c0be76687bc2e42fe3fbec5fdf4cb9e0892 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 17:20:50 -0400
Subject: [PATCH 44/48] Preserve authoritative bind sources during Compose
 creation

---
 .../scripts/train_health_deployment_guard.py  | 161 +++++--
 .../test_train_health_deployment_guard.py     | 417 +++++++++++++++++-
 2 files changed, 540 insertions(+), 38 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index fb499f59fc..d57c3a2376 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -35,6 +35,7 @@
 import tempfile
 import time
 from collections.abc import Iterator
+from copy import deepcopy
 from decimal import Decimal, InvalidOperation
 from pathlib import Path
 from typing import Any, NamedTuple
@@ -156,8 +157,8 @@ class ComposeSnapshot(NamedTuple):
     # Maps a rendered snapshot-relative bind source back to the authoritative
     # project-tree path it represented when the snapshot was frozen.  Compose
     # providers are allowed to retain the snapshot spelling in ``config``
-    # output, but that temporary spelling must never become the mount identity
-    # compared with the running service.
+    # output, but creation must receive the authoritative source in an explicit
+    # bind override as well as use it for comparison with the running service.
     bind_sources: dict[str, str]
 
     @property
@@ -1146,9 +1147,8 @@ def normalize_snapshot_bind_sources(
     """Map provider-rendered snapshot bind paths to authoritative host paths.
 
     Compose still reads all includes, env files, and overrides from the frozen
-    snapshot.  This changes only rendered bind identities used for guard
-    comparison, so a temporary snapshot path cannot be mistaken for a changed
-    persistent mount after its temporary directory is removed.
+    snapshot.  The normalized model supplies both guard comparison and the
+    explicit bind-source override passed to container creation.
     """
     services = config.get("services")
     if not isinstance(services, dict):
@@ -1196,6 +1196,88 @@ def normalize_snapshot_bind_sources(
     return config
 
 
+@contextlib.contextmanager
+def stable_bind_compose_config(
+    args: list[str],
+    snapshot: ComposeSnapshot,
+    *,
+    env: dict[str, str],
+    deadline: OperationDeadline | None,
+    pass_fds: tuple[int, ...],
+) -> Iterator[tuple[list[str], dict[str, Any]]]:
+    """Give Compose the same stable bind sources that the guard compares.
+
+    Frozen includes and env files must retain their snapshot path base.  A
+    provider can retain that base in a bind source even when it names a
+    symlink to live data, so resolving only the comparison model is not enough.
+    Override only manifest-authorized bind sources, preserve every mount
+    option, and require a second provider render to match the intended model
+    exactly before the caller can use these same files for ``up``.
+    """
+
+    def render(launch_args: list[str]) -> dict[str, Any]:
+        result = run(
+            [*launch_args, "config", "--format", "json"],
+            cwd=snapshot.root,
+            env=env,
+            deadline=deadline,
+            pass_fds=pass_fds,
+        )
+        model = parse_json_output(result, "Compose config")
+        if not isinstance(model, dict):
+            raise GuardError("Compose config did not return an object")
+        return model
+
+    original = render(args)
+    normalized = normalize_snapshot_bind_sources(deepcopy(original), snapshot)
+    provider_model = deepcopy(normalized)
+    services: dict[str, Any] = {}
+    definitions = normalized.get("services")
+    if not isinstance(definitions, dict):
+        raise GuardError("Compose config services are invalid")
+    for service, definition in definitions.items():
+        if not isinstance(definition, dict):
+            continue
+        original_mounts = original["services"][service].get("volumes") or []
+        mounts = definition.get("volumes") or []
+        changed = []
+        for index, (before, mount) in enumerate(zip(original_mounts, mounts)):
+            if before.get("source") == mount.get("source"):
+                continue
+            override = deepcopy(mount)
+            # Compose's JSON config retains replay escaping for literal dollar
+            # signs; its create request decodes that escaping. Only the new
+            # authoritative source is an unescaped runtime value. Every other
+            # mount field was already serialized by the provider, so preserve
+            # it exactly instead of escaping it a second time.
+            override["source"] = mount["source"].replace("$", "$$")
+            provider_model["services"][service]["volumes"][index]["source"] = override["source"]
+            changed.append(override)
+        if changed:
+            services[service] = {"volumes": changed}
+    if not services:
+        yield args, normalized
+        return
+
+    with tempfile.TemporaryDirectory(prefix=".unstract-compose-binds-") as temporary:
+        root = Path(temporary)
+        relative = Path("bind-sources.override.json")
+        digest = _write_snapshot_file(
+            root,
+            relative,
+            json.dumps({"services": services}).encode("utf-8"),
+            private=True,
+        )
+        os.chmod(root, 0o500)
+        launch_args = [*args, "-f", str(root / relative)]
+        rendered = render(launch_args)
+        if rendered != provider_model:
+            raise GuardError("Compose bind-source override changed the effective config")
+        _validate_snapshot_file(root / relative, digest, private=True)
+        _validate_snapshot_parents(root / relative, root)
+        yield launch_args, normalized
+
+
 def parse_json_output(result: subprocess.CompletedProcess[str], description: str) -> Any:
     try:
         return json.loads(result.stdout)
@@ -2312,7 +2394,6 @@ def compose_config(
             live_env_file=live_env_file,
             snapshot=active_snapshot,
         )
-        args.extend(["config", "--format", "json"])
         bound_args = [bound_paths.get(argument, argument) for argument in args]
         final_settings = verify_compose_inputs_before_run(
             project_dir=project_dir,
@@ -2337,17 +2418,14 @@ def compose_config(
             if final_settings
             else env["UNSTRACT_HEALTHCHECK_SOURCE"]
         )
-        result = run(
+        with stable_bind_compose_config(
             bound_args,
-            cwd=active_snapshot.root,
+            active_snapshot,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
-        )
-        rendered = parse_json_output(result, "Compose config")
-        if not isinstance(rendered, dict):
-            raise GuardError("Compose config did not return an object")
-        return normalize_snapshot_bind_sources(rendered, active_snapshot)
+        ) as (_, rendered):
+            return rendered
 
 
 def compose_environment(value: Any) -> dict[str, Any]:
@@ -3493,20 +3571,8 @@ def targeted_up(
             live_env_file=live_env_file,
             snapshot=active_snapshot,
         )
-        args.extend(
-            [
-                "up",
-                "-d",
-                "--no-deps",
-                "--force-recreate",
-                "--no-build",
-                "--pull",
-                "never",
-                *services,
-            ]
-        )
         bound_args = [bound_paths.get(argument, argument) for argument in args]
-        final_settings = verify_compose_inputs_before_run(
+        verification = dict(
             project_dir=project_dir,
             live_env_file=live_env_file,
             expected_source=expected_source,
@@ -3522,6 +3588,7 @@ def targeted_up(
             settings_file_sha256=settings_file_sha256,
             candidate_version=candidate_version,
         )
+        final_settings = verify_compose_inputs_before_run(**verification)
         if final_settings:
             env["VERSION"] = final_settings["VERSION"]
         env["UNSTRACT_HEALTHCHECK_SOURCE"] = bound_paths.get(
@@ -3529,13 +3596,31 @@ def targeted_up(
             if final_settings
             else env["UNSTRACT_HEALTHCHECK_SOURCE"]
         )
-        run(
+        with stable_bind_compose_config(
             bound_args,
-            cwd=active_snapshot.root,
+            active_snapshot,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
-        )
+        ) as (launch_args, _):
+            verify_compose_inputs_before_run(**verification)
+            run(
+                [
+                    *launch_args,
+                    "up",
+                    "-d",
+                    "--no-deps",
+                    "--force-recreate",
+                    "--no-build",
+                    "--pull",
+                    "never",
+                    *services,
+                ],
+                cwd=active_snapshot.root,
+                env=env,
+                deadline=deadline,
+                pass_fds=pass_fds,
+            )
 
 
 def compose_start(
@@ -3600,9 +3685,8 @@ def compose_start(
             live_env_file=live_env_file,
             snapshot=active_snapshot,
         )
-        args.extend(["up", "-d", "--no-build", "--pull", "never"])
         bound_args = [bound_paths.get(argument, argument) for argument in args]
-        final_settings = verify_compose_inputs_before_run(
+        verification = dict(
             project_dir=project_dir,
             live_env_file=live_env_file,
             expected_source=expected_source,
@@ -3618,6 +3702,7 @@ def compose_start(
             settings_file_sha256=settings_file_sha256,
             candidate_version=candidate_version,
         )
+        final_settings = verify_compose_inputs_before_run(**verification)
         if final_settings:
             env["VERSION"] = final_settings["VERSION"]
         env["UNSTRACT_HEALTHCHECK_SOURCE"] = bound_paths.get(
@@ -3625,13 +3710,21 @@ def compose_start(
             if final_settings
             else env["UNSTRACT_HEALTHCHECK_SOURCE"]
         )
-        run(
+        with stable_bind_compose_config(
             bound_args,
-            cwd=active_snapshot.root,
+            active_snapshot,
             env=env,
             deadline=deadline,
             pass_fds=pass_fds,
-        )
+        ) as (launch_args, _):
+            verify_compose_inputs_before_run(**verification)
+            run(
+                [*launch_args, "up", "-d", "--no-build", "--pull", "never"],
+                cwd=active_snapshot.root,
+                env=env,
+                deadline=deadline,
+                pass_fds=pass_fds,
+            )
 
 
 def wait_healthy(
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index e107656640..896512e067 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -2,15 +2,19 @@
 
 import contextlib
 import importlib.util
+import http.server
 import json
 import os
+import re
 import shlex
 import shutil
 import subprocess
 import sys
+import threading
 from copy import deepcopy
 from pathlib import Path
 from types import SimpleNamespace
+from urllib.parse import parse_qs, urlsplit
 
 import pytest
 import yaml
@@ -691,7 +695,7 @@ def fake_run(
         snapshot=snapshot,
     )
 
-    assert len(calls) == 2
+    assert len(calls) == 3
     config_args, config_env = calls[0]
     assert config_args[:3] == ["docker", "compose", "--project-directory"]
     assert str(snapshot.root) in config_args
@@ -702,7 +706,8 @@ def fake_run(
     assert sum(argument == "-f" for argument in config_args) == 3
     assert config_env and config_env["VERSION"] == "goal09-test"
     assert config_env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(snapshot.probe_path)
-    assert calls[1][0][-1] == "runner"
+    assert calls[1][0][-3:] == ["config", "--format", "json"]
+    assert calls[2][0][-1] == "runner"
 
     monkeypatch.setattr(guard, "verify_candidate_source_state", lambda *_: None)
     guard.compose_start(
@@ -723,7 +728,8 @@ def fake_run(
         probe_source_sha256=probe_sha256,
         snapshot=snapshot,
     )
-    assert calls[2][0][-5:] == ["up", "-d", "--no-build", "--pull", "never"]
+    assert calls[3][0][-3:] == ["config", "--format", "json"]
+    assert calls[4][0][-5:] == ["up", "-d", "--no-build", "--pull", "never"]
 
     settings.write_text(settings.read_text(encoding="utf-8").replace("goal09-test", "tampered"), encoding="utf-8")
     with pytest.raises(guard.GuardError, match="Compose settings changed"):
@@ -937,6 +943,298 @@ def test_real_compose_provider_preserves_snapshot_runner_paths_and_env(
     assert Path(mounts["/data/tool_registry_config"]["source"]).resolve() == tool_registry.resolve()
 
 
+@contextlib.contextmanager
+def _recording_docker_api():
+    """Record real Compose create requests without connecting to any daemon."""
+    created: dict[str, dict] = {}
+    creation_lock = threading.Lock()
+    started: set[str] = set()
+    unexpected: list[tuple[str, str]] = []
+    image_id = "sha256:" + "1" * 64
+
+    class Engine(http.server.BaseHTTPRequestHandler):
+        protocol_version = "HTTP/1.1"
+
+        def log_message(self, *_args):
+            pass
+
+        def do_HEAD(self):
+            self.respond()
+
+        def do_GET(self):
+            self.respond()
+
+        def do_POST(self):
+            self.respond()
+
+        def do_DELETE(self):
+            self.respond()
+
+        def respond(self):
+            url = urlsplit(self.path)
+            path = re.sub(r"^/v[0-9.]+", "", url.path)
+            query = parse_qs(url.query)
+            length = int(self.headers.get("Content-Length", "0"))
+            body = json.loads(self.rfile.read(length)) if length else None
+            code, headers = 200, {}
+            if path == "/_ping":
+                payload = b"OK"
+                headers = {"API-Version": "1.47", "OSType": "linux"}
+            elif path == "/version":
+                payload = {"ApiVersion": "1.47", "Version": "27.0.0", "Os": "linux", "Arch": "amd64"}
+            elif path.startswith("/images/") and path.endswith("/json"):
+                payload = {
+                    "Id": image_id,
+                    "Architecture": "amd64",
+                    "Os": "linux",
+                    "Config": {"Env": ["IMAGE_FIXTURE=present"], "Labels": {}},
+                    "RootFS": {"Type": "layers", "Layers": []},
+                    "Size": 1,
+                }
+            elif path == "/containers/json":
+                filters = json.loads(query.get("filters", ["{}"])[0])
+                payload = []
+                for identifier, record in created.copy().items():
+                    labels = record["request"]["Labels"]
+                    if any(
+                        (labels.get(key) != value if separator else key not in labels)
+                        for key, separator, value in (
+                            expression.partition("=")
+                            for expression in filters.get("label", [])
+                        )
+                    ):
+                        continue
+                    payload.append({
+                        "Id": identifier,
+                        "Names": ["/" + record["name"]],
+                        "Image": record["request"]["Image"],
+                        "ImageID": image_id,
+                        "State": "running" if identifier in started else "created",
+                        "Labels": labels,
+                        "HostConfig": {"NetworkMode": "none"},
+                        "NetworkSettings": {"Networks": {}},
+                        "Mounts": [],
+                    })
+            elif path == "/containers/create" and self.command == "POST":
+                with creation_lock:
+                    identifier = f"{len(created) + 1:064x}"
+                    created[identifier] = {"name": query["name"][0], "request": body}
+                code, payload = 201, {"Id": identifier, "Warnings": []}
+            elif path.startswith("/containers/") and path.endswith("/json"):
+                identifier = path.split("/")[2]
+                request = created[identifier]["request"]
+                payload = {
+                    "Id": identifier,
+                    "Name": "/" + created[identifier]["name"],
+                    "Image": image_id,
+                    "Config": {key: value for key, value in request.items() if key not in ("HostConfig", "NetworkingConfig")},
+                    "HostConfig": request.get("HostConfig", {}),
+                    "State": {"Status": "running" if identifier in started else "created", "Running": identifier in started, "ExitCode": 0},
+                    "Mounts": [],
+                    "NetworkSettings": {"Networks": {}},
+                }
+            elif path.startswith("/containers/") and path.endswith("/start"):
+                started.add(path.split("/")[2])
+                code, payload = 204, None
+            else:
+                unexpected.append((self.command, path))
+                code, payload = 404, {"message": "unexpected isolated fixture endpoint"}
+            raw = payload if isinstance(payload, bytes) else json.dumps(payload).encode() if payload is not None else b""
+            self.send_response(code)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(raw)))
+            for key, value in headers.items():
+                self.send_header(key, value)
+            self.end_headers()
+            if self.command != "HEAD" and raw:
+                self.wfile.write(raw)
+
+    class RecordingServer(http.server.ThreadingHTTPServer):
+        # Compose may open requests for every target together even when its
+        # dependency traversal is serial. Keep the local fixture backlog large
+        # enough for all 24 without dropping a creation request.
+        request_queue_size = 128
+
+    server = RecordingServer(("127.0.0.1", 0), Engine)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield f"tcp://127.0.0.1:{server.server_port}", created, unexpected
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=5)
+
+
+@pytest.mark.parametrize("action", ["targeted_up", "compose_start"])
+def test_real_compose_creation_uses_stable_sources_for_all_24_targets(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, action: str
+) -> None:
+    provider = _real_compose_provider()
+    project = tmp_path / "project-$literal"
+    docker = project / "docker"
+    data = docker / "workflow_data"
+    data.mkdir(parents=True)
+    registry = project / "tool-registry"
+    registry.mkdir()
+    socket = docker / "runtime.sock"
+    socket.touch()
+    bind_files = {
+        "db": ("./scripts/db-setup/db_setup.sh", "/docker-entrypoint-initdb.d/db_setup.sh"),
+        "reverse-proxy": ("./proxy_overrides.yaml", "/proxy_overrides.yaml"),
+    }
+    for relative, _ in bind_files.values():
+        path = docker / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text("fixture bind file\n")
+    runner_env = project / "runner" / ".env"
+    runner_env.parent.mkdir()
+    runner_env.write_text("ENV_FILE_VALUE=from-frozen-env-file\n")
+
+    def service_lines(service: str) -> list[str]:
+        result = [
+            f"  {service}:",
+            f"    image: fixture/{service}:original",
+            "    network_mode: none",
+            "    command: [sleep, '600']",
+            "    env_file:",
+            "      - ../runner/.env",
+            "    environment:",
+            "      DOTENV_VALUE: ${SOURCE_VALUE}",
+            "      PRIVATE_VALUE: source-value",
+            "    volumes:",
+            "      - ./workflow_data:/data:rw",
+            "      - ./workflow_data:/literal-$$target:ro",
+            "      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config:ro",
+            "      - ${SOCKET_SOURCE}:/var/run/docker.sock:ro",
+        ]
+        if service in bind_files:
+            relative, target = bind_files[service]
+            result.append(f"      - {relative}:{target}:ro")
+        return result
+
+    main = ["include:", "  - docker-compose-dev-essentials.yaml", "services:"]
+    included = ["services:"]
+    for service in guard.TARGET_SERVICES:
+        (included if service in guard.CORE_SERVICES else main).extend(service_lines(service))
+    (docker / "docker-compose.yaml").write_text("\n".join(main) + "\n")
+    (docker / "docker-compose-dev-essentials.yaml").write_text("\n".join(included) + "\n")
+    (docker / "compose.train.yaml").write_text("services: {}\n")
+    shutil.copy2(GUARD_PATH.parents[1] / "compose.train.healthchecks.yaml", docker / "health.yaml")
+    env_file = docker / ".env"
+    env_file.write_text(
+        "COMPOSE_PROJECT_NAME=stable-bind-fixture\n"
+        "SOURCE_VALUE=from-frozen-dotenv\n"
+        "TOOL_REGISTRY_CONFIG_SRC_PATH=${PWD}/../tool-registry\n"
+        "SOCKET_SOURCE=${PWD}/runtime.sock\n"
+    )
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n")
+    probe.chmod(0o755)
+    lock = {"images": {service: {"reference": f"fixture/{service}:locked"} for service in guard.TARGET_SERVICES}}
+    image_override = guard.candidate_image_override(lock, tmp_path / guard.CANDIDATE_IMAGE_FILENAME)
+    private_value = "opaque-$value ${literal} # fixture\nsecond line"
+    environment_override = tmp_path / guard.RUNTIME_ENVIRONMENT_FILENAME
+    guard.write_runtime_environment_override(
+        {service: {"PRIVATE_VALUE": private_value} for service in guard.TARGET_SERVICES},
+        environment_override,
+    )
+    settings = tmp_path / guard.COMPOSE_SETTINGS_FILENAME
+    guard.write_compose_settings("fixture", probe, settings, probe_source_sha256=guard.sha256_file(probe))
+    docker_config = tmp_path / "docker-config"
+    docker_config.mkdir()
+    snapshots: set[Path] = set()
+    launch_paths: list[Path] = []
+    with _recording_docker_api() as (endpoint, created, unexpected):
+        def provider_run(args: list[str], *, cwd: Path, env: dict[str, str], **kwargs) -> subprocess.CompletedProcess[str]:
+            assert args[:2] == ["docker", "compose"]
+            assert (cwd / "docker" / "workflow_data").is_symlink()
+            snapshots.add(cwd)
+            frozen_env = Path(args[args.index("--env-file") + 1])
+            assert frozen_env.is_relative_to(cwd)
+            assert frozen_env.read_bytes() == env_file.read_bytes()
+            if "up" in args:
+                override = next(
+                    (Path(value) for value in args if value.endswith("bind-sources.override.json")),
+                    None,
+                )
+                if override is not None:
+                    assert override.stat().st_mode & 0o777 == 0o400
+                    assert override.parent.stat().st_mode & 0o777 == 0o500
+                    assert all(set(value) == {"volumes"} for value in json.loads(override.read_text())["services"].values())
+                    launch_paths.append(override)
+            # The actual provider can reach only the isolated recording API.
+            # Its environment contains no ambient Docker context or secrets.
+            isolated_env = {
+                "PATH": os.defpath,
+                "DOCKER_HOST": endpoint,
+                "DOCKER_API_VERSION": "1.47",
+                "DOCKER_CONFIG": str(docker_config),
+                "COMPOSE_PARALLEL_LIMIT": "1",
+                "COMPOSE_ANSI": "never",
+                "COMPOSE_PROGRESS": "plain",
+                "PWD": str(docker),
+                "VERSION": env["VERSION"],
+                "UNSTRACT_HEALTHCHECK_SOURCE": env["UNSTRACT_HEALTHCHECK_SOURCE"],
+            }
+            result = subprocess.run([*provider, "--parallel", "1", *args[2:]], cwd=cwd, env=isolated_env, text=True, capture_output=True, timeout=45)
+            assert result.returncode == 0, result.stderr
+            return result
+
+        monkeypatch.setattr(guard, "run", provider_run)
+        monkeypatch.setattr(guard, "verify_candidate_source_state", lambda *_: None)
+        common = dict(
+            candidate_version="fixture",
+            probe_source=probe,
+            probe_source_sha256=guard.sha256_file(probe),
+            live_env_file=env_file,
+            image_override=image_override,
+            image_override_sha256=guard.sha256_file(image_override),
+            environment_override=environment_override,
+            environment_override_sha256=guard.sha256_file(environment_override),
+            settings_file=settings,
+            settings_file_sha256=guard.sha256_file(settings),
+        )
+        files = ("docker/docker-compose.yaml", "docker/health.yaml")
+        if action == "targeted_up":
+            guard.targeted_up(project, files, guard.TARGET_SERVICES, **common)
+        else:
+            guard.compose_start(project, files, expected_source=None, candidate_source=project, candidate_lock=lock, **common)
+
+    assert not unexpected
+    assert len(created) == 24
+    observed = {}
+    for record in created.values():
+        request = record["request"]
+        service = request["Labels"]["com.docker.compose.service"]
+        assert service not in observed
+        observed[service] = request
+        assert request["Image"] == lock["images"][service]["reference"]
+        assert request["HostConfig"]["NetworkMode"] == "none"
+        environment = dict(value.split("=", 1) for value in request["Env"])
+        assert environment["DOTENV_VALUE"] == "from-frozen-dotenv"
+        assert environment["ENV_FILE_VALUE"] == "from-frozen-env-file"
+        assert environment["PRIVATE_VALUE"] == private_value
+        # Compare raw provider creation strings. Resolving symlinks here would
+        # hide precisely the snapshot alias that caused the real failure.
+        mounts = {value.split(":")[1]: value.split(":") for value in request["HostConfig"]["Binds"]}
+        assert mounts["/data"] == [str(data), "/data", "rw"]
+        assert mounts["/literal-$target"] == [str(data), "/literal-$target", "ro"]
+        assert mounts["/data/tool_registry_config"] == [str(docker / ".." / "tool-registry"), "/data/tool_registry_config", "ro"]
+        assert mounts["/var/run/docker.sock"] == [str(socket), "/var/run/docker.sock", "ro"]
+        if service in bind_files:
+            relative, target = bind_files[service]
+            assert mounts[target] == [str(docker / relative), target, "ro"]
+        if service in guard.CORE_SERVICES:
+            source, target, mode = mounts[guard.PROBE_MOUNT_TARGET]
+            assert Path(source).is_relative_to(next(iter(snapshots)))
+            assert target == guard.PROBE_MOUNT_TARGET and mode == "ro"
+            assert request["Healthcheck"]["Test"] == guard._probe_test(service)
+    assert set(observed) == set(guard.TARGET_SERVICES)
+    assert len(launch_paths) == 1
+    assert all(not path.exists() for path in snapshots | set(launch_paths))
+
+
 def test_compose_config_maps_temporary_runner_data_bind_to_project_tree(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -986,6 +1284,22 @@ def fake_run(
         assert env["UNSTRACT_HEALTHCHECK_SOURCE"] == str(
             project_directory.parent / "__helper__" / "unstract-services.sh"
         )
+        override = next(
+            (Path(value) for value in args if value.endswith("bind-sources.override.json")),
+            None,
+        )
+        if override:
+            assert json.loads(override.read_text()) == {
+                "services": {
+                    "runner": {
+                        "volumes": [{
+                            "type": "bind",
+                            "source": str(workflow_data),
+                            "target": "/data",
+                        }]
+                    }
+                }
+            }
         return subprocess.CompletedProcess(
             args,
             0,
@@ -996,7 +1310,7 @@ def fake_run(
                             "volumes": [
                                 {
                                     "type": "bind",
-                                    "source": str(project_directory / "workflow_data"),
+                                    "source": str(workflow_data if override else project_directory / "workflow_data"),
                                     "target": "/data",
                                 },
                                 {
@@ -1103,6 +1417,101 @@ def fake_run(args: list[str], **_: object) -> subprocess.CompletedProcess[str]:
         )
 
 
+@pytest.mark.parametrize("failure", [None, "temporary-source", "options", "environment", "tamper"])
+def test_creation_bind_override_preserves_24_targets_and_rejects_drift(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str | None
+) -> None:
+    root = tmp_path / "snapshot"
+    root.mkdir()
+    live_data = tmp_path / "project-$literal" / "docker" / "workflow_data"
+    live_data.mkdir(parents=True)
+    snapshot = guard.ComposeSnapshot(
+        root=root,
+        manifest_path=root / "manifest.json",
+        project_dir=live_data.parents[1],
+        paths={"__probe_source__": str(root / "probe.sh")},
+        bind_sources={"docker/workflow_data": str(live_data)},
+    )
+    original = {
+        "services": {
+            service: {
+                "image": f"fixture/{service}:locked",
+                "environment": {"PRIVATE_VALUE": "literal-${untouched}"},
+                "volumes": [
+                    {
+                        "type": "bind",
+                        "source": str(root / "docker" / "workflow_data"),
+                        "target": "/data",
+                        "read_only": False,
+                        "bind": {"propagation": "rshared", "create_host_path": False},
+                    },
+                    {"type": "volume", "source": "persistent", "target": "/named"},
+                ],
+            }
+            for service in guard.TARGET_SERVICES
+        }
+    }
+    expected = deepcopy(original)
+    for definition in expected["services"].values():
+        definition["volumes"][0]["source"] = str(live_data)
+    overrides: list[Path] = []
+
+    def fake_run(args: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]:
+        assert args[-3:] == ["config", "--format", "json"]
+        override = next(
+            (Path(value) for value in args if value.endswith("bind-sources.override.json")),
+            None,
+        )
+        if override is None:
+            return subprocess.CompletedProcess(args, 0, json.dumps(original), "")
+        overrides.append(override)
+        assert override.stat().st_mode & 0o777 == 0o400
+        assert override.parent.stat().st_mode & 0o777 == 0o500
+        # This file carries only changed typed bind mounts, never private env
+        # values, images, named volumes, or the separately frozen probe mount.
+        written = json.loads(override.read_text())
+        assert set(written) == {"services"}
+        assert set(written["services"]) == set(guard.TARGET_SERVICES)
+        for service, definition in written["services"].items():
+            mount = deepcopy(expected["services"][service]["volumes"][0])
+            mount["source"] = mount["source"].replace("$", "$$")
+            assert definition == {"volumes": [mount]}
+        rendered = deepcopy(expected)
+        for definition in rendered["services"].values():
+            definition["volumes"][0]["source"] = str(live_data).replace("$", "$$")
+        if failure == "temporary-source":
+            rendered = original
+        elif failure == "options":
+            rendered["services"]["runner"]["volumes"][0]["read_only"] = True
+        elif failure == "environment":
+            rendered["services"]["runner"]["environment"]["PRIVATE_VALUE"] = "changed"
+        elif failure == "tamper":
+            os.chmod(override, 0o600)
+            override.write_text("{}")
+            os.chmod(override, 0o400)
+        return subprocess.CompletedProcess(args, 0, json.dumps(rendered), "")
+
+    monkeypatch.setattr(guard, "run", fake_run)
+    manager = guard.stable_bind_compose_config(
+        ["docker", "compose", "-f", str(root / "compose.yaml")],
+        snapshot,
+        env={},
+        deadline=None,
+        pass_fds=(),
+    )
+    if failure:
+        with pytest.raises(guard.GuardError, match="effective config|snapshot file changed"):
+            with manager:
+                pytest.fail("invalid creation input was made available to up")
+    else:
+        with manager as (creation_args, config):
+            assert config == expected
+            assert creation_args[-2:] == ["-f", str(overrides[0])]
+            assert overrides[0].is_file()
+    assert len(overrides) == 1
+    assert not overrides[0].exists()
+
+
 def test_compose_config_rejects_unrecognized_snapshot_bind_source(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:

From f3199e65d7c7564dae1d04a3363027e8dfb68393 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Wed, 9 Sep 2026 18:50:05 -0400
Subject: [PATCH 45/48] Preserve retained probes and runtime stop signals
 during guarded replay

---
 .../scripts/train_health_deployment_guard.py  |  16 ++
 .../test_train_health_deployment_guard.py     | 141 +++++++++++++++++-
 2 files changed, 151 insertions(+), 6 deletions(-)

diff --git a/docker/scripts/train_health_deployment_guard.py b/docker/scripts/train_health_deployment_guard.py
index d57c3a2376..fda62d531c 100644
--- a/docker/scripts/train_health_deployment_guard.py
+++ b/docker/scripts/train_health_deployment_guard.py
@@ -972,6 +972,10 @@ def load_compose_snapshot(path: Path) -> ComposeSnapshot:
         paths[key] = str(candidate)
     if "__probe_source__" not in paths:
         raise GuardError("Compose snapshot lacks the immutable probe source")
+    # This bind is executed by both root and non-root service users. A retained
+    # inode with the right bytes but owner-only permissions is not usable.
+    if stat.S_IMODE(Path(paths["__probe_source__"]).stat().st_mode) != 0o555:
+        raise GuardError("Compose snapshot probe must remain readable and executable with mode 0555")
     project_dir_value = manifest.get("project_dir")
     if not isinstance(project_dir_value, str) or not project_dir_value:
         raise GuardError("Compose snapshot project directory is invalid")
@@ -3917,10 +3921,17 @@ def rollback_override(
         reference = record.get("reference")
         if not reference:
             raise GuardError(f"backup image reference missing for {service}")
+        # Podman commit can omit image StopSignal metadata. Replaying that
+        # image alone then changes the container's shutdown behavior to TERM.
+        # The original runtime capture is authoritative for this option.
+        stop_signal = (record.get("old_options") or {}).get("stop_signal")
+        if type(stop_signal) is not int or not 1 <= stop_signal <= 64:
+            raise GuardError(f"backup runtime stop signal is missing or invalid for {service}")
         lines.extend(
             [
                 f"  {service}:",
                 f"    image: {json.dumps(reference)}",
+                f"    stop_signal: {json.dumps(str(stop_signal))}",
                 '    healthcheck: {test: ["NONE"]}',
             ]
         )
@@ -4321,6 +4332,10 @@ def compensating_rollback(
         quiescence_max_wait_seconds=POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS,
         quiescence_phase=f"post-compensating-rollback:{services[0]}",
     )
+    # Keep the actual comparison input even if a preserved contract fails.
+    # This observation does not claim verified compensation or quiescence
+    # beyond what the captured evidence itself establishes.
+    write_json(backup_dir / "post-compensating-rollback-observed.json", final)
     verify_rollback_result(
         baseline,
         final,
@@ -4437,6 +4452,7 @@ def apply_batch(
             settings_file=settings_file,
             settings_file_sha256=settings_file_sha256,
             probe_source_sha256=probe_source_sha256,
+            snapshot=snapshot,
             deadline=operation_deadline,
         )
         observed = capture(
diff --git a/tests/healthchecks/test_train_health_deployment_guard.py b/tests/healthchecks/test_train_health_deployment_guard.py
index 896512e067..0d21c4a72b 100644
--- a/tests/healthchecks/test_train_health_deployment_guard.py
+++ b/tests/healthchecks/test_train_health_deployment_guard.py
@@ -475,6 +475,7 @@ def test_durable_compose_inputs_are_private_and_reusable(tmp_path: Path) -> None
 def test_replay_manifest_binds_private_runtime_override_hash(tmp_path: Path) -> None:
     probe = tmp_path / "probe.sh"
     probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    probe.chmod(0o700)
     probe_sha256 = guard.sha256_file(probe)
     lock = {
         "schema": "unstract-health-candidate/v1",
@@ -566,6 +567,7 @@ def test_compose_replay_consumes_durable_settings_and_overrides(
 ) -> None:
     probe = tmp_path / "probe.sh"
     probe.write_text("#!/bin/sh\nprintf probe\n", encoding="utf-8")
+    probe.chmod(0o700)
     probe_sha256 = guard.sha256_file(probe)
     lock = {
         "images": {
@@ -848,6 +850,27 @@ def test_compose_snapshot_rejects_tampered_retained_input(tmp_path: Path) -> Non
         guard.load_compose_snapshot(snapshot.manifest_path)
 
 
+@pytest.mark.parametrize("mode", [0o500, 0o444, 0o700])
+def test_compose_snapshot_rejects_probe_permissions_unusable_by_service_users(
+    tmp_path: Path, mode: int
+) -> None:
+    (tmp_path / "compose.yaml").write_text("services: {}\n")
+    probe = tmp_path / "probe.sh"
+    probe.write_text("#!/bin/sh\nexit 0\n")
+    probe.chmod(0o700)
+    snapshot = guard.create_compose_snapshot(
+        state_dir=tmp_path / "state", project_dir=tmp_path,
+        compose_files=("compose.yaml",), live_env_file=None,
+        probe_source=probe, probe_source_sha256=guard.sha256_file(probe),
+        image_override=None, image_override_sha256=None,
+        environment_override=None, environment_override_sha256=None,
+    )
+    assert snapshot.probe_path.stat().st_mode & 0o777 == 0o555
+    snapshot.probe_path.chmod(mode)
+    with pytest.raises(guard.GuardError, match="writable or not regular|mode 0555"):
+        guard.load_compose_snapshot(snapshot.manifest_path)
+
+
 def test_real_compose_provider_preserves_snapshot_runner_paths_and_env(
     tmp_path: Path,
 ) -> None:
@@ -1066,7 +1089,7 @@ class RecordingServer(http.server.ThreadingHTTPServer):
         thread.join(timeout=5)
 
 
-@pytest.mark.parametrize("action", ["targeted_up", "compose_start"])
+@pytest.mark.parametrize("action", ["targeted_up", "compose_start", "rollback", "apply_batch"])
 def test_real_compose_creation_uses_stable_sources_for_all_24_targets(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch, action: str
 ) -> None:
@@ -1131,8 +1154,17 @@ def service_lines(service: str) -> list[str]:
     probe = tmp_path / "probe.sh"
     probe.write_text("#!/bin/sh\nexit 0\n")
     probe.chmod(0o755)
-    lock = {"images": {service: {"reference": f"fixture/{service}:locked"} for service in guard.TARGET_SERVICES}}
+    lock = {"candidate_version": "fixture", "images": {service: {"reference": f"fixture/{service}:locked"} for service in guard.TARGET_SERVICES}}
     image_override = guard.candidate_image_override(lock, tmp_path / guard.CANDIDATE_IMAGE_FILENAME)
+    stop_signals = {service: 2 if service == "db" else 3 if service == "frontend" else 15 for service in guard.TARGET_SERVICES}
+    if action == "rollback":
+        # The recording engine's backup-image Config has no StopSignal, just
+        # like the committed images that caused the actual recovery failure.
+        guard.rollback_override(
+            {"services": {service: {"reference": lock["images"][service]["reference"], "old_options": {"stop_signal": stop_signals[service]}} for service in guard.TARGET_SERVICES}},
+            image_override,
+            guard.TARGET_SERVICES,
+        )
     private_value = "opaque-$value ${literal} # fixture\nsecond line"
     environment_override = tmp_path / guard.RUNTIME_ENVIRONMENT_FILENAME
     guard.write_runtime_environment_override(
@@ -1195,9 +1227,35 @@ def provider_run(args: list[str], *, cwd: Path, env: dict[str, str], **kwargs) -
             settings_file=settings,
             settings_file_sha256=guard.sha256_file(settings),
         )
-        files = ("docker/docker-compose.yaml", "docker/health.yaml")
-        if action == "targeted_up":
+        files = ("docker/docker-compose.yaml",) if action == "rollback" else ("docker/docker-compose.yaml", "docker/health.yaml")
+        if action in {"targeted_up", "rollback"}:
             guard.targeted_up(project, files, guard.TARGET_SERVICES, **common)
+        elif action == "apply_batch":
+            retained = guard.create_compose_snapshot(
+                state_dir=tmp_path / "retained", project_dir=project,
+                compose_files=files, live_env_file=env_file,
+                probe_source=probe, probe_source_sha256=guard.sha256_file(probe),
+                image_override=image_override, image_override_sha256=guard.sha256_file(image_override),
+                environment_override=environment_override, environment_override_sha256=guard.sha256_file(environment_override),
+            )
+            retained_probe_before = retained.probe_path.stat()
+            baseline = {"source": {"live_inputs": guard.live_compose_inputs(project, live_env_file=env_file)}}
+            monkeypatch.setattr(guard, "advisory_lock", lambda **_kwargs: contextlib.nullcontext())
+            monkeypatch.setattr(guard, "capture", lambda *_args, **_kwargs: {"job_quiescence": {"quiescent": True, "stability": {"stable": True}}})
+            monkeypatch.setattr(guard, "candidate_image_snapshot", lambda *_args, **_kwargs: {})
+            for operation in ("compare_untargeted_runtime", "compare_source_and_quiescence", "verify_untouched_targets", "check_candidate_config", "wait_healthy", "compare_post_apply"):
+                monkeypatch.setattr(guard, operation, lambda *_args, **_kwargs: None)
+            monkeypatch.setattr(guard, "settled_queue_snapshot", lambda *_args, **_kwargs: {"stability": {"stable": True}})
+            monkeypatch.setattr(guard, "record_replacements", lambda *_args, **_kwargs: {"services": {service: {} for service in guard.TARGET_SERVICES}})
+            guard.apply_batch(
+                SimpleNamespace(project_dir=str(project), candidate_source=str(project), probe_source=str(probe), compose_file=files),
+                baseline, lock, tmp_path / "backup", image_override,
+                guard.TARGET_SERVICES, (), (),
+                settings_file=settings, settings_file_sha256=guard.sha256_file(settings),
+                image_override_sha256=guard.sha256_file(image_override), probe_source_sha256=guard.sha256_file(probe),
+                runtime_environment_override=environment_override, runtime_environment_sha256=guard.sha256_file(environment_override),
+                reviewed_environment_keys={}, snapshot=retained, operation_deadline=guard.OperationDeadline(120),
+            )
         else:
             guard.compose_start(project, files, expected_source=None, candidate_source=project, candidate_lock=lock, **common)
 
@@ -1225,14 +1283,43 @@ def provider_run(args: list[str], *, cwd: Path, env: dict[str, str], **kwargs) -
         if service in bind_files:
             relative, target = bind_files[service]
             assert mounts[target] == [str(docker / relative), target, "ro"]
-        if service in guard.CORE_SERVICES:
+        if action == "rollback":
+            assert request["StopSignal"] == str(stop_signals[service])
+            assert request["Healthcheck"]["Test"] == ["NONE"]
+            assert guard.PROBE_MOUNT_TARGET not in mounts
+        elif service in guard.CORE_SERVICES:
             source, target, mode = mounts[guard.PROBE_MOUNT_TARGET]
             assert Path(source).is_relative_to(next(iter(snapshots)))
+            if action == "apply_batch":
+                assert Path(source) == retained.probe_path
             assert target == guard.PROBE_MOUNT_TARGET and mode == "ro"
             assert request["Healthcheck"]["Test"] == guard._probe_test(service)
     assert set(observed) == set(guard.TARGET_SERVICES)
     assert len(launch_paths) == 1
-    assert all(not path.exists() for path in snapshots | set(launch_paths))
+    assert all(not path.exists() for path in launch_paths)
+    if action == "apply_batch":
+        assert snapshots == {retained.root}
+        assert guard.sha256_file(retained.probe_path) == guard.sha256_file(probe)
+        retained_probe_after = retained.probe_path.stat()
+        assert retained_probe_after.st_mode & 0o777 == 0o555
+        assert (retained_probe_after.st_dev, retained_probe_after.st_ino) == (retained_probe_before.st_dev, retained_probe_before.st_ino)
+        assert guard.load_compose_snapshot(retained.manifest_path) == retained
+    else:
+        assert all(not path.exists() for path in snapshots)
+
+
+@pytest.mark.parametrize("stop_signal", [None, False, 0, -1, 65, "SIGTERM"])
+def test_rollback_override_requires_recorded_runtime_stop_signal(
+    tmp_path: Path, stop_signal: object
+) -> None:
+    path = tmp_path / "rollback.yaml"
+    with pytest.raises(guard.GuardError, match="backup runtime stop signal is missing or invalid"):
+        guard.rollback_override(
+            {"services": {"db": {"reference": "fixture/db:backup", "old_options": {"stop_signal": stop_signal}}}},
+            path,
+            ("db",),
+        )
+    assert not path.exists()
 
 
 def test_compose_config_maps_temporary_runner_data_bind_to_project_tree(
@@ -1580,6 +1667,7 @@ def test_compose_snapshot_reloads_authoritative_bind_source_mapping(
     )
     probe = tmp_path / "probe.sh"
     probe.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+    probe.chmod(0o700)
 
     snapshot = guard.create_compose_snapshot(
         state_dir=tmp_path / "state",
@@ -2120,3 +2208,44 @@ def fake_capture(*_args, **kwargs):
         guard.POST_RECREATION_QUIESCENCE_MAX_WAIT_SECONDS
     )
     assert capture_kwargs[2]["quiescence_phase"] == "post-recreation:runner"
+
+
+def test_compensating_rollback_retains_actual_observation_when_verification_fails(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    actual = {"captured_at": "actual-post-compensation", "containers": [{"id": "actual-restored-id"}]}
+    captures = iter([{"phase": "before"}, {"phase": "locked"}, actual])
+    snapshot = SimpleNamespace(manifest_path=tmp_path / "retained-manifest.json")
+    environment = tmp_path / "runtime.yaml"
+    guard.write_private_text(environment, "services: {}\n", replace=False, description="fixture runtime")
+    monkeypatch.setattr(guard, "capture", lambda *_args, **_kwargs: next(captures))
+    monkeypatch.setattr(guard, "advisory_lock", lambda **_kwargs: contextlib.nullcontext())
+    monkeypatch.setattr(guard, "compare_source_and_quiescence", lambda *_args: None)
+    monkeypatch.setattr(guard, "verify_replacement_ids", lambda *_args: None)
+    monkeypatch.setattr(guard, "create_compose_snapshot", lambda **_kwargs: snapshot)
+    monkeypatch.setattr(guard, "wait_running", lambda *_args, **_kwargs: None)
+
+    def recreate(*_args, **kwargs):
+        assert kwargs["snapshot"] is snapshot
+
+    def fail_verification(_baseline, final, *_args, **_kwargs):
+        assert final is actual
+        assert json.loads((tmp_path / "post-compensating-rollback-observed.json").read_text()) == actual
+        raise guard.GuardError("runtime options changed for db")
+
+    monkeypatch.setattr(guard, "targeted_up", recreate)
+    monkeypatch.setattr(guard, "verify_rollback_result", fail_verification)
+    with pytest.raises(guard.GuardError, match="runtime options changed for db"):
+        guard.compensating_rollback(
+            SimpleNamespace(project_dir=str(tmp_path), probe_source=str(tmp_path / "probe.sh")),
+            {"source": {}},
+            {"services": {"db": {"reference": "fixture/db:backup", "old_options": {"stop_signal": 2}}}},
+            {"services": {"db": {"replacement_container_id": "candidate-id"}}},
+            tmp_path,
+            runtime_environment_override=environment,
+            runtime_environment_sha256=guard.sha256_file(environment),
+            operation_deadline=guard.OperationDeadline(60),
+        )
+    assert json.loads((tmp_path / "post-compensating-rollback-observed.json").read_text()) == actual
+    assert not (tmp_path / "post-compensating-rollback.json").exists()
+    assert not (tmp_path / "compensating-rollback.json").exists()

From bff5142c15bfdc21b4906dd8793b9a75df3d2b30 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 10 Sep 2026 08:34:12 -0400
Subject: [PATCH 46/48] runner: classify bounded runtime health failures

---
 .../src/unstract/runner/controller/health.py  | 120 +++++++++++--
 runner/tests/test_health.py                   | 168 ++++++++++++++++++
 2 files changed, 269 insertions(+), 19 deletions(-)

diff --git a/runner/src/unstract/runner/controller/health.py b/runner/src/unstract/runner/controller/health.py
index 1fbefaf4b5..ddb016e8a8 100644
--- a/runner/src/unstract/runner/controller/health.py
+++ b/runner/src/unstract/runner/controller/health.py
@@ -1,48 +1,130 @@
 import logging
+import re
 from typing import Any
 
+from docker import DockerClient
+from docker.errors import APIError
 from flask import Blueprint, jsonify
+from requests.exceptions import ConnectionError as RequestsConnectionError
+from requests.exceptions import Timeout as RequestsTimeout
 
 logger = logging.getLogger(__name__)
 
+RuntimeProbeFailure = tuple[str, str]
+
+# Docker SDK client construction negotiates the daemon API version before
+# ``ping`` runs. Keep both per-request budgets small enough that construction
+# plus ping remains inside the runner's three-second HTTP healthcheck timeout.
+_RUNTIME_PROBE_TIMEOUT_SECONDS = 1
+_MAX_ERROR_DETAIL_LENGTH = 32
+_SAFE_DETAIL = re.compile(r"[^a-z0-9_]+")
+
 # Define a Blueprint with a root URL path
 health_bp = Blueprint("health", __name__)
 
 
-def _container_runtime_ready() -> tuple[bool, str | None]:
-    """Perform a bounded, read-only ping against the mounted container socket."""
+def _safe_error_detail(value: str) -> str:
+    """Return a short identifier that cannot contain daemon-provided text."""
+    detail = re.sub(r"(?<!^)(?=[A-Z])", "_", value)
+    detail = _SAFE_DETAIL.sub("_", detail.lower()).strip("_")
+    return (detail or "unknown")[:_MAX_ERROR_DETAIL_LENGTH]
+
+
+def _exception_chain(exc: BaseException):
+    """Yield a bounded exception chain without exposing exception messages."""
+    current: BaseException | None = exc
+    seen: set[int] = set()
+    for _ in range(3):
+        if current is None or id(current) in seen:
+            return
+        seen.add(id(current))
+        yield current
+        current = current.__cause__ or current.__context__
+
+
+def _safe_exception_message(exc: BaseException) -> str:
+    """Read an exception marker without allowing a broken ``__str__`` to escape."""
     try:
-        # Import lazily so importing the Flask blueprint does not create a Docker
-        # client or touch the socket. ``ping`` only asks the daemon for liveness;
-        # it does not list, create, publish, or remove a tool container.
-        from docker import DockerClient
+        return str(exc).lower()[:256]
+    except Exception:
+        return ""
+
+
+def _runtime_probe_failure(exc: BaseException) -> RuntimeProbeFailure:
+    """Classify a Docker probe error using only bounded, safe identifiers."""
+    chain = tuple(_exception_chain(exc))
 
-        client = DockerClient.from_env(timeout=2)
+    for candidate in chain:
+        if isinstance(candidate, APIError):
+            status_code = candidate.status_code
+            if isinstance(status_code, int) and 100 <= status_code <= 599:
+                return "api", f"status_{status_code}"
+            return "api", "status_unknown"
+
+    for candidate in chain:
+        if isinstance(candidate, (RequestsTimeout, TimeoutError)):
+            return "timeout", _safe_error_detail(type(candidate).__name__)
+
+    for candidate in chain:
+        if isinstance(candidate, (RequestsConnectionError, ConnectionError)):
+            return "connection", _safe_error_detail(type(candidate).__name__)
+
+    # Docker SDK releases have wrapped transport failures in DockerException
+    # in different layers. Inspect only lower-cased text for fixed markers and
+    # never return or log the original message, which may contain socket paths.
+    message = _safe_exception_message(exc)
+    if "timeout" in message or "timed out" in message:
+        return "timeout", "docker_exception"
+    if "connection" in message or "connect" in message:
+        return "connection", "docker_exception"
+
+    return "docker", _safe_error_detail(type(exc).__name__)
+
+
+def _container_runtime_ready() -> tuple[bool, RuntimeProbeFailure | None]:
+    """Perform a bounded, read-only ping against the mounted container socket."""
+    try:
+        # Importing DockerClient does not create a client or touch the socket.
+        # ``ping`` only asks the daemon for liveness; it does not list, create,
+        # publish, or remove a tool container.
+        client = DockerClient.from_env(timeout=_RUNTIME_PROBE_TIMEOUT_SECONDS)
         try:
             client.ping()
         finally:
             client.close()
     except Exception as exc:
         # Keep credentials, socket paths, and daemon error text out of the HTTP
-        # body. The exception class is enough for an operator to identify the
-        # failed dependency while logs retain only the same sanitized class name.
-        logger.warning("Runner container runtime probe failed: %s", type(exc).__name__)
-        return False, type(exc).__name__
+        # body and logs. The fixed category/detail pair distinguishes transport
+        # failures without weakening the fail-closed readiness decision.
+        category, detail = _runtime_probe_failure(exc)
+        logger.warning(
+            "Runner container runtime probe failed category=%s detail=%s",
+            category,
+            detail,
+        )
+        return False, (category, detail)
     return True, None
 
 
 @health_bp.route("/health", methods=["GET"])
 def health_check() -> str | tuple[Any, int]:
-    runtime_ready, error_type = _container_runtime_ready()
+    runtime_ready, failure = _container_runtime_ready()
     if not runtime_ready:
+        if isinstance(failure, tuple):
+            error, error_detail = failure
+        else:
+            # Keep compatibility with callers that replace the private probe
+            # in tests or integrations with the original string result.
+            error, error_detail = failure or "unavailable", None
+        payload = {
+            "status": "unhealthy",
+            "dependency": "container_runtime",
+            "error": error,
+        }
+        if error_detail:
+            payload["error_detail"] = error_detail
         return (
-            jsonify(
-                {
-                    "status": "unhealthy",
-                    "dependency": "container_runtime",
-                    "error": error_type or "unavailable",
-                }
-            ),
+            jsonify(payload),
             503,
         )
     return "OK"
diff --git a/runner/tests/test_health.py b/runner/tests/test_health.py
index 846281b155..a8e79f9cbf 100644
--- a/runner/tests/test_health.py
+++ b/runner/tests/test_health.py
@@ -2,7 +2,12 @@
 
 from __future__ import annotations
 
+import time
+from types import SimpleNamespace
+
+from docker.errors import DockerException
 from flask import Flask
+from requests.exceptions import ConnectionError, ReadTimeout
 
 from unstract.runner.controller import health as health_module
 
@@ -37,3 +42,166 @@ def test_health_returns_503_without_runtime_readiness(monkeypatch):
         "dependency": "container_runtime",
         "error": "PermissionError",
     }
+
+
+def test_runtime_probe_reports_bounded_timeout_detail(monkeypatch):
+    client = SimpleNamespace(
+        ping=lambda: (_ for _ in ()).throw(ReadTimeout("/run/secrets/token")),
+        close=lambda: None,
+    )
+    monkeypatch.setattr(
+        health_module.DockerClient,
+        "from_env",
+        lambda *, timeout: client,
+    )
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert not ready
+    assert failure == ("timeout", "read_timeout")
+
+
+def test_runtime_probe_reports_connection_detail_and_keeps_fail_closed(monkeypatch):
+    client = SimpleNamespace(
+        ping=lambda: (_ for _ in ()).throw(ConnectionError("unix:///run/docker.sock")),
+        close=lambda: None,
+    )
+    monkeypatch.setattr(
+        health_module.DockerClient,
+        "from_env",
+        lambda *, timeout: client,
+    )
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert not ready
+    assert failure == ("connection", "connection_error")
+
+
+def test_health_exposes_api_status_without_exception_text(monkeypatch):
+    class FakeAPIError(health_module.APIError):
+        @property
+        def status_code(self):
+            return 503
+
+    monkeypatch.setattr(
+        health_module,
+        "_container_runtime_ready",
+        lambda: (False, health_module._runtime_probe_failure(FakeAPIError("secret"))),
+    )
+
+    response = _client().get("/v1/api/health")
+
+    assert response.status_code == 503
+    assert response.get_json() == {
+        "status": "unhealthy",
+        "dependency": "container_runtime",
+        "error": "api",
+        "error_detail": "status_503",
+    }
+
+
+def test_runtime_probe_sanitizes_unknown_exception_detail(monkeypatch):
+    class SecretTransportFailure(Exception):
+        pass
+
+    client = SimpleNamespace(
+        ping=lambda: (_ for _ in ()).throw(
+            SecretTransportFailure("password=do-not-return")
+        ),
+        close=lambda: None,
+    )
+    monkeypatch.setattr(
+        health_module.DockerClient,
+        "from_env",
+        lambda *, timeout: client,
+    )
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert not ready
+    assert failure == ("docker", "secret_transport_failure")
+    assert "password" not in str(failure)
+
+
+def test_runtime_probe_classifies_wrapped_docker_timeout_without_message(monkeypatch):
+    client = SimpleNamespace(
+        ping=lambda: (_ for _ in ()).throw(
+            DockerException("Error while pinging /run/secrets/token: timed out")
+        ),
+        close=lambda: None,
+    )
+    monkeypatch.setattr(
+        health_module.DockerClient,
+        "from_env",
+        lambda *, timeout: client,
+    )
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert not ready
+    assert failure == ("timeout", "docker_exception")
+    assert "/run/secrets/token" not in str(failure)
+
+
+def test_runtime_probe_passes_fixed_timeout_to_docker_client(monkeypatch):
+    client = SimpleNamespace(ping=lambda: None, close=lambda: None)
+    observed = {}
+
+    def from_env(*, timeout):
+        observed["timeout"] = timeout
+        return client
+
+    monkeypatch.setattr(health_module.DockerClient, "from_env", from_env)
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert (ready, failure) == (True, None)
+    assert observed == {"timeout": 1}
+
+
+def test_runtime_probe_construction_and_ping_fit_healthcheck_budget(monkeypatch):
+    observed = []
+
+    class SlowButHealthyClient:
+        def ping(self):
+            time.sleep(observed[-1] + 0.02)
+
+        def close(self):
+            return None
+
+    def from_env(*, timeout):
+        observed.append(timeout)
+        time.sleep(timeout + 0.02)
+        return SlowButHealthyClient()
+
+    monkeypatch.setattr(health_module.DockerClient, "from_env", from_env)
+
+    started = time.monotonic()
+    ready, failure = health_module._container_runtime_ready()
+    elapsed = time.monotonic() - started
+
+    assert (ready, failure) == (True, None)
+    assert observed == [1]
+    assert elapsed < 3
+
+
+def test_runtime_probe_survives_exception_with_broken_string_conversion(monkeypatch):
+    class BrokenStringFailure(Exception):
+        def __str__(self):
+            raise RuntimeError("string conversion failed")
+
+    client = SimpleNamespace(
+        ping=lambda: (_ for _ in ()).throw(BrokenStringFailure()),
+        close=lambda: None,
+    )
+    monkeypatch.setattr(
+        health_module.DockerClient,
+        "from_env",
+        lambda *, timeout: client,
+    )
+
+    ready, failure = health_module._container_runtime_ready()
+
+    assert not ready
+    assert failure == ("docker", "broken_string_failure")

From 2821ad066b1bea5ea0a83f0ad057960738e648f1 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 10 Sep 2026 20:14:37 -0400
Subject: [PATCH 47/48] Fix Unstract healthcheck timeout reaping

---
 docker/healthchecks/unstract-services.sh     | 22 +++----------
 tests/healthchecks/test_unstract_services.py | 33 ++++++++++++++++++++
 2 files changed, 38 insertions(+), 17 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index c6b88e6e48..fa08076d70 100755
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -77,25 +77,13 @@ terminate_process_group() {
     kill -KILL "$terminate_process_pid" >/dev/null 2>&1 || :
 }
 
-# A trapped signal can remain pending while a POSIX shell is blocked in a
-# foreground `wait`. Poll the child instead, so the shell gets a chance to run
-# its cleanup trap between short sleeps. The final wait only runs after the
-# child has exited and is therefore bounded even when a healthcheck is
-# terminated while its client owns one of the capture FIFOs.
+# The timeout wrapper owns the client deadline, so waiting for that wrapper is
+# bounded by the same deadline and, crucially, reaps it in this shell. Do not
+# poll with `kill -0`: on BusyBox, a child that has exited but is still a
+# zombie continues to satisfy `kill -0`, which can discard its PID without a
+# wait and leak one zombie on every health invocation.
 wait_for_child() {
     wait_for_child_pid=$1
-    wait_for_child_ticks=0
-    wait_for_child_limit=$((timeout_seconds * 20 + 40))
-    while kill -0 "$wait_for_child_pid" >/dev/null 2>&1; do
-        wait_for_child_ticks=$((wait_for_child_ticks + 1))
-        if [ "$wait_for_child_ticks" -ge "$wait_for_child_limit" ]; then
-            terminate_process_group "$wait_for_child_pid"
-            return 124
-        fi
-        # GNU and BusyBox sleep both support sub-second intervals; keeping the
-        # interval short bounds signal latency without a busy loop.
-        sleep 0.05
-    done
     wait "$wait_for_child_pid"
 }
 
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index 9c5f03cbe4..a4eac203e0 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -46,6 +46,39 @@ def test_probe_script_is_valid_posix_shell() -> None:
     assert result.returncode == 0, result.stderr
 
 
+def test_wait_for_child_reaps_timeout_wrapper() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+    function_start = source.index("wait_for_child() {")
+    function_end = source.index("\n}\n", function_start) + 3
+    function = source[function_start:function_end]
+
+    # The timeout command already bounds the client.  The shell must wait for
+    # that exact child rather than polling kill(0), which treats BusyBox
+    # zombies as live and leaks one unreaped timeout per health run.
+    assert 'wait "$wait_for_child_pid"' in function
+    assert "kill -0" not in function
+    assert "sleep" not in function
+
+
+def test_successful_probe_reaps_early_timeout_wrapper(tmp_path: Path) -> None:
+    timeout_pid = tmp_path / "timeout.pid"
+    timeout = write_fake(
+        tmp_path,
+        "timeout",
+        f'printf "%s" "$$" > "{timeout_pid}"; shift; "$@"',
+    )
+    redis_cli = write_fake(tmp_path, "redis-cli", 'printf "PONG\\n"')
+
+    result = run_probe(
+        "redis",
+        {"TIMEOUT_BIN": str(timeout), "REDIS_CLI_BIN": str(redis_cli)},
+    )
+    assert result.returncode == 0, result.stderr
+    child_pid = int(timeout_pid.read_text(encoding="utf-8"))
+    with pytest.raises(ProcessLookupError):
+        os.kill(child_pid, 0)
+
+
 def test_weaviate_requires_metadata_and_ready_status(tmp_path: Path) -> None:
     wget = write_fake(
         tmp_path,

From aac206a1f51aa8a8631d4be3a947137a26275e20 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Thu, 10 Sep 2026 20:25:20 -0400
Subject: [PATCH 48/48] Bound health probe clients with SIGKILL

---
 docker/healthchecks/unstract-services.sh     | 21 +++++++-----
 tests/healthchecks/test_unstract_services.py | 36 ++++++++++++++++++--
 2 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/docker/healthchecks/unstract-services.sh b/docker/healthchecks/unstract-services.sh
index fa08076d70..4ef2f39f0d 100755
--- a/docker/healthchecks/unstract-services.sh
+++ b/docker/healthchecks/unstract-services.sh
@@ -62,6 +62,9 @@ rabbitmq_diagnostics_bin=${RABBITMQ_DIAGNOSTICS_BIN:-rabbitmq-diagnostics}
 pg_isready_bin=${PG_ISREADY_BIN:-pg_isready}
 psql_bin=${PSQL_BIN:-psql}
 
+# Use an uncatchable deadline signal. The default TERM signal can be ignored by
+# a client, leaving the timeout wrapper alive while wait_for_child waits for it;
+# GNU and BusyBox timeout both support the portable `-s KILL` form.
 # The timeout wrapper starts client commands in their own process group on the
 # supported GNU and BusyBox implementations.  A healthcheck can be signalled
 # while its shell is waiting for that wrapper; terminate both the group and
@@ -158,7 +161,7 @@ bounded_wget() {
     bounded_wget_body_reader_pid=$!
     "$head_bin" -c 16385 <"$bounded_wget_header_fifo" >"$bounded_wget_headers" &
     bounded_wget_header_reader_pid=$!
-    "$timeout_bin" "$timeout_seconds" "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" \
+    "$timeout_bin" -s KILL "$timeout_seconds" "$wget_bin" -qS -O- -t 1 -T "$timeout_seconds" "$bounded_wget_url" \
         >"$bounded_wget_body_fifo" 2>"$bounded_wget_header_fifo" &
     bounded_wget_client_pid=$!
     if wait_for_child "$bounded_wget_client_pid"; then
@@ -249,7 +252,7 @@ bounded_curl() {
     "$mkfifo_bin" "$bounded_curl_body_fifo" >/dev/null 2>&1 || return 1
     "$head_bin" -c "$((bounded_curl_limit + 1))" <"$bounded_curl_body_fifo" >"$bounded_curl_body_file" &
     bounded_curl_body_reader_pid=$!
-    "$timeout_bin" "$timeout_seconds" "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
+    "$timeout_bin" -s KILL "$timeout_seconds" "$curl_bin" -fsS --location --max-redirs 0 --max-filesize "$bounded_curl_limit" \
         --max-time "$timeout_seconds" "$bounded_curl_url" >"$bounded_curl_body_fifo" &
     bounded_curl_client_pid=$!
     if wait_for_child "$bounded_curl_client_pid"; then
@@ -342,7 +345,7 @@ bounded_exec() {
     "$head_bin" -c "$((bounded_exec_limit + 1))" <"$bounded_exec_body_fifo" \
         >"$bounded_exec_body_file" &
     bounded_exec_reader_pid=$!
-    "$timeout_bin" "$timeout_seconds" "$@" >"$bounded_exec_body_fifo" 2>/dev/null &
+    "$timeout_bin" -s KILL "$timeout_seconds" "$@" >"$bounded_exec_body_fifo" 2>/dev/null &
     bounded_exec_client_pid=$!
     if wait_for_child "$bounded_exec_client_pid"; then
         bounded_exec_status=0
@@ -393,7 +396,7 @@ probe_vector_db() {
     # The Qdrant image does not ship curl/wget. Its Debian base does ship Bash,
     # so use Bash's TCP client to exercise the real REST health endpoint. The
     # response is bounded and matched on both HTTP status and body semantics.
-    "$timeout_bin" "$timeout_seconds" "$qdrant_bash_bin" -ec '
+    "$timeout_bin" -s KILL "$timeout_seconds" "$qdrant_bash_bin" -ec '
         host=$1
         port=$2
         case "$host" in
@@ -439,8 +442,8 @@ probe_proxy() {
 }
 
 probe_rabbitmq() {
-    "$timeout_bin" "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_running >/dev/null 2>&1 || fail
-    "$timeout_bin" "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_local_alarms >/dev/null 2>&1 || fail
+    "$timeout_bin" -s KILL "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_running >/dev/null 2>&1 || fail
+    "$timeout_bin" -s KILL "$timeout_seconds" "$rabbitmq_diagnostics_bin" -q check_local_alarms >/dev/null 2>&1 || fail
 }
 
 probe_minio() {
@@ -454,7 +457,7 @@ probe_minio() {
 probe_db() {
     db_user=${POSTGRES_USER:-postgres}
     db_name=${POSTGRES_DB:-postgres}
-    "$timeout_bin" "$timeout_seconds" "$pg_isready_bin" -t "$timeout_seconds" -U "$db_user" -d "$db_name" >/dev/null 2>&1 || fail
+    "$timeout_bin" -s KILL "$timeout_seconds" "$pg_isready_bin" -t "$timeout_seconds" -U "$db_user" -d "$db_name" >/dev/null 2>&1 || fail
     bounded_exec 16 "$psql_bin" -XAtqc 'SELECT 1' -U "$db_user" -d "$db_name" || fail
     result=$("$head_bin" -c 16 "$bounded_exec_result_file") || {
         bounded_exec_finish
@@ -467,7 +470,7 @@ probe_db() {
 probe_python_body() {
     url=$1
     expected=$2
-    "$timeout_bin" "$timeout_seconds" "$python_bin" -c '
+    "$timeout_bin" -s KILL "$timeout_seconds" "$python_bin" -c '
 import sys
 import urllib.request
 
@@ -494,7 +497,7 @@ probe_platform() {
 }
 
 probe_backend() {
-    "$timeout_bin" "$timeout_seconds" "$python_bin" -c '
+    "$timeout_bin" -s KILL "$timeout_seconds" "$python_bin" -c '
 import json
 import os
 import sys
diff --git a/tests/healthchecks/test_unstract_services.py b/tests/healthchecks/test_unstract_services.py
index a4eac203e0..cb1af85fd0 100644
--- a/tests/healthchecks/test_unstract_services.py
+++ b/tests/healthchecks/test_unstract_services.py
@@ -65,7 +65,14 @@ def test_successful_probe_reaps_early_timeout_wrapper(tmp_path: Path) -> None:
     timeout = write_fake(
         tmp_path,
         "timeout",
-        f'printf "%s" "$$" > "{timeout_pid}"; shift; "$@"',
+        f'''
+if [ "$1" = "-s" ] && [ "$2" = "KILL" ]; then
+    shift 2
+fi
+printf "%s" "$$" > "{timeout_pid}"
+shift
+"$@"
+''',
     )
     redis_cli = write_fake(tmp_path, "redis-cli", 'printf "PONG\\n"')
 
@@ -114,7 +121,14 @@ def test_timeout_configuration_is_capped(tmp_path: Path) -> None:
     timeout = write_fake(
         tmp_path,
         "timeout",
-        f'printf "%s" "$1" > "{timeout_record}"; shift; "$@"',
+        f'''
+if [ "$1" = "-s" ] && [ "$2" = "KILL" ]; then
+    shift 2
+fi
+printf "%s" "$1" > "{timeout_record}"
+shift
+"$@"
+''',
     )
     redis_cli = write_fake(tmp_path, "redis-cli", 'printf "PONG\\n"')
     result = run_probe(
@@ -129,6 +143,24 @@ def test_timeout_configuration_is_capped(tmp_path: Path) -> None:
     assert timeout_record.read_text(encoding="utf-8") == "30"
 
 
+def test_native_probe_hard_deadline_kills_term_ignoring_client(tmp_path: Path) -> None:
+    redis_cli = write_fake(
+        tmp_path,
+        "redis-cli",
+        'trap "" TERM\nwhile :; do :; done',
+    )
+    started = time.monotonic()
+    result = run_probe(
+        "redis",
+        {
+            "REDIS_CLI_BIN": str(redis_cli),
+            "HEALTHCHECK_TIMEOUT_SECONDS": "1",
+        },
+    )
+    assert result.returncode != 0
+    assert time.monotonic() - started < 3
+
+
 def test_redis_response_and_total_deadline_are_bounded(tmp_path: Path) -> None:
     redis_cli = write_fake(
         tmp_path,