From 372f8a30e9361bcbe9bef06822c822f62241e1fb Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Fri, 4 Sep 2026 19:45:31 -0400
Subject: [PATCH 1/2] Start rootless Unstract through bounded Compose readiness
 phases

---
 docker/scripts/start-rootless-stack.sh        | 35 ++++++++
 .../tests/test_start_rootless_stack.py        | 82 +++++++++++++++++++
 docker/systemd/README.md                      | 78 ++++++++++++++++++
 docker/systemd/unstract-compose.service       | 14 ++++
 4 files changed, 209 insertions(+)
 create mode 100755 docker/scripts/start-rootless-stack.sh
 create mode 100644 docker/scripts/tests/test_start_rootless_stack.py
 create mode 100644 docker/systemd/README.md
 create mode 100644 docker/systemd/unstract-compose.service

diff --git a/docker/scripts/start-rootless-stack.sh b/docker/scripts/start-rootless-stack.sh
new file mode 100755
index 0000000000..ed14438d48
--- /dev/null
+++ b/docker/scripts/start-rootless-stack.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Start a reviewed, already deployed Compose stack without rebuilding or replacing it.
+set -euo pipefail
+
+: "${UNSTRACT_PROJECT_DIRECTORY:?Set the absolute directory containing the deployed Compose files}"
+: "${COMPOSE_FILE:?Set the deployed Compose file list}"
+: "${COMPOSE_PROJECT_NAME:?Set the existing Compose project name}"
+case "$UNSTRACT_PROJECT_DIRECTORY" in
+  /*) ;;
+  *) echo "UNSTRACT_PROJECT_DIRECTORY must be absolute" >&2; exit 2 ;;
+esac
+case "${UNSTRACT_START_TIMEOUT:-300}" in
+  ''|*[!0-9]*|0) echo "UNSTRACT_START_TIMEOUT must be a positive number of seconds" >&2; exit 2 ;;
+esac
+export DOCKER_HOST="${DOCKER_HOST:-unix://${XDG_RUNTIME_DIR:?}/podman/podman.sock}"
+compose=(docker compose --project-directory "$UNSTRACT_PROJECT_DIRECTORY")
+"${compose[@]}" config --quiet
+services=$("${compose[@]}" config --services)
+
+# Database-dependent application imports can fail before the worker starts.
+# Gate those imports on dependency readiness, not Podman's arbitrary ID order.
+dependencies=()
+for service in db redis rabbitmq minio milvus-etcd milvus-minio; do
+  if grep -Fxq "$service" <<< "$services"; then
+    dependencies+=("$service")
+  fi
+done
+if ((${#dependencies[@]} == 0)); then
+  echo "No expected Unstract dependencies in the selected Compose project" >&2
+  exit 2
+fi
+up=(up --detach --no-recreate --no-build --pull never --wait
+    --wait-timeout "${UNSTRACT_START_TIMEOUT:-300}")
+"${compose[@]}" "${up[@]}" "${dependencies[@]}"
+"${compose[@]}" "${up[@]}"
diff --git a/docker/scripts/tests/test_start_rootless_stack.py b/docker/scripts/tests/test_start_rootless_stack.py
new file mode 100644
index 0000000000..4e0385ff74
--- /dev/null
+++ b/docker/scripts/tests/test_start_rootless_stack.py
@@ -0,0 +1,82 @@
+"""Exercise startup sequencing without starting containers or contacting a host."""
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).resolve().parents[1] / "start-rootless-stack.sh"
+FAKE_DOCKER = """#!/usr/bin/env python3
+import json, os, sys
+args = sys.argv[1:]
+with open(os.environ['CALL_LOG'], 'a') as log:
+    log.write(json.dumps(args) + '\\n')
+if args[-2:] == ['config', '--services']:
+    print(os.environ.get('TEST_SERVICES', 'db\\nredis\\nx2text-service'))
+if 'up' in args and args[-1] == 'redis':
+    sys.exit(int(os.environ.get('DEPENDENCY_EXIT', '0')))
+"""
+
+
+class StartupTests(unittest.TestCase):
+    def run_start(self, **overrides):
+        import json
+
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            executable = root / "docker"
+            executable.write_text(FAKE_DOCKER)
+            executable.chmod(0o700)
+            log = root / "calls"
+            env = {
+                **os.environ,
+                "PATH": f"{root}:{os.environ['PATH']}",
+                "UNSTRACT_PROJECT_DIRECTORY": "/existing/docker",
+                "COMPOSE_FILE": "/existing/docker/docker-compose.yaml",
+                "COMPOSE_PROJECT_NAME": "existing-project",
+                "DOCKER_HOST": "unix:///run/user/1000/podman/podman.sock",
+                "CALL_LOG": str(log),
+                **overrides,
+            }
+            result = subprocess.run(
+                ["bash", str(SCRIPT)], env=env, capture_output=True, text=True
+            )
+            calls = (
+                [json.loads(line) for line in log.read_text().splitlines()]
+                if log.exists()
+                else []
+            )
+            return result, calls
+
+    def test_waits_for_dependencies_before_application(self):
+        result, calls = self.run_start()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        starts = [call for call in calls if "up" in call]
+        self.assertEqual(len(starts), 2)
+        self.assertEqual(starts[0][-2:], ["db", "redis"])
+        self.assertEqual(starts[1][-2:], ["--wait-timeout", "300"])
+        for command in starts:
+            self.assertIn("--no-recreate", command)
+            self.assertIn("--no-build", command)
+            self.assertIn("--wait", command)
+            self.assertEqual(command[command.index("--pull") + 1], "never")
+
+    def test_dependency_failure_never_starts_application(self):
+        result, calls = self.run_start(DEPENDENCY_EXIT="17")
+        self.assertEqual(result.returncode, 17)
+        self.assertEqual(len([call for call in calls if "up" in call]), 1)
+
+    def test_unknown_project_fails_before_starting(self):
+        result, calls = self.run_start(TEST_SERVICES="unrelated-service")
+        self.assertEqual(result.returncode, 2)
+        self.assertFalse(any("up" in call for call in calls))
+
+    def test_bad_timeout_fails_before_contacting_runtime(self):
+        result, calls = self.run_start(UNSTRACT_START_TIMEOUT="unbounded")
+        self.assertEqual(result.returncode, 2)
+        self.assertEqual(calls, [])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/docker/systemd/README.md b/docker/systemd/README.md
new file mode 100644
index 0000000000..3f810d26d3
--- /dev/null
+++ b/docker/systemd/README.md
@@ -0,0 +1,78 @@
+# Rootless startup
+
+Podman's generic restart service starts containers in container-ID order. It
+does not interpret Compose `depends_on` or wait for database readiness. A Python
+service that opens its database while importing its application can exit before
+the database has joined the network. A standalone vector database can likewise
+fail during dependency startup.
+
+`unstract-compose.service` delegates startup to the existing Docker Compose
+provider connected to the user's Podman socket. The script first waits for the
+configured database, broker, cache, and object-storage healthchecks, then starts
+the rest of the configured project. Dependency failure aborts startup. Its two
+phases each have a 300-second default convergence budget.
+If `UNSTRACT_START_TIMEOUT` is increased, raise the unit's `TimeoutStartSec`
+above twice that value plus configuration/startup overhead. The unit does not
+retry a failed stack automatically: inspect the failing dependency, resolve it,
+then explicitly restart this unit. Repeated blind starts can obscure the cause.
+
+## Install after reviewing the deployment
+
+Keep the unit disabled until the host's generic restart helper excludes this
+exact Compose project from its start path. Preserve the existing shutdown path
+and every other project's startup behavior. Otherwise two supervisors race each other
+and the original dependency-order problem remains possible. This repository does
+not overwrite a host-wide helper. User lingering and the Podman user socket must
+already be enabled; installing this unit does not change host policy.
+
+Copy `scripts/start-rootless-stack.sh` into
+`~/.local/libexec/unstract/start-rootless-stack.sh` and the unit into
+`~/.config/systemd/user/unstract-compose.service`. Create a mode-0600
+`~/.config/unstract/compose.env` containing the *existing* project's values:
+
+```ini
+UNSTRACT_PROJECT_DIRECTORY=/absolute/path/to/checkout/docker
+COMPOSE_FILE=/absolute/path/to/checkout/docker/docker-compose.yaml:/absolute/path/to/checkout/docker/compose.override.yaml
+COMPOSE_PROJECT_NAME=existing-project-name
+VERSION=existing-deployed-image-tag
+```
+
+Preserve any additional deployment variables, image overrides and profiles used
+by the established deployment. Do not copy secret values into this document or
+publish the environment file. Compose reads the existing service environment
+files from the selected checkout. Confirm the selected files with `docker
+compose config --quiet`, without logging rendered secrets.
+
+Deploy the reviewed healthcheck definitions through the established narrow
+service rollout first. `--no-recreate` deliberately does not apply new checks to
+existing containers. It also cannot repair stale OCI runtime state: inspect and
+recover that container separately before enabling startup. Keep application
+image tags aligned with the actual deployed images.
+
+When deployment overrides replace Traefik's command, retain `--ping=true` so its
+local healthcheck is enabled. When Nginx listens on an internal port other than
+80, set `FRONTEND_HEALTH_PORT` in the frontend service's environment to that
+internal port (for example, `8080`). Changing only a published host port does
+not change the internal healthcheck port.
+
+The startup script uses `up --no-recreate --no-build --pull never --wait`:
+existing containers retain their images and settings, while a missing service
+can be created only from an already available image. This is a startup command,
+not an upgrade command. Inspect the configured profiles and service set before
+running it. A completed bootstrap service is expected to exit successfully;
+other services must converge according to their configured readiness checks.
+
+After reviewing the generic-helper exclusion, run `systemctl --user
+daemon-reload`, then `systemctl --user enable --now unstract-compose.service`.
+Check the unit result, service readiness, three scheduled healthcheck results,
+and the application route. Report boot configuration review separately from an
+actual reboot test. No reboot is needed to install the unit.
+
+## Rollback
+
+Back up any prior unit and configuration before replacement. To roll back this
+startup integration, disable and stop `unstract-compose.service`, restore those
+files and the previous generic-helper configuration, and reload the user daemon.
+Stopping this oneshot unit does not stop or remove application containers. Check
+the restored startup configuration and live readiness. Retain all volumes and
+existing images; never use `down -v` or pruning for this rollback.
diff --git a/docker/systemd/unstract-compose.service b/docker/systemd/unstract-compose.service
new file mode 100644
index 0000000000..f35ed79e99
--- /dev/null
+++ b/docker/systemd/unstract-compose.service
@@ -0,0 +1,14 @@
+[Unit]
+Description=Start the deployed Unstract rootless Compose stack in dependency order
+Requires=podman.socket
+After=podman.socket
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+EnvironmentFile=%h/.config/unstract/compose.env
+ExecStart=/bin/bash %h/.local/libexec/unstract/start-rootless-stack.sh
+TimeoutStartSec=700
+
+[Install]
+WantedBy=default.target

From 22e85c57b3036f62d2c09e8588d12eb325359651 Mon Sep 17 00:00:00 2001
From: Timothy Wayne Gregg <5861166+CompleteDotTech@users.noreply.github.com>
Date: Fri, 4 Sep 2026 19:49:23 -0400
Subject: [PATCH 2/2] Check service readiness and worker progress in Compose

---
 docker/HEALTHCHECKS.md                        | 80 +++++++++++++++++
 docker/docker-compose-dev-essentials.yaml     | 48 +++++++++-
 docker/docker-compose.yaml                    | 73 ++++++++++++++-
 .../tests/test_compose_healthchecks.py        | 88 +++++++++++++++++++
 workers/log_consumer/heartbeat.py             | 62 +++++++++++++
 workers/log_consumer/redis_stream_consumer.py |  5 +-
 workers/log_consumer/scheduler.sh             |  8 +-
 workers/tests/test_log_consumer_heartbeat.py  | 36 ++++++++
 workers/tests/test_log_stream_consumer.py     | 31 +++++--
 9 files changed, 418 insertions(+), 13 deletions(-)
 create mode 100644 docker/HEALTHCHECKS.md
 create mode 100644 docker/scripts/tests/test_compose_healthchecks.py
 create mode 100644 workers/log_consumer/heartbeat.py
 create mode 100644 workers/tests/test_log_consumer_heartbeat.py

diff --git a/docker/HEALTHCHECKS.md b/docker/HEALTHCHECKS.md
new file mode 100644
index 0000000000..7c6d062de0
--- /dev/null
+++ b/docker/HEALTHCHECKS.md
@@ -0,0 +1,80 @@
+# Compose health checks
+
+The Compose files report application HTTP liveness, database readiness, and
+worker progress separately. A healthy container does not prove a successful
+document extraction, authenticated API operation, or delivery to every log sink.
+Unhealthy status alone does not restart a container; these probes also do not
+repair stale Podman process metadata.
+
+| Services | Signal |
+| --- | --- |
+| Backend | `/api/v1/health` returns 200 or its expected unauthenticated 401. This is HTTP liveness, not database or authentication readiness. |
+| Frontend | Nginx serves `/` successfully on its internal port. |
+| Platform, x2text, runner | Their existing local HTTP health endpoint answers successfully. |
+| PG queue workers and reaper | Existing `/health` endpoint, which checks queue-loop freshness and, for prefork workers, child progress. Existing long-task stale thresholds remain in effect. |
+| Log stream consumer | A successful Redis poll or completed envelope disposition refreshes a local heartbeat. Redis errors do not refresh it. Poison-envelope disposal counts as loop progress; individual sink delivery is not proven. |
+| Log history scheduler | Both history processing and notification-buffer processing must finish successfully and refresh separate heartbeats. |
+| PostgreSQL instances | `pg_isready` over TCP, excluding the initialization-only Unix-socket server. |
+| Redis | `redis-cli ping` must return `PONG`. |
+| RabbitMQ | The application must be running and its configured listener ports reachable. |
+| MinIO | The existing local readiness endpoint must return success. |
+| Traefik | Built-in `traefik healthcheck --ping`. This checks the proxy itself, not every routed upstream. |
+| Qdrant | `/readyz` must return HTTP 200, using the Bash shipped in the image. |
+| Weaviate | `/v1/.well-known/ready` via the pinned Alpine image's BusyBox wget. |
+| Milvus, its etcd and MinIO | Existing service-specific health probes are retained. |
+
+Log heartbeats allow a 120-second processing grace beyond the configured poll
+interval. Repeated failed or blocked processing is intentionally unhealthy;
+process existence does not refresh a heartbeat. Probe startup grace is 90 seconds
+for workers. Backend migration startup receives 120 seconds. PostgreSQL health
+gates backend, platform-service, and x2text startup.
+
+## Deployment requirements
+
+- Rebuild and deploy `worker-unified` from this source before enabling the log
+  heartbeat probes. They require `log_consumer/heartbeat.py` and the updated
+  scheduler/consumer in the image. Merely changing Compose on an old image will
+  fail these probes.
+- Frontend health defaults to internal port 80. A rootless deployment with an
+  Nginx override on port 8080 must set `FRONTEND_HEALTH_PORT=8080` in the frontend
+  container environment. Host published-port numbers are not health ports.
+- A deployment overlay replacing Traefik's `command` must retain `--ping=true`.
+  The base Compose command now enables it.
+- Apply changes to the final merged Compose configuration and recreate affected
+  containers. Container restart alone does not install new health configuration.
+- Preserve named volumes, encryption keys, credentials, and unrelated deployment
+  overlays. Never use `down -v` to apply health checks.
+
+The `minio-bootstrap` service is an intentional one-shot job and has no health
+probe. Its successful exit is required by the backend dependency. Optional
+`celery-flower` and `unstructured-io` profiles are outside the observed active
+fleet. `feature-flag` was absent from that fleet; no unverified check is added for
+it. These services must not be described as newly health-verified.
+
+## Verification evidence and limits
+
+Read-only probes on Train verified backend's expected 401, frontend HTTP success
+on port 8080, platform and runner HTTP 200, representative PG worker/reaper HTTP
+200, Redis PONG, PostgreSQL TCP readiness, MinIO readiness, and Qdrant HTTP 200.
+Several containers had stale runtime process metadata, preventing execution of
+RabbitMQ, Traefik, and Weaviate probes during this inspection. Weaviate tooling
+is supported by the [pinned upstream Dockerfile](https://github.com/weaviate/weaviate/blob/v1.39.2/Dockerfile);
+it still requires verification in the recovered live container. x2text was
+unavailable, so its endpoint was checked in source only.
+
+Local tests execute the actual Compose HTTP commands against isolated servers,
+including HTTP failures and a closed listener. Worker tests cover missing,
+stale, future-dated, and partially successful scheduler heartbeats, existing log
+delivery behavior, and Redis failures that must not refresh progress. These are
+component checks, not live deployment or end-to-end extraction evidence.
+
+```sh
+python3 -m unittest discover -s docker/scripts/tests -p test_compose_healthchecks.py
+PYTHONPATH=workers:unstract/core/src uv run --no-project --with pytest --with redis \
+  pytest --noconftest -o addopts= -q \
+  workers/tests/test_log_consumer_heartbeat.py workers/tests/test_log_stream_consumer.py
+bash -n workers/log_consumer/scheduler.sh
+```
+
+The first command requires PyYAML; alternatively use
+`uv run --no-project --with pyyaml python -m unittest discover -s docker/scripts/tests -p test_compose_healthchecks.py`.
diff --git a/docker/docker-compose-dev-essentials.yaml b/docker/docker-compose-dev-essentials.yaml
index 8e8937c93c..f15b8f57ab 100644
--- a/docker/docker-compose-dev-essentials.yaml
+++ b/docker/docker-compose-dev-essentials.yaml
@@ -2,6 +2,13 @@ services:
   db:
     image: "pgvector/pgvector:pg15"
     container_name: unstract-db
+    healthcheck:
+      # TCP avoids reporting the entrypoint's temporary Unix-socket server ready.
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U \"$${POSTGRES_USER:-postgres}\" -d \"$${POSTGRES_DB:-postgres}\""]
+      interval: 10s
+      timeout: 5s
+      start_period: 30s
+      retries: 5
     restart: unless-stopped
     # set shared memory limit when using docker-compose
     shm_size: 128mb
@@ -18,6 +25,11 @@ services:
   redis:
     image: "redis:7.2.3"
     container_name: unstract-redis
+    healthcheck:
+      test: ["CMD-SHELL", "test \"$$(redis-cli ping)\" = PONG"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
     restart: unless-stopped
     # uncomment below command if persistance required.
     #command: redis-server --save 20 1 --loglevel warning --
@@ -31,6 +43,12 @@ services:
   minio:
     image: "minio/minio:latest"
     container_name: unstract-minio
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "--max-time", "5", "-o", "/dev/null", "http://127.0.0.1:9000/minio/health/ready"]
+      interval: 30s
+      timeout: 10s
+      start_period: 30s
+      retries: 3
     hostname: minio
     restart: unless-stopped
     ports:
@@ -65,6 +83,12 @@ services:
     # The official v2 Traefik docker image
     image: traefik:v3.6.2
     container_name: unstract-proxy
+    healthcheck:
+      test: ["CMD", "traefik", "healthcheck", "--ping"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     restart: unless-stopped
     # - Enables the web UI.
     # - Tells Traefik to use docker and file providers.
@@ -73,6 +97,7 @@ services:
     #   round-robin fashion. With multiple providers, services can be on multiple
     #   networks causing 504 Gateway Timeout.
     command: --api.insecure=true
+      --ping=true
       --accesslog=true --log.level=INFO
       --providers.docker=true --providers.docker.network=unstract-network
       --providers.file.filename=/proxy_overrides.yaml --providers.file.watch=true
@@ -122,6 +147,14 @@ services:
     # the 1.16 client calls endpoints the 1.8 server lacks and returns 404 with empty body.
     image: "qdrant/qdrant:v1.16.1"
     container_name: unstract-vector-db
+    healthcheck:
+      # This image ships bash but neither curl nor wget. Check the HTTP status,
+      # rather than accepting an open TCP socket or any /readyz response.
+      test: ["CMD", "bash", "-ec", "exec 3<>/dev/tcp/127.0.0.1/6333; printf 'GET /readyz HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n' >&3; read -r status <&3; [[ $$status == 'HTTP/1.0 200 '* || $$status == 'HTTP/1.1 200 '* ]]"]
+      interval: 30s
+      timeout: 5s
+      start_period: 30s
+      retries: 3
     restart: unless-stopped
     ports:
       - "127.0.0.1:6333:6333"
@@ -154,7 +187,7 @@ services:
       # retaining a separate data directory and container.
       - ./essentials.env
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U \"$${POSTGRES_USER:-postgres}\" -d \"$${POSTGRES_DB:-postgres}\""]
       interval: 10s
       timeout: 5s
       retries: 5
@@ -167,6 +200,13 @@ services:
   weaviate:
     image: "docker.io/semitechnologies/weaviate:1.39.2"
     container_name: unstract-weaviate
+    healthcheck:
+      # The pinned Alpine image includes BusyBox wget; this is server readiness.
+      test: ["CMD", "wget", "-q", "-T", "5", "-O", "/dev/null", "http://127.0.0.1:8080/v1/.well-known/ready"]
+      interval: 30s
+      timeout: 10s
+      start_period: 60s
+      retries: 3
     restart: unless-stopped
     command:
       - --host
@@ -272,6 +312,12 @@ services:
   rabbitmq:
     image: rabbitmq:4.1.0-management
     container_name: unstract-rabbitmq
+    healthcheck:
+      test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_connectivity"]
+      interval: 30s
+      timeout: 20s
+      start_period: 60s
+      retries: 3
     hostname: unstract-rabbit
     restart: unless-stopped
     env_file:
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
index 9f6ca7789d..d0536a63a6 100644
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@@ -13,12 +13,40 @@ x-host-gateway: &host_gateway
   extra_hosts:
     - "host.docker.internal:host-gateway"
 
+# PG workers expose poll-loop freshness (including prefork children), not just
+# a listening socket. The existing per-worker stale thresholds cover long tasks.
+x-pg-health: &pg_health
+  test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8090/health', timeout=5)"]
+  interval: 30s
+  timeout: 10s
+  start_period: 90s
+  retries: 3
+
 services:
   # Backend service
   backend:
     <<: *host_gateway
     image: unstract/backend:${VERSION}
     container_name: unstract-backend
+    # This protected route returns 401 without credentials: HTTP liveness only.
+    # Database readiness is checked on db below.
+    healthcheck:
+      test:
+        - CMD
+        - python
+        - -c
+        - |
+          import urllib.request, urllib.error
+          try:
+              response = urllib.request.urlopen('http://127.0.0.1:8000/api/v1/health', timeout=5)
+              assert response.status == 200
+          except urllib.error.HTTPError as error:
+              if error.code != 401:
+                  raise
+      interval: 30s
+      timeout: 10s
+      start_period: 120s
+      retries: 3
     restart: unless-stopped
     command: --migrate
     ports:
@@ -27,7 +55,7 @@ services:
       - ../backend/.env
     depends_on:
       db:
-        condition: service_started
+        condition: service_healthy
       redis:
         condition: service_started
       rabbitmq:
@@ -99,6 +127,13 @@ services:
   frontend:
     image: unstract/frontend:${VERSION}
     container_name: unstract-frontend
+    healthcheck:
+      # Rootless deployments that remap Nginx must set its internal health port.
+      test: ["CMD-SHELL", "curl -fsS --max-time 5 -o /dev/null http://127.0.0.1:$${FRONTEND_HEALTH_PORT:-80}/"]
+      interval: 30s
+      timeout: 10s
+      start_period: 30s
+      retries: 3
     restart: unless-stopped
     ports:
       - "3000:80"
@@ -118,27 +153,36 @@ services:
   platform-service:
     image: unstract/platform-service:${VERSION}
     container_name: unstract-platform-service
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:3001/health', timeout=5)"]
     restart: unless-stopped
     ports:
       - "3001:3001"
     env_file:
       - ../platform-service/.env
     depends_on:
-      - redis
-      - db
+      redis:
+        condition: service_healthy
+      db:
+        condition: service_healthy
     labels:
       - traefik.enable=false
 
   x2text-service:
     image: unstract/x2text-service:${VERSION}
     container_name: unstract-x2text-service
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:3004/api/v1/x2text/health', timeout=5)"]
     restart: unless-stopped
     ports:
       - "3004:3004"
     env_file:
       - ../x2text-service/.env
     depends_on:
-      - db
+      db:
+        condition: service_healthy
     labels:
       - traefik.enable=false
 
@@ -146,6 +190,9 @@ services:
     <<: *host_gateway
     image: unstract/runner:${VERSION}
     container_name: unstract-runner
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5002/v1/api/health', timeout=5)"]
     restart: unless-stopped
     ports:
       - 5002:5002
@@ -170,6 +217,9 @@ services:
     <<: *host_gateway
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-log-history-scheduler-v2
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "/app/.venv/bin/python", "-m", "log_consumer.heartbeat", "check", "scheduler"]
     restart: unless-stopped
     entrypoint: ["/bin/bash"]
     command: ["/app/log_consumer/scheduler.sh"]
@@ -194,6 +244,7 @@ services:
   worker-pg-orchestrator-api:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-orchestrator-api
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -230,6 +281,7 @@ services:
   worker-pg-orchestrator-general:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-orchestrator-general
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -263,6 +315,7 @@ services:
   worker-pg-fileproc:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-fileproc
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -304,6 +357,7 @@ services:
   worker-pg-callback:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-callback
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -343,6 +397,7 @@ services:
   worker-pg-scheduler:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-scheduler
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -391,6 +446,7 @@ services:
   worker-pg-metrics:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-metrics
+    healthcheck: *pg_health
     restart: unless-stopped
     # The GENERIC consumer command, with identity carried by the env below — the
     # container entrypoint (run-worker-docker.sh) is env-driven and its `pg-*`
@@ -442,6 +498,9 @@ services:
   worker-log-stream-consumer:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-log-stream-consumer
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "/app/.venv/bin/python", "-m", "log_consumer.heartbeat", "check", "log-stream"]
     restart: unless-stopped
     command: ["log-stream-consumer"]
     env_file:
@@ -471,6 +530,7 @@ services:
   worker-pg-executor:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-executor
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -523,6 +583,7 @@ services:
   worker-pg-ide-callback:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-ide-callback
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -564,6 +625,7 @@ services:
   worker-pg-notification:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-notification
+    healthcheck: *pg_health
     restart: unless-stopped
     command: ["pg-queue-consumer"]
     ports:
@@ -604,6 +666,9 @@ services:
   worker-pg-reaper:
     image: unstract/worker-unified:${VERSION}
     container_name: unstract-worker-pg-reaper
+    healthcheck:
+      <<: *pg_health
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8086/health', timeout=5)"]
     restart: unless-stopped
     command: ["pg-queue-reaper"]
     ports:
diff --git a/docker/scripts/tests/test_compose_healthchecks.py b/docker/scripts/tests/test_compose_healthchecks.py
new file mode 100644
index 0000000000..9e0a1db2e4
--- /dev/null
+++ b/docker/scripts/tests/test_compose_healthchecks.py
@@ -0,0 +1,88 @@
+"""Exercise actual Compose probe commands against isolated HTTP responses.
+
+Run with: uv run --no-project --with pyyaml python -m unittest discover \
+    -s docker/scripts/tests -p test_compose_healthchecks.py
+"""
+
+import http.server
+import pathlib
+import subprocess
+import sys
+import threading
+import unittest
+
+import yaml
+
+DOCKER = pathlib.Path(__file__).resolve().parents[2]
+
+
+class Response(http.server.BaseHTTPRequestHandler):
+    status = 200
+
+    def do_GET(self):
+        self.send_response(self.status)
+        self.end_headers()
+
+    def log_message(self, *_args):
+        pass
+
+
+class ComposeHealthTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.main = yaml.safe_load((DOCKER / "docker-compose.yaml").read_text())
+        cls.essentials = yaml.safe_load(
+            (DOCKER / "docker-compose-dev-essentials.yaml").read_text()
+        )
+        cls.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Response)
+        cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
+        cls.thread.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.server.shutdown()
+        cls.server.server_close()
+        cls.thread.join()
+
+    def run_probe(self, probe, old_port, status):
+        Response.status = status
+        args = [
+            part.replace(str(old_port), str(self.server.server_port))
+            for part in probe[1:]
+        ]
+        args = [part.replace("$$", "$") for part in args]
+        if args[0] == "python":
+            args[0] = sys.executable
+        return subprocess.run(args, capture_output=True, timeout=10).returncode
+
+    def test_pg_worker_rejects_http_failure(self):
+        probe = self.main["services"]["worker-pg-executor"]["healthcheck"]["test"]
+        self.assertEqual(self.run_probe(probe, 8090, 200), 0)
+        self.assertNotEqual(self.run_probe(probe, 8090, 503), 0)
+
+    def test_backend_allows_only_documented_statuses(self):
+        probe = self.main["services"]["backend"]["healthcheck"]["test"]
+        for status in (200, 401):
+            self.assertEqual(self.run_probe(probe, 8000, status), 0)
+        for status in (403, 404, 500):
+            self.assertNotEqual(self.run_probe(probe, 8000, status), 0)
+
+    def test_qdrant_requires_ready_status(self):
+        probe = self.essentials["services"]["qdrant"]["healthcheck"]["test"]
+        self.assertEqual(self.run_probe(probe, 6333, 200), 0)
+        self.assertNotEqual(self.run_probe(probe, 6333, 503), 0)
+
+    def test_pg_worker_rejects_closed_listener(self):
+        probe = self.main["services"]["worker-pg-executor"]["healthcheck"]["test"]
+        listener = http.server.HTTPServer(("127.0.0.1", 0), Response)
+        port = listener.server_port
+        listener.server_close()
+        command = probe[-1].replace(":8090/", f":{port}/")
+        result = subprocess.run(
+            [sys.executable, "-c", command], capture_output=True, timeout=10
+        )
+        self.assertNotEqual(result.returncode, 0)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/workers/log_consumer/heartbeat.py b/workers/log_consumer/heartbeat.py
new file mode 100644
index 0000000000..56475a617c
--- /dev/null
+++ b/workers/log_consumer/heartbeat.py
@@ -0,0 +1,62 @@
+"""Local progress signals for workers without an HTTP health server.
+
+Only successful polling/task cycles refresh these files; process existence or a
+separate successful dependency probe must not hide a wedged worker loop.
+Log-stream progress does not prove that every downstream sink accepted a log;
+the consumer can intentionally discard poison envelopes and retry sink work.
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+DIRECTORY = Path("/tmp")
+NAMES = {"log-stream", "log-history", "notification-buffer"}
+
+
+def mark(name: str) -> None:
+    if name not in NAMES:
+        raise ValueError("Unknown heartbeat")
+    (DIRECTORY / f"unstract-{name}.heartbeat").touch()
+
+
+def healthy(names: list[str], max_age: float) -> bool:
+    now = time.time()
+    try:
+        return bool(names) and all(
+            name in NAMES
+            and 0
+            <= now - (DIRECTORY / f"unstract-{name}.heartbeat").stat().st_mtime
+            <= max_age
+            for name in names
+        )
+    except OSError:
+        return False
+
+
+def main() -> int:
+    if len(sys.argv) == 3 and sys.argv[1] == "mark" and sys.argv[2] in NAMES:
+        mark(sys.argv[2])
+        return 0
+    if sys.argv[1:] == ["check", "log-stream"]:
+        max_age = max(1, float(os.getenv("LOG_STREAM_BLOCK_TIMEOUT", "5"))) + 120
+        return 0 if healthy(["log-stream"], max_age) else 1
+    if sys.argv[1:] == ["check", "scheduler"]:
+        bounds = (
+            ("log-history", "LOG_HISTORY_CONSUMER_INTERVAL", 5),
+            ("notification-buffer", "NOTIFICATION_BUFFER_POLL_INTERVAL", 10),
+        )
+        return (
+            0
+            if all(
+                healthy([name], max(1, float(os.getenv(variable, str(default)))) + 120)
+                for name, variable, default in bounds
+            )
+            else 1
+        )
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/workers/log_consumer/redis_stream_consumer.py b/workers/log_consumer/redis_stream_consumer.py
index a2242f11f2..f23707890c 100644
--- a/workers/log_consumer/redis_stream_consumer.py
+++ b/workers/log_consumer/redis_stream_consumer.py
@@ -37,7 +37,6 @@
 from shared.enums.worker_enums import WorkerType
 from shared.infrastructure.config.builder import WorkerBuilder
 from shared.infrastructure.logging import WorkerLogger
-
 from unstract.core.cache.redis_client import create_redis_client
 from unstract.core.constants import LogProcessingTask
 
@@ -111,6 +110,8 @@ def _dispatch(raw: bytes | str) -> None:
 
 
 def run() -> int:
+    from log_consumer.heartbeat import mark
+
     signal.signal(signal.SIGTERM, _handle_signal)
     signal.signal(signal.SIGINT, _handle_signal)
 
@@ -141,6 +142,7 @@ def run() -> int:
             continue
 
         if raw is None:  # timeout, no work — loop so shutdown can be observed
+            mark("log-stream")
             continue
 
         try:
@@ -154,6 +156,7 @@ def run() -> int:
             # Remove exactly one copy, whether it succeeded or was discarded — leaving it
             # parked would have it re-queued on the next restart and replayed forever.
             redis_client.lrem(processing, 1, raw)
+            mark("log-stream")
 
     logger.info("Log stream consumer stopped")
     return 0
diff --git a/workers/log_consumer/scheduler.sh b/workers/log_consumer/scheduler.sh
index e09a82dbd7..9832c7943d 100755
--- a/workers/log_consumer/scheduler.sh
+++ b/workers/log_consumer/scheduler.sh
@@ -75,13 +75,17 @@ while true; do
 
     if [[ $((now - last_log_run)) -ge "${LOG_HISTORY_INTERVAL}" ]]; then
         run_count=$((run_count + 1))
-        run_task "process_log_history" "${LOG_HISTORY_CMD}" "${run_count}"
+        if run_task "process_log_history" "${LOG_HISTORY_CMD}" "${run_count}"; then
+            /app/.venv/bin/python -m log_consumer.heartbeat mark log-history
+        fi
         last_log_run="${now}"
     fi
 
     if [[ $((now - last_buffer_run)) -ge "${NOTIFICATION_BUFFER_INTERVAL}" ]]; then
         run_count=$((run_count + 1))
-        run_task "process_notification_buffer" "${BUFFER_FLUSH_CMD}" "${run_count}"
+        if run_task "process_notification_buffer" "${BUFFER_FLUSH_CMD}" "${run_count}"; then
+            /app/.venv/bin/python -m log_consumer.heartbeat mark notification-buffer
+        fi
         last_buffer_run="${now}"
     fi
 
diff --git a/workers/tests/test_log_consumer_heartbeat.py b/workers/tests/test_log_consumer_heartbeat.py
new file mode 100644
index 0000000000..a68ac2c407
--- /dev/null
+++ b/workers/tests/test_log_consumer_heartbeat.py
@@ -0,0 +1,36 @@
+"""A fresh process alone must not pass worker progress checks."""
+
+import os
+import time
+
+from log_consumer import heartbeat
+
+
+def test_missing_fresh_and_stale_heartbeat(tmp_path, monkeypatch):
+    monkeypatch.setattr(heartbeat, "DIRECTORY", tmp_path)
+    assert not heartbeat.healthy(["log-stream"], 120)
+    heartbeat.mark("log-stream")
+    assert heartbeat.healthy(["log-stream"], 120)
+    stale = time.time() - 121
+    os.utime(tmp_path / "unstract-log-stream.heartbeat", (stale, stale))
+    assert not heartbeat.healthy(["log-stream"], 120)
+
+
+def test_both_scheduler_tasks_must_report_success(tmp_path, monkeypatch):
+    monkeypatch.setattr(heartbeat, "DIRECTORY", tmp_path)
+    monkeypatch.setattr(heartbeat.sys, "argv", ["heartbeat", "check", "scheduler"])
+    heartbeat.mark("log-history")
+    assert heartbeat.main() == 1
+    heartbeat.mark("notification-buffer")
+    assert heartbeat.main() == 0
+    stale = time.time() - 500
+    os.utime(tmp_path / "unstract-notification-buffer.heartbeat", (stale, stale))
+    assert heartbeat.main() == 1
+
+
+def test_future_timestamp_cannot_mask_failure(tmp_path, monkeypatch):
+    monkeypatch.setattr(heartbeat, "DIRECTORY", tmp_path)
+    heartbeat.mark("log-stream")
+    future = time.time() + 500
+    os.utime(tmp_path / "unstract-log-stream.heartbeat", (future, future))
+    assert not heartbeat.healthy(["log-stream"], 120)
diff --git a/workers/tests/test_log_stream_consumer.py b/workers/tests/test_log_stream_consumer.py
index 8c3f3091e6..8b6c53e56b 100644
--- a/workers/tests/test_log_stream_consumer.py
+++ b/workers/tests/test_log_stream_consumer.py
@@ -58,6 +58,7 @@ def _mod(name, **attrs):
             WorkerLogger=types.SimpleNamespace(setup=lambda _t: MagicMock()),
         ),
         "log_consumer": _mod("log_consumer"),
+        "log_consumer.heartbeat": _mod("log_consumer.heartbeat", mark=MagicMock()),
         "log_consumer.tasks": _mod("log_consumer.tasks", logs_consumer=logs_consumer),
     }
     for name, mod in stubs.items():
@@ -69,6 +70,7 @@ def _mod(name, **attrs):
     mod = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(mod)
     mod._test_logs_consumer = logs_consumer
+    mod._test_heartbeat = stubs["log_consumer.heartbeat"].mark
     return mod
 
 
@@ -78,7 +80,9 @@ def consumer(monkeypatch):
 
 
 def _envelope(task="logs_consumer", **kwargs):
-    return json.dumps({"task": task, "kwargs": kwargs or {"event": "logs:c", "room": "c"}})
+    return json.dumps(
+        {"task": task, "kwargs": kwargs or {"event": "logs:c", "room": "c"}}
+    )
 
 
 class TestDispatch:
@@ -108,7 +112,12 @@ def test_startup_requeues_what_the_previous_run_left_in_flight(self, consumer):
         consumer._recover_in_flight(redis, "proc")
         assert redis.lmove.call_count == 3
         # Back to the HEAD of the source list, so recovered logs precede newer ones.
-        assert redis.lmove.call_args_list[0][0] == ("proc", "log_stream_queue", "RIGHT", "LEFT")
+        assert redis.lmove.call_args_list[0][0] == (
+            "proc",
+            "log_stream_queue",
+            "RIGHT",
+            "LEFT",
+        )
 
     def _one_shot_redis(self, consumer, raw):
         """A redis mock that yields exactly one envelope, then ends the loop.
@@ -142,6 +151,20 @@ def test_envelope_is_removed_only_after_the_handler_returns(self, consumer):
         # a crash, which is exactly the acks_late behaviour this replaces.
         assert order == ["handled", "lrem"]
         redis.lrem.assert_called_once_with("log_stream_queue:processing:pod-abc", 1, raw)
+        consumer._test_heartbeat.assert_called_with("log-stream")
+
+    def test_redis_failure_does_not_refresh_progress(self, consumer):
+        redis = MagicMock()
+        redis.lmove.return_value = None
+
+        def fail_poll(*_args):
+            consumer._shutdown = True
+            raise ConnectionError("Redis unavailable")
+
+        redis.blmove.side_effect = fail_poll
+        with patch.object(consumer, "create_redis_client", return_value=redis):
+            consumer.run()
+        consumer._test_heartbeat.assert_not_called()
 
     def test_a_poison_envelope_is_dropped_not_replayed_forever(self, consumer):
         raw = b"not-json"
@@ -186,9 +209,7 @@ def _blmove(*_a, **_k):
 
         redis.blmove.side_effect = _blmove
 
-        with patch.object(
-            consumer, "create_redis_client", return_value=redis
-        ) as factory:
+        with patch.object(consumer, "create_redis_client", return_value=redis) as factory:
             consumer.run()
 
         assert factory.call_args.kwargs["socket_timeout"] == (