From 5a30daf759a18e7225c53bc4fff1d4d64a9fe68b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 13:43:51 +0000 Subject: [PATCH 1/2] Give shepherd_db's healthcheck a start_period so slow boots don't fail the stack `docker compose up` intermittently died with "dependency shepherd_db failed to start". The DB healthcheck had no start_period, so Docker counted probe failures from container start: 5 retries x 2s marked Postgres unhealthy about 10s in, which is often before it has even begun accepting connections. The "FATAL: the database system is starting up" line in the logs is a probe being correctly rejected mid-recovery, and it lands on the last retry. Also probe 127.0.0.1 instead of the default Unix socket. On a first run the postgres entrypoint starts a temporary socket-only server to apply init_db.sql; a socket probe reports ready against that, dependents connect, and the entrypoint then shuts it down under them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PQeAtdPFUgZ1BTB3UB6oXc --- compose.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index bd6d77d..4a80742 100644 --- a/compose.yml +++ b/compose.yml @@ -33,11 +33,33 @@ services: ports: - "5432:5432" restart: unless-stopped + # Two things were wrong here, and together they made ``docker compose up`` + # fail intermittently with "dependency shepherd_db failed to start". + # + # 1. No ``start_period``. Docker counts healthcheck failures from the moment + # the container starts, so 5 retries x 2s flipped the DB to *unhealthy* + # barely 10s in -- and Postgres routinely needs longer than that. Before + # it even logs "starting PostgreSQL" the entrypoint stats/chowns the + # bind-mounted ./pgdata, and then crash recovery still has to run; with + # ~25 containers booting at once and contending for IO, that easily eats + # the whole budget. The "FATAL: the database system is starting up" line + # in the logs is exactly that: a probe correctly rejected mid-recovery, + # which then happened to be the last retry. Inside ``start_period`` + # failures don't count against ``retries`` (and one success ends it + # early), so a slow-but-healthy boot no longer trips the check while a + # genuinely broken DB still fails, just 60s later. + # 2. ``pg_isready`` over the default Unix socket. On a first run (empty + # ./pgdata) the entrypoint boots a *temporary* server to apply + # docker-entrypoint-initdb.d/init_db.sql; it listens on the socket only. + # A socket probe reports ready against that temp server, dependents + # connect, and the entrypoint then shuts it down under them. Probing + # 127.0.0.1 can only ever see the real server. healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d postgres"] interval: 2s timeout: 2s retries: 5 + start_period: 60s volumes: - ./pgdata:/var/lib/postgresql/data shepherd_broker: From 7dda5e8f39c473ea3814307735e7a9d07ef134db Mon Sep 17 00:00:00 2001 From: Max Wang Date: Mon, 24 Aug 2026 10:12:12 -0400 Subject: [PATCH 2/2] Remove unnecessary comments --- compose.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/compose.yml b/compose.yml index 4a80742..7a048ef 100644 --- a/compose.yml +++ b/compose.yml @@ -33,27 +33,6 @@ services: ports: - "5432:5432" restart: unless-stopped - # Two things were wrong here, and together they made ``docker compose up`` - # fail intermittently with "dependency shepherd_db failed to start". - # - # 1. No ``start_period``. Docker counts healthcheck failures from the moment - # the container starts, so 5 retries x 2s flipped the DB to *unhealthy* - # barely 10s in -- and Postgres routinely needs longer than that. Before - # it even logs "starting PostgreSQL" the entrypoint stats/chowns the - # bind-mounted ./pgdata, and then crash recovery still has to run; with - # ~25 containers booting at once and contending for IO, that easily eats - # the whole budget. The "FATAL: the database system is starting up" line - # in the logs is exactly that: a probe correctly rejected mid-recovery, - # which then happened to be the last retry. Inside ``start_period`` - # failures don't count against ``retries`` (and one success ends it - # early), so a slow-but-healthy boot no longer trips the check while a - # genuinely broken DB still fails, just 60s later. - # 2. ``pg_isready`` over the default Unix socket. On a first run (empty - # ./pgdata) the entrypoint boots a *temporary* server to apply - # docker-entrypoint-initdb.d/init_db.sql; it listens on the socket only. - # A socket probe reports ready against that temp server, dependents - # connect, and the entrypoint then shuts it down under them. Probing - # 127.0.0.1 can only ever see the real server. healthcheck: test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres -d postgres"] interval: 2s