From f9fe70dd202c74fbca4d498a6f923db9a0b385e1 Mon Sep 17 00:00:00 2001
From: dougc95 <dougcr95@gmail.com>
Date: Mon, 28 Sep 2026 06:18:20 -0400
Subject: [PATCH 1/4] ci(bench): benchmark all six storage backends (#1475)

Extends fhir-benchmark.yml from sqlite + postgres to every storage
backend with and without Elasticsearch: sqlite, sqlite-elasticsearch,
postgres, postgres-elasticsearch, mongodb and mongodb-elasticsearch.
Bare s3 is left out because it has no search, so the import and
search suites cannot run on it; s3-elasticsearch is a follow-up.

Dispatch:
- backend: core (default; sqlite + postgres), all (6 legs),
  elasticsearch (the 3 composites), or any single backend.
- max_parallel (default 1, max 2): legs share one 12 GB / 4-CPU
  Docker host, so running them together skews every leg's numbers.
- es_heap, es_sync_mode (asynchronous = HFS default | synchronous),
  mongo_wt_cache_gb, hfs_mongo_max_connections. The setup job
  validates them all before the build starts.

Per leg:
- MongoDB runs as a single-member replica set (transaction bundles
  need one), with a capped WiredTiger cache and a 900 s transaction
  lifetime to match HFS_REQUEST_TIMEOUT.
- Elasticsearch runs single-node. Yellow health is expected, because
  HFS creates every index with one replica.
- A capacity gate waits up to 10 minutes for enough free memory on
  the Docker host and fails the leg rather than risk an OOM.
- Containers and volumes carry the hfs-bench, hfs-ci and leg labels,
  so the reaper, docker-host-gc and the leak check all cover them.

Measurement:
- On ES legs, a drain gate runs before the search suite: a
  conditional DELETE that matches nothing acts as a barrier on the
  composite's sync queue, then ES counters must settle, then live
  primary and ES resource counts are compared.
- Import completeness, per-type crud leftovers and a _summary=count
  cross-check are recorded, so legs that did not load the same data
  are flagged rather than silently compared.
- Backend stats, a "how to read this leg" note and the tuning knobs
  go into the step summary and runner-info.txt.

The new bash and Python live in .github/scripts/fhir-bench/, following
.github/scripts/obs-ab, so the workflow keeps its orchestration and
the scripts can be linted directly.

Closes #1475

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013NudzWDu2yTGExYaxdTQYJ
---
 .github/scripts/fhir-bench/capacity-gate.sh   | 150 ++++
 .github/scripts/fhir-bench/crud_residue.py    |  62 ++
 .github/scripts/fhir-bench/diagnose.sh        |  72 ++
 .github/scripts/fhir-bench/resolve-matrix.sh  | 182 +++++
 .../scripts/fhir-bench/start-elasticsearch.sh | 113 +++
 .github/scripts/fhir-bench/start-mongodb.sh   | 183 +++++
 .github/scripts/fhir-bench/suite-lib.sh       | 536 ++++++++++++
 .../scripts/fhir-bench/summary_backends.py    | 284 +++++++
 .github/workflows/fhir-benchmark.yml          | 763 +++++++++++++++---
 9 files changed, 2247 insertions(+), 98 deletions(-)
 create mode 100755 .github/scripts/fhir-bench/capacity-gate.sh
 create mode 100755 .github/scripts/fhir-bench/crud_residue.py
 create mode 100755 .github/scripts/fhir-bench/diagnose.sh
 create mode 100755 .github/scripts/fhir-bench/resolve-matrix.sh
 create mode 100755 .github/scripts/fhir-bench/start-elasticsearch.sh
 create mode 100755 .github/scripts/fhir-bench/start-mongodb.sh
 create mode 100644 .github/scripts/fhir-bench/suite-lib.sh
 create mode 100755 .github/scripts/fhir-bench/summary_backends.py

diff --git a/.github/scripts/fhir-bench/capacity-gate.sh b/.github/scripts/fhir-bench/capacity-gate.sh
new file mode 100755
index 000000000..33461d94e
--- /dev/null
+++ b/.github/scripts/fhir-bench/capacity-gate.sh
@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+#
+# Docker host capacity gate for one benchmark leg.
+#
+# Called from: the `benchmark` job's "Docker host capacity gate" step.
+#
+# `max_parallel`'s clamp (1..2, see the `setup` job / resolve-matrix.sh) only
+# bounds THIS run's own legs — it cannot see the rest of CI sharing the same
+# 4-CPU / 11 GB Docker host, which is exactly what has OOM-killed a mongod
+# here before. NEED_MB is a rough per-backend model, not a real cgroup
+# budget:
+#   postgres family: shared_buffers, +1.5G for the rest of the server
+#     (autovacuum workers' maintenance_work_mem, connections), or a flat
+#     3584 when pg_shared_buffers isn't a plain "<N>GB" value (e.g.
+#     "auto" — resolved for real in "Start ephemeral Postgres").
+#   mongo family: --wiredTigerCacheSizeGB + ~1G mongod/OS overhead.
+#   sqlite family: 512 (HFS itself; no extra container).
+#   *-elasticsearch legs ADD heap*2 (heap plus JVM off-heap/direct
+#     memory, ballparked at another heap's worth) + 512 (ES process
+#     overhead).
+# A combination that could never fit even with the WHOLE host free
+# fails immediately, naming the inputs to lower. Otherwise poll
+# MemAvailable (host-wide, not per-container — this is what protects
+# OTHER CI too) every 60s for up to 10 minutes; still short after that,
+# fail the leg rather than risk taking down a neighbour's container.
+#
+# Required environment (exported by the workflow step's env:):
+#   BACKEND                  matrix.backend
+#   RUN_ID                   github.run_id
+#   IN_PG_SHARED_BUFFERS     inputs.pg_shared_buffers
+#   IN_MONGO_WT_CACHE_GB     inputs.mongo_wt_cache_gb
+#   ES_HEAP_MB               needs.setup.outputs.es_heap_mb
+#
+# Outputs: CAPACITY_NEED_MB / CAPACITY_AVAIL_MB / CAPACITY_WAIT_S appended to
+# $GITHUB_ENV (read by "Run benchmark suites" for runner-info.txt, and by
+# summary_backends.py as its fallback Capacity gate row for a leg that
+# failed this gate before runner-info.txt was ever written).
+set -euo pipefail
+
+case "$BACKEND" in
+  postgres|postgres-elasticsearch)
+    PG_SHARED_BUFFERS="${IN_PG_SHARED_BUFFERS:-2GB}"
+    # A plain or fractional GB value (e.g. "2GB", "1.5GB") is parsed
+    # here via awk (bash arithmetic can't do fractions); anything
+    # else — "auto" or a bad value — falls back to the flat 3584
+    # estimate instead of handing non-numeric text to bash
+    # arithmetic, which aborts this step with a raw "arithmetic
+    # syntax error" rather than naming the input (the pre-#1475
+    # shm-size calc in the YAML's "Start ephemeral Postgres" step has
+    # the same shape and the same gap, just later in the run).
+    if [[ "$PG_SHARED_BUFFERS" =~ ^([0-9]+(\.[0-9]+)?)GB$ ]]; then
+      NEED_PRIMARY_MB=$(awk -v g="${BASH_REMATCH[1]}" 'BEGIN { printf "%d", g * 1024 + 1536 }')
+    else
+      NEED_PRIMARY_MB=3584
+    fi
+    ;;
+  mongodb|mongodb-elasticsearch)
+    MONGO_WT_CACHE_GB="${IN_MONGO_WT_CACHE_GB:-2}"
+    # WT cache can be fractional (0.25..6) — let awk do the math.
+    NEED_PRIMARY_MB=$(awk -v g="$MONGO_WT_CACHE_GB" 'BEGIN { printf "%d", g * 1024 + 1024 }')
+    ;;
+  sqlite|sqlite-elasticsearch)
+    NEED_PRIMARY_MB=512
+    ;;
+  *)
+    echo "::error::Docker host capacity gate has no memory model for backend '$BACKEND'"
+    exit 1
+    ;;
+esac
+
+NEED_ES_MB=0
+case "$BACKEND" in
+  *-elasticsearch)
+    NEED_ES_MB=$(( ${ES_HEAP_MB:-1024} * 2 + 512 ))
+    ;;
+esac
+
+CAPACITY_NEED_MB=$(( NEED_PRIMARY_MB + NEED_ES_MB ))
+echo "Capacity need for $BACKEND: primary=${NEED_PRIMARY_MB}MB elasticsearch=${NEED_ES_MB}MB total=${CAPACITY_NEED_MB}MB"
+
+MEM_TOTAL_BYTES=$(timeout 60 docker info --format '{{.MemTotal}}' 2>/dev/null) || MEM_TOTAL_BYTES=0
+case "$MEM_TOTAL_BYTES" in ''|*[!0-9]*) MEM_TOTAL_BYTES=0 ;; esac
+MEM_TOTAL_MB=$(( MEM_TOTAL_BYTES / 1024 / 1024 ))
+echo "Docker host MemTotal: ${MEM_TOTAL_MB}MB"
+
+# A `docker info` hiccup reads as MEM_TOTAL_MB=0, which would
+# otherwise always trip the impossible-fit check below and tell the
+# user to lower their inputs when the real problem is that the
+# daemon couldn't be read. Skip straight to the poll loop instead —
+# it re-reads memory through a separate `docker run`, so a
+# transient `docker info` failure alone doesn't fail the leg.
+if [ "$MEM_TOTAL_MB" -eq 0 ]; then
+  echo "::warning::could not read Docker host MemTotal (docker info failed) — skipping the impossible-fit check; the poll below still guards capacity"
+elif [ $(( CAPACITY_NEED_MB + 2048 )) -gt "$MEM_TOTAL_MB" ]; then
+  # Record what this leg needed even though no suite will run, so
+  # the summary (which reads these from $GITHUB_ENV when
+  # runner-info.txt was never written) can still show a Capacity
+  # gate row instead of nothing.
+  {
+    echo "CAPACITY_NEED_MB=$CAPACITY_NEED_MB"
+    echo "CAPACITY_AVAIL_MB=skipped"
+    echo "CAPACITY_WAIT_S=0"
+  } >> "$GITHUB_ENV"
+  echo "::error::backend=$BACKEND needs ~${CAPACITY_NEED_MB}MB (plus a 2048MB margin), but the Docker host only reports ${MEM_TOTAL_MB}MB total RAM. Lower es_heap / mongo_wt_cache_gb / pg_shared_buffers, or pick a lighter backend — this combination can never fit, even with the whole host free."
+  exit 1
+fi
+
+echo "── Waiting for MemAvailable >= $(( CAPACITY_NEED_MB + 2048 ))MB (poll 60s, timeout 10min) ──"
+CAPACITY_START=$SECONDS
+CAPACITY_AVAIL_MB=""
+CAPACITY_OK=0
+while :; do
+  # shellcheck disable=SC2016 # single-quoted deliberately: $2 is awk's field
+  # reference, evaluated inside the container, not a shell variable here.
+  CAPACITY_AVAIL_MB=$(timeout 60 docker run --rm --name "hfs-bench-mem-$BACKEND-$RUN_ID" \
+      --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=$BACKEND" \
+      alpine:3 awk '/^MemAvailable:/{print int($2 / 1024)}' /proc/meminfo 2>/dev/null) || CAPACITY_AVAIL_MB=""
+  # `timeout 60` only kills the local docker CLI, not a container
+  # still starting on the daemon — force it gone so the same
+  # `--name` doesn't collide on the next poll (every 60s).
+  docker rm -f "hfs-bench-mem-$BACKEND-$RUN_ID" >/dev/null 2>&1 || true
+  CAPACITY_WAIT_S=$((SECONDS - CAPACITY_START))
+  if [ -n "$CAPACITY_AVAIL_MB" ] && [ "$CAPACITY_AVAIL_MB" -ge $(( CAPACITY_NEED_MB + 2048 )) ]; then
+    CAPACITY_OK=1
+    echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB}MB — capacity OK"
+    break
+  fi
+  echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB:-unknown}MB, need $(( CAPACITY_NEED_MB + 2048 ))MB — waiting"
+  # Never sleep past the 600s budget: a plain `sleep 60` here could
+  # carry the last iteration well beyond it (e.g. wait_s=590 -> next
+  # check at 650s). Cap the sleep to whatever is actually left.
+  CAPACITY_REMAIN_S=$((600 - CAPACITY_WAIT_S))
+  if [ "$CAPACITY_REMAIN_S" -le 0 ]; then
+    break
+  fi
+  CAPACITY_SLEEP_S=$CAPACITY_REMAIN_S
+  [ "$CAPACITY_SLEEP_S" -gt 60 ] && CAPACITY_SLEEP_S=60
+  sleep "$CAPACITY_SLEEP_S"
+done
+
+{
+  echo "CAPACITY_NEED_MB=$CAPACITY_NEED_MB"
+  echo "CAPACITY_AVAIL_MB=${CAPACITY_AVAIL_MB:-unknown}"
+  echo "CAPACITY_WAIT_S=$CAPACITY_WAIT_S"
+} >> "$GITHUB_ENV"
+
+if [ "$CAPACITY_OK" -ne 1 ]; then
+  echo "::error::Docker host still short of memory for $BACKEND after ${CAPACITY_WAIT_S}s (needed $(( CAPACITY_NEED_MB + 2048 ))MB, last MemAvailable=${CAPACITY_AVAIL_MB:-unknown}MB). Failing this leg rather than risk an OOM on the shared host."
+  exit 1
+fi
diff --git a/.github/scripts/fhir-bench/crud_residue.py b/.github/scripts/fhir-bench/crud_residue.py
new file mode 100755
index 000000000..4f121d83a
--- /dev/null
+++ b/.github/scripts/fhir-bench/crud_residue.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+#
+# CRUD residue (search-counts comparability) — sums, per FHIR resource type,
+# how many rows a benchmark leg's prewarm + crud suites left behind live.
+#
+# Called from: the `benchmark` job's "Run benchmark suites" step
+# (fhir-benchmark.yml), via the step's own `$RESIDUE_PY` (python3, falling
+# back to python — that resolution stays inline in the workflow), invoked as:
+#   "$RESIDUE_PY" crud_residue.py <prewarm.json> <crud.json> > crud-residue.txt
+#
+# crud.js (upstream: HealthSamurai/fhir-server-performance-benchmark
+# k6/crud.js) creates 9 resources per iteration then deletes them in reverse
+# order; the checks it runs are named literally `${rt} created` / `${rt}
+# delete` (verified against crud.js's `check(x, { [...]: ... })` calls).
+# prewarm.js re-exports crud.js's SAME default/setup at 10 VUs for 30s, so a
+# check failure on delete during EITHER suite (e.g. one 409/500) leaves a
+# live row that the search-counts table would otherwise attribute to import
+# instead of to crud/prewarm leftovers. created - delete, summed over both
+# files' k6 --summary-export JSON, is that leftover per resource type. Walks
+# root_group's checks/groups recursively rather than assuming a fixed
+# nesting depth.
+#
+# Input: sys.argv[1:] — one or more k6 --summary-export JSON paths (a
+# missing or unparsable file is skipped, not fatal).
+# Output: stdout — one `<resourceType>=<created - deleted>` line per type
+# that had a "created" or "delete" check, sorted by resource type name.
+import json, re, sys
+
+
+def collect_checks(node, out):
+    if not isinstance(node, dict):
+        return
+    for name, c in (node.get("checks") or {}).items():
+        if isinstance(c, dict):
+            slot = out.setdefault(name, {"passes": 0, "fails": 0})
+            slot["passes"] += c.get("passes", 0) or 0
+            slot["fails"] += c.get("fails", 0) or 0
+    for g in (node.get("groups") or {}).values():
+        collect_checks(g, out)
+
+
+checks = {}
+for path in sys.argv[1:]:
+    try:
+        with open(path) as fh:
+            root = json.load(fh).get("root_group", {})
+    except (OSError, ValueError):
+        continue
+    collect_checks(root, checks)
+
+totals = {}
+for name, c in checks.items():
+    m = re.match(r"^(.+) created$", name)
+    if m:
+        totals.setdefault(m.group(1), {"created": 0, "deleted": 0})["created"] += c["passes"]
+        continue
+    m = re.match(r"^(.+) delete$", name)
+    if m:
+        totals.setdefault(m.group(1), {"created": 0, "deleted": 0})["deleted"] += c["passes"]
+
+for rt in sorted(totals):
+    print(f"{rt}={totals[rt]['created'] - totals[rt]['deleted']}")
diff --git a/.github/scripts/fhir-bench/diagnose.sh b/.github/scripts/fhir-bench/diagnose.sh
new file mode 100755
index 000000000..1414ceaab
--- /dev/null
+++ b/.github/scripts/fhir-bench/diagnose.sh
@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+#
+# Why a backend container died (host OOM vs. cgroup OOM vs. crash), captured
+# before the Stop steps remove the evidence. Based on inferno-us-core.yml's
+# "Diagnose MongoDB failure", widened to every container this leg owns.
+#
+# Called from: the `benchmark` job's "Diagnose backend container failure"
+# step (fhir-benchmark.yml), which runs on
+# `failure() || cancelled() || env.LEG_CONTAINER_DIED == 'true'` — see that
+# step's own `if:` for why (that rationale stays in the YAML, not here).
+#
+# The original step's `run:` had no explicit `set` line, so it ran under
+# GitHub's bare default of `bash -e {0}` (errexit only, no pipefail, no -u).
+# `bash <script>` does not inherit that, so it is reproduced explicitly
+# below rather than assumed.
+#
+# Required environment (exported by the workflow step's env:):
+#   BACKEND   matrix.backend
+#   RUN_ID    github.run_id
+#
+# Also reads PG_CONTAINER / MONGO_CONTAINER / ES_CONTAINER, already exported
+# to $GITHUB_ENV by "Configure backend env" (whichever of the three this leg
+# sets — unset ones are fine, see the loop below).
+#
+# Outputs: none (stdout/`::group::`/`::endgroup::` only — this never fails
+# the step, every command is `|| true`).
+set -e
+
+# `timeout 15`, not the usual 60: this step's own timeout-minutes is 3, and
+# on cancellation it competes with Upload results/server log and the Stop
+# steps for the 5-minute cancellation window. Up to 2 containers (a
+# *-elasticsearch leg) x 2 calls each (4), plus the 5 standalone calls below
+# (dmesg run + rm, meminfo run + rm, docker stats), keeps the worst case
+# near 135s (9 calls x 15s) instead of eating the full 3-minute budget on
+# ~60s-per-call timeouts.
+for c in ${PG_CONTAINER:-} ${MONGO_CONTAINER:-} ${ES_CONTAINER:-}; do
+  echo "::group::$c state + last 60 log lines"
+  timeout 15 docker inspect "$c" --format 'OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Status={{.State.Status}} Error={{.State.Error}} FinishedAt={{.State.FinishedAt}}' 2>&1 || true
+  timeout 15 docker logs "$c" --tail 60 2>&1 || true
+  echo "::endgroup::"
+done
+echo "::group::Host OOM-killer (kernel ring buffer)"
+DMESG_NAME="hfs-bench-dmesg-${BACKEND}-${RUN_ID}"
+timeout 15 docker run --rm --privileged --name "$DMESG_NAME" \
+  --label hfs-bench=1 --label "hfs-bench-run=${RUN_ID}" --label "hfs-bench-leg=${BACKEND}" \
+  busybox dmesg 2>/dev/null \
+  | grep -iE "out of memory|killed process|oom" | tail -25 || true
+# `timeout 15` only kills the local docker CLI, not a container still
+# starting on the daemon — force it gone rather than trust `--rm`
+# alone, same reasoning as the voldf cleanups in the YAML's Start
+# Postgres step, start-mongodb.sh and start-elasticsearch.sh.
+timeout 15 docker rm -f "$DMESG_NAME" >/dev/null 2>&1 || true
+echo "(end of OOM grep — empty means no OOM lines found)"
+echo "::endgroup::"
+
+# This host is shared with the rest of CI, so a container that
+# died here is not necessarily this leg's own doing — a per-leg
+# OOMKilled=false plus a healthy-looking log can still be a
+# neighbour's memory pressure. These two numbers are what tell the
+# two apart after the fact.
+echo "::group::Docker host memory + containers"
+MEMINFO_NAME="hfs-bench-meminfo-${BACKEND}-${RUN_ID}"
+timeout 15 docker run --rm --name "$MEMINFO_NAME" \
+  --label hfs-bench=1 --label "hfs-bench-run=${RUN_ID}" --label "hfs-bench-leg=${BACKEND}" \
+  busybox sh -c 'grep -E "MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree" /proc/meminfo' 2>&1 || true
+# `timeout 15` only kills the local docker CLI, not a container still
+# starting on the daemon — force it gone rather than trust `--rm`
+# alone, same reasoning as the voldf cleanups in the YAML's Start
+# Postgres step, start-mongodb.sh and start-elasticsearch.sh.
+timeout 15 docker rm -f "$MEMINFO_NAME" >/dev/null 2>&1 || true
+timeout 15 docker stats --no-stream --format 'table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.CPUPerc}}' 2>&1 || true
+echo "::endgroup::"
diff --git a/.github/scripts/fhir-bench/resolve-matrix.sh b/.github/scripts/fhir-bench/resolve-matrix.sh
new file mode 100755
index 000000000..1842b85ba
--- /dev/null
+++ b/.github/scripts/fhir-bench/resolve-matrix.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+#
+# Resolve the `backend` workflow_dispatch input into a job matrix for
+# fhir-benchmark.yml, and validate the free-text tuning inputs in one place
+# before the ~13-minute build runs — once, not once per leg; its single
+# binary serves every leg in the resolved matrix.
+#
+# Called from: the `setup` job's "Resolve matrix from backend input" step.
+#
+# Free-text/choice inputs reach this script only through environment
+# variables, never inlined as ${{ }} inside the step's `run:` (a value
+# containing `"` or `$(...)` would otherwise be substituted into the script
+# text before bash runs it, and validated only after — same risk the pg_*
+# inputs already carry, limited today to people with write access).
+#
+# Required environment (exported by the workflow step's env:):
+#   IN_BACKEND                      inputs.backend
+#   IN_MAX_PARALLEL                 inputs.max_parallel
+#   IN_ES_HEAP                      inputs.es_heap
+#   IN_ES_SYNC_MODE                 inputs.es_sync_mode
+#   IN_MONGO_WT_CACHE_GB            inputs.mongo_wt_cache_gb
+#   IN_HFS_MONGO_MAX_CONNECTIONS    inputs.hfs_mongo_max_connections
+#
+# Outputs (via $GITHUB_OUTPUT): matrix, max_parallel, es_heap_mb,
+# leg_timeout_min.
+#
+# This combines GitHub's default `bash -e {0}` step wrapping with the
+# original step's own `set -uo pipefail` line — the net effect the step ran
+# under was `-euo pipefail`. A script invoked as `bash <script>` does NOT
+# inherit the calling step's shell options, so this script sets that
+# combination explicitly to reproduce the same behavior.
+set -euo pipefail
+FAIL=0
+BACKEND="${IN_BACKEND:-core}"
+
+port_for() {
+  case "$1" in
+    sqlite)                 echo 8081 ;;
+    postgres)               echo 8082 ;;
+    mongodb)                echo 8083 ;;
+    sqlite-elasticsearch)   echo 8084 ;;
+    postgres-elasticsearch) echo 8085 ;;
+    mongodb-elasticsearch)  echo 8086 ;;
+  esac
+}
+
+case "$BACKEND" in
+  core)          LEGS="sqlite postgres" ;;
+  all)           LEGS="sqlite postgres mongodb sqlite-elasticsearch postgres-elasticsearch mongodb-elasticsearch" ;;
+  elasticsearch) LEGS="sqlite-elasticsearch postgres-elasticsearch mongodb-elasticsearch" ;;
+  sqlite|postgres|mongodb|sqlite-elasticsearch|postgres-elasticsearch|mongodb-elasticsearch)
+                 LEGS="$BACKEND" ;;
+  *)
+    echo "::error::Unknown backend selection '$BACKEND'"
+    FAIL=1
+    LEGS="" ;;
+esac
+
+ITEMS=""
+for b in $LEGS; do
+  ITEMS="${ITEMS:+$ITEMS,}{\"backend\":\"$b\",\"port\":$(port_for "$b")}"
+done
+MATRIX="{\"include\":[$ITEMS]}"
+LEG_COUNT=$(echo "$LEGS" | wc -w)
+
+# max_parallel: job outputs are strings, so the benchmark job's
+# strategy.max-parallel reads this through fromJSON(). Clamped to
+# 1..2, not higher: the heaviest leg (postgres-elasticsearch, at
+# default inputs) needs about 6 GB (the capacity gate's own formula:
+# 2048+1536 primary, 1024*2+512 ES = 6144MB, so 8 GB free with its
+# 2 GB margin), so 2 concurrent legs already use most of the shared
+# 4-CPU / 11 GB Docker host that the rest of CI also runs on.
+MAX_PARALLEL="${IN_MAX_PARALLEL:-1}"
+case "$MAX_PARALLEL" in
+  *[!0-9]*)
+    echo "::error::max_parallel must be a positive integer (got '$MAX_PARALLEL')"
+    FAIL=1
+    MAX_PARALLEL=1 ;;
+  *)
+    # Reject an overlong digit string BEFORE the 10# arithmetic below
+    # — bash arithmetic on a huge value doesn't error, it silently
+    # wraps, which could land back in 1..2 and pass clamping that
+    # was meant to reject it. 3 digits (<=999) is well inside a safe
+    # range for the arithmetic and the `-gt 2` clamp that follows.
+    if [ "${#MAX_PARALLEL}" -gt 3 ]; then
+      echo "::error::max_parallel must be a positive integer, at most 3 digits (got '$MAX_PARALLEL')"
+      FAIL=1
+      MAX_PARALLEL=1
+    else
+      # 10# forces base-10 so a leading zero ("01") isn't read as octal.
+      MAX_PARALLEL=$((10#$MAX_PARALLEL))
+      if [ "$MAX_PARALLEL" -lt 1 ]; then
+        echo "::error::max_parallel must be >= 1 (got '$IN_MAX_PARALLEL')"
+        FAIL=1
+        MAX_PARALLEL=1
+      elif [ "$MAX_PARALLEL" -gt 2 ]; then
+        echo "::warning::max_parallel=$MAX_PARALLEL clamped to 2 (shared 4-CPU / 11 GB Docker host)"
+        MAX_PARALLEL=2
+      fi
+    fi
+    ;;
+esac
+
+# es_heap: -Xms = -Xmx, 256m..4g. Emitted as es_heap_mb so later
+# steps never have to re-parse the unit suffix.
+ES_HEAP="${IN_ES_HEAP:-1g}"
+ES_HEAP_MB=""
+if [[ "$ES_HEAP" =~ ^([0-9]+)([mMgG])$ ]]; then
+  ES_HEAP_MB=$((10#${BASH_REMATCH[1]}))
+  case "${BASH_REMATCH[2]}" in g|G) ES_HEAP_MB=$((ES_HEAP_MB * 1024)) ;; esac
+else
+  echo "::error::es_heap must match ^[0-9]+[mMgG]\$ (got '$ES_HEAP')"
+  FAIL=1
+fi
+if [ -n "$ES_HEAP_MB" ] && { [ "$ES_HEAP_MB" -lt 256 ] || [ "$ES_HEAP_MB" -gt 4096 ]; }; then
+  echo "::error::es_heap must be 256m..4g (got '$ES_HEAP')"
+  FAIL=1
+fi
+
+# es_sync_mode: defence in depth — workflow_dispatch's UI already
+# restricts this to the choice options, but an API dispatch can send
+# anything.
+ES_SYNC_MODE="${IN_ES_SYNC_MODE:-asynchronous}"
+case "$ES_SYNC_MODE" in
+  asynchronous|synchronous) ;;
+  *)
+    echo "::error::es_sync_mode must be 'asynchronous' or 'synchronous' (got '$ES_SYNC_MODE')"
+    FAIL=1 ;;
+esac
+
+# mongo_wt_cache_gb: decimal, 0.25..6 (parity with Postgres's
+# shared_buffers default; mongod's own default risks a host OOM).
+MONGO_WT_CACHE_GB="${IN_MONGO_WT_CACHE_GB:-2}"
+if ! [[ "$MONGO_WT_CACHE_GB" =~ ^[0-9]+(\.[0-9]+)?$ ]] \
+   || ! awk -v v="$MONGO_WT_CACHE_GB" 'BEGIN { exit !(v >= 0.25 && v <= 6) }'; then
+  echo "::error::mongo_wt_cache_gb must be a number in 0.25..6 (got '$MONGO_WT_CACHE_GB')"
+  FAIL=1
+fi
+
+# hfs_mongo_max_connections: positive integer, 1..1000, no leading
+# zeros. HFS silently falls back to 10 on a non-numeric value —
+# reject it here instead. This is the ONLY validation of
+# hfs_mongo_max_connections: "Start HFS server" re-derives
+# HFS_MONGO_POOL from the same raw input (env:
+# IN_HFS_MONGO_MAX_CONNECTIONS) without re-checking it, trusting
+# this step to have already rejected a bad value before the
+# ~13-minute build runs on every leg, including sqlite/postgres.
+HFS_MONGO_POOL="${IN_HFS_MONGO_MAX_CONNECTIONS:-32}"
+# At most 4 digits (matched BEFORE the -gt 1000 test below): an
+# overlong digit string handed straight to `[ -gt ]` risks "value
+# too great for base"/an unparseable-integer failure there instead
+# of the clear error this step is supposed to give.
+if ! [[ "$HFS_MONGO_POOL" =~ ^[1-9][0-9]{0,3}$ ]]; then
+  echo "::error::hfs_mongo_max_connections must be a positive integer with no leading zeros, at most 4 digits (got '$HFS_MONGO_POOL')"
+  FAIL=1
+elif [ "$HFS_MONGO_POOL" -gt 1000 ]; then
+  echo "::error::hfs_mongo_max_connections must be 1..1000 (got '$HFS_MONGO_POOL')"
+  FAIL=1
+fi
+
+if [ "$FAIL" -ne 0 ]; then
+  exit 1
+fi
+
+{
+  echo "matrix=$MATRIX"
+  echo "max_parallel=$MAX_PARALLEL"
+  echo "es_heap_mb=$ES_HEAP_MB"
+} >> "$GITHUB_OUTPUT"
+echo "Backend matrix: $MATRIX"
+echo "::notice title=Benchmark legs::backend=$BACKEND -> $LEG_COUNT leg(s) [$LEGS], max_parallel=$MAX_PARALLEL. A leg takes ~30-130 min; legs beyond max_parallel queue."
+
+# The benchmark job's timeout-minutes, and the base the "Reap stale
+# benchmark containers and verify Docker host storage" step (in the
+# `benchmark` job, fhir-benchmark.yml) sizes its age guard from. Must stay <= 175:
+# .github/actions/docker-host-gc (input container-max-age-min,
+# default 180; used by ci.yml / ci-extended / hts / keycloak-smoke
+# / ui-tests-matrix) reaps any `hfs-ci=true` container whose
+# `.State.StartedAt` is >= 180 min old, and GitHub gives a
+# cancelled job up to 5 more minutes to exit. Do not raise this
+# without re-checking that reaper's default.
+echo "leg_timeout_min=150" >> "$GITHUB_OUTPUT"
diff --git a/.github/scripts/fhir-bench/start-elasticsearch.sh b/.github/scripts/fhir-bench/start-elasticsearch.sh
new file mode 100755
index 000000000..20786efc2
--- /dev/null
+++ b/.github/scripts/fhir-bench/start-elasticsearch.sh
@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+#
+# Start an ephemeral single-node Elasticsearch for the *-elasticsearch legs.
+#
+# Called from: the `benchmark` job's "Start ephemeral Elasticsearch" step
+# (if: endsWith(matrix.backend, '-elasticsearch')).
+#
+# Heap from the es_heap input, already resolved to MB by "Resolve
+# matrix from backend input" (setup job / resolve-matrix.sh) so this step
+# never re-parses the unit suffix. The -Xlog flags match the rest of this
+# repo's Elasticsearch containers: the default jvm.options otherwise writes a
+# rotating gc.log into the container's writable layer — a resource
+# #453 (a repo-wide outage) exhausted.
+# Data lives in a labelled NAMED volume, same reaper reasoning as
+# Postgres/Mongo. Health gates accept YELLOW everywhere: HFS creates
+# every index with number_of_replicas=1
+# (elasticsearch/backend.rs default_replicas(), applied in schema.rs),
+# which cannot be satisfied on a single node.
+#
+# Required environment (exported by the workflow step's env:):
+#   BACKEND     matrix.backend
+#   RUN_ID      github.run_id
+#   ES_HEAP_MB  needs.setup.outputs.es_heap_mb
+#
+# Also reads, already exported to $GITHUB_ENV by earlier steps:
+#   ES_CONTAINER, ES_VOLUME                                "Configure backend env"
+#   DOCKER_HOST_IP                                         "Determine runner / Docker host IP"
+#   HFS_COMPOSITE_SYNC_MODE, HFS_ELASTICSEARCH_WRITE_REFRESH,
+#   HFS_ELASTICSEARCH_REFRESH_INTERVAL                     "Configure backend env" (summary only)
+#
+# Outputs (via $GITHUB_ENV): ES_VOL_FREE_MB, ES_PORT, HFS_ELASTICSEARCH_NODES.
+# Also appends a tuning table to $GITHUB_STEP_SUMMARY.
+set -euo pipefail
+ES_HEAP_MB="${ES_HEAP_MB:-1024}"
+ES_IMAGE="elasticsearch:8.15.0"
+
+docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
+docker volume rm -f "$ES_VOLUME" 2>/dev/null || true
+docker volume create --label hfs-bench=1 \
+  --label "hfs-bench-run=$RUN_ID" \
+  --label "hfs-bench-leg=$BACKEND" "$ES_VOLUME" >/dev/null
+
+# Volume filesystem free space at start, and (ES only) how full it
+# already is: indices go read-only at Elasticsearch's 95%
+# flood-stage watermark, so warn well before that. Never fails the
+# leg — this is a signal for calibration, not a gate.
+ES_VOLDF_NAME="hfs-bench-voldf-$BACKEND-$RUN_ID-es"
+ES_VOL_DF=$(timeout 60 docker run --rm --name "$ES_VOLDF_NAME" \
+    --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=$BACKEND" \
+    -v "$ES_VOLUME:/v" alpine:3 df -Pm /v 2>/dev/null | awk 'NR==2{print $4, $5}') || ES_VOL_DF=""
+# See the Postgres step's identical note: `timeout` alone cannot be
+# trusted to leave `--rm` a clean container to remove.
+timeout 15 docker rm -f "$ES_VOLDF_NAME" >/dev/null 2>&1 || true
+ES_VOL_FREE_MB=$(echo "$ES_VOL_DF" | awk '{print $1}')
+ES_VOL_PCT_USED=$(echo "$ES_VOL_DF" | awk '{gsub("%","",$2); print $2}')
+echo "ES_VOL_FREE_MB=${ES_VOL_FREE_MB:-unknown}" >> "$GITHUB_ENV"
+echo "Elasticsearch volume free space: ${ES_VOL_FREE_MB:-unknown} MB (${ES_VOL_PCT_USED:-unknown}% used)"
+if [ -n "$ES_VOL_PCT_USED" ] && [[ "$ES_VOL_PCT_USED" =~ ^[0-9]+$ ]] && [ "$ES_VOL_PCT_USED" -gt 85 ]; then
+  echo "::warning::Elasticsearch data volume $ES_VOLUME filesystem is ${ES_VOL_PCT_USED}% used — indices go read-only at Elasticsearch's 95% flood-stage watermark."
+fi
+
+docker run -d \
+  --name "$ES_CONTAINER" \
+  --label hfs-bench=1 \
+  --label hfs-ci=true \
+  --label "hfs-bench-run=$RUN_ID" \
+  --label "hfs-bench-leg=$BACKEND" \
+  --log-driver local \
+  -v "$ES_VOLUME:/usr/share/elasticsearch/data" \
+  -p 0:9200 \
+  -e "discovery.type=single-node" \
+  -e "xpack.security.enabled=false" \
+  -e "ES_JAVA_OPTS=-Xms${ES_HEAP_MB}m -Xmx${ES_HEAP_MB}m -Xlog:disable -Xlog:all=warning:stderr" \
+  "$ES_IMAGE" >/dev/null
+
+echo "Waiting for Elasticsearch..."
+ES_PORT=""
+ES_READY_T0=$SECONDS
+while :; do
+  ES_PORT=$(timeout 30 docker port "$ES_CONTAINER" 9200 2>/dev/null | head -1 | sed 's/.*://') || ES_PORT=""
+  # --max-time 5, not 30: this loop enforces its own 120s wall-clock
+  # deadline below, so one slow/unresponsive curl must not itself
+  # eat most of that budget before the deadline is checked again.
+  if [ -n "$ES_PORT" ] && curl -sf --max-time 5 "http://$DOCKER_HOST_IP:$ES_PORT/_cluster/health?wait_for_status=yellow&timeout=1s" -o /dev/null 2>/dev/null; then
+    echo "Elasticsearch ready on $DOCKER_HOST_IP:$ES_PORT after $((SECONDS - ES_READY_T0))s"
+    break
+  fi
+  if [ $((SECONDS - ES_READY_T0)) -ge 120 ]; then
+    echo "ERROR: Elasticsearch did not become ready within 120s"
+    timeout 60 docker inspect -f 'OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Status={{.State.Status}}' "$ES_CONTAINER" || true
+    timeout 60 docker logs --tail 100 "$ES_CONTAINER" 2>&1 || true
+    exit 1
+  fi
+  sleep 2
+done
+
+echo "ES_PORT=$ES_PORT" >> "$GITHUB_ENV"
+echo "HFS_ELASTICSEARCH_NODES=http://$DOCKER_HOST_IP:$ES_PORT" >> "$GITHUB_ENV"
+ES_VERSION=$(curl -sf --max-time 30 "http://$DOCKER_HOST_IP:$ES_PORT/" | jq -r '.version.number' 2>/dev/null) || ES_VERSION='?'
+{
+  echo "### Elasticsearch tuning ($BACKEND)"
+  echo ""
+  echo "| setting | value |"
+  echo "|---|---|"
+  echo "| image | \`$ES_IMAGE\` (reports $ES_VERSION), single node |"
+  echo "| JVM heap (-Xms = -Xmx) | \`${ES_HEAP_MB}m\` |"
+  echo "| HFS_COMPOSITE_SYNC_MODE | \`${HFS_COMPOSITE_SYNC_MODE:-}\` |"
+  echo "| HFS_ELASTICSEARCH_WRITE_REFRESH | \`${HFS_ELASTICSEARCH_WRITE_REFRESH:-}\` |"
+  echo "| HFS_ELASTICSEARCH_REFRESH_INTERVAL | \`${HFS_ELASTICSEARCH_REFRESH_INTERVAL:-}\` |"
+  echo "| HFS_COMPOSITE_SYNC_REPAIR_INTERVAL | \`60\` |"
+  echo "| index replicas | 1 (hardcoded default in HFS) — cluster **yellow** on one node is expected |"
+  echo "| data volume free space (start) | \`${ES_VOL_FREE_MB:-unknown} MB (${ES_VOL_PCT_USED:-unknown}% used)\` |"
+} >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/scripts/fhir-bench/start-mongodb.sh b/.github/scripts/fhir-bench/start-mongodb.sh
new file mode 100755
index 000000000..2a6e34452
--- /dev/null
+++ b/.github/scripts/fhir-bench/start-mongodb.sh
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+#
+# Start an ephemeral single-member MongoDB replica set for the mongodb /
+# mongodb-elasticsearch legs.
+#
+# Called from: the `benchmark` job's "Start ephemeral MongoDB (single-member
+# replica set)" step (if: startsWith(matrix.backend, 'mongodb')).
+#
+# Single-member replica set: HFS runs each FHIR transaction Bundle as
+# ONE multi-document transaction, which MongoDB only supports on a
+# replica set. The member is registered as localhost:27017 —
+# unreachable from the runner — so the client URL below uses
+# directConnection=true (skips replica-set topology discovery), the
+# same pattern as inferno-us-core.yml's Mongo wiring.
+#
+# Explicit engine caps, because this runs on the shared 4-CPU / 11 GB
+# host:
+#   --wiredTigerCacheSizeGB  mongod's own default is ~50% of (host RAM
+#                            - 1GB), which risks an OOM on this host
+#                            (crates/persistence/tests/mongodb_tests.rs
+#                            has a regression test for that failure
+#                            mode).
+#   transactionLifetimeLimitSeconds=900  mongod's default is 60s; import
+#                            bundles run into the thousands of entries
+#                            and already need HFS_REQUEST_TIMEOUT=900
+#                            on the Postgres leg.
+#   --oplogSize 2048         default is ~5% of free disk (~8GB here).
+# Data lives in a labelled NAMED volume for the same reaper reason as
+# Postgres's. mongo:7.0 also declares an anonymous /data/configdb
+# volume; that one is deliberately left unnamed — the workflow's "Stop
+# ephemeral MongoDB" step's `docker rm -fv` and docker-host-gc's
+# anonymous-volume sweep both already remove it, so a named/labelled
+# volume for it would add reaper surface for no reclaim benefit.
+#
+# Required environment (exported by the workflow step's env:):
+#   BACKEND                        matrix.backend
+#   RUN_ID                         github.run_id
+#   IN_MONGO_WT_CACHE_GB           inputs.mongo_wt_cache_gb
+#   IN_HFS_MONGO_MAX_CONNECTIONS   inputs.hfs_mongo_max_connections
+#
+# Also reads, already exported to $GITHUB_ENV by earlier steps:
+#   MONGO_CONTAINER, MONGO_VOLUME, HFS_MONGODB_DATABASE   "Configure backend env"
+#   DOCKER_HOST_IP                                        "Determine runner / Docker host IP"
+#
+# Outputs (via $GITHUB_ENV): MONGO_VOL_FREE_MB, MONGO_PORT, HFS_DATABASE_URL,
+# MONGO_TXN_LIFETIME_S. Also appends a tuning table to $GITHUB_STEP_SUMMARY.
+set -euo pipefail
+
+MONGO_WT_CACHE_GB="${IN_MONGO_WT_CACHE_GB:-2}"
+if ! [[ "$MONGO_WT_CACHE_GB" =~ ^[0-9]+(\.[0-9]+)?$ ]] \
+   || ! awk -v v="$MONGO_WT_CACHE_GB" 'BEGIN { exit !(v >= 0.25 && v <= 6) }'; then
+  echo "::error::mongo_wt_cache_gb must be a number in 0.25..6 (got '$MONGO_WT_CACHE_GB'); the Docker host has 11 GB shared with all CI"
+  exit 1
+fi
+MONGO_IMAGE="mongo:7.0"
+MONGO_TXN_LIFETIME_S=900
+MONGO_OPLOG_MB=2048
+
+docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
+docker volume rm -f "$MONGO_VOLUME" 2>/dev/null || true
+docker volume create --label hfs-bench=1 \
+  --label "hfs-bench-run=$RUN_ID" \
+  --label "hfs-bench-leg=$BACKEND" "$MONGO_VOLUME" >/dev/null
+
+# Volume filesystem free space at start (see the Postgres step for
+# why this is the volume's own filesystem, not `docker system df`'s
+# numbers). Never fails the leg.
+MONGO_VOLDF_NAME="hfs-bench-voldf-$BACKEND-$RUN_ID-mongo"
+MONGO_VOL_DF=$(timeout 60 docker run --rm --name "$MONGO_VOLDF_NAME" \
+    --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=$BACKEND" \
+    -v "$MONGO_VOLUME:/v" alpine:3 df -Pm /v 2>/dev/null | awk 'NR==2{print $4}') || MONGO_VOL_DF=""
+# See the Postgres step's identical note: `timeout` alone cannot be
+# trusted to leave `--rm` a clean container to remove.
+timeout 15 docker rm -f "$MONGO_VOLDF_NAME" >/dev/null 2>&1 || true
+echo "MONGO_VOL_FREE_MB=${MONGO_VOL_DF:-unknown}" >> "$GITHUB_ENV"
+echo "Mongo volume free space: ${MONGO_VOL_DF:-unknown} MB"
+
+docker run -d \
+  --name "$MONGO_CONTAINER" \
+  --label hfs-bench=1 \
+  --label hfs-ci=true \
+  --label "hfs-bench-run=$RUN_ID" \
+  --label "hfs-bench-leg=$BACKEND" \
+  --log-driver local \
+  -v "$MONGO_VOLUME:/data/db" \
+  -p 0:27017 \
+  "$MONGO_IMAGE" \
+  --replSet rs0 \
+  --bind_ip_all \
+  --wiredTigerCacheSizeGB "$MONGO_WT_CACHE_GB" \
+  --oplogSize "$MONGO_OPLOG_MB" \
+  --setParameter "transactionLifetimeLimitSeconds=$MONGO_TXN_LIFETIME_S" >/dev/null
+
+# 10s, not 60s: this is called from wall-clock-deadline loops below,
+# and a mongod that accepts TCP but never answers (host memory
+# pressure) would otherwise let a single call eat most of the
+# loop's own budget before the deadline check ever runs again.
+mongo_eval() { timeout 10 docker exec "$MONGO_CONTAINER" mongosh --quiet --eval "$1"; }
+
+echo "Waiting for mongod..."
+MONGO_PING_T0=$SECONDS
+until mongo_eval 'db.adminCommand({ ping: 1 }).ok' >/dev/null 2>&1; do
+  if [ $((SECONDS - MONGO_PING_T0)) -ge 60 ]; then
+    echo "ERROR: mongod did not answer within 60s"
+    timeout 60 docker logs --tail 100 "$MONGO_CONTAINER" 2>&1 || true
+    exit 1
+  fi
+  sleep 2
+done
+
+mongo_eval 'try { rs.status(); } catch (e) { rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); }' >/dev/null 2>&1 || true
+
+echo "Waiting for replica-set PRIMARY..."
+MONGO_PORT=""
+MONGO_PRIMARY_T0=$SECONDS
+while :; do
+  STATE=$(mongo_eval 'try { rs.status().myState } catch (e) { 0 }' 2>/dev/null | tr -d '\r\n ' || true)
+  if [ "$STATE" = "1" ]; then
+    MONGO_PORT=$(timeout 30 docker port "$MONGO_CONTAINER" 27017 2>/dev/null | head -1 | sed 's/.*://') || MONGO_PORT=""
+    if [ -n "$MONGO_PORT" ] && timeout 2 bash -c "cat < /dev/null > /dev/tcp/$DOCKER_HOST_IP/$MONGO_PORT" 2>/dev/null; then
+      echo "MongoDB PRIMARY reachable on $DOCKER_HOST_IP:$MONGO_PORT after $((SECONDS - MONGO_PRIMARY_T0))s"
+      break
+    fi
+  fi
+  if [ $((SECONDS - MONGO_PRIMARY_T0)) -ge 120 ]; then
+    echo "ERROR: MongoDB replica set did not become a reachable PRIMARY within 120s"
+    timeout 60 docker logs --tail 100 "$MONGO_CONTAINER" 2>&1 || true
+    exit 1
+  fi
+  sleep 2
+done
+
+# Prove the caps took effect rather than trusting the flags. A single
+# mongo_eval right after PRIMARY comes up can hit mongod still
+# settling — reading that hiccup as "the setting is wrong" would be
+# a false failure, so retry up to 5 times, 2s apart, and only treat
+# it as an error once it is still unreadable after that (or once a
+# value WAS read and definitely disagrees).
+mongo_read_int() {
+  local expr="$1" attempt out
+  for attempt in 1 2 3 4 5; do
+    out=$(mongo_eval "$expr" 2>/dev/null | tr -d '\r\n ') || out=""
+    if [[ "$out" =~ ^[0-9]+$ ]]; then
+      echo "$out"
+      return 0
+    fi
+    [ "$attempt" -lt 5 ] && sleep 2
+  done
+  echo '?'
+  return 1
+}
+WT_MAX_BYTES=$(mongo_read_int 'Number(db.serverStatus().wiredTiger.cache["maximum bytes configured"])') || true
+TXN_LIFETIME=$(mongo_read_int 'Number(db.adminCommand({ getParameter: 1, transactionLifetimeLimitSeconds: 1 }).transactionLifetimeLimitSeconds)') || true
+echo "WiredTiger cache max: $WT_MAX_BYTES bytes; transactionLifetimeLimitSeconds: $TXN_LIFETIME"
+if [ "$TXN_LIFETIME" = '?' ]; then
+  echo "::error::could not read transactionLifetimeLimitSeconds after 5 attempts (2s apart) — cannot confirm import transactions won't abort at mongod's 60s default"
+  exit 1
+fi
+if [ "$TXN_LIFETIME" != "$MONGO_TXN_LIFETIME_S" ]; then
+  echo "::error::transactionLifetimeLimitSeconds is '$TXN_LIFETIME', expected $MONGO_TXN_LIFETIME_S — import transactions would abort at mongod's 60s default"
+  exit 1
+fi
+
+{
+  echo "MONGO_PORT=$MONGO_PORT"
+  echo "HFS_DATABASE_URL=mongodb://$DOCKER_HOST_IP:$MONGO_PORT/?replicaSet=rs0&directConnection=true"
+  # Recorded so "Run benchmark suites" can put it in runner-info.txt
+  # — this step's own local $MONGO_TXN_LIFETIME_S isn't visible there.
+  echo "MONGO_TXN_LIFETIME_S=$MONGO_TXN_LIFETIME_S"
+} >> "$GITHUB_ENV"
+{
+  echo "### MongoDB tuning ($BACKEND)"
+  echo ""
+  echo "| setting | value |"
+  echo "|---|---|"
+  echo "| image | \`$MONGO_IMAGE\` — single-member replica set \`rs0\`, directConnection |"
+  echo "| --wiredTigerCacheSizeGB | \`$MONGO_WT_CACHE_GB\` (mongod reports $WT_MAX_BYTES bytes) |"
+  echo "| transactionLifetimeLimitSeconds | \`$TXN_LIFETIME\` |"
+  echo "| --oplogSize | \`${MONGO_OPLOG_MB}MB\` |"
+  echo "| HFS_MONGODB_MAX_CONNECTIONS | \`${IN_HFS_MONGO_MAX_CONNECTIONS:-32}\` |"
+  echo "| HFS_MONGODB_DATABASE | \`$HFS_MONGODB_DATABASE\` |"
+  echo "| data volume free space (start) | \`${MONGO_VOL_DF:-unknown} MB\` |"
+} >> "$GITHUB_STEP_SUMMARY"
diff --git a/.github/scripts/fhir-bench/suite-lib.sh b/.github/scripts/fhir-bench/suite-lib.sh
new file mode 100644
index 000000000..af401c4f1
--- /dev/null
+++ b/.github/scripts/fhir-bench/suite-lib.sh
@@ -0,0 +1,536 @@
+#!/usr/bin/env bash
+#
+# Sourced library of "Run benchmark suites" helper functions (issue #1475's
+# six-leg matrix instrumentation): the Elasticsearch drain gate, the
+# per-suite ES/Mongo/composite stats snapshot, the per-suite import
+# completeness and Mongo transaction-error writers, the post-suite
+# search-counts cross-check, and the end-of-step container/volume state dump
+# plus liveness gate. Moved out of that step's inline `run:` block so the
+# step's own YAML diff stays small and these functions can be shellchecked
+# directly.
+#
+# Sourced, never executed: keep it non-executable and never run it as
+# `bash suite-lib.sh`. Also do NOT add a `set` line here. It is read with
+#   source "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/suite-lib.sh"
+# near the top of the `benchmark` job's "Run benchmark suites" step
+# (fhir-benchmark.yml), so its function bodies run in THAT step's own shell
+# and inherit whatever options are already active there: GitHub's own
+# default for a `run:` step (`bash -e {0}`) plus that step's own
+# `set -uo pipefail` line — together, errexit + nounset + pipefail. Setting
+# options in this file would instead change them for the CALLING step from
+# the `source` line onward, silently altering behaviour the workflow author
+# did not touch.
+#
+# Because it runs in the step's own shell (not a subshell), every function
+# below also sees whatever plain, non-exported shell variables that step has
+# already set BY THE TIME THE FUNCTION IS CALLED (not by the time it is
+# defined) — in particular RESULTS_DIR (the leg's output directory) and
+# ES_DRAIN_TIMEOUT_S (the ES drain deadline), both set once near the top of
+# the step, and, inside its per-suite loop, SUITE_WALL (the current suite's
+# wall-clock seconds, set right before each call this file's functions are
+# used from — the current suite NAME is instead passed as an explicit
+# argument, see "Functions" below). It also sees every real environment
+# variable earlier steps already exported via $GITHUB_ENV — DOCKER_HOST_IP,
+# ES_PORT, ES_PREFIX, ES_CONTAINER, PG_CONTAINER, MONGO_CONTAINER,
+# TGZ_CONTAINER, PG_VOLUME, MONGO_VOLUME, ES_VOLUME, HFS_MONGODB_DATABASE,
+# HFS_DATABASE_URL, HFS_COMPOSITE_SYNC_MODE, HFS_ELASTICSEARCH_WRITE_REFRESH —
+# the normal way a later step's shell inherits an earlier step's $GITHUB_ENV
+# writes, so none of them are re-declared here.
+#
+# Required environment (plain shell variables the "Run benchmark suites"
+# step sets for itself near the top of its `run:` block, because a sourced
+# file — unlike that step's inline `run:` block — never sees ${{ }} GitHub
+# Actions expression substitution; they need not be exported, since sourcing
+# runs this file's functions in that same shell):
+#   BACKEND        matrix.backend
+#   BENCH_PORT     matrix.port (the HFS port this leg's server listens on)
+#   BENCH_RUN_ID   github.run_id — deliberately NOT the step's own $RUN_ID
+#                  shell variable, which holds the possibly-user-overridden
+#                  inputs.run_id tag used for k6 --tag runid=. Container and
+#                  volume names, and the drain-probe identifier below, all
+#                  need the actual run id, not that tag.
+#
+# Functions (call order/sites in the step are unchanged from before this
+# move — see fhir-benchmark.yml's pointer comments for exactly where each is
+# called):
+#   es_drain_gate                    barrier + settle poll run before
+#                                     `search` on a *-elasticsearch leg (and
+#                                     again after search, if it never ran).
+#                                     Writes es-drain.txt and the
+#                                     es-sync-metrics-{before,after}-drain.txt
+#                                     pair.
+#   capture_backend_stats SUITE      per-suite ES/Mongo/composite snapshot.
+#                                     Writes <suite>-esstat.txt,
+#                                     <suite>-mongostat.txt and (ES legs)
+#                                     <suite>-syncmetrics.txt.
+#   write_import_completeness SUITE  import-completeness.txt plus the
+#                                     <1000-bundles ::warning:: (import
+#                                     suite only; a no-op for any other
+#                                     suite).
+#   write_mongo_txn_errors SUITE     import-mongo-txn-errors.txt (mongo*
+#                                     legs, import suite only).
+#   write_search_counts              post-suite search-counts.txt
+#                                     cross-check; re-runs es_drain_gate
+#                                     first if it never ran this leg.
+#   write_containers_state           containers-state.txt (container
+#                                     OOM/exit state + per-volume `du`) plus
+#                                     the end-of-run Elasticsearch volume
+#                                     fill check.
+#   inspect_running CONTAINER        echoes true/false/unknown; used by
+#                                     check_leg_containers_alive below.
+#   check_leg_containers_alive       fails the step (GITHUB_ENV +
+#                                     ::error:: + exit 1) if a backend
+#                                     container died during the suites.
+
+# ── Elasticsearch drain gate (search suite, *-elasticsearch legs) ──
+# In asynchronous mode a write acks once the PRIMARY commits and ONE
+# background worker forwards it to Elasticsearch one at a time
+# (composite/sync.rs, mpsc::channel(1000)) — so up to ~1000 events
+# from crud/import can still be queued when search starts. Two
+# checks, both bounded by the ONE $ES_DRAIN_TIMEOUT_S deadline set
+# in the "Run benchmark suites" step (fhir-benchmark.yml), not here
+# (barrier + settle together, not 600s each):
+#   1. Barrier (exact). A conditional DELETE matching nothing goes
+#      through conditional_delete_handler -> ensure_writes_visible
+#      -> SyncManager::barrier() (composite/storage.rs,
+#      composite/sync.rs): it returns only once the single sync
+#      worker has processed every event queued before it, then
+#      refreshes Patient. It writes nothing (204/200).
+#   2. Settle (catches what the barrier cannot see — e.g. a request
+#      k6 already abandoned that is still running server-side, or
+#      the periodic sync-repair sweep): poll until the ES doc count
+#      AND the primaries' indexing index_total/delete_total are
+#      unchanged across 2 consecutive 5s polls, then _refresh.
+# Completeness compares live resources in the PRIMARY against
+# top-level ES docs (must_not is_contained — contained resources
+# are indexed separately) and reads
+# composite_secondary_sync_needs_reindex from /metrics. A primary
+# count that cannot complete inside its own timeout is recorded as
+# status=incomplete reason=count-timeout, distinct from a genuine
+# mismatch. Never fails the leg: on any status other than "drained"
+# it warns and search still runs anyway, with the reason recorded
+# so the numbers are not mistaken for clean.
+es_drain_gate() {
+  local es="http://$DOCKER_HOST_IP:$ES_PORT"
+  local poll_s=5 need_stable=2 stable=0 last="" cur="" docs="" ops=""
+  local gate_start=$SECONDS t0 barrier_s=0 barrier_http=000
+  local es_live=unknown primary_live=unknown missing=unknown
+  local needs_reindex=unknown primary_count_s=0 status=timeout reason=""
+  local settle_until min_window min_settle_until
+
+  curl -s --max-time 30 "http://localhost:$BENCH_PORT/metrics" 2>/dev/null \
+    | grep -E '^composite_secondary_sync_' > "$RESULTS_DIR/es-sync-metrics-before-drain.txt" || true
+
+  echo "── Elasticsearch drain gate (timeout ${ES_DRAIN_TIMEOUT_S}s) ──"
+  t0=$SECONDS
+  barrier_http=$(curl -s -o "$RESULTS_DIR/es-drain-probe.txt" -w '%{http_code}' \
+      --max-time "$ES_DRAIN_TIMEOUT_S" -X DELETE \
+      "http://localhost:$BENCH_PORT/Patient?identifier=urn:hfs-bench:drain-probe%7C$BENCH_RUN_ID") || barrier_http=000
+  barrier_s=$((SECONDS - t0))
+  echo "  barrier probe: HTTP $barrier_http after ${barrier_s}s"
+
+  # The barrier above shares $ES_DRAIN_TIMEOUT_S with the settle
+  # poll below (one deadline across both — see where it's set, in
+  # the "Run benchmark suites" step). A barrier that itself takes most of
+  # that budget would otherwise leave the settle loop no time to
+  # observe even $need_stable consecutive unchanged polls, and a
+  # genuinely drained index would misreport as status=timeout.
+  # Once the barrier ITSELF succeeds (2xx — proof the sync worker
+  # was caught up as of that moment), guarantee a floor: enough
+  # polls to reach need_stable, plus one initial read, plus a 5s
+  # margin — even if that pushes past the nominal deadline.
+  settle_until=$((gate_start + ES_DRAIN_TIMEOUT_S))
+  if [ "${barrier_http:0:1}" = "2" ]; then
+    min_window=$(( (need_stable + 1) * poll_s + 5 ))
+    min_settle_until=$((SECONDS + min_window))
+    [ "$min_settle_until" -gt "$settle_until" ] && settle_until=$min_settle_until
+  fi
+
+  while [ "$SECONDS" -lt "$settle_until" ]; do
+    docs=$(curl -sf --max-time 30 "$es/_cat/count/${ES_PREFIX}_*?h=count" 2>/dev/null | tr -d '[:space:]') || docs=""
+    ops=$(curl -sf --max-time 30 "$es/${ES_PREFIX}_*/_stats/indexing?filter_path=_all.primaries.indexing.index_total,_all.primaries.indexing.delete_total" 2>/dev/null \
+          | jq -r '"\(._all.primaries.indexing.index_total // 0)/\(._all.primaries.indexing.delete_total // 0)"' 2>/dev/null) || ops=""
+    if [ -z "$docs" ] || [ -z "$ops" ]; then
+      stable=0; last=""
+      echo "  t=$((SECONDS - gate_start))s Elasticsearch did not answer — retrying"
+    else
+      cur="$docs|$ops"
+      if [ "$cur" = "$last" ]; then stable=$((stable + 1)); else stable=0; last="$cur"; fi
+      echo "  t=$((SECONDS - gate_start))s docs=$docs index/delete=$ops unchanged=$stable/$need_stable"
+      [ "$stable" -ge "$need_stable" ] && break
+    fi
+    sleep "$poll_s"
+  done
+  curl -s -o /dev/null --max-time 120 -X POST "$es/${ES_PREFIX}_*/_refresh?allow_no_indices=true" || true
+
+  es_live=$(curl -sf --max-time 30 -H 'Content-Type: application/json' \
+      "$es/${ES_PREFIX}_*/_count?allow_no_indices=true" \
+      -d '{"query":{"bool":{"must_not":{"term":{"is_contained":true}}}}}' 2>/dev/null \
+      | jq -r '.count // "unknown"' 2>/dev/null) || es_live=unknown
+
+  t0=$SECONDS
+  case "$BACKEND" in
+    postgres-elasticsearch)
+      # `timeout 300` only kills the local `docker exec` CLI process
+      # — psql itself keeps counting inside the container and would
+      # otherwise overlap the search suite that follows. Bound it
+      # server-side too (290s, just under the client-side timeout)
+      # with statement_timeout.
+      # -q suppresses the `SET` command tag. Since psql 15,
+      # SHOW_ALL_RESULTS defaults on, so every statement in a -c
+      # string prints a result; without -q stdout would be
+      # "SET\n<n>" (postgres:18 ships psql 18), which
+      # tr -d '[:space:]' turns into "SET<n>" — never a bare
+      # integer, so this always fell through to primary_live=unknown.
+      primary_live=$(timeout 300 docker exec "$PG_CONTAINER" psql -U postgres -d postgres -qtAc \
+          "SET statement_timeout='290s'; SELECT count(*) FROM resources WHERE NOT is_deleted" 2>/dev/null | tr -d '[:space:]') || primary_live=""
+      ;;
+    mongodb-elasticsearch)
+      # idx_resources_type_scan (mongodb/schema.rs
+      # RESOURCES_TYPE_SCAN_INDEX) covers (tenant_id, resource_type,
+      # is_deleted, last_updated, id) — a valid hint for this filter.
+      # maxTimeMS bounds the query server-side for the same reason
+      # as statement_timeout above: `timeout 300` alone only kills
+      # the local mongosh client.
+      primary_live=$(timeout 300 docker exec "$MONGO_CONTAINER" mongosh --quiet "${HFS_MONGODB_DATABASE:-hfs_bench}" --eval \
+          'db.resources.countDocuments({is_deleted:false},{hint:"idx_resources_type_scan",maxTimeMS:290000})' 2>/dev/null | tr -d '[:space:]') || primary_live=""
+      ;;
+    sqlite-elasticsearch)
+      # Prefer python3 (this runner is Linux), but don't assume it's
+      # there; fall back to python, else skip (recorded as unknown).
+      local pybin
+      pybin=$(command -v python3 || command -v python || true)
+      if [ -n "$pybin" ]; then
+        primary_live=$(timeout 300 "$pybin" -c '
+import sqlite3, sys
+c = sqlite3.connect("file:" + sys.argv[1] + "?mode=ro", uri=True, timeout=60)
+print(c.execute("select count(*) from resources where is_deleted=0").fetchone()[0])
+' "$HFS_DATABASE_URL" 2>/dev/null | tr -d '[:space:]') || primary_live=""
+      else
+        primary_live=""
+      fi
+      ;;
+  esac
+  primary_count_s=$((SECONDS - t0))
+  # Reject anything that isn't a bare integer, not just empty —
+  # e.g. a psql/mongosh warning line on stdout would otherwise be
+  # compared byte-for-byte against $es_live below and read as a
+  # (wrong) "mismatch" instead of "the count is unknown".
+  [[ "$primary_live" =~ ^[0-9]+$ ]] || primary_live=unknown
+
+  needs_reindex=$(curl -sf --max-time 30 "http://localhost:$BENCH_PORT/metrics" 2>/dev/null \
+      | awk '/^composite_secondary_sync_needs_reindex/{v=$NF} END{print v+0}') || needs_reindex=unknown
+  curl -s --max-time 30 "http://localhost:$BENCH_PORT/metrics" 2>/dev/null \
+    | grep -E '^composite_secondary_sync_' > "$RESULTS_DIR/es-sync-metrics-after-drain.txt" || true
+
+  if [[ "$primary_live" =~ ^[0-9]+$ ]] && [[ "$es_live" =~ ^[0-9]+$ ]]; then
+    missing=$((primary_live - es_live))
+  fi
+
+  # Ordered by which signal is the strongest evidence of a real
+  # problem, so e.g. a failed barrier is never reported as a plain
+  # "mismatch". count-timeout vs count-failed is a heuristic: the
+  # `$(docker exec ... | tr ...) || primary_live=""` fallback
+  # discards the pipeline's exit status (so the inner `timeout
+  # 300`'s 124 is never inspected) — elapsed time (primary_count_s)
+  # stands in for "the server-side timeout fired" instead.
+  if [ "${barrier_http:0:1}" != "2" ]; then
+    status=incomplete; reason=barrier-failed
+  elif [ "$stable" -lt "$need_stable" ]; then
+    status=timeout
+  elif [ "$es_live" = unknown ]; then
+    status=incomplete; reason=index-count-failed
+  elif [ "$primary_live" = unknown ]; then
+    # The server-side bounds above (statement_timeout / maxTimeMS)
+    # fire at ~290s, well under the 300s client-side `timeout`, so
+    # a real server-side timeout is observed here as ~290-291s
+    # elapsed, not ~299s. 285 separates that from a near-instant
+    # failure (bad container, missing client) without relying on
+    # the discarded exit code (see the pipefail note above).
+    if [ "$primary_count_s" -ge 285 ]; then
+      status=incomplete; reason=count-timeout
+    else
+      status=incomplete; reason=count-failed
+    fi
+  elif [ "$primary_live" = "$es_live" ]; then
+    status=drained
+  else
+    status=incomplete; reason=mismatch
+  fi
+
+  {
+    echo "status=$status"
+    echo "reason=$reason"
+    echo "barrier_http=$barrier_http"
+    echo "barrier_seconds=$barrier_s"
+    echo "primary_live=$primary_live"
+    echo "es_live=$es_live"
+    echo "missing=$missing"
+    echo "primary_count_seconds=$primary_count_s"
+    echo "needs_reindex_after=$needs_reindex"
+    echo "gate_seconds=$((SECONDS - gate_start))"
+    echo "sync_mode=${HFS_COMPOSITE_SYNC_MODE:-unknown}"
+    echo "write_refresh=${HFS_ELASTICSEARCH_WRITE_REFRESH:-unknown}"
+  } > "$RESULTS_DIR/es-drain.txt"
+  cat "$RESULTS_DIR/es-drain.txt"
+  if [ "$status" != drained ]; then
+    echo "::warning::Elasticsearch drain gate status=$status${reason:+ reason=$reason} (barrier_http=$barrier_http primary_live=$primary_live es_live=$es_live) — search may run against an incomplete index (es-drain.txt)."
+  fi
+  return 0
+}
+
+# ── Per-suite backend snapshot — the ES/Mongo/composite analogue of
+# the pgstat block (still inline in fhir-benchmark.yml). Counters are
+# CUMULATIVE where noted; bound every call and never fail the leg over a
+# stats capture.
+capture_backend_stats() {
+  local suite="$1"
+  if [ -n "${ES_PORT:-}" ]; then
+    local es="http://$DOCKER_HOST_IP:$ES_PORT"
+    {
+      echo "suite=$suite wall_seconds=$SUITE_WALL"
+      echo "── cluster health (yellow expected: 1 replica, 1 node) ──"
+      curl -s --max-time 30 "$es/_cluster/health?filter_path=status,active_shards,unassigned_shards" 2>/dev/null || true
+      echo
+      echo "── ${ES_PREFIX}_* indices by size ──"
+      curl -s --max-time 30 "$es/_cat/indices/${ES_PREFIX}_*?v&h=index,health,docs.count,docs.deleted,store.size&s=store.size:desc" 2>/dev/null | head -25 || true
+      echo "── node: jvm heap, indexing, write/search thread-pool rejections ──"
+      curl -s --max-time 30 "$es/_nodes/stats/jvm,indices,thread_pool?filter_path=nodes.*.jvm.mem.heap_used_percent,nodes.*.indices.indexing.index_total,nodes.*.indices.indexing.index_time_in_millis,nodes.*.thread_pool.write.rejected,nodes.*.thread_pool.search.rejected" 2>/dev/null || true
+      echo
+      echo "── container ──"
+      timeout 60 docker stats --no-stream --format '{{.Name}} mem={{.MemUsage}} cpu={{.CPUPerc}}' "$ES_CONTAINER" 2>/dev/null || true
+    } > "$RESULTS_DIR/${suite}-esstat.txt" 2>&1 || echo "::warning::ES stats capture failed for $suite"
+  fi
+  if [ -n "${MONGO_CONTAINER:-}" ]; then
+    {
+      echo "suite=$suite wall_seconds=$SUITE_WALL"
+      timeout 60 docker exec "$MONGO_CONTAINER" mongosh --quiet "${HFS_MONGODB_DATABASE:-hfs_bench}" --eval '
+        const s = db.serverStatus(), c = s.wiredTiger.cache;
+        printjson({
+          wt_cache_bytes: c["bytes currently in the cache"],
+          wt_cache_max_bytes: c["maximum bytes configured"],
+          wt_dirty_bytes: c["tracked dirty bytes in the cache"],
+          wt_app_thread_evictions: c["pages evicted by application threads"],
+          opcounters: s.opcounters,
+          txn: { started: s.transactions.totalStarted, committed: s.transactions.totalCommitted, aborted: s.transactions.totalAborted },
+          connections: s.connections.current
+        });' 2>/dev/null || true
+      timeout 60 docker stats --no-stream --format '{{.Name}} mem={{.MemUsage}} cpu={{.CPUPerc}}' "$MONGO_CONTAINER" 2>/dev/null || true
+    } > "$RESULTS_DIR/${suite}-mongostat.txt" 2>&1 || echo "::warning::Mongo stats capture failed for $suite"
+  fi
+  case "$BACKEND" in
+    *-elasticsearch)
+      {
+        echo "suite=$suite wall_seconds=$SUITE_WALL"
+        curl -s --max-time 30 "http://localhost:$BENCH_PORT/metrics" 2>/dev/null | grep -E '^composite_secondary_sync_' \
+          || echo "(none recorded at this point)"
+      } > "$RESULTS_DIR/${suite}-syncmetrics.txt" 2>&1 || echo "::warning::composite sync metrics capture failed for $suite"
+      ;;
+  esac
+}
+
+# ── Import completeness (F1a), all legs ──────────────────────────
+# k6 caps import at 60m (import.js maxDuration); an async
+# *-elasticsearch leg is expected to hit that cap (one sequential
+# sync worker, two ES requests per resource). A truncated corpus
+# makes crud/search look better than a full 1000/1000 leg, so flag
+# it rather than let the numbers pass as comparable.
+write_import_completeness() {
+  local suite="$1"
+  if [ "$suite" = import ] && [ -f "$RESULTS_DIR/import.json" ]; then
+    local IMP_OK IMP_ITERS IMP_ENTRIES IMP_OK_INT
+    IMP_OK=$(jq -r '.metrics.checks.passes // 0' "$RESULTS_DIR/import.json" 2>/dev/null) || IMP_OK=0
+    IMP_ITERS=$(jq -r '.metrics.iterations.count // 0' "$RESULTS_DIR/import.json" 2>/dev/null) || IMP_ITERS=0
+    IMP_ENTRIES=$(jq -r '.metrics.bundle_size.count // 0' "$RESULTS_DIR/import.json" 2>/dev/null) || IMP_ENTRIES=0
+    {
+      echo "bundles_ok=$IMP_OK"
+      echo "iterations=$IMP_ITERS"
+      echo "entries=$IMP_ENTRIES"
+      echo "wall_seconds=$SUITE_WALL"
+    } > "$RESULTS_DIR/import-completeness.txt"
+    IMP_OK_INT="${IMP_OK%.*}"
+    case "$IMP_OK_INT" in ''|*[!0-9]*) IMP_OK_INT=0 ;; esac
+    if [ "$IMP_OK_INT" -lt 1000 ]; then
+      echo "::warning::import completed ${IMP_OK}/1000 bundles (${IMP_ENTRIES} entries) — k6's 60-min cap or failed bundles. crud/search on this leg ran on a smaller corpus and are not comparable to a 1000/1000 leg."
+    fi
+  elif [ "$suite" = import ]; then
+    # k6 died or was killed before writing any summary at all — the
+    # most incomplete import there is. Without this branch the
+    # completeness file (and its warning) are silently skipped
+    # entirely, and crud/search look like any other leg.
+    printf 'bundles_ok=0\niterations=0\nentries=0\nwall_seconds=%s\nnote=no-k6-summary\n' \
+      "$SUITE_WALL" > "$RESULTS_DIR/import-completeness.txt"
+    echo "::warning::import produced no k6 summary (import.json missing — k6 died or was killed) — crud/search on this leg are not comparable to any leg that completed an import."
+  fi
+}
+
+# ── Mongo transaction-abort count (F8), mongo legs, after import ──
+# A bundle is one multi-document transaction over sequential
+# entries with no whole-transaction retry (mongodb/storage.rs) —
+# any abort fails the bundle. 20 concurrent transactions against a
+# capped WiredTiger cache can plausibly hit these.
+write_mongo_txn_errors() {
+  local suite="$1"
+  if [ "$suite" = import ]; then
+    case "$BACKEND" in
+      mongodb*)
+        local MONGO_TXN_ERR
+        MONGO_TXN_ERR=$(grep -ciE 'WriteConflict|TransientTransactionError|TransactionTooLargeForCache|NoSuchTransaction|Commit failed' \
+            "/tmp/hfs-bench-$BACKEND.log" 2>/dev/null) || MONGO_TXN_ERR=0
+        echo "$MONGO_TXN_ERR" > "$RESULTS_DIR/import-mongo-txn-errors.txt"
+        ;;
+    esac
+  fi
+}
+
+# ── Result-size cross-check (F5), all legs ─────────────────────────
+# Runs AFTER search so it cannot warm caches for the measured suite.
+# k6's search checks are status-only, so a composite search that
+# silently degrades to matching on the parameter name alone (or a
+# Mongo sa/eb range mistranslation) would otherwise pass unnoticed —
+# different result-set sizes change latency and nothing else would
+# show it. Single-quoted so `$gt100`/`$gt140` reach the query string
+# literally instead of being shell-expanded.
+#
+# On an ES leg run WITHOUT 'search' in tests (e.g. tests=prewarm,import),
+# the suite loop never called es_drain_gate, so up to ~1000 queued
+# async sync events could still be unprocessed — these counts would
+# then read low and look like a cross-leg mismatch that isn't one.
+# Run the gate here too when it hasn't already run this leg.
+write_search_counts() {
+  if [ -n "${ES_PORT:-}" ] && [ ! -f "$RESULTS_DIR/es-drain.txt" ]; then
+    es_drain_gate || echo '::warning::drain gate errored'
+  fi
+  {
+    echo "query|total|http|seconds"
+    # shellcheck disable=SC2016 # single-quoted deliberately: $gt100/$gt140
+    # must reach the query string literally, not shell-expand.
+    for Q in 'Patient?_summary=count' 'Observation?_summary=count' 'Encounter?_summary=count' \
+             'Patient?name:contains=an&_summary=count' 'Observation?date=ge2020-01-01&_summary=count' \
+             'Observation?date=sa2020-01-01&_summary=count' \
+             'Observation?code=8302-2,29463-7&_summary=count' 'Observation?value-quantity=gt100&_summary=count' \
+             'Observation?value-quantity=sa100&_summary=count' \
+             'Observation?code-value-quantity=http://loinc.org%7C8867-4$gt100&_summary=count' \
+             'Observation?combo-code-value-quantity=8480-6$gt140&_summary=count' 'Encounter?class=AMB,EMER&_summary=count'; do
+      local SC_T0 SC_OUT SC_TOTAL SC_HTTP
+      SC_T0=$SECONDS
+      SC_OUT=$(curl -s --max-time 30 -w '\n%{http_code}' "http://localhost:$BENCH_PORT/$Q") || SC_OUT=$'\n000'
+      SC_TOTAL=$(printf '%s\n' "$SC_OUT" | head -n -1 | jq -r '.total // "?"' 2>/dev/null) || SC_TOTAL="?"
+      SC_HTTP=$(printf '%s\n' "$SC_OUT" | tail -n1)
+      echo "$Q|$SC_TOTAL|$SC_HTTP|$((SECONDS - SC_T0))"
+    done
+  } > "$RESULTS_DIR/search-counts.txt" || echo "::warning::search-counts cross-check failed"
+}
+
+# ── Container / volume final state (end of step) ───────────────────
+# k6 runs with `|| true` throughout, so a mid-run OOM would otherwise
+# only ever show up as mysteriously bad numbers, never as a failed
+# step. This is the step's own final action: every result file above
+# is already written, so a fail-fast exit here loses nothing.
+write_containers_state() {
+  : > "$RESULTS_DIR/containers-state.txt"
+  local c CS_OUT
+  for c in ${PG_CONTAINER:-} ${MONGO_CONTAINER:-} ${ES_CONTAINER:-} ${TGZ_CONTAINER:-}; do
+    # Tell a Docker daemon hiccup apart from proof the container is
+    # actually gone (Docker's own "no such container"), same
+    # distinction inspect_running() draws below — "Generate step
+    # summary" reads "gone" as died and "unknown" as a neutral note,
+    # not a death.
+    if CS_OUT=$(timeout 60 docker inspect -f "$c OOMKilled={{.State.OOMKilled}} Status={{.State.Status}} ExitCode={{.State.ExitCode}} StartedAt={{.State.StartedAt}}" "$c" 2>&1); then
+      printf '%s\n' "$CS_OUT" >> "$RESULTS_DIR/containers-state.txt"
+    elif printf '%s' "$CS_OUT" | grep -qi 'no such'; then
+      echo "$c inspect-failed gone" >> "$RESULTS_DIR/containers-state.txt"
+    else
+      echo "$c inspect-failed unknown" >> "$RESULTS_DIR/containers-state.txt"
+    fi
+  done
+  local VOLDU_I=0 v VOLDU_NAME VOLDU_MB
+  for v in ${PG_VOLUME:-} ${MONGO_VOLUME:-} ${ES_VOLUME:-}; do
+    VOLDU_I=$((VOLDU_I + 1))
+    VOLDU_NAME="hfs-bench-voldu-$BACKEND-$BENCH_RUN_ID-$VOLDU_I"
+    # The outer `timeout 150` only kills the local docker CLI, not a
+    # container still running on the daemon — an outer-timeout kill
+    # would otherwise leave `du` running with this volume's :ro mount
+    # still open, and the later `docker volume rm -f` in "Stop
+    # ephemeral ..." would then fail and leak the volume until the
+    # next run's reaper. The inner `timeout -s KILL 120 du` bounds du
+    # ITSELF, well inside the outer timeout, so (with --rm) the
+    # container always exits and is removed on its own; the distinct
+    # per-volume name avoids a stale container from one volume
+    # blocking the next volume's `--name`.
+    VOLDU_MB=$(timeout 150 docker run --rm --name "$VOLDU_NAME" \
+        --label hfs-bench=1 --label "hfs-bench-run=$BENCH_RUN_ID" --label "hfs-bench-leg=$BACKEND" \
+        -v "$v:/v:ro" alpine:3 timeout -s KILL 120 du -sm /v 2>/dev/null | awk '{print $1}') || VOLDU_MB=""
+    docker rm -f "$VOLDU_NAME" >/dev/null 2>&1 || true
+    echo "$v du_mb=${VOLDU_MB:-unknown}" >> "$RESULTS_DIR/containers-state.txt"
+  done
+  # Elasticsearch indices go read-only at the 95% flood-stage
+  # watermark; the only check so far ran at leg start, before ES had
+  # written anything. Check again now that import/crud/search have
+  # actually filled the volume.
+  if [ -n "${ES_VOLUME:-}" ]; then
+    local ES_VOLDF_END_NAME ES_VOL_PCT_USED_END
+    ES_VOLDF_END_NAME="hfs-bench-voldf-$BACKEND-$BENCH_RUN_ID-esend"
+    ES_VOL_PCT_USED_END=$(timeout 60 docker run --rm --name "$ES_VOLDF_END_NAME" \
+        --label hfs-bench=1 --label "hfs-bench-run=$BENCH_RUN_ID" --label "hfs-bench-leg=$BACKEND" \
+        -v "$ES_VOLUME:/v" alpine:3 df -Pm /v 2>/dev/null | awk 'NR==2{gsub("%","",$5); print $5}') || ES_VOL_PCT_USED_END=""
+    # See the Postgres step's identical note: `timeout` alone cannot be
+    # trusted to leave `--rm` a clean container to remove.
+    timeout 15 docker rm -f "$ES_VOLDF_END_NAME" >/dev/null 2>&1 || true
+    if [[ "$ES_VOL_PCT_USED_END" =~ ^[0-9]+$ ]] && [ "$ES_VOL_PCT_USED_END" -gt 85 ]; then
+      echo "::warning::Elasticsearch data volume $ES_VOLUME filesystem is ${ES_VOL_PCT_USED_END}% used at end of run — indices go read-only at Elasticsearch's 95% flood-stage watermark."
+    fi
+  fi
+  cat "$RESULTS_DIR/containers-state.txt"
+}
+
+# Only the actual backend (primary/ES) containers gate the leg —
+# the tgz corpus server dying mid-run is logged above but does not,
+# by itself, invalidate this leg's numbers. A container that is not
+# Running (OOM-killed, exited, or gone entirely) fails the gate.
+#
+# `docker inspect` failing is ambiguous by itself — it also fails on
+# a daemon hiccup, which must NOT be read as "died". Tell them apart
+# by stderr: Docker's own "No such container" is unambiguous proof
+# the container is gone (never coming back), so it fails the gate
+# immediately; any other failure is a possible hiccup and gets one
+# retry before being read as "unknown" (warn only, not counted as
+# died).
+inspect_running() {
+  local out
+  if out=$(timeout 60 docker inspect -f '{{.State.Running}}' "$1" 2>&1); then
+    # `2>&1` above also captures a Docker CLI stderr warning printed
+    # ahead of the value (e.g. "DOCKER_HOST environment variable
+    # overrides the active context") — an unfiltered `echo "$out"`
+    # would then return "<warning>\ntrue" (never == "true") and read
+    # a live container as died. Only the last line is the actual
+    # `{{.State.Running}}` value.
+    printf '%s\n' "$out" | tail -n1
+  elif printf '%s' "$out" | grep -qi 'no such'; then
+    echo "false"
+  else
+    echo "unknown"
+  fi
+}
+
+check_leg_containers_alive() {
+  local c RUNNING
+  LEG_DIED=false
+  for c in ${PG_CONTAINER:-} ${MONGO_CONTAINER:-} ${ES_CONTAINER:-}; do
+    RUNNING=$(inspect_running "$c")
+    if [ "$RUNNING" = unknown ]; then
+      sleep 2
+      RUNNING=$(inspect_running "$c")
+    fi
+    if [ "$RUNNING" = unknown ]; then
+      echo "::warning::could not inspect container $c to confirm it is still running (Docker host hiccup?) — not counted as died"
+    elif [ "$RUNNING" != "true" ]; then
+      LEG_DIED=true
+    fi
+  done
+  if [ "$LEG_DIED" = true ]; then
+    echo "LEG_CONTAINER_DIED=true" >> "$GITHUB_ENV"
+    echo "::error::A backend container for leg '$BACKEND' died or was OOM-killed during the suites — this leg's numbers are not valid (containers-state.txt)."
+    exit 1
+  fi
+}
diff --git a/.github/scripts/fhir-bench/summary_backends.py b/.github/scripts/fhir-bench/summary_backends.py
new file mode 100755
index 000000000..f4acb1822
--- /dev/null
+++ b/.github/scripts/fhir-bench/summary_backends.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+#
+# Per-leg benchmark diagnostics for the step summary: tuning table, import
+# completeness, ES drain status, host load at crud start, "how to read this
+# leg" guidance, the result-size cross-check + crud-residue caption, Mongo
+# transaction-error heuristic, composite ES sync-failure count, and
+# dead-container warnings.
+#
+# Called from: the `benchmark` job's "Generate step summary" step
+# (fhir-benchmark.yml), via:
+#   python3 "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/summary_backends.py"
+# runs before the Results table (after the header lines) inside that step's
+# `{ ... } >> "$GITHUB_STEP_SUMMARY"` group, so its stdout lands at the same
+# position this block originally printed at: after the leg header table,
+# before the step's own (still inline) Results table and Environment
+# paragraph — this prints unconditionally, same as before, regardless of
+# whether any *.json summary file exists.
+#
+# Required environment (exported by the workflow step's env:):
+#   BACKEND               matrix.backend
+#   IN_PG_SHARED_BUFFERS  inputs.pg_shared_buffers — used only for the
+#                         mongodb "How to read this leg" shared_buffers
+#                         comparison below; nothing else in the calling
+#                         step (including its own still-inline heredoc)
+#                         reads it
+# Also reads GITHUB_WORKSPACE (set by Actions on every runner, not just this
+# step) and, when present, CAPACITY_NEED_MB / CAPACITY_AVAIL_MB /
+# CAPACITY_WAIT_S — $GITHUB_ENV values "Docker host capacity gate" writes
+# before its own early exit, inherited automatically like any other
+# $GITHUB_ENV value (no env: mapping needed for those three).
+#
+# Input: the *.txt files "Run benchmark suites" wrote under
+# bench-results/<backend>/ — runner-info.txt, import-completeness.txt,
+# es-drain.txt, host-contention.txt, search-counts.txt, crud-residue.txt,
+# import-mongo-txn-errors.txt, es-sync-metrics-after-drain.txt,
+# containers-state.txt. Each section below is skipped, not fatal, when its
+# file is absent (e.g. a leg that died before that file was ever written).
+# Output: stdout — Markdown, appended to $GITHUB_STEP_SUMMARY by the caller.
+import os
+import re
+
+results_dir = os.path.join(os.environ["GITHUB_WORKSPACE"], "bench-results", os.environ["BACKEND"])
+backend = os.environ["BACKEND"]
+
+# ── #1475 leg detail ────────────────────────────────────────────────
+# Everything below reads files "Run benchmark suites" already wrote to
+# results_dir, plus (for the Capacity gate row alone, via the ri
+# fallback just below) a couple of $GITHUB_ENV values "Docker host
+# capacity gate" writes even on its own early exit. So this prints
+# even when a leg produced no *.json summary at all (capacity gate
+# failure, a backend container that died before any suite ran) —
+# exactly when a reader most needs the context.
+
+def read_kv(path, sep):
+    kv = {}
+    if os.path.exists(path):
+        for line in open(path):
+            if sep in line:
+                k, v = line.strip().split(sep, 1)
+                kv[k.strip()] = v.strip()
+    return kv
+
+# Tuning table: heap, sync mode, WT cache, Mongo pool, max_parallel,
+# leg timeout and the capacity gate's own numbers — all already
+# recorded in runner-info.txt by "Run benchmark suites".
+ri = read_kv(f"{results_dir}/runner-info.txt", ":")
+if not ri:
+    # A capacity-gate failure exits before "Run benchmark suites"
+    # ever runs, so runner-info.txt is never written. Fall back to
+    # the $GITHUB_ENV values "Docker host capacity gate" wrote
+    # before its own early exit, so this leg still gets a Capacity
+    # gate row instead of an empty summary.
+    cap_need = os.environ.get("CAPACITY_NEED_MB")
+    if cap_need is not None:
+        ri = {
+            "capacity_need_mb": cap_need,
+            "capacity_avail_mb": os.environ.get("CAPACITY_AVAIL_MB", "unknown"),
+            "capacity_wait_s": os.environ.get("CAPACITY_WAIT_S", "unknown"),
+        }
+if ri:
+    print("### Leg configuration\n")
+    print("| | |")
+    print("|---|---|")
+    print(f"| **Backend** | `{backend}` |")
+    if ri.get("es_heap", "n/a") != "n/a":
+        print(f"| **ES heap** | `{ri.get('es_heap', '?')}` |")
+        print(f"| **ES sync mode** | `{ri.get('es_sync_mode', '?')}` |")
+    if ri.get("mongo_wt_cache_gb", "n/a") != "n/a":
+        print(f"| **Mongo WT cache** | `{ri.get('mongo_wt_cache_gb', '?')}` GB |")
+        print(f"| **Mongo pool max** | `{ri.get('mongo_pool_max', '?')}` |")
+    print(f"| **Legs at once (max_parallel)** | {ri.get('max_parallel', '?')} |")
+    print(f"| **Leg timeout** | {ri.get('leg_timeout_min', '?')} min |")
+    need = ri.get("capacity_need_mb")
+    if need is not None:
+        print(f"| **Capacity gate** | need {need} MB, available "
+              f"{ri.get('capacity_avail_mb', '?')} MB, waited {ri.get('capacity_wait_s', '?')} s |")
+    print()
+
+# Import completeness (F1b): k6 caps import at 60m, so an async
+# *-elasticsearch leg is expected to hit that cap — flag a leg whose
+# crud/search ran on fewer than 1000 imported bundles.
+ic = read_kv(f"{results_dir}/import-completeness.txt", "=")
+if ic:
+    try:
+        imp_ok = int(float(ic.get("bundles_ok", "0")))
+    except ValueError:
+        imp_ok = 0
+    try:
+        imp_entries = int(float(ic.get("entries", "0")))
+    except ValueError:
+        imp_entries = 0
+    imp_warn = "" if imp_ok >= 1000 else (
+        " ⚠ **import incomplete — crud/search on this leg ran on a smaller corpus "
+        "and are not comparable to a 1000/1000 leg** (and abandoned import requests "
+        "may still have been running server-side)")
+    print(f"\n**Import:** {imp_ok:,}/1000 bundles, {imp_entries:,} entries in "
+          f"{ic.get('wall_seconds', '?')} s.{imp_warn}")
+
+# ES drain (F2/F4): the barrier + settle + primary-vs-ES-count gate
+# "Run benchmark suites" ran before the search suite.
+drain = read_kv(f"{results_dir}/es-drain.txt", "=")
+if drain:
+    d_status = drain.get("status", "?")
+    d_missing = drain.get("missing", "unknown")
+    try:
+        d_needs_reindex = float(drain.get("needs_reindex_after", "0"))
+    except ValueError:
+        d_needs_reindex = 0.0
+    d_warn = " ⚠" if (d_status != "drained" or d_missing not in ("0", "unknown")
+                       or d_needs_reindex > 0) else ""
+    print(f"\n**ES drain:** status=`{d_status}` (barrier HTTP {drain.get('barrier_http', '?')} "
+          f"after {drain.get('barrier_seconds', '?')} s); primary_live={drain.get('primary_live', '?')} "
+          f"es_live={drain.get('es_live', '?')} missing={d_missing} "
+          f"needs_reindex={drain.get('needs_reindex_after', '?')}.{d_warn}")
+
+# Host load (F9), every leg — not just Postgres ones (see the
+# Environment paragraph further below, which stays Postgres-only).
+contention_path = f"{results_dir}/host-contention.txt"
+if os.path.exists(contention_path):
+    hc = re.search(
+        r"suite=crud phase=start host_loadavg=(\S+).*host_containers=(\d+).*host_mem_avail_kb=(\S+)",
+        open(contention_path).read())
+    if hc:
+        print(f"\n**Host load at crud start:** loadavg {hc.group(1)}, {hc.group(2)} containers, "
+              f"{hc.group(3)} kB MemAvailable on the Docker host.")
+
+# "How to read this leg" (F4): what the ES/Mongo numbers actually
+# measure, so they are not misread as directly comparable to a bare
+# sqlite/postgres leg.
+if backend.endswith("-elasticsearch"):
+    if ri.get("es_sync_mode") == "synchronous":
+        print("\n**How to read this leg.** Every search is answered by Elasticsearch; the primary's "
+              "own search index is not written. In `synchronous` mode a write returns only after "
+              "Elasticsearch has indexed it — bundles go through a per-resource-type `_bulk` request "
+              "with `refresh=wait_for` against a 200ms `refresh_interval`, so **import/crud latency "
+              "on this leg includes Elasticsearch indexing plus that refresh wait**, not just the "
+              "primary. Every update and delete also runs a `_delete_by_query` with a forced refresh "
+              "across all of the tenant's indices. Cluster health `yellow` is expected (1 replica per "
+              "index, 1 node). Compare search latency with another leg only if both imported the same "
+              "entry count and the ES drain above says `drained`.")
+    else:
+        print("\n**How to read this leg.** Every search is answered by Elasticsearch; the primary's "
+              "own search index is not written. Writes commit on the primary first. In `asynchronous` "
+              "mode ONE background worker then forwards each resource to Elasticsearch individually "
+              "(an index-exists check plus an index request per resource) through a 1,000-event queue "
+              "that blocks writers when full, so **import and crud throughput on this leg is that "
+              "worker's rate, not the primary's**. Every update and delete also runs a "
+              "`_delete_by_query` with a forced refresh across all of the tenant's indices. Cluster "
+              "health `yellow` is expected (1 replica per index, 1 node). Compare search latency with "
+              "another leg only if both imported the same entry count and the ES drain above says "
+              "`drained`.")
+if backend.startswith("mongodb"):
+    pg_shared_buffers = os.environ.get("IN_PG_SHARED_BUFFERS") or "2GB"
+    print(f"\n**How to read this leg.** MongoDB 7.0 single-member replica set (directConnection); "
+          f"WiredTiger cache **{ri.get('mongo_wt_cache_gb', '?')} GB** (Postgres: shared_buffers "
+          f"{pg_shared_buffers}; both also use the host page cache). Each FHIR "
+          "transaction bundle is one multi-document transaction (lifetime "
+          f"{ri.get('mongo_txn_lifetime_s', '900')} s); an aborted transaction fails that bundle's "
+          "`Bundle import` check — see Checks ✗ on the import row (in Results, below) and "
+          "`txn.aborted` in `import-mongostat.txt`.")
+
+# Result-size cross-check (F5): a composite search that silently
+# degrades to matching on the parameter name alone (or a Mongo sa/eb
+# mistranslation) changes the result-set size, and nothing else
+# would show it.
+sc_path = f"{results_dir}/search-counts.txt"
+if os.path.exists(sc_path):
+    sc_lines = [l.rstrip("\n") for l in open(sc_path) if l.strip()]
+    sc_rows = [l.split("|") for l in sc_lines[1:] if l.count("|") == 3]
+    if sc_rows:
+        print("\n### Result-size cross-check\n")
+        print("| Query | Total | HTTP | Seconds |")
+        print("|---|---:|---:|---:|")
+        for q, total, http, secs in sc_rows:
+            print(f"| `{q}` | {total} | {http} | {secs} |")
+
+        # Comparability caption: crud.js creates 9 resources per
+        # iteration and deletes them in the same one; prewarm.js
+        # runs the identical script at low concurrency first. A
+        # failed delete check on EITHER leaves a live row this
+        # table's totals cannot tell apart from import.
+        # crud-residue.txt (written by "Run benchmark suites" from
+        # the upstream k6/crud.js check names) is that count, per
+        # resource type, created minus deleted.
+        cr = read_kv(f"{results_dir}/crud-residue.txt", "=")
+        try:
+            residue_total = sum(int(v) for v in cr.values())
+        except ValueError:
+            residue_total = 0
+        patient_residue = cr.get("Patient", "0")
+        print(f"\n_totals include {residue_total} crud/prewarm leftovers "
+              f"(Patient: {patient_residue}) and any import shortfall; a composite "
+              "total equal to the Observation total means the name-only fallback; "
+              "seconds after a 000 row overlap the previous query (HFS keeps "
+              "executing after curl's --max-time)._")
+
+# Mongo transaction-abort count (F8): this counts matching HFS log
+# LINES across the whole log (startup, prewarm and import), not
+# bundles — a heuristic, not an exact bundle-failure count (a
+# WriteConflict retried into a VersionConflict, for instance, won't
+# match this grep). The exact per-bundle failure count is the
+# import row's Checks ✗ above.
+mte_path = f"{results_dir}/import-mongo-txn-errors.txt"
+if os.path.exists(mte_path):
+    try:
+        mte_n = int(open(mte_path).read().strip() or "0")
+    except ValueError:
+        mte_n = 0
+    if mte_n > 0:
+        print(f"\n⚠ **Mongo transaction errors (heuristic):** {mte_n} HFS log line(s) matching "
+              "WriteConflict/TransientTransactionError/TransactionTooLargeForCache/NoSuchTransaction/"
+              "commit-failed — bundle failures are the import row's Checks ✗ (in Results, below), "
+              "not this count.")
+
+# Final ES sync failures (spec §15d): composite_secondary_sync_failures_total
+# (observability/composite_metrics.rs SECONDARY_SYNC_FAILURES) is a
+# counter that only ever goes up (record_secondary_sync_failure ->
+# increment(1)) — a later successful reindex by the sync-repair
+# sweep does NOT decrement it. So a nonzero count here is cumulative
+# over the whole leg, not necessarily still outstanding at the end;
+# what's still outstanding is composite_secondary_sync_needs_reindex
+# (the gauge already shown on the ES drain line above), captured
+# into es-sync-metrics-after-drain.txt by the drain gate.
+esf_path = f"{results_dir}/es-sync-metrics-after-drain.txt"
+if os.path.exists(esf_path):
+    # One line per {backend,operation} label combination — sum all
+    # of them, not just the first match, or a failure recorded
+    # under one operation (e.g. "update") would be missed while
+    # "create" reads zero.
+    esf_vals = re.findall(r"^composite_secondary_sync_failures_total\S*\s+(\S+)",
+                           open(esf_path).read(), re.M)
+    esf_n = 0.0
+    for v in esf_vals:
+        try:
+            esf_n += float(v)
+        except ValueError:
+            pass
+    if esf_n > 0:
+        print(f"\n⚠ **Composite sync:** {esf_n:g} final ES sync failure(s) recorded "
+              "(cumulative — includes any the sync-repair sweep later fixed; what is "
+              "still outstanding is `needs_reindex` in the ES drain line).")
+
+# Dead-container warning: a backend container that died or was
+# OOM-killed mid-suite invalidates this leg's numbers outright. The
+# tgz corpus server is excluded — "Run benchmark suites" deliberately
+# does not gate the leg on it (only the primary/ES containers do).
+# "inspect-failed gone" (Docker's own "no such container" — see
+# suite-lib.sh's write_containers_state, which writes this file) is
+# unambiguous proof of death; "inspect-failed
+# unknown" is a possible daemon hiccup and gets a neutral note
+# instead of being folded into the same warning.
+cs_path = f"{results_dir}/containers-state.txt"
+if os.path.exists(cs_path):
+    cs_lines = [l.strip() for l in open(cs_path) if not l.startswith("hfs-bench-tgz-")]
+    cs_bad = [l for l in cs_lines
+              if re.search(r"OOMKilled=true|Status=(exited|dead)|inspect-failed gone", l)]
+    cs_unknown = [l for l in cs_lines if "inspect-failed unknown" in l]
+    if cs_bad:
+        print("\n⚠ **A backend container died during the run — do not use these numbers:** "
+              + "; ".join(f"`{b}`" for b in cs_bad))
+    if cs_unknown:
+        print("\n_Could not confirm final state for: "
+              + "; ".join(f"`{b}`" for b in cs_unknown)
+              + " (Docker host hiccup at inspect time — not counted as died)._")
diff --git a/.github/workflows/fhir-benchmark.yml b/.github/workflows/fhir-benchmark.yml
index 8b1992e7a..fe0fefd10 100644
--- a/.github/workflows/fhir-benchmark.yml
+++ b/.github/workflows/fhir-benchmark.yml
@@ -13,19 +13,38 @@ name: FHIR Server Benchmark
 # Topology (mirrors hts-benchmark.yml — HFS and k6 run natively ON the runner;
 # only the stateful dependencies live in containers on the REMOTE Docker daemon):
 #
-#   k6 (runner) ──HTTP──▶ HFS (runner, :8081/:8082)
-#        │                     │
-#        │                     └──SQL──▶ postgres:18 container  (postgres backend only)
+#   k6 (runner) ──HTTP──▶ HFS (runner, :8081-:8086, one port per backend)
+#        │                  ├──SQL────▶ postgres:18             (postgres, postgres-elasticsearch)
+#        │                  ├──Mongo──▶ mongo:7.0, 1-member rs0 (mongodb, mongodb-elasticsearch)
+#        │                  └──HTTP───▶ elasticsearch:8.15.0    (*-elasticsearch: serves every search)
 #        └──HTTP──▶ tgz bundle server container  (Synthea import corpus)
 #
-#   • HFS is built once with both storage backends compiled in and the backend
-#     is chosen at runtime via HFS_STORAGE_BACKEND:
-#       - sqlite   — file-backed DB on the runner.
-#       - postgres — ephemeral postgres:18 container, tuned to approximate the
+#   • HFS is built once with sqlite, postgres, mongodb and elasticsearch
+#     compiled in; the leg picks one via HFS_STORAGE_BACKEND:
+#       - sqlite (8081)   — file-backed DB on the runner.
+#       - postgres (8082) — ephemeral postgres:18, tuned to approximate the
 #         benchmark's infra/postgres/postgres.conf.
-#       - all      — both, as parallel matrix legs (default).
+#       - mongodb (8083)  — ephemeral single-member replica set (FHIR
+#         transaction Bundles need one), WiredTiger cache capped.
+#       - sqlite-/postgres-/mongodb-elasticsearch (8084/8085/8086) — the same
+#         primary plus a single-node Elasticsearch that serves every search; the
+#         primary's own search index is not written (set_search_offloaded).
+#   • `backend`: core (default) = sqlite + postgres; all = all six;
+#     elasticsearch = the three composites; or any single leg. `max_parallel`
+#     (default 1, clamped to 1..2) caps how many legs share the Docker host at
+#     once.
+#   • s3 / s3-elasticsearch are not benchmarked: bare s3 has no search
+#     (persistence/src/backends/s3/storage.rs:1763-1775) — its only
+#     conditional support is identifier-scoped conditional create
+#     (persistence/src/backends/s3/backend.rs:94), not enough for import's
+#     conditional references or the search suite. s3-elasticsearch is a
+#     follow-up issue.
 #   • The `tgz` bundle server (benchmark's infra/tgz) serves the Synthea
 #     `bulk_1k` corpus consumed by the import suite.
+#   • Every named volume this workflow creates is
+#     `hfs-bench-<kind>-<backend>-<run_id>`, labelled hfs-bench=1 +
+#     hfs-bench-run=<run_id> + hfs-bench-leg=<backend>, so the start-of-job
+#     reaper and the end-of-job leak check both cover it.
 #
 # The upstream suites pin their own k6 `scenarios` blocks (CRUD 300 VUs / 5m,
 # search 30 VUs / 2m, import 20 VUs × 1000 iters) so results stay comparable to
@@ -51,14 +70,20 @@ on:
   workflow_dispatch:
     inputs:
       backend:
-        description: "Storage backend(s) to benchmark"
+        description: "Backend legs: core = sqlite+postgres (default); all = all 6 legs (~3x a core run's runner time); elasticsearch = the 3 *-elasticsearch composites; or one backend"
         type: choice
         required: false
-        default: all
+        default: core
         options:
+          - core
           - all
+          - elasticsearch
           - sqlite
           - postgres
+          - mongodb
+          - sqlite-elasticsearch
+          - postgres-elasticsearch
+          - mongodb-elasticsearch
       tests:
         description: "Suites to run: 'all' | comma-separated (prewarm,import,crud,search)"
         required: false
@@ -111,6 +136,31 @@ on:
         description: "HFS Postgres pool size (crud drives 300 VUs; 32 leaves most of each request queued)"
         required: false
         default: "32"
+      max_parallel:
+        description: "Backend legs running at once (1..2 — every leg shares one 4-CPU / 11 GB Docker host, so 2 skews both legs' numbers)"
+        type: number
+        required: false
+        default: 1
+      es_heap:
+        description: "*-elasticsearch legs: ES JVM heap, -Xms = -Xmx (256m..4g; 512m is the conformance workflows' value)"
+        required: false
+        default: "1g"
+      es_sync_mode:
+        description: "asynchronous (HFS default): ack on primary commit; ONE worker forwards each resource individually (import/crud throughput = that worker) | synchronous: bundles batched via _bulk + refresh=wait_for, 200ms"
+        type: choice
+        required: false
+        default: asynchronous
+        options:
+          - asynchronous
+          - synchronous
+      mongo_wt_cache_gb:
+        description: "mongodb legs: --wiredTigerCacheSizeGB (0.25..6). 2 = parity with Postgres shared_buffers; mongod's own default (~5 GB here) risks a host OOM"
+        required: false
+        default: "2"
+      hfs_mongo_max_connections:
+        description: "HFS MongoDB pool size (HFS default 10 starves crud's 300 VUs; 32 matches hfs_pg_max_connections)"
+        required: false
+        default: "32"
 
 # Self-hosted runner has finite Docker port and disk capacity; superseded
 # runs on the same branch yield to fresh ones.
@@ -125,8 +175,9 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_BUILD_JOBS: 2
   CARGO_PROFILE_DEV_DEBUG: 0
-  # Remote Docker daemon wiring — postgres + tgz containers run here, and the
-  # runner reaches their published ports via DOCKER_HOST_IP.
+  # Remote Docker daemon wiring — postgres / mongo / elasticsearch + tgz
+  # containers run here, and the runner reaches their published ports via
+  # DOCKER_HOST_IP.
   DOCKER_HOST: ${{ secrets.DOCKER_HOST }}
   DOCKER_HOST_IP: ${{ secrets.DOCKER_HOST_IP }}
   # Pin the benchmark repo/ref so a run is reproducible.
@@ -135,49 +186,72 @@ env:
 
 jobs:
   # ────────────────────────────────────────────────────────────────────────────
-  # Resolve the `backend` input into a job matrix. Each leg carries the HFS
-  # server port — distinct per backend so sqlite (8081) and postgres (8082)
-  # can run concurrently on the same self-hosted runner.
+  # Resolve the `backend` input into a job matrix. Each leg carries a fixed HFS
+  # port (8081-8086), distinct per backend so legs never collide even with
+  # max_parallel > 1 (clamped to 1..2 below — the shared 4-CPU / 11 GB Docker
+  # host). Include order = intended run order: bare backends first. Also
+  # validates the free-text tuning inputs here, in one place, before the
+  # ~13-minute build.
   # ────────────────────────────────────────────────────────────────────────────
   setup:
     name: Resolve backend matrix
     runs-on: [self-hosted, Linux]
+    timeout-minutes: 10
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
+      max_parallel: ${{ steps.matrix.outputs.max_parallel }}
+      es_heap_mb: ${{ steps.matrix.outputs.es_heap_mb }}
       leg_timeout_min: ${{ steps.matrix.outputs.leg_timeout_min }}
     steps:
+      # Needed only so the next step can read its own script. `setup`,
+      # `build` and `benchmark` all run on the same `[self-hosted, Linux]`
+      # pool and therefore share one on-disk workspace per runner — they are
+      # separate JOBS, not separate runners. actions/checkout@v5 defaults to
+      # `clean: true` (git clean -ffdx + reset --hard) and a sparse-checkout
+      # narrows the working tree to what it's given, and either effect
+      # landing in that shared root would strip `build`'s warm `target/`
+      # (kept via that job's own `clean: false` — see its Checkout step) plus
+      # every file outside `.github/scripts` whenever `setup` and `build`
+      # land on the same runner. Checking out to a dedicated `path:` keeps
+      # this entirely out of the shared root instead. Sparse (and, via
+      # non-cone mode, narrowed to only this workflow's own scripts under
+      # .github/scripts/fhir-bench rather than cone mode's whole-ancestor-
+      # directory listing) to keep this as cheap as it was before it had any
+      # checkout at all (same idea as ci.yml's "Check out the Docker host GC
+      # action", which sparse-checks-out into the shared root because that
+      # job owns no cache worth protecting there — this one does).
+      - name: Check out benchmark scripts
+        uses: actions/checkout@v5
+        with:
+          path: .fhir-bench-setup
+          persist-credentials: false
+          sparse-checkout: .github/scripts/fhir-bench/
+          sparse-checkout-cone-mode: false
+
       - name: Resolve matrix from backend input
         id: matrix
-        run: |
-          case "${{ inputs.backend || 'all' }}" in
-            sqlite)
-              MATRIX='{"include":[{"backend":"sqlite","port":8081}]}' ;;
-            postgres)
-              MATRIX='{"include":[{"backend":"postgres","port":8082}]}' ;;
-            *)
-              MATRIX='{"include":[{"backend":"sqlite","port":8081},{"backend":"postgres","port":8082}]}' ;;
-          esac
-          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
-          echo "Backend matrix: $MATRIX"
-
-          # The benchmark job's timeout-minutes, and the base the reaper
-          # below sizes its age guard from. Must stay <= 175:
-          # .github/actions/docker-host-gc (input container-max-age-min,
-          # default 180; used by ci.yml / ci-extended / hts / keycloak-smoke
-          # / ui-tests-matrix) reaps any `hfs-ci=true` container whose
-          # `.State.StartedAt` is >= 180 min old, and GitHub gives a
-          # cancelled job up to 5 more minutes to exit. Do not raise this
-          # without re-checking that reaper's default.
-          echo "leg_timeout_min=150" >> "$GITHUB_OUTPUT"
+        # Body (including why inputs reach it only via env:) moved to
+        # .github/scripts/fhir-bench/resolve-matrix.sh.
+        env:
+          IN_BACKEND: ${{ inputs.backend }}
+          IN_MAX_PARALLEL: ${{ inputs.max_parallel }}
+          IN_ES_HEAP: ${{ inputs.es_heap }}
+          IN_ES_SYNC_MODE: ${{ inputs.es_sync_mode }}
+          IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
+          IN_HFS_MONGO_MAX_CONNECTIONS: ${{ inputs.hfs_mongo_max_connections }}
+        run: bash "$GITHUB_WORKSPACE/.fhir-bench-setup/.github/scripts/fhir-bench/resolve-matrix.sh"
 
   # ────────────────────────────────────────────────────────────────────────────
-  # Compile one debug `hfs` binary with BOTH backends linked in — the backend is
-  # selected at runtime via HFS_STORAGE_BACKEND, so a single build serves every
-  # matrix leg. `ui` is dropped so the benchmark exercises the FHIR API only.
+  # Compile one debug `hfs` binary with every benchmarked backend (sqlite,
+  # postgres, mongodb, elasticsearch) linked in — the backend is selected at
+  # runtime via HFS_STORAGE_BACKEND, so a single build serves every matrix
+  # leg. `ui` is dropped so the benchmark exercises the FHIR API only.
   # ────────────────────────────────────────────────────────────────────────────
   build:
     name: Build HFS binary
+    needs: setup
     runs-on: [self-hosted, Linux]
+    timeout-minutes: 120
 
     steps:
       - name: Checkout
@@ -210,18 +284,21 @@ jobs:
       # postgres, so it cannot explain why *postgres specifically* collapsed — but
       # any absolute number taken before this change is not comparable to anything,
       # including our own earlier runs.
-      - name: Build release binary (R4 + sqlite + postgres)
+      # mongodb pulls in mongocrypt-sys (native libmongocrypt, needs cmake);
+      # these runners already build it via ci.yml's --all-features. s3 is not
+      # compiled: no s3 leg is benchmarked (see header).
+      - name: Build release binary (R4 + sqlite + postgres + mongodb + elasticsearch)
         run: |
           cargo build --release -p helios-hfs \
             --no-default-features \
-            --features "R4,sqlite,postgres"
+            --features "R4,sqlite,postgres,mongodb,elasticsearch"
 
       - name: Upload binary
         uses: actions/upload-artifact@v7
         with:
           name: hfs-bench-binary-${{ github.run_id }}
           path: target/release/hfs
-          retention-days: 1
+          retention-days: 3
 
   # ────────────────────────────────────────────────────────────────────────────
   # Run the benchmark suites per backend.
@@ -249,6 +326,7 @@ jobs:
     timeout-minutes: ${{ fromJSON(needs.setup.outputs.leg_timeout_min) }}
     strategy:
       fail-fast: false
+      max-parallel: ${{ fromJSON(needs.setup.outputs.max_parallel) }}
       matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
 
     steps:
@@ -320,8 +398,8 @@ jobs:
           #
           #   1. name starts with `hfs-bench-`   → only ever our own containers
           #   2. name does NOT contain this run id → never a live sibling. The
-          #      matrix legs (sqlite, postgres) share a run id and run
-          #      concurrently, so this stops one leg killing the other.
+          #      matrix legs share a run id (and run concurrently when
+          #      max_parallel > 1), so this stops one leg killing another.
           #   3. older than MAX_AGE_MIN          → never a *different* run that
           #      is still going.
           #
@@ -433,8 +511,9 @@ jobs:
           # RUN_ID and start within the same second, so the plain "$RUN_ID"
           # name collided between them.
           echo "── Canary: create a container and write 256 MiB ──"
-          if ! docker run --rm --name "hfs-bench-canary-${{ matrix.backend }}-$RUN_ID" alpine:3 \
-                 sh -c 'dd if=/dev/zero of=/canary bs=1M count=256 2>&1 | tail -1'; then
+          if ! docker run --rm --name "hfs-bench-canary-${{ matrix.backend }}-$RUN_ID" \
+                 --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=${{ matrix.backend }}" \
+                 alpine:3 sh -c 'dd if=/dev/zero of=/canary bs=1M count=256 2>&1 | tail -1'; then
             echo "::error::Docker host cannot create a container or has no writable space."
             echo "The benchmark writes several GB (Synthea corpus + Postgres data), so it"
             echo "would fail slowly and leave more debris. Failing fast instead."
@@ -454,9 +533,26 @@ jobs:
           # of WAL segments on top of the Synthea corpus, and this host is
           # shared with every other testcontainers job in the repo.
           echo "── Docker host storage ──"
-          docker run --rm --name "hfs-bench-df-${{ matrix.backend }}-$RUN_ID" -v /:/hostroot:ro alpine:3 \
+          docker run --rm --name "hfs-bench-df-${{ matrix.backend }}-$RUN_ID" \
+            --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=${{ matrix.backend }}" \
+            -v /:/hostroot:ro alpine:3 \
             df -h /hostroot 2>/dev/null || docker system df 2>/dev/null || true
 
+      - name: Docker host capacity gate
+        # A floor under the 600s poll loop's own budget (plus the impossible-
+        # fit preflight and the two `docker`/`awk` calls around it) — never
+        # let this step itself become the thing that overruns the leg.
+        timeout-minutes: 12
+        # Body (including the per-backend memory model and the poll-loop
+        # rationale) moved to .github/scripts/fhir-bench/capacity-gate.sh.
+        env:
+          BACKEND: ${{ matrix.backend }}
+          RUN_ID: ${{ github.run_id }}
+          IN_PG_SHARED_BUFFERS: ${{ inputs.pg_shared_buffers }}
+          IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
+          ES_HEAP_MB: ${{ needs.setup.outputs.es_heap_mb }}
+        run: bash "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/capacity-gate.sh"
+
       # ── tgz bundle server ────────────────────────────────────────────────
       # The import suite pulls the Synthea `bulk_1k` corpus from this service
       # (`GET /reset`, `GET /<bundle>.json`). Built from the benchmark repo's
@@ -483,20 +579,21 @@ jobs:
             --label hfs-ci=true \
             --label "hfs-bench-run=${{ github.run_id }}" \
             --label "hfs-bench-leg=${{ matrix.backend }}" \
+            --log-driver local \
             -p 0:8080 "$TGZ_IMAGE" \
             "https://storage.googleapis.com/aidbox-public/synthea/performance/bulk_1k.tar.gz" >/dev/null
 
           echo "Waiting for tgz bundle server..."
           TGZ_PORT=""
           for i in $(seq 1 60); do
-            TGZ_PORT=$(docker port "$TGZ_CONTAINER" 8080 2>/dev/null | head -1 | sed 's/.*://')
+            TGZ_PORT=$(timeout 30 docker port "$TGZ_CONTAINER" 8080 2>/dev/null | head -1 | sed 's/.*://')
             if [ -n "$TGZ_PORT" ] && curl -sf "http://$DOCKER_HOST_IP:$TGZ_PORT/hospitalInformation.json" -o /dev/null 2>/dev/null; then
               echo "tgz ready on $DOCKER_HOST_IP:$TGZ_PORT after $((i * 2))s"
               break
             fi
             if [ "$i" -eq 60 ]; then
               echo "ERROR: tgz bundle server did not become ready (still downloading corpus?)"
-              docker logs "$TGZ_CONTAINER" | tail -50 || true
+              timeout 60 docker logs --tail 50 "$TGZ_CONTAINER" 2>&1 || true
               exit 1
             fi
             sleep 2
@@ -505,36 +602,116 @@ jobs:
           echo "BUNDLE_URL=http://$DOCKER_HOST_IP:$TGZ_PORT" >> "$GITHUB_ENV"
 
       # ── Backend wiring ───────────────────────────────────────────────────
+      # One explicit branch per backend family — never an if/else. An if/else
+      # keyed only on "sqlite" would send sqlite-elasticsearch down the
+      # Postgres branch: no HFS_DATABASE_URL, so HFS would open the
+      # never-cleaned $GITHUB_WORKSPACE default SQLite path and could replay a
+      # stale WAL. Every container/volume name carries the backend, because
+      # e.g. postgres and postgres-elasticsearch share a run id and each
+      # start step `docker rm -fv`s its name first — a shared name would let
+      # one leg destroy the other's data. All names are written here, before
+      # any container starts.
       - name: Configure backend env
+        env:
+          IN_ES_SYNC_MODE: ${{ inputs.es_sync_mode }}
         run: |
-          echo "HFS_STORAGE_BACKEND=${{ matrix.backend }}" >> "$GITHUB_ENV"
-          if [ "${{ matrix.backend }}" = "sqlite" ]; then
-            DB_PATH="$GITHUB_WORKSPACE/fhir-bench.db"
-            # Remove the WAL sidecars too, not just the database file. The
-            # runner workspace is reused across runs, and a benchmark killed
-            # mid-import (as in run 30550776427) leaves `-wal`/`-shm` behind.
-            # Deleting only the database and then opening a fresh empty one
-            # next to a stale WAL makes SQLite replay that WAL into it, which
-            # both resurrects the previous run's data and stalls startup long
-            # enough to blow the 120s readiness gate.
-            if ls "$DB_PATH"* >/dev/null 2>&1; then
-              echo "Leftover SQLite state from a previous run:"
-              ls -la "$DB_PATH"*
-            fi
-            rm -f "$DB_PATH" "$DB_PATH-wal" "$DB_PATH-shm" "$DB_PATH-journal"
-            echo "HFS_DATABASE_URL=$DB_PATH" >> "$GITHUB_ENV"
-            echo "SQLite DB: $DB_PATH"
-          else
-            PG_CONTAINER="hfs-bench-pg-${{ github.run_id }}"
-            PG_VOLUME="hfs-bench-pgdata-${{ github.run_id }}"
-            echo "PG_CONTAINER=$PG_CONTAINER" >> "$GITHUB_ENV"
-            echo "PG_VOLUME=$PG_VOLUME" >> "$GITHUB_ENV"
-            echo "Container name: $PG_CONTAINER"
-            echo "Data volume:    $PG_VOLUME"
-          fi
+          set -euo pipefail
+          BACKEND="${{ matrix.backend }}"
+          RUN="${{ github.run_id }}"
+          echo "HFS_STORAGE_BACKEND=$BACKEND" >> "$GITHUB_ENV"
+
+          case "$BACKEND" in
+            sqlite|sqlite-elasticsearch)
+              DB_PATH="$GITHUB_WORKSPACE/fhir-bench-$BACKEND.db"
+              # Remove the WAL sidecars too, not just the database file. The
+              # runner workspace is reused across runs, and a benchmark killed
+              # mid-import (as in run 30550776427) leaves `-wal`/`-shm`
+              # behind. Deleting only the database and then opening a fresh
+              # empty one next to a stale WAL makes SQLite replay that WAL
+              # into it, which both resurrects the previous run's data and
+              # stalls startup long enough to blow the readiness gate. The
+              # pre-#1475 shared path (fhir-bench.db) is cleared too so an old
+              # corpus is never stranded there.
+              for P in "$GITHUB_WORKSPACE/fhir-bench.db" "$DB_PATH"; do
+                if ls "$P"* >/dev/null 2>&1; then
+                  echo "Leftover SQLite state from a previous run:"
+                  ls -la "$P"*
+                fi
+                rm -f "$P" "$P-wal" "$P-shm" "$P-journal"
+              done
+              echo "HFS_DATABASE_URL=$DB_PATH" >> "$GITHUB_ENV"
+              echo "SQLite DB: $DB_PATH"
+              ;;
+            postgres|postgres-elasticsearch)
+              PG_CONTAINER="hfs-bench-pg-$BACKEND-$RUN"
+              PG_VOLUME="hfs-bench-pgdata-$BACKEND-$RUN"
+              echo "PG_CONTAINER=$PG_CONTAINER" >> "$GITHUB_ENV"
+              echo "PG_VOLUME=$PG_VOLUME" >> "$GITHUB_ENV"
+              echo "Postgres container: $PG_CONTAINER"
+              echo "Postgres volume:    $PG_VOLUME"
+              ;;
+            mongodb|mongodb-elasticsearch)
+              MONGO_CONTAINER="hfs-bench-mongo-$BACKEND-$RUN"
+              MONGO_VOLUME="hfs-bench-mongodata-$BACKEND-$RUN"
+              echo "MONGO_CONTAINER=$MONGO_CONTAINER" >> "$GITHUB_ENV"
+              echo "MONGO_VOLUME=$MONGO_VOLUME" >> "$GITHUB_ENV"
+              echo "HFS_MONGODB_DATABASE=hfs_bench" >> "$GITHUB_ENV"
+              echo "Mongo container: $MONGO_CONTAINER"
+              echo "Mongo volume:    $MONGO_VOLUME"
+              ;;
+            *)
+              echo "::error::Configure backend env has no wiring for backend '$BACKEND'"
+              exit 1
+              ;;
+          esac
+
+          case "$BACKEND" in
+            *-elasticsearch)
+              ES_CONTAINER="hfs-bench-es-$BACKEND-$RUN"
+              ES_VOLUME="hfs-bench-esdata-$BACKEND-$RUN"
+              # HFS's own default (config.rs elasticsearch_index_prefix); no
+              # workflow input overrides it today, but reading it through the
+              # env var (rather than hardcoding "hfs") keeps this correct if
+              # HFS_ELASTICSEARCH_INDEX_PREFIX is ever set in this workflow's
+              # top-level env:.
+              ES_PREFIX="${HFS_ELASTICSEARCH_INDEX_PREFIX:-hfs}"
+              echo "ES_CONTAINER=$ES_CONTAINER" >> "$GITHUB_ENV"
+              echo "ES_VOLUME=$ES_VOLUME" >> "$GITHUB_ENV"
+              echo "ES_PREFIX=$ES_PREFIX" >> "$GITHUB_ENV"
+              echo "ES container:    $ES_CONTAINER"
+              echo "ES volume:       $ES_VOLUME"
+              echo "ES index prefix: $ES_PREFIX"
+              # asynchronous = HFS's own defaults, set explicitly so the run
+              # records them: a write acks once the PRIMARY commits and one
+              # background worker forwards it to Elasticsearch; the search
+              # suite's drain gate waits for that backlog before measuring.
+              # synchronous = the conformance workflows' read-your-writes
+              # setting (inferno-us-core.yml — wait_for makes the refresh
+              # interval the price of every write, hence 200ms).
+              case "${IN_ES_SYNC_MODE:-asynchronous}" in
+                synchronous)  SYNC_MODE=synchronous;  WRITE_REFRESH=wait_for; REFRESH_INTERVAL=200ms ;;
+                asynchronous) SYNC_MODE=asynchronous; WRITE_REFRESH=false;    REFRESH_INTERVAL=1s ;;
+                *) echo "::error::Unknown es_sync_mode '$IN_ES_SYNC_MODE'"; exit 1 ;;
+              esac
+              {
+                echo "HFS_COMPOSITE_SYNC_MODE=$SYNC_MODE"
+                echo "HFS_ELASTICSEARCH_WRITE_REFRESH=$WRITE_REFRESH"
+                echo "HFS_ELASTICSEARCH_REFRESH_INTERVAL=$REFRESH_INTERVAL"
+                # Explicit even though 60 is HFS's own default, so the run
+                # records it: the periodic sweep that retries secondary-sync
+                # failures (HFS_COMPOSITE_SYNC_REPAIR_INTERVAL).
+                echo "HFS_COMPOSITE_SYNC_REPAIR_INTERVAL=60"
+              } >> "$GITHUB_ENV"
+              echo "ES sync: mode=$SYNC_MODE write_refresh=$WRITE_REFRESH refresh_interval=$REFRESH_INTERVAL repair_interval=60"
+              ;;
+          esac
 
       - name: Start ephemeral Postgres
-        if: matrix.backend == 'postgres'
+        # startsWith, not ==: postgres-elasticsearch also uses Postgres as its
+        # primary (this step still uses the backend-qualified $PG_CONTAINER /
+        # $PG_VOLUME names written by "Configure backend env" — nothing here
+        # recomputes them).
+        if: startsWith(matrix.backend, 'postgres')
         run: |
           set -euo pipefail
           docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
@@ -543,6 +720,22 @@ jobs:
             --label "hfs-bench-run=${{ github.run_id }}" \
             --label "hfs-bench-leg=${{ matrix.backend }}" "$PG_VOLUME" >/dev/null
 
+          # Volume filesystem free space at start — the volume's OWN
+          # filesystem, not `docker system df`'s aggregate/compressed numbers
+          # (run 34187014872 logged "Local Volumes ... 76.67GB" against a host
+          # root filesystem that was only 19% used — the two don't
+          # reconcile). Recorded for the summary; never fails the leg.
+          PG_VOLDF_NAME="hfs-bench-voldf-${{ matrix.backend }}-${{ github.run_id }}-pg"
+          PG_VOL_DF=$(timeout 60 docker run --rm --name "$PG_VOLDF_NAME" \
+              --label hfs-bench=1 --label "hfs-bench-run=${{ github.run_id }}" --label "hfs-bench-leg=${{ matrix.backend }}" \
+              -v "$PG_VOLUME:/v" alpine:3 df -Pm /v 2>/dev/null | awk 'NR==2{print $4}') || PG_VOL_DF=""
+          # `timeout 60` only kills the local docker CLI, not a container still
+          # starting on the daemon — force it gone so `--rm` never loses a race
+          # against a `timeout` kill and leaks a name/volume-mount hold.
+          timeout 15 docker rm -f "$PG_VOLDF_NAME" >/dev/null 2>&1 || true
+          echo "PG_VOL_FREE_MB=${PG_VOL_DF:-unknown}" >> "$GITHUB_ENV"
+          echo "Postgres volume free space: ${PG_VOL_DF:-unknown} MB"
+
           # ── Memory sizing ────────────────────────────────────────────────
           # Postgres runs on the REMOTE Docker host, not the runner, so it is
           # that host's RAM that bounds shared_buffers — read it from the daemon
@@ -641,6 +834,7 @@ jobs:
             echo "| min_wal_size | \`$PG_MIN_WAL_SIZE\` |"
             echo "| wal_buffers | \`$PG_WAL_BUFFERS\` |"
             echo "| checkpoint_timeout | \`$PG_CHECKPOINT_TIMEOUT\` |"
+            echo "| data volume free space (start) | \`${PG_VOL_DF:-unknown} MB\` |"
           } >> "$GITHUB_STEP_SUMMARY"
           #
           # The data directory is a **named volume**, not the container's
@@ -660,6 +854,7 @@ jobs:
             --label hfs-ci=true \
             --label "hfs-bench-run=${{ github.run_id }}" \
             --label "hfs-bench-leg=${{ matrix.backend }}" \
+            --log-driver local \
             --shm-size=${SHM_GB}g \
             -v "$PG_VOLUME:/var/lib/postgresql" \
             -e POSTGRES_PASSWORD=postgres \
@@ -689,7 +884,7 @@ jobs:
           PG_PORT=""
           for i in $(seq 1 30); do
             if docker exec "$PG_CONTAINER" pg_isready -U postgres -d postgres >/dev/null 2>&1; then
-              PG_PORT=$(docker port "$PG_CONTAINER" 5432 | head -1 | sed 's/.*://')
+              PG_PORT=$(timeout 30 docker port "$PG_CONTAINER" 5432 | head -1 | sed 's/.*://')
               if [ -n "$PG_PORT" ] && timeout 2 bash -c "cat < /dev/null > /dev/tcp/$DOCKER_HOST_IP/$PG_PORT" 2>/dev/null; then
                 echo "Postgres ready on $DOCKER_HOST_IP:$PG_PORT after $((i * 2))s"
                 break
@@ -697,7 +892,7 @@ jobs:
             fi
             if [ "$i" -eq 30 ]; then
               echo "ERROR: Postgres did not become reachable within 60s"
-              docker logs "$PG_CONTAINER" | tail -100 || true
+              timeout 60 docker logs --tail 100 "$PG_CONTAINER" 2>&1 || true
               exit 1
             fi
             sleep 2
@@ -706,6 +901,27 @@ jobs:
           echo "PG_PORT=$PG_PORT" >> "$GITHUB_ENV"
           echo "HFS_DATABASE_URL=postgresql://postgres:postgres@$DOCKER_HOST_IP:$PG_PORT/postgres" >> "$GITHUB_ENV"
 
+      - name: Start ephemeral MongoDB (single-member replica set)
+        if: startsWith(matrix.backend, 'mongodb')
+        # Body (including the single-member-replica-set and engine-cap
+        # rationale) moved to .github/scripts/fhir-bench/start-mongodb.sh.
+        env:
+          BACKEND: ${{ matrix.backend }}
+          RUN_ID: ${{ github.run_id }}
+          IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
+          IN_HFS_MONGO_MAX_CONNECTIONS: ${{ inputs.hfs_mongo_max_connections }}
+        run: bash "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/start-mongodb.sh"
+
+      - name: Start ephemeral Elasticsearch
+        if: endsWith(matrix.backend, '-elasticsearch')
+        # Body (including the heap/health/volume rationale) moved to
+        # .github/scripts/fhir-bench/start-elasticsearch.sh.
+        env:
+          BACKEND: ${{ matrix.backend }}
+          RUN_ID: ${{ github.run_id }}
+          ES_HEAP_MB: ${{ needs.setup.outputs.es_heap_mb }}
+        run: bash "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/start-elasticsearch.sh"
+
       # ── Tools (k6) ───────────────────────────────────────────────────────
       - name: Install k6
         uses: grafana/setup-k6-action@v1
@@ -714,7 +930,34 @@ jobs:
       - name: Kill any leftover process on port ${{ matrix.port }}
         run: fuser -k "${{ matrix.port }}/tcp" 2>/dev/null || true
 
+      # "Start ephemeral Elasticsearch" only proved ES answered right after
+      # creation; the k6 install and the port-kill step above open a gap
+      # after that, and HFS's own readiness loop only polls /metadata — it
+      # never contacts Elasticsearch — so a dead ES would surface only as HFS
+      # burning its whole readiness budget retrying bootstrap writes against
+      # a backend that will never answer. Re-reads the port (not just the
+      # env var) in case it moved. No restart, deliberately: nothing has been
+      # written to ES yet on this leg, so failing loudly here — where the
+      # logs are still attached to this run — is simpler to read than a
+      # silent recovery would be.
+      - name: Check Elasticsearch is still healthy
+        if: endsWith(matrix.backend, '-elasticsearch')
+        run: |
+          set -euo pipefail
+          ES_PORT=$(timeout 30 docker port "$ES_CONTAINER" 9200 2>/dev/null | head -1 | sed 's/.*://') || ES_PORT=""
+          if [ -n "$ES_PORT" ] && curl -sf --max-time 30 "http://$DOCKER_HOST_IP:$ES_PORT/_cluster/health?wait_for_status=yellow&timeout=5s" >/dev/null 2>&1; then
+            echo "Elasticsearch healthy on $DOCKER_HOST_IP:$ES_PORT"
+            echo "ES_PORT=$ES_PORT" >> "$GITHUB_ENV"
+            echo "HFS_ELASTICSEARCH_NODES=http://$DOCKER_HOST_IP:$ES_PORT" >> "$GITHUB_ENV"
+            exit 0
+          fi
+          echo "::error::Elasticsearch is not answering on port ${ES_PORT:-unknown} — failing here rather than let HFS burn its readiness budget against a dead search backend."
+          timeout 60 docker logs --tail 100 "$ES_CONTAINER" 2>&1 || true
+          exit 1
+
       - name: Start HFS server
+        env:
+          IN_HFS_MONGO_MAX_CONNECTIONS: ${{ inputs.hfs_mongo_max_connections }}
         run: |
           # HFS_STORAGE_BACKEND + HFS_DATABASE_URL already exported. init_schema()
           # runs on startup, so an empty Postgres DB / fresh SQLite file is fine.
@@ -750,12 +993,24 @@ jobs:
           # pool was pinned at 10 and, on the HFS_DATABASE_URL path, no environment
           # variable could change it). 32 covers the 30 VUs while staying within a
           # few multiples of the container's ~8 cores.
+          #
+          # HFS_MONGODB_MAX_CONNECTIONS: same reasoning as
+          # HFS_PG_MAX_CONNECTIONS above — Mongo's own pool default is 10.
+          # HFS silently falls back to 10 on a non-numeric value
+          # (mongodb/backend.rs, config.max_connections). Already validated
+          # in setup's "Resolve matrix from backend input" (same regex, plus
+          # the 1..1000 range check) before the ~13-minute build — no re-check
+          # here, since re-validating the raw input here too (as this step
+          # used to) is redundant on every leg, including sqlite/postgres.
+          HFS_MONGO_POOL="${IN_HFS_MONGO_MAX_CONNECTIONS:-32}"
+
           HFS_SERVER_HOST="127.0.0.1" \
           HFS_SERVER_PORT="${{ matrix.port }}" \
           HFS_LOG_LEVEL="${{ inputs.log_level || 'info' }}" \
           HFS_MAX_BODY_SIZE="209715200" \
           HFS_REQUEST_TIMEOUT="900" \
           HFS_PG_MAX_CONNECTIONS="${{ inputs.hfs_pg_max_connections || '32' }}" \
+          HFS_MONGODB_MAX_CONNECTIONS="$HFS_MONGO_POOL" \
           HFS_DATA_DIR="$GITHUB_WORKSPACE/data" \
             ./hfs > "/tmp/hfs-bench-${{ matrix.backend }}.log" 2>&1 &
 
@@ -775,11 +1030,17 @@ jobs:
           # line every 10s so a slow start is distinguishable from a hung one.
           echo "Waiting for HFS to become ready..."
           READY_TIMEOUT=300
+          # *-elasticsearch legs seed the SearchParameters bundle THROUGH the
+          # composite (the same "Composite storage initialized: ... +
+          # Elasticsearch (search)" path checked in "Assert search parameters
+          # loaded"), which is slower than seeding the primary alone.
+          if [[ "${{ matrix.backend }}" == *-elasticsearch ]]; then READY_TIMEOUT=600; fi
           START_TS=$SECONDS
           READY=0
           while [ $((SECONDS - START_TS)) -lt "$READY_TIMEOUT" ]; do
-            if curl -sf "http://localhost:${{ matrix.port }}/metadata" -o /dev/null 2>/dev/null; then
+            if curl -sf --max-time 10 "http://localhost:${{ matrix.port }}/metadata" -o /dev/null 2>/dev/null; then
               echo "HFS ready after $((SECONDS - START_TS))s"
+              echo "HFS_READY_SECONDS=$((SECONDS - START_TS))" >> "$GITHUB_ENV"
               READY=1
               break
             fi
@@ -806,10 +1067,22 @@ jobs:
             cat "/proc/$HFS_PID/stack" 2>/dev/null || echo "(stack unavailable)"
             echo "── wait channel ──"
             cat "/proc/$HFS_PID/wchan" 2>/dev/null && echo || true
-            if [ "${{ matrix.backend }}" = "sqlite" ]; then
+            if [[ "${{ matrix.backend }}" == "sqlite" || "${{ matrix.backend }}" == "sqlite-elasticsearch" ]]; then
               echo "── sqlite database state ──"
               ls -la "$HFS_DATABASE_URL"* 2>/dev/null || echo "(no database files)"
             fi
+            if [ -n "${ES_CONTAINER:-}" ]; then
+              echo "── elasticsearch ($ES_CONTAINER) ──"
+              curl -s --max-time 30 "http://$DOCKER_HOST_IP:$ES_PORT/_cluster/health?pretty" \
+                || echo "::error::Elasticsearch on port $ES_PORT is NOT answering — HFS may have been retrying bootstrap writes against a dead search backend."
+              timeout 60 docker inspect -f 'OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Status={{.State.Status}}' "$ES_CONTAINER" || true
+              timeout 60 docker logs --tail 30 "$ES_CONTAINER" 2>&1 || true
+            fi
+            if [ -n "${MONGO_CONTAINER:-}" ]; then
+              echo "── mongodb ($MONGO_CONTAINER) ──"
+              timeout 60 docker inspect -f 'OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Status={{.State.Status}}' "$MONGO_CONTAINER" || true
+              timeout 60 docker logs --tail 30 "$MONGO_CONTAINER" 2>&1 || true
+            fi
             exit 1
           fi
 
@@ -826,11 +1099,62 @@ jobs:
             echo "Search parameters loaded from spec bundle."
           fi
 
+          # *-elasticsearch legs: confirm HFS actually started the composite
+          # (SQLite/PostgreSQL/MongoDB primary + Elasticsearch search) rather
+          # than silently falling back to the primary's own search index.
+          # Every composite start logs "Composite storage initialized: ... +
+          # Elasticsearch (search)" (crates/hfs/src/main.rs); grep only the
+          # common prefix so it matches regardless of which primary this leg
+          # uses.
+          if [[ "${{ matrix.backend }}" == *-elasticsearch ]]; then
+            if grep -m1 "Composite storage initialized" "/tmp/hfs-bench-${{ matrix.backend }}.log"; then
+              :
+            else
+              echo "::warning::No 'Composite storage initialized' line in the HFS log — this leg may not be running the Elasticsearch composite (or log_level is above info)."
+            fi
+          fi
+
       # ── Benchmark ────────────────────────────────────────────────────────
       - name: Run benchmark suites
         working-directory: fhir-benchmark
+        env:
+          # Free-text inputs used below reach bash only through env:, never
+          # inlined as ${{ }} inside run: — same reasoning as "Resolve matrix
+          # from backend input" and "Configure backend env". es_sync_mode
+          # needs no entry here: it is already ambient as
+          # $HFS_COMPOSITE_SYNC_MODE, written to $GITHUB_ENV by "Configure
+          # backend env".
+          IN_ES_HEAP: ${{ inputs.es_heap }}
+          IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
+          IN_HFS_MONGO_MAX_CONNECTIONS: ${{ inputs.hfs_mongo_max_connections }}
         run: |
           set -uo pipefail
+          # es_drain_gate, capture_backend_stats, write_import_completeness,
+          # write_mongo_txn_errors, write_search_counts,
+          # write_containers_state, inspect_running and
+          # check_leg_containers_alive (issue #1475's ES/Mongo/composite
+          # per-suite instrumentation) live in suite-lib.sh, sourced here so
+          # they run in THIS shell and inherit its options (see that file's
+          # own header) instead of needing GitHub Actions expression
+          # substitution.
+          source "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/suite-lib.sh"
+
+          # Plain (non-exported) shell variables, not `env:` above: this
+          # inline block, unlike the sourced suite-lib.sh functions, still
+          # gets GitHub Actions expression substitution, so there is no need
+          # to export these to the process environment — doing so would leak
+          # them into k6's __ENV and every other child process this step
+          # spawns. Sourcing puts suite-lib.sh's functions in THIS shell, so
+          # they see these like any other shell variable already set by the
+          # time they're called; no export needed for that either.
+          # BENCH_RUN_ID is deliberately NOT named RUN_ID: this step's own
+          # $RUN_ID (below) holds the possibly-user-overridden
+          # inputs.run_id tag used for k6 --tag runid=, not the actual
+          # github.run_id that container/volume names and the drain-probe
+          # identifier need.
+          BACKEND="${{ matrix.backend }}"
+          BENCH_PORT="${{ matrix.port }}"
+          BENCH_RUN_ID="${{ github.run_id }}"
           RUN_ID="${{ inputs.run_id }}"
           [ -z "$RUN_ID" ] && RUN_ID="${{ github.run_id }}"
 
@@ -850,6 +1174,25 @@ jobs:
           # the agent's size; the artifact outlives it and travels with the
           # numbers. Anyone reading these results later can tell a 2x from a
           # bigger box apart from a 2x from the code.
+          # #1475 tuning knobs, recorded next to the results so a leg's
+          # numbers can always be read back against what produced them.
+          # Mongo/ES values read n/a off their own legs. CAPACITY_* were
+          # written to $GITHUB_ENV by "Docker host capacity gate", above.
+          case "${{ matrix.backend }}" in
+            mongodb*) MONGO_POOL_REC="${IN_HFS_MONGO_MAX_CONNECTIONS:-32}"; MONGO_WT_REC="${IN_MONGO_WT_CACHE_GB:-2}" ;;
+            *)        MONGO_POOL_REC=n/a; MONGO_WT_REC=n/a ;;
+          esac
+          case "${{ matrix.backend }}" in
+            *-elasticsearch) ES_HEAP_REC="${IN_ES_HEAP:-1g}"
+                             ES_WRITE_REFRESH_REC="${HFS_ELASTICSEARCH_WRITE_REFRESH:-unknown}"
+                             ES_REFRESH_INTERVAL_REC="${HFS_ELASTICSEARCH_REFRESH_INTERVAL:-unknown}" ;;
+            *)               ES_HEAP_REC=n/a; ES_WRITE_REFRESH_REC=n/a; ES_REFRESH_INTERVAL_REC=n/a ;;
+          esac
+          case "${{ matrix.backend }}" in
+            mongodb*) MONGO_TXN_LIFETIME_REC="${MONGO_TXN_LIFETIME_S:-unknown}" ;;
+            *)        MONGO_TXN_LIFETIME_REC=n/a ;;
+          esac
+
           {
             echo "runner_name:  ${RUNNER_NAME:-unknown}"
             echo "runner_cpus:  $(nproc 2>/dev/null || echo '?')"
@@ -861,6 +1204,25 @@ jobs:
             echo "git_ref:      ${{ github.ref_name }}"
             echo "backend:      ${{ matrix.backend }}"
             echo "pg_pool_max:  ${{ inputs.hfs_pg_max_connections || '32' }}"
+            echo "es_heap:            $ES_HEAP_REC"
+            echo "es_sync_mode:       ${HFS_COMPOSITE_SYNC_MODE:-n/a}"
+            echo "es_write_refresh:   $ES_WRITE_REFRESH_REC"
+            echo "es_refresh_interval: $ES_REFRESH_INTERVAL_REC"
+            echo "mongo_wt_cache_gb:  $MONGO_WT_REC"
+            echo "mongo_pool_max:     $MONGO_POOL_REC"
+            echo "mongo_txn_lifetime_s: $MONGO_TXN_LIFETIME_REC"
+            echo "hfs_ready_s:        ${HFS_READY_SECONDS:-unknown}"
+            echo "max_parallel:       ${{ needs.setup.outputs.max_parallel }}"
+            echo "leg_timeout_min:    ${{ needs.setup.outputs.leg_timeout_min }}"
+            echo "capacity_need_mb:   ${CAPACITY_NEED_MB:-unknown}"
+            echo "capacity_avail_mb:  ${CAPACITY_AVAIL_MB:-unknown}"
+            echo "capacity_wait_s:    ${CAPACITY_WAIT_S:-unknown}"
+            # PG_VOL_FREE_MB / MONGO_VOL_FREE_MB / ES_VOL_FREE_MB were written
+            # to $GITHUB_ENV by the container-start steps above (they also
+            # print into that step's own $GITHUB_STEP_SUMMARY block, but
+            # weren't otherwise carried into the artifact until now).
+            echo "vol_free_mb_start:  ${PG_VOL_FREE_MB:-${MONGO_VOL_FREE_MB:-n/a}}"
+            echo "es_vol_free_mb_start: ${ES_VOL_FREE_MB:-n/a}"
           } > "$RESULTS_DIR/runner-info.txt"
           cat "$RESULTS_DIR/runner-info.txt"
 
@@ -893,7 +1255,24 @@ jobs:
           # clock: if PG's total is a small fraction of the wall time, the
           # bottleneck is on the HFS/k6 side (extraction CPU, serialization,
           # round trips) and not in the database at all.
-          if [ "${{ matrix.backend }}" = "postgres" ] && [ -n "${PG_CONTAINER:-}" ]; then
+          # Postgres-family legs: pg_stat_statements and the per-suite pgstat
+          # capture below run on BOTH postgres and postgres-elasticsearch
+          # (crud/import still hit resources / resource_history on that leg).
+          # The search_index VACUUM + plan capture and index-usage.txt stay
+          # EXACT 'postgres' further down: with search offloaded,
+          # postgres-elasticsearch never writes search_index
+          # (main.rs set_search_offloaded), so they would measure an empty
+          # table.
+          PG_FAMILY=false
+          case "${{ matrix.backend }}" in postgres|postgres-elasticsearch) PG_FAMILY=true ;; esac
+
+          # ES drain gate: ONE deadline shared across the barrier probe and
+          # the settle poll (see es_drain_gate in suite-lib.sh) — not 15 min budgeted to
+          # each, so a wedged Elasticsearch cannot burn 2x this before search
+          # still runs against a possibly-incomplete index.
+          ES_DRAIN_TIMEOUT_S=600
+
+          if [ "$PG_FAMILY" = true ] && [ -n "${PG_CONTAINER:-}" ]; then
             docker exec "$PG_CONTAINER" psql -U postgres -d postgres -q \
               -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements" >/dev/null 2>&1 \
               || echo "::warning::pg_stat_statements unavailable"
@@ -914,17 +1293,51 @@ jobs:
           # cannot be trusted for throughput now says so in its own artifact.
           probe_host() {
             local phase="$1" suite="$2"
-            local la cn
-            la=$(docker run --rm alpine:3 cat /proc/loadavg 2>/dev/null | cut -d' ' -f1-3)
-            cn=$(docker ps -q 2>/dev/null | wc -l)
-            echo "$(date -u +%H:%M:%SZ) suite=$suite phase=$phase host_loadavg=${la:-unknown} host_containers=${cn:-unknown}" \
+            local la cn mem out
+            # One alpine run answers both questions: how loaded is the host,
+            # and (#1475) how much memory does it have left — the same
+            # MemAvailable the capacity gate polls, recorded in kB so a
+            # suite's own numbers can be read against host memory pressure,
+            # not just load average and container count.
+            local probe_name="hfs-bench-probe-${{ matrix.backend }}-${{ github.run_id }}"
+            out=$(timeout 60 docker run --rm --name "$probe_name" \
+                --label hfs-bench=1 --label "hfs-bench-run=${{ github.run_id }}" --label "hfs-bench-leg=${{ matrix.backend }}" \
+                alpine:3 sh -c 'cat /proc/loadavg; grep MemAvailable /proc/meminfo' 2>/dev/null) || out=""
+            # `timeout 60` only kills the local docker CLI; force the
+            # container gone so repeated probe_host calls (every suite,
+            # every phase) never collide on this fixed --name.
+            docker rm -f "$probe_name" >/dev/null 2>&1 || true
+            la=$(printf '%s\n' "$out" | sed -n '1p' | cut -d' ' -f1-3)
+            mem=$(printf '%s\n' "$out" | sed -n '2p' | awk '{print $2}')
+            cn=$(docker ps -q 2>/dev/null | wc -l) || cn=""
+            echo "$(date -u +%H:%M:%SZ) suite=$suite phase=$phase host_loadavg=${la:-unknown} host_containers=${cn:-unknown} host_mem_avail_kb=${mem:-unknown}" \
               >> "$RESULTS_DIR/host-contention.txt"
           }
 
+          # es_drain_gate() and capture_backend_stats() (the Elasticsearch
+          # drain gate and the per-suite ES/Mongo/composite snapshot) moved
+          # to .github/scripts/fhir-bench/suite-lib.sh, sourced above.
+
           for SUITE in $SUITES; do
             SUITE="${SUITE// /}"
             [ -z "$SUITE" ] && continue
             SCRIPT="k6/${SUITE}.js"
+
+            # Drain before the gate's own probe_host/SUITE_START, so ES
+            # backlog time is not folded into the search suite's own wall
+            # clock or host-contention reading.
+            if [ "$SUITE" = search ] && [ -n "${ES_PORT:-}" ] && [ -f "$SCRIPT" ]; then
+              es_drain_gate || echo '::warning::drain gate errored'
+              # The gate's own primary-count query (postgres-elasticsearch)
+              # ran inside crud's post-suite reset window and would otherwise
+              # be folded into search-pgstat.txt as if it were search traffic
+              # — reset again here so search's pgstat capture starts clean.
+              if [ "$PG_FAMILY" = true ] && [ -n "${PG_CONTAINER:-}" ]; then
+                timeout 60 docker exec "$PG_CONTAINER" psql -U postgres -d postgres -q \
+                  -c "SELECT pg_stat_statements_reset()" >/dev/null 2>&1 || true
+              fi
+            fi
+
             SUITE_START=$(date +%s)
             probe_host start "$SUITE"
             if [ ! -f "$SCRIPT" ]; then
@@ -995,6 +1408,12 @@ jobs:
             SUITE_WALL=$(( $(date +%s) - SUITE_START ))
             probe_host end "$SUITE"
 
+            # write_import_completeness() and write_mongo_txn_errors() (F1a /
+            # F8) moved to .github/scripts/fhir-bench/suite-lib.sh, sourced
+            # above.
+            write_import_completeness "$SUITE"
+            write_mongo_txn_errors "$SUITE"
+
             # Attribute this suite's time. `wall` is the suite's own elapsed
             # seconds; `pg_exec_total_s` is everything Postgres executed during
             # it. The ratio is the point: a small ratio means the database was
@@ -1025,8 +1444,14 @@ jobs:
             # you tell whether v26's `autovacuum_analyze_scale_factor = 0.4`
             # actually cut the number of those, and `n_tup_ins` is the row count
             # every index-insertion estimate is scaled from.
-            if [ "${{ matrix.backend }}" = "postgres" ] && [ -n "${PG_CONTAINER:-}" ]; then
+            if [ "$PG_FAMILY" = true ] && [ -n "${PG_CONTAINER:-}" ]; then
               {
+                # postgres-elasticsearch writes no search_index (main.rs
+                # set_search_offloaded), so its wall time includes ES sync
+                # backpressure, not just Postgres — say so before the numbers.
+                if [ "${{ matrix.backend }}" = "postgres-elasticsearch" ]; then
+                  echo "note: postgres-elasticsearch — search_index is not written and wall time includes Elasticsearch sync backpressure; a low pg_exec/wall ratio does NOT mean HFS-CPU-bound"
+                fi
                 echo "suite=$SUITE wall_seconds=$SUITE_WALL"
                 docker exec "$PG_CONTAINER" psql -U postgres -d postgres \
                   -c "SELECT round((sum(total_exec_time)/1000.0)::numeric, 1) AS pg_exec_total_s,
@@ -1055,8 +1480,24 @@ jobs:
               docker exec "$PG_CONTAINER" psql -U postgres -d postgres -q \
                 -c "SELECT pg_stat_statements_reset()" >/dev/null 2>&1 || true
             fi
+
+            capture_backend_stats "$SUITE"
           done
 
+          # CRUD residue (search-counts comparability): algorithm and the
+          # upstream check-name rationale are documented in
+          # .github/scripts/fhir-bench/crud_residue.py, moved with the code
+          # it explains. $RESIDUE_PY resolution (python3, falling back to
+          # python) is unchanged from before the move.
+          : > "$RESULTS_DIR/crud-residue.txt"
+          RESIDUE_PY=$(command -v python3 || command -v python || true)
+          if [ -n "$RESIDUE_PY" ]; then
+            "$RESIDUE_PY" "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/crud_residue.py" \
+              "$RESULTS_DIR/prewarm.json" "$RESULTS_DIR/crud.json" \
+              > "$RESULTS_DIR/crud-residue.txt" 2>/dev/null || echo "::warning::crud-residue computation failed"
+          fi
+          cat "$RESULTS_DIR/crud-residue.txt" 2>/dev/null || true
+
           # Which search_index indexes the suites actually read.
           #
           # The plan capture above runs before the load, so its idx_scan counters
@@ -1079,6 +1520,17 @@ jobs:
               echo "::warning::index usage capture failed"
           fi
 
+          # write_search_counts() (Result-size cross-check, F5) moved to
+          # .github/scripts/fhir-bench/suite-lib.sh, sourced above.
+          write_search_counts
+
+          # write_containers_state(), inspect_running() and
+          # check_leg_containers_alive() (end-of-step container/volume state
+          # dump + liveness gate) moved to
+          # .github/scripts/fhir-bench/suite-lib.sh, sourced above.
+          write_containers_state
+          check_leg_containers_alive
+
       # ── Artifacts ────────────────────────────────────────────────────────
       # Deliberately placed immediately after the suites and BEFORE the
       # post-processing steps below. When run 30550776427's runner was killed
@@ -1088,16 +1540,19 @@ jobs:
       # that can be recomputed from these artifacts; the artifacts cannot be
       # recovered from anything. Upload first, analyse second.
       #
-      # Teardown (Stop HFS server / Stop tgz bundle server / Stop ephemeral
-      # Postgres, below) is placed right after these uploads for the same
-      # reason. On run 33515369645 the runner was lost during "Attribute search
-      # latency per query shape", so every step after it — Generate step
-      # summary, Stop HFS server, Stop tgz bundle server, Stop ephemeral
-      # Postgres — was skipped, and its containers survived 6.4 days until the
-      # next reaper caught them. Teardown now runs immediately after the
-      # uploads, before any post-processing step (which only reads the files
-      # already on disk), so a runner lost during any later post-processing
-      # step still leaves cleanup done.
+      # Teardown (Diagnose backend container failure, then Stop HFS server /
+      # Stop tgz bundle server / Stop ephemeral Postgres / Stop ephemeral
+      # MongoDB / Stop ephemeral Elasticsearch / Remove SQLite database
+      # files, below) is placed right after these uploads for the same
+      # reason. On run 33515369645 the runner was
+      # lost during "Attribute search latency per query shape", so every step
+      # after it — Generate step summary, Stop HFS server, Stop tgz bundle
+      # server, Stop ephemeral Postgres — was skipped, and its containers
+      # survived 6.4 days until the next reaper caught them. Diagnose and
+      # every Stop step now run immediately after the uploads, before any
+      # post-processing step (which only reads the files already on disk), so
+      # a runner lost during any later post-processing step still leaves
+      # diagnostics captured and cleanup done.
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@v7
@@ -1116,6 +1571,41 @@ jobs:
           retention-days: 7
           if-no-files-found: ignore
 
+      # Why a backend container died (host OOM vs. cgroup OOM vs. crash),
+      # captured before the Stop steps below remove the evidence. Based on
+      # inferno-us-core.yml's "Diagnose MongoDB failure", widened to every
+      # container this leg owns. `cancelled()` matters because a job-timeout
+      # cancellation re-evaluates `if:` for steps not yet run and is not
+      # `failure()` — a hung leg hitting the job timeout is exactly the case
+      # that most needs these diagnostics. `env.LEG_CONTAINER_DIED` catches
+      # the case "Run benchmark suites" already detected on its own (k6 runs
+      # with `|| true`, so a mid-run OOM would not otherwise fail the job).
+      - name: Diagnose backend container failure
+        if: failure() || cancelled() || env.LEG_CONTAINER_DIED == 'true'
+        timeout-minutes: 3
+        # Body (including the timeout-budget math, the OOM-killer grep and
+        # the shared-host caveat) moved to
+        # .github/scripts/fhir-bench/diagnose.sh.
+        env:
+          BACKEND: ${{ matrix.backend }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          # This step runs on failure()/cancelled(), which includes "Checkout
+          # HFS" itself never completing — in that case the script this
+          # moved to isn't on disk, and no backend container was started
+          # either ("Checkout HFS" runs before any Start-backend step), so
+          # there is nothing here to diagnose. The guard turns that missing
+          # script into a ::warning:: instead of letting a bare `bash
+          # missing-file` (exit 127) fail this step outright; it does not
+          # preserve any dmesg/meminfo/docker-stats diagnostics — there are
+          # none to preserve in that case.
+          D="$GITHUB_WORKSPACE/.github/scripts/fhir-bench/diagnose.sh"
+          if [ -f "$D" ]; then
+            bash "$D"
+          else
+            echo "::warning::diagnose.sh missing (checkout did not complete)"
+          fi
+
       # ── Cleanup ──────────────────────────────────────────────────────────
       - name: Stop HFS server
         if: always()
@@ -1138,7 +1628,9 @@ jobs:
       # this step never runs at all (killed runner, as in run 33515369645 — see
       # the Artifacts comment above).
       - name: Stop ephemeral Postgres
-        if: always() && matrix.backend == 'postgres'
+        # startsWith, not ==: postgres-elasticsearch also uses this Postgres
+        # as its primary and must have its container/volume reclaimed too.
+        if: always() && startsWith(matrix.backend, 'postgres')
         run: |
           if [ -n "${PG_CONTAINER:-}" ]; then
             docker rm -fv "$PG_CONTAINER" 2>/dev/null || true
@@ -1147,6 +1639,36 @@ jobs:
             docker volume rm -f "$PG_VOLUME" 2>/dev/null || true
           fi
 
+      - name: Stop ephemeral MongoDB
+        if: always() && startsWith(matrix.backend, 'mongodb')
+        run: |
+          if [ -n "${MONGO_CONTAINER:-}" ]; then
+            docker rm -fv "$MONGO_CONTAINER" 2>/dev/null || true
+          fi
+          if [ -n "${MONGO_VOLUME:-}" ]; then
+            docker volume rm -f "$MONGO_VOLUME" 2>/dev/null || true
+          fi
+
+      - name: Stop ephemeral Elasticsearch
+        if: always() && endsWith(matrix.backend, '-elasticsearch')
+        run: |
+          if [ -n "${ES_CONTAINER:-}" ]; then
+            docker rm -fv "$ES_CONTAINER" 2>/dev/null || true
+          fi
+          if [ -n "${ES_VOLUME:-}" ]; then
+            docker volume rm -f "$ES_VOLUME" 2>/dev/null || true
+          fi
+
+      # The sqlite-family DB is several GB and the workspace is never wiped
+      # between runs, so a leg's database (and any WAL sidecars a killed run
+      # left behind) must be removed explicitly rather than left to rot.
+      - name: Remove SQLite database files
+        if: always() && startsWith(matrix.backend, 'sqlite')
+        run: |
+          if [ -n "${HFS_DATABASE_URL:-}" ]; then
+            rm -f "$HFS_DATABASE_URL" "$HFS_DATABASE_URL-wal" "$HFS_DATABASE_URL-shm" "$HFS_DATABASE_URL-journal"
+          fi
+
       # ── Per-query-shape attribution ──────────────────────────────────────
       # Without this the search suite reports a single aggregate latency and we
       # cannot tell which of the ~21 query shapes is slow. Produces a table sorted
@@ -1203,6 +1725,17 @@ jobs:
       # ── Summary ──────────────────────────────────────────────────────────
       - name: Generate step summary
         if: always()
+        env:
+          # pg_shared_buffers is a free-text input; it reaches
+          # summary_backends.py (see the pointer comment below) only
+          # through env:, never inlined as ${{ }} — a value containing a
+          # `"` would otherwise break the Python string literal it lands
+          # in (and, in principle, inject code). Nothing else in this step
+          # reads it.
+          IN_PG_SHARED_BUFFERS: ${{ inputs.pg_shared_buffers }}
+          # Also consumed by summary_backends.py (see the pointer comment
+          # below) — that script cannot use ${{ }} itself.
+          BACKEND: ${{ matrix.backend }}
         run: |
           {
             echo "## FHIR Benchmark — \`${{ github.ref_name }}\` (${{ matrix.backend }})"
@@ -1214,10 +1747,33 @@ jobs:
             echo "| **Suites**  | ${{ inputs.tests || 'all' }} |"
             echo ""
 
+            # PR-1 per-leg diagnostics (tuning table, import completeness, ES
+            # drain, host load, "how to read this leg", search-counts +
+            # crud-residue caption, Mongo txn errors, composite sync
+            # failures, container-state notes) moved to
+            # .github/scripts/fhir-bench/summary_backends.py. Runs here, as
+            # its own python3 process, so its output lands at the same
+            # position it did before: ahead of the Results table below.
+            # Guarded (this step runs on always(), including "Checkout HFS"
+            # never completing): under this step's default `bash -e {0}`, a
+            # missing script would abort the `{ ... }` group here and lose
+            # the "_No summary files were produced._" Results line below
+            # too. A missing script here is only possible when "Checkout
+            # HFS" itself never completed, in which case skipping the whole
+            # leg-detail section — including the "How to read this leg"
+            # paragraph that otherwise prints unconditionally on ES/Mongo
+            # legs — is acceptable: no suite ran, so there is nothing for
+            # that section to report.
+            SUMMARY_PY="$GITHUB_WORKSPACE/.github/scripts/fhir-bench/summary_backends.py"
+            if [ -f "$SUMMARY_PY" ]; then
+              python3 "$SUMMARY_PY"
+            fi
+
             python3 - <<'PYEOF'
           import json, glob, os, sys
 
           results_dir = os.path.join(os.environ["GITHUB_WORKSPACE"], "bench-results", "${{ matrix.backend }}")
+          backend = "${{ matrix.backend }}"
           files = sorted(glob.glob(f"{results_dir}/*.json"))
 
           if not files:
@@ -1281,6 +1837,17 @@ jobs:
                       if c:
                           load = f", host loadavg {c.group(1)} over {c.group(2)} containers at crud start"
                   flag = " ⚠ **contended — do not compare this run's crud to a quiet one**" if per_call > 3.0 else ""
+                  if backend != "postgres":
+                      # postgres-elasticsearch also writes crud-pgstat.txt
+                      # (crud/import still hit resources/resource_history
+                      # there), but it writes no search_index — the 3ms/call
+                      # contention threshold was calibrated on bare postgres,
+                      # where Postgres is the only backend under load. On
+                      # this leg Elasticsearch shares the same 4-CPU host and
+                      # can itself add to per-call cost, so a high number
+                      # here is not necessarily a noisy NEIGHBOUR.
+                      flag = (" _(postgres-elasticsearch writes no search_index, so this is not comparable"
+                              " to bare postgres; contention threshold not applied)_")
                   print(f"\n**Environment:** crud Postgres cost **{per_call:.2f} ms/call** "
                         f"({exec_s:,.0f} s over {calls:,} statements{load}).{flag}")
           PYEOF
@@ -1290,8 +1857,8 @@ jobs:
       # The only way to confirm cleanup actually reclaimed this leg's
       # containers/volumes, since developers cannot reach the shared Docker
       # host directly to check by hand. Filtered on the run id AND the leg
-      # label together (multiple --filter label= are ANDed): sqlite and
-      # postgres legs of one run share the run id and run concurrently, so a
+      # label together (multiple --filter label= are ANDed): the matrix legs
+      # share a run id (and run concurrently when max_parallel > 1), so a
       # run-id-only filter would report a sibling leg's still-live containers
       # as if they were this leg's leftovers.
       - name: Docker host usage after cleanup

From 9badfe635f3111962e68f9f581f04c78e5a3140e Mon Sep 17 00:00:00 2001
From: dougc95 <dougcr95@gmail.com>
Date: Mon, 28 Sep 2026 08:06:53 -0400
Subject: [PATCH 2/4] ci(bench): size the capacity gate from docker stats, not
 /proc/meminfo

Run 36410157709 showed the gate measuring the wrong machine:
`docker info` reported MemTotal 12000 MB while a plain container's
/proc/meminfo showed MemAvailable 61324 MB, so the gate would pass
almost every time. The Docker daemon evidently runs inside a 12 GB
limit that containers' /proc/meminfo does not reflect (likely an
lxcfs-virtualised LXC, where even a bind-mounted meminfo would report
free memory relative to the reading container's own cgroup).

host-mem.sh now derives available memory from one source it can
trust: docker info MemTotal minus the usage `docker stats` reports for
every running container, minus 1024 MB for dockerd and non-container
processes. If `docker stats` fails the reading is `unknown` rather
than "nothing running", so the gate keeps polling and fails at its
10-minute budget instead of passing on a guess.

The capacity gate, probe_host (host_mem_avail_mb + host_mem_source in
host-contention.txt), runner-info.txt (capacity_mem_source), the step
summary and diagnose.sh all use it. probe_host now runs before
SUITE_START, so suite wall time no longer includes the probe itself.

Refs #1475

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013NudzWDu2yTGExYaxdTQYJ
---
 .github/scripts/fhir-bench/capacity-gate.sh   |  85 +++++++----
 .github/scripts/fhir-bench/diagnose.sh        |  64 ++++++++-
 .github/scripts/fhir-bench/host-mem.sh        | 136 ++++++++++++++++++
 .../scripts/fhir-bench/summary_backends.py    |  21 ++-
 .github/workflows/fhir-benchmark.yml          |  68 +++++++--
 5 files changed, 320 insertions(+), 54 deletions(-)
 create mode 100755 .github/scripts/fhir-bench/host-mem.sh

diff --git a/.github/scripts/fhir-bench/capacity-gate.sh b/.github/scripts/fhir-bench/capacity-gate.sh
index 33461d94e..d1fa92424 100755
--- a/.github/scripts/fhir-bench/capacity-gate.sh
+++ b/.github/scripts/fhir-bench/capacity-gate.sh
@@ -24,19 +24,33 @@
 # OTHER CI too) every 60s for up to 10 minutes; still short after that,
 # fail the leg rather than risk taking down a neighbour's container.
 #
+# Both MemTotal and every MemAvailable poll go through host-mem.sh, not a
+# direct `docker info`/`docker run ... /proc/meminfo` read — see that
+# script's header for why (run 36410157709: this host's own /proc/meminfo
+# read from inside a plain container disagreed with `docker info` by more
+# than 5x, most likely an LXC/VM-like daemon host whose /proc/meminfo is
+# virtualised, e.g. lxcfs). The source host-mem.sh actually trusted for a
+# given poll is logged on that poll's line and recorded as
+# CAPACITY_MEM_SOURCE alongside the other CAPACITY_* outputs below.
+#
 # Required environment (exported by the workflow step's env:):
 #   BACKEND                  matrix.backend
-#   RUN_ID                   github.run_id
 #   IN_PG_SHARED_BUFFERS     inputs.pg_shared_buffers
 #   IN_MONGO_WT_CACHE_GB     inputs.mongo_wt_cache_gb
 #   ES_HEAP_MB               needs.setup.outputs.es_heap_mb
 #
-# Outputs: CAPACITY_NEED_MB / CAPACITY_AVAIL_MB / CAPACITY_WAIT_S appended to
-# $GITHUB_ENV (read by "Run benchmark suites" for runner-info.txt, and by
-# summary_backends.py as its fallback Capacity gate row for a leg that
-# failed this gate before runner-info.txt was ever written).
+# Also reads host-mem.sh (same directory) for MemTotal/MemAvailable.
+#
+# Outputs: CAPACITY_NEED_MB / CAPACITY_AVAIL_MB / CAPACITY_WAIT_S /
+# CAPACITY_MEM_SOURCE appended to $GITHUB_ENV (read by "Run benchmark
+# suites" for runner-info.txt, and by summary_backends.py as its fallback
+# Capacity gate row for a leg that failed this gate before
+# runner-info.txt was ever written).
 set -euo pipefail
 
+SCRIPT_DIR="$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+HOST_MEM_SH="$SCRIPT_DIR/host-mem.sh"
+
 case "$BACKEND" in
   postgres|postgres-elasticsearch)
     PG_SHARED_BUFFERS="${IN_PG_SHARED_BUFFERS:-2GB}"
@@ -78,19 +92,35 @@ esac
 CAPACITY_NEED_MB=$(( NEED_PRIMARY_MB + NEED_ES_MB ))
 echo "Capacity need for $BACKEND: primary=${NEED_PRIMARY_MB}MB elasticsearch=${NEED_ES_MB}MB total=${CAPACITY_NEED_MB}MB"
 
-MEM_TOTAL_BYTES=$(timeout 60 docker info --format '{{.MemTotal}}' 2>/dev/null) || MEM_TOTAL_BYTES=0
-case "$MEM_TOTAL_BYTES" in ''|*[!0-9]*) MEM_TOTAL_BYTES=0 ;; esac
-MEM_TOTAL_MB=$(( MEM_TOTAL_BYTES / 1024 / 1024 ))
-echo "Docker host MemTotal: ${MEM_TOTAL_MB}MB"
+# read_host_mem: runs host-mem.sh and splits its one guaranteed output
+# line into HOSTMEM_SOURCE / HOSTMEM_TOTAL_MB / HOSTMEM_AVAIL_MB (each
+# "unknown"/"none" on any failure, including host-mem.sh itself being
+# unreadable — a case host-mem.sh's own `set -uo pipefail` guarding
+# cannot cover).
+read_host_mem() {
+  local line
+  line=$(bash "$HOST_MEM_SH" 2>/dev/null) || line=""
+  HOSTMEM_SOURCE=$(printf '%s\n' "$line" | sed -n 's/^source=\([^ ]*\) .*/\1/p')
+  HOSTMEM_TOTAL_MB=$(printf '%s\n' "$line" | sed -n 's/.*total_mb=\([^ ]*\) avail_mb=.*/\1/p')
+  HOSTMEM_AVAIL_MB=$(printf '%s\n' "$line" | sed -n 's/.*avail_mb=\(.*\)$/\1/p')
+  [ -n "$HOSTMEM_SOURCE" ] || HOSTMEM_SOURCE=none
+  [ -n "$HOSTMEM_TOTAL_MB" ] || HOSTMEM_TOTAL_MB=unknown
+  [ -n "$HOSTMEM_AVAIL_MB" ] || HOSTMEM_AVAIL_MB=unknown
+}
+
+read_host_mem
+MEM_TOTAL_MB="$HOSTMEM_TOTAL_MB"
+case "$MEM_TOTAL_MB" in ''|*[!0-9]*) MEM_TOTAL_MB=0 ;; esac
+echo "Docker host MemTotal: ${MEM_TOTAL_MB}MB (source=${HOSTMEM_SOURCE})"
 
-# A `docker info` hiccup reads as MEM_TOTAL_MB=0, which would
+# A host-mem.sh miss reads as MEM_TOTAL_MB=0 (source=none), which would
 # otherwise always trip the impossible-fit check below and tell the
-# user to lower their inputs when the real problem is that the
-# daemon couldn't be read. Skip straight to the poll loop instead —
-# it re-reads memory through a separate `docker run`, so a
-# transient `docker info` failure alone doesn't fail the leg.
+# user to lower their inputs when the real problem is that memory
+# couldn't be read at all. Skip straight to the poll loop instead — it
+# re-reads memory through the same script, so a transient miss alone
+# doesn't fail the leg.
 if [ "$MEM_TOTAL_MB" -eq 0 ]; then
-  echo "::warning::could not read Docker host MemTotal (docker info failed) — skipping the impossible-fit check; the poll below still guards capacity"
+  echo "::warning::could not determine Docker host MemTotal (host-mem.sh source=${HOSTMEM_SOURCE}) — skipping the impossible-fit check; the poll below still guards capacity"
 elif [ $(( CAPACITY_NEED_MB + 2048 )) -gt "$MEM_TOTAL_MB" ]; then
   # Record what this leg needed even though no suite will run, so
   # the summary (which reads these from $GITHUB_ENV when
@@ -100,32 +130,32 @@ elif [ $(( CAPACITY_NEED_MB + 2048 )) -gt "$MEM_TOTAL_MB" ]; then
     echo "CAPACITY_NEED_MB=$CAPACITY_NEED_MB"
     echo "CAPACITY_AVAIL_MB=skipped"
     echo "CAPACITY_WAIT_S=0"
+    echo "CAPACITY_MEM_SOURCE=$HOSTMEM_SOURCE"
   } >> "$GITHUB_ENV"
-  echo "::error::backend=$BACKEND needs ~${CAPACITY_NEED_MB}MB (plus a 2048MB margin), but the Docker host only reports ${MEM_TOTAL_MB}MB total RAM. Lower es_heap / mongo_wt_cache_gb / pg_shared_buffers, or pick a lighter backend — this combination can never fit, even with the whole host free."
+  echo "::error::backend=$BACKEND needs ~${CAPACITY_NEED_MB}MB (plus a 2048MB margin), but the Docker host only reports ${MEM_TOTAL_MB}MB total RAM (source=${HOSTMEM_SOURCE}). Lower es_heap / mongo_wt_cache_gb / pg_shared_buffers, or pick a lighter backend — this combination can never fit, even with the whole host free."
   exit 1
 fi
 
 echo "── Waiting for MemAvailable >= $(( CAPACITY_NEED_MB + 2048 ))MB (poll 60s, timeout 10min) ──"
 CAPACITY_START=$SECONDS
 CAPACITY_AVAIL_MB=""
+CAPACITY_MEM_SOURCE=""
 CAPACITY_OK=0
 while :; do
-  # shellcheck disable=SC2016 # single-quoted deliberately: $2 is awk's field
-  # reference, evaluated inside the container, not a shell variable here.
-  CAPACITY_AVAIL_MB=$(timeout 60 docker run --rm --name "hfs-bench-mem-$BACKEND-$RUN_ID" \
-      --label hfs-bench=1 --label "hfs-bench-run=$RUN_ID" --label "hfs-bench-leg=$BACKEND" \
-      alpine:3 awk '/^MemAvailable:/{print int($2 / 1024)}' /proc/meminfo 2>/dev/null) || CAPACITY_AVAIL_MB=""
-  # `timeout 60` only kills the local docker CLI, not a container
-  # still starting on the daemon — force it gone so the same
-  # `--name` doesn't collide on the next poll (every 60s).
-  docker rm -f "hfs-bench-mem-$BACKEND-$RUN_ID" >/dev/null 2>&1 || true
+  read_host_mem
+  CAPACITY_MEM_SOURCE="$HOSTMEM_SOURCE"
+  CAPACITY_AVAIL_MB="$HOSTMEM_AVAIL_MB"
+  case "$CAPACITY_AVAIL_MB" in ''|*[!0-9]*) CAPACITY_AVAIL_MB="" ;; esac
   CAPACITY_WAIT_S=$((SECONDS - CAPACITY_START))
   if [ -n "$CAPACITY_AVAIL_MB" ] && [ "$CAPACITY_AVAIL_MB" -ge $(( CAPACITY_NEED_MB + 2048 )) ]; then
     CAPACITY_OK=1
-    echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB}MB — capacity OK"
+    echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB}MB (source=${CAPACITY_MEM_SOURCE}) — capacity OK"
     break
   fi
-  echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB:-unknown}MB, need $(( CAPACITY_NEED_MB + 2048 ))MB — waiting"
+  # source=none reads the same as any other unknown MemAvailable above
+  # (CAPACITY_AVAIL_MB blanked by the numeric guard) — keep waiting and
+  # let the 600s budget below be the only thing that fails the leg.
+  echo "  t=${CAPACITY_WAIT_S}s MemAvailable=${CAPACITY_AVAIL_MB:-unknown}MB (source=${CAPACITY_MEM_SOURCE}), need $(( CAPACITY_NEED_MB + 2048 ))MB — waiting"
   # Never sleep past the 600s budget: a plain `sleep 60` here could
   # carry the last iteration well beyond it (e.g. wait_s=590 -> next
   # check at 650s). Cap the sleep to whatever is actually left.
@@ -142,6 +172,7 @@ done
   echo "CAPACITY_NEED_MB=$CAPACITY_NEED_MB"
   echo "CAPACITY_AVAIL_MB=${CAPACITY_AVAIL_MB:-unknown}"
   echo "CAPACITY_WAIT_S=$CAPACITY_WAIT_S"
+  echo "CAPACITY_MEM_SOURCE=${CAPACITY_MEM_SOURCE:-none}"
 } >> "$GITHUB_ENV"
 
 if [ "$CAPACITY_OK" -ne 1 ]; then
diff --git a/.github/scripts/fhir-bench/diagnose.sh b/.github/scripts/fhir-bench/diagnose.sh
index 1414ceaab..7fcd4c0d8 100755
--- a/.github/scripts/fhir-bench/diagnose.sh
+++ b/.github/scripts/fhir-bench/diagnose.sh
@@ -20,7 +20,9 @@
 #
 # Also reads PG_CONTAINER / MONGO_CONTAINER / ES_CONTAINER, already exported
 # to $GITHUB_ENV by "Configure backend env" (whichever of the three this leg
-# sets — unset ones are fine, see the loop below).
+# sets — unset ones are fine, see the loop below), and host-mem.sh (same
+# directory) for the docker-info/docker-stats-derived memory reading
+# captured below.
 #
 # Outputs: none (stdout/`::group::`/`::endgroup::` only — this never fails
 # the step, every command is `|| true`).
@@ -30,9 +32,18 @@ set -e
 # on cancellation it competes with Upload results/server log and the Stop
 # steps for the 5-minute cancellation window. Up to 2 containers (a
 # *-elasticsearch leg) x 2 calls each (4), plus the 5 standalone calls below
-# (dmesg run + rm, meminfo run + rm, docker stats), keeps the worst case
-# near 135s (9 calls x 15s) instead of eating the full 3-minute budget on
-# ~60s-per-call timeouts.
+# (dmesg run + rm, meminfo run + rm, docker stats), keeps that part's worst
+# case near 135s (9 calls x 15s) instead of eating the whole step budget on
+# ~60s-per-call timeouts. The host-mem.sh call further below is bounded the
+# same way, but via ITS OWN internal per-call timeout (HFS_BENCH_MEM_TIMEOUT
+# below), not an external `timeout` wrapper — see the comment at that call
+# for why an external wrapper is the wrong tool here. host-mem.sh no longer
+# starts any helper containers (it derives MemAvailable from `docker info` +
+# `docker stats` alone, see its header), so at the HFS_BENCH_MEM_TIMEOUT=10
+# override used below its own worst case is ~20s (10s docker info + 10s
+# docker stats) — down from the ~50s the three-candidate-read era needed.
+# Step total: ~155s against the 180s (3min) ceiling — 25s of margin for
+# GitHub's own overhead and cancellation-window contention.
 for c in ${PG_CONTAINER:-} ${MONGO_CONTAINER:-} ${ES_CONTAINER:-}; do
   echo "::group::$c state + last 60 log lines"
   timeout 15 docker inspect "$c" --format 'OOMKilled={{.State.OOMKilled}} ExitCode={{.State.ExitCode}} Status={{.State.Status}} Error={{.State.Error}} FinishedAt={{.State.FinishedAt}}' 2>&1 || true
@@ -56,10 +67,24 @@ echo "::endgroup::"
 # This host is shared with the rest of CI, so a container that
 # died here is not necessarily this leg's own doing — a per-leg
 # OOMKilled=false plus a healthy-looking log can still be a
-# neighbour's memory pressure. These two numbers are what tell the
-# two apart after the fact.
+# neighbour's memory pressure. These numbers are what tell the two
+# apart after the fact.
+#
+# Two readings, deliberately: a plain container's own /proc/meminfo
+# (below, labelled "container view" — NOT the daemon's real limit) AND
+# host-mem.sh's derived estimate (further below, from `docker info` +
+# `docker stats`; no /proc/meminfo read at all). Run 36410157709 is why
+# both are worth keeping in a post-mortem — that run's plain /proc/meminfo
+# read (MemAvailable 61324MB) disagreed with `docker info`'s MemTotal
+# (12000MB) by over 5x, most likely (see host-mem.sh's header for the
+# fuller reasoning) an lxcfs-virtualised Docker host resolving the reading
+# container's own near-empty cgroup rather than the host's. Keeping the
+# raw, unvalidated figure alongside host-mem.sh's derived number lets a
+# future post-mortem see both the symptom and the corrected number,
+# instead of only ever seeing the corrected one.
 echo "::group::Docker host memory + containers"
 MEMINFO_NAME="hfs-bench-meminfo-${BACKEND}-${RUN_ID}"
+echo "container view (not the daemon's 12 GB limit — see host-mem.sh):"
 timeout 15 docker run --rm --name "$MEMINFO_NAME" \
   --label hfs-bench=1 --label "hfs-bench-run=${RUN_ID}" --label "hfs-bench-leg=${BACKEND}" \
   busybox sh -c 'grep -E "MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree" /proc/meminfo' 2>&1 || true
@@ -68,5 +93,32 @@ timeout 15 docker run --rm --name "$MEMINFO_NAME" \
 # alone, same reasoning as the voldf cleanups in the YAML's Start
 # Postgres step, start-mongodb.sh and start-elasticsearch.sh.
 timeout 15 docker rm -f "$MEMINFO_NAME" >/dev/null 2>&1 || true
+# `docker stats` BEFORE host-mem.sh, deliberately: host-mem.sh's own worst
+# case (see below) is the biggest single chunk of this step's remaining
+# budget, so capture the cheap, bounded reading first — a step killed by
+# its own timeout-minutes mid-host-mem.sh still leaves this one behind,
+# instead of losing it too (pre-#1475-followup-round-2 ordering lost this
+# capture whenever host-mem.sh ran long).
 timeout 15 docker stats --no-stream --format 'table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}\t{{.CPUPerc}}' 2>&1 || true
+echo "-- host-mem.sh (docker info + docker stats derived estimate; no /proc/meminfo read) --"
+# No OUTER `timeout` wrapper here: host-mem.sh no longer starts any helper
+# containers (it derives MemAvailable from `docker info` + `docker stats`
+# alone — see its header), so the old risk of an external wrapper killing
+# it mid-candidate and leaking an orphaned container is gone. This still
+# passes host-mem.sh's OWN internal per-call knob (HFS_BENCH_MEM_TIMEOUT) a
+# smaller value instead of wrapping the call, so each of its two docker
+# calls is individually bounded rather than the whole script being cut off
+# mid-call by an outer `timeout` (which would just report whatever
+# partial/no output host-mem.sh had produced, instead of host-mem.sh's own
+# clean "source=none"/"avail_mb=unknown" fallback). A diagnostic capture
+# gains little from waiting out the full default 60s-per-call budget; a
+# reading it can get quickly is just as useful as one it spends longer
+# confirming, and "avail_mb=unknown" here (this step's other captures above
+# already have the raw numbers) is a fine outcome for a post-mortem.
+HOST_MEM_SH="$(dirname "${BASH_SOURCE[0]}")/host-mem.sh"
+if [ -f "$HOST_MEM_SH" ]; then
+  HFS_BENCH_MEM_TIMEOUT=10 bash "$HOST_MEM_SH" 2>&1 || true
+else
+  echo "(host-mem.sh missing)"
+fi
 echo "::endgroup::"
diff --git a/.github/scripts/fhir-bench/host-mem.sh b/.github/scripts/fhir-bench/host-mem.sh
new file mode 100755
index 000000000..cd7155f06
--- /dev/null
+++ b/.github/scripts/fhir-bench/host-mem.sh
@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+#
+# Single-source Docker host memory reading — never reads /proc/meminfo.
+#
+# Called from: capacity-gate.sh (MemTotal preflight and every MemAvailable
+# poll), probe_host() (inline in the `benchmark` job's "Run benchmark
+# suites" step, fhir-benchmark.yml), and diagnose.sh (post-mortem capture).
+#
+# Why /proc/meminfo is not used, from anywhere, any more (run 36410157709):
+# "Docker host capacity gate" logged `docker info --format '{{.MemTotal}}'`
+# as 12000MB, then polled MemAvailable via
+# `docker run alpine:3 awk '/MemAvailable/' /proc/meminfo` and got 61324MB
+# back — over 5x the host's own reported total, on a host that has
+# OOM-killed a mongod before (#453) at the 12GB figure. The likely
+# mechanism, not proven against this specific host: the Docker daemon here
+# runs inside an LXC-like environment whose /proc/meminfo is virtualised by
+# lxcfs, and lxcfs resolves MemAvailable against the READING PROCESS's own
+# cgroup rather than the host's. If that is right, no container we spin up
+# to read /proc/meminfo can ever be trusted for this — even a container
+# that bind-mounts the daemon side's /proc/meminfo would still be read by a
+# process sitting in ITS OWN (near-empty, just-started) cgroup, so it would
+# report its own usage rather than the host's regardless of which
+# /proc/meminfo path it opened. `docker info` itself is unaffected because
+# it asks the daemon directly, not a file read from inside a container.
+# So this script never reads /proc/meminfo, from any container, for any
+# purpose — it derives MemAvailable from `docker info` (MemTotal) and
+# `docker stats` (per-container usage) instead, neither of which involves a
+# nested container reporting its own view of memory.
+#
+# Method:
+#   total_mb = `docker info --format '{{.MemTotal}}'` (bytes -> MB), bounded
+#     by `timeout ${HFS_BENCH_MEM_TIMEOUT:-60}`. Empty, non-numeric, or 0 is
+#     "unknown" (source=none — nothing left to derive an estimate from).
+#   avail_mb = total_mb
+#              - (sum of every running container's current memory usage,
+#                from `docker stats --no-stream --format '{{.MemUsage}}'`,
+#                same timeout)
+#              - 1024 (flat allowance for dockerd itself and any
+#                non-container process on the host)
+#              clamped at 0 if that goes negative.
+#   `docker stats` exiting non-zero, or timing out, means "couldn't ask" —
+#   NOT "nothing is running" — so avail_mb is reported "unknown" in that
+#   case; it is never guessed as total_mb minus just the flat allowance.
+#   Empty stdout with exit 0 legitimately means no containers are running,
+#   i.e. 0 used.
+#
+# What the estimate includes/excludes: container memory exactly as `docker
+# stats` accounts it for each container — this excludes page cache
+# attributed to a container's cgroup that the kernel could reclaim under
+# pressure, so a real host is often somewhat MORE available than this
+# number says. The flat 1024MB is not measured; it is just assumed to
+# cover dockerd plus whatever else runs directly on the host outside a
+# container. Both choices make this estimate deliberately conservative: it
+# is meant to never tell a caller more memory is free than actually is, at
+# the cost of occasionally under-reporting.
+#
+# This never fails its caller: set -uo pipefail (deliberately no -e), every
+# docker call bounded by `timeout`, always exits 0.
+#
+# Optional environment:
+#   HFS_BENCH_MEM_TIMEOUT   per-docker-call timeout in seconds (default
+#                           60). Up to 2 such calls run per invocation
+#                           (docker info, docker stats), so this script's
+#                           own worst case is about 2x this value.
+#                           diagnose.sh, with its own tight step budget,
+#                           passes a smaller value; capacity-gate.sh, which
+#                           mainly wants correctness, leaves it at the
+#                           default.
+#
+# Output: stdout, exactly one line, always:
+#   source=<docker-stats|none> total_mb=<n|unknown> avail_mb=<n|unknown>
+# source=none only when total_mb itself could not be determined. Once
+# total_mb is known, source is always docker-stats, even when avail_mb
+# ends up "unknown" because the docker-stats call itself failed.
+#
+# Exit status: always 0.
+set -uo pipefail
+
+DOCKER_TIMEOUT_S="${HFS_BENCH_MEM_TIMEOUT:-60}"
+
+emit() {
+  echo "source=$1 total_mb=$2 avail_mb=$3"
+  exit 0
+}
+
+# ---- total_mb: docker info MemTotal, in MB -------------------------------
+T_BYTES=$(timeout "$DOCKER_TIMEOUT_S" docker info --format '{{.MemTotal}}' 2>/dev/null) || T_BYTES=""
+case "$T_BYTES" in ''|*[!0-9]*) T_BYTES="" ;; esac
+T_MB=""
+if [ -n "$T_BYTES" ] && [ "$T_BYTES" -gt 0 ]; then
+  T_MB=$(( T_BYTES / 1024 / 1024 ))
+  [ "$T_MB" -gt 0 ] || T_MB=""
+fi
+
+if [ -z "$T_MB" ]; then
+  emit "none" "unknown" "unknown"
+fi
+
+# ---- avail_mb: total_mb minus every running container's current usage
+# (docker stats), minus a flat 1024MB allowance -----------------------------
+STATS_OUT=""
+if ! STATS_OUT=$(timeout "$DOCKER_TIMEOUT_S" docker stats --no-stream --format '{{.MemUsage}}' 2>/dev/null); then
+  # "Couldn't ask" (timeout, daemon hiccup) — NOT "nothing running". T is
+  # still known; there is nothing left to derive an availability figure
+  # from.
+  emit "docker-stats" "$T_MB" "unknown"
+fi
+
+USED_MB="0"
+while IFS= read -r line; do
+  [ -z "$line" ] && continue
+  used_part="${line%% / *}"
+  num=$(printf '%s' "$used_part" | sed -E 's/^([0-9.]+).*/\1/')
+  unit=$(printf '%s' "$used_part" | sed -E 's/^[0-9.]+//')
+  case "$num" in ''|*[!0-9.]*) continue ;; esac
+  line_mb=$(awk -v n="$num" -v u="$unit" 'BEGIN {
+    u = tolower(u)
+    if (u == "b")          m = 1 / 1048576
+    else if (u == "kb")    m = 1000 / 1048576
+    else if (u == "kib")   m = 1024 / 1048576
+    else if (u == "mb")    m = 1000000 / 1048576
+    else if (u == "mib")   m = 1
+    else if (u == "gb")    m = 1000000000 / 1048576
+    else if (u == "gib")   m = 1024
+    else { print ""; exit }
+    printf "%.6f", n * m
+  }')
+  [ -z "$line_mb" ] && continue
+  USED_MB=$(awk -v a="$USED_MB" -v b="$line_mb" 'BEGIN{printf "%.6f", a+b}')
+done <<< "$STATS_OUT"
+
+USED_INT_MB=$(awk -v v="$USED_MB" 'BEGIN{printf "%d", v + 0.5}')
+AVAIL_MB=$(( T_MB - USED_INT_MB - 1024 ))
+[ "$AVAIL_MB" -lt 0 ] && AVAIL_MB=0
+
+emit "docker-stats" "$T_MB" "$AVAIL_MB"
diff --git a/.github/scripts/fhir-bench/summary_backends.py b/.github/scripts/fhir-bench/summary_backends.py
index f4acb1822..3d042e834 100755
--- a/.github/scripts/fhir-bench/summary_backends.py
+++ b/.github/scripts/fhir-bench/summary_backends.py
@@ -25,9 +25,14 @@
 #                         reads it
 # Also reads GITHUB_WORKSPACE (set by Actions on every runner, not just this
 # step) and, when present, CAPACITY_NEED_MB / CAPACITY_AVAIL_MB /
-# CAPACITY_WAIT_S — $GITHUB_ENV values "Docker host capacity gate" writes
-# before its own early exit, inherited automatically like any other
-# $GITHUB_ENV value (no env: mapping needed for those three).
+# CAPACITY_WAIT_S / CAPACITY_MEM_SOURCE — $GITHUB_ENV values "Docker host
+# capacity gate" writes before its own early exit, inherited automatically
+# like any other $GITHUB_ENV value (no env: mapping needed for those four).
+# CAPACITY_MEM_SOURCE (and host-contention.txt's host_mem_source below) name
+# which of host-mem.sh's sources (docker-stats / none) that reading actually
+# came from — see host-mem.sh's header for why it derives MemAvailable from
+# `docker info` + `docker stats` instead of ever reading /proc/meminfo
+# (run 36410157709).
 #
 # Input: the *.txt files "Run benchmark suites" wrote under
 # bench-results/<backend>/ — runner-info.txt, import-completeness.txt,
@@ -76,6 +81,7 @@ def read_kv(path, sep):
             "capacity_need_mb": cap_need,
             "capacity_avail_mb": os.environ.get("CAPACITY_AVAIL_MB", "unknown"),
             "capacity_wait_s": os.environ.get("CAPACITY_WAIT_S", "unknown"),
+            "capacity_mem_source": os.environ.get("CAPACITY_MEM_SOURCE", "unknown"),
         }
 if ri:
     print("### Leg configuration\n")
@@ -93,7 +99,9 @@ def read_kv(path, sep):
     need = ri.get("capacity_need_mb")
     if need is not None:
         print(f"| **Capacity gate** | need {need} MB, available "
-              f"{ri.get('capacity_avail_mb', '?')} MB, waited {ri.get('capacity_wait_s', '?')} s |")
+              f"{ri.get('capacity_avail_mb', '?')} MB "
+              f"(source `{ri.get('capacity_mem_source', 'unknown')}`), "
+              f"waited {ri.get('capacity_wait_s', '?')} s |")
     print()
 
 # Import completeness (F1b): k6 caps import at 60m, so an async
@@ -138,11 +146,12 @@ def read_kv(path, sep):
 contention_path = f"{results_dir}/host-contention.txt"
 if os.path.exists(contention_path):
     hc = re.search(
-        r"suite=crud phase=start host_loadavg=(\S+).*host_containers=(\d+).*host_mem_avail_kb=(\S+)",
+        r"suite=crud phase=start host_loadavg=(\S+).*host_containers=(\d+).*"
+        r"host_mem_avail_mb=(\S+).*host_mem_source=(\S+)",
         open(contention_path).read())
     if hc:
         print(f"\n**Host load at crud start:** loadavg {hc.group(1)}, {hc.group(2)} containers, "
-              f"{hc.group(3)} kB MemAvailable on the Docker host.")
+              f"{hc.group(3)} MB MemAvailable on the Docker host (source `{hc.group(4)}`).")
 
 # "How to read this leg" (F4): what the ES/Mongo numbers actually
 # measure, so they are not misread as directly comparable to a bare
diff --git a/.github/workflows/fhir-benchmark.yml b/.github/workflows/fhir-benchmark.yml
index fe0fefd10..7898c1a39 100644
--- a/.github/workflows/fhir-benchmark.yml
+++ b/.github/workflows/fhir-benchmark.yml
@@ -539,15 +539,23 @@ jobs:
             df -h /hostroot 2>/dev/null || docker system df 2>/dev/null || true
 
       - name: Docker host capacity gate
-        # A floor under the 600s poll loop's own budget (plus the impossible-
-        # fit preflight and the two `docker`/`awk` calls around it) — never
-        # let this step itself become the thing that overruns the leg.
-        timeout-minutes: 12
+        # A floor under the 600s poll loop's own budget — never let this
+        # step itself become the thing that overruns the leg. Every
+        # MemTotal/MemAvailable read now goes through host-mem.sh, whose own
+        # worst case (see its header) is ~120s per call (docker info +
+        # docker stats, each up to the default 60s timeout) — it no longer
+        # starts or cleans up any meminfo-candidate helper containers.
+        # Worst case for the whole step: ~120s preflight MemTotal read +
+        # the 600s poll budget (which already counts each poll's own
+        # host-mem.sh time against it, via bash's SECONDS) + one more
+        # ~120s poll that pushes past that budget before the loop notices
+        # = ~840s (14min). 15 minutes (900s) covers that with a little
+        # margin.
+        timeout-minutes: 15
         # Body (including the per-backend memory model and the poll-loop
         # rationale) moved to .github/scripts/fhir-bench/capacity-gate.sh.
         env:
           BACKEND: ${{ matrix.backend }}
-          RUN_ID: ${{ github.run_id }}
           IN_PG_SHARED_BUFFERS: ${{ inputs.pg_shared_buffers }}
           IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
           ES_HEAP_MB: ${{ needs.setup.outputs.es_heap_mb }}
@@ -1217,6 +1225,7 @@ jobs:
             echo "capacity_need_mb:   ${CAPACITY_NEED_MB:-unknown}"
             echo "capacity_avail_mb:  ${CAPACITY_AVAIL_MB:-unknown}"
             echo "capacity_wait_s:    ${CAPACITY_WAIT_S:-unknown}"
+            echo "capacity_mem_source: ${CAPACITY_MEM_SOURCE:-unknown}"
             # PG_VOL_FREE_MB / MONGO_VOL_FREE_MB / ES_VOL_FREE_MB were written
             # to $GITHUB_ENV by the container-start steps above (they also
             # print into that step's own $GITHUB_STEP_SUMMARY block, but
@@ -1293,24 +1302,43 @@ jobs:
           # cannot be trusted for throughput now says so in its own artifact.
           probe_host() {
             local phase="$1" suite="$2"
-            local la cn mem out
-            # One alpine run answers both questions: how loaded is the host,
-            # and (#1475) how much memory does it have left — the same
-            # MemAvailable the capacity gate polls, recorded in kB so a
-            # suite's own numbers can be read against host memory pressure,
-            # not just load average and container count.
+            local la cn out mem_line mem_mb mem_source
+            # Loadavg still comes from a plain container's own /proc/loadavg,
+            # unvalidated — NOT because lxcfs-style virtualisation is known
+            # to leave /proc/loadavg alone (it doesn't; lxcfs virtualises
+            # loadavg the same way it can virtualise meminfo, scoped to
+            # whatever cgroup it resolves the reader to). What actually
+            # matters here is what a plain nested container's own read
+            # happens to give us on THIS host: run 36410157709 showed it
+            # returning the physical machine's real, unvirtualised figures
+            # for memory, not a cgroup-scoped view — the whole reason
+            # host-mem.sh exists — and there is no reason to expect loadavg
+            # to behave differently on the same host. That is the number
+            # this probe actually wants: the WHOLE box's contention,
+            # including neighbours outside this leg's own cgroup, not a
+            # scoped-down view of just this leg. If that ever turns out
+            # false here (loadavg reads a cgroup-scoped value while meminfo
+            # doesn't), this reading would need the same kind of
+            # cross-validation host-mem.sh does for memory — nothing
+            # currently checks that assumption for loadavg.
             local probe_name="hfs-bench-probe-${{ matrix.backend }}-${{ github.run_id }}"
             out=$(timeout 60 docker run --rm --name "$probe_name" \
                 --label hfs-bench=1 --label "hfs-bench-run=${{ github.run_id }}" --label "hfs-bench-leg=${{ matrix.backend }}" \
-                alpine:3 sh -c 'cat /proc/loadavg; grep MemAvailable /proc/meminfo' 2>/dev/null) || out=""
+                alpine:3 cat /proc/loadavg 2>/dev/null) || out=""
             # `timeout 60` only kills the local docker CLI; force the
             # container gone so repeated probe_host calls (every suite,
             # every phase) never collide on this fixed --name.
             docker rm -f "$probe_name" >/dev/null 2>&1 || true
             la=$(printf '%s\n' "$out" | sed -n '1p' | cut -d' ' -f1-3)
-            mem=$(printf '%s\n' "$out" | sed -n '2p' | awk '{print $2}')
             cn=$(docker ps -q 2>/dev/null | wc -l) || cn=""
-            echo "$(date -u +%H:%M:%SZ) suite=$suite phase=$phase host_loadavg=${la:-unknown} host_containers=${cn:-unknown} host_mem_avail_kb=${mem:-unknown}" \
+            # Memory (#1475 follow-up, run 36410157709): the same
+            # host-mem.sh the capacity gate polls — derived from `docker
+            # info` + `docker stats`, not a bare container /proc/meminfo
+            # read — see that script's header.
+            mem_line=$(bash "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/host-mem.sh" 2>/dev/null) || mem_line=""
+            mem_mb=$(printf '%s\n' "$mem_line" | sed -n 's/.*avail_mb=\(.*\)$/\1/p')
+            mem_source=$(printf '%s\n' "$mem_line" | sed -n 's/^source=\([^ ]*\) .*/\1/p')
+            echo "$(date -u +%H:%M:%SZ) suite=$suite phase=$phase host_loadavg=${la:-unknown} host_containers=${cn:-unknown} host_mem_avail_mb=${mem_mb:-unknown} host_mem_source=${mem_source:-none}" \
               >> "$RESULTS_DIR/host-contention.txt"
           }
 
@@ -1338,8 +1366,14 @@ jobs:
               fi
             fi
 
-            SUITE_START=$(date +%s)
+            # probe_host itself makes two docker calls via host-mem.sh
+            # (docker info + docker stats, up to ~120s worst case at the
+            # default 60s per-call timeout, see that script's header) —
+            # call it BEFORE starting the clock, so a slow host reading
+            # isn't folded into this suite's own wall time / the
+            # pg_exec-vs-wall attribution below.
             probe_host start "$SUITE"
+            SUITE_START=$(date +%s)
             if [ ! -f "$SCRIPT" ]; then
               echo "Skip '$SUITE' — $SCRIPT not found"
               continue
@@ -1582,6 +1616,10 @@ jobs:
       # with `|| true`, so a mid-run OOM would not otherwise fail the job).
       - name: Diagnose backend container failure
         if: failure() || cancelled() || env.LEG_CONTAINER_DIED == 'true'
+        # 3: host-mem.sh no longer starts helper containers to read
+        # /proc/meminfo (just docker info + docker stats), so its capture
+        # below adds only a small, bounded amount on top of this step's
+        # other calls — see the timeout-budget math in diagnose.sh.
         timeout-minutes: 3
         # Body (including the timeout-budget math, the OOM-killer grep and
         # the shared-host caveat) moved to

From cffc076b1329520d0d289a70ce020a20cf722a92 Mon Sep 17 00:00:00 2001
From: dougc95 <dougcr95@gmail.com>
Date: Mon, 28 Sep 2026 08:33:11 -0400
Subject: [PATCH 3/4] ci(bench): default Elasticsearch legs to synchronous sync
 mode

Run 36412022609 (sqlite-elasticsearch, asynchronous, prewarm+import+
search) showed that async mode cannot finish the import corpus inside
k6's 60-minute cap: 313 of 1000 bundles succeeded, 14 hit the 900 s
client timeout, and bundle latency p95 was 489 s. The drain gate then
reported status=incomplete: SQLite held 562,649 live resources and
Elasticsearch 367,985, with composite_secondary_sync_needs_reindex=0.

As agreed for this case, es_sync_mode now defaults to synchronous
(bundles batched via _bulk, refresh=wait_for). asynchronous, HFS's own
out-of-the-box mode, stays available as an input, and its
"How to read this leg" note now says import is expected to hit the cap.

Refs #1475

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013NudzWDu2yTGExYaxdTQYJ
---
 .github/scripts/fhir-bench/resolve-matrix.sh  |  7 +++--
 .../scripts/fhir-bench/summary_backends.py    |  5 +++-
 .github/workflows/fhir-benchmark.yml          | 29 ++++++++++++-------
 3 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/scripts/fhir-bench/resolve-matrix.sh b/.github/scripts/fhir-bench/resolve-matrix.sh
index 1842b85ba..8e041855a 100755
--- a/.github/scripts/fhir-bench/resolve-matrix.sh
+++ b/.github/scripts/fhir-bench/resolve-matrix.sh
@@ -119,8 +119,11 @@ fi
 
 # es_sync_mode: defence in depth — workflow_dispatch's UI already
 # restricts this to the choice options, but an API dispatch can send
-# anything.
-ES_SYNC_MODE="${IN_ES_SYNC_MODE:-asynchronous}"
+# anything. Default here matches the workflow input's default
+# (synchronous): async import cannot finish the 1000-bundle corpus
+# inside k6's 60-minute cap (run 36412022609 — 313/1000 bundles, ES
+# drain status=incomplete, 194,664 resources never indexed).
+ES_SYNC_MODE="${IN_ES_SYNC_MODE:-synchronous}"
 case "$ES_SYNC_MODE" in
   asynchronous|synchronous) ;;
   *)
diff --git a/.github/scripts/fhir-bench/summary_backends.py b/.github/scripts/fhir-bench/summary_backends.py
index 3d042e834..aea6bf654 100755
--- a/.github/scripts/fhir-bench/summary_backends.py
+++ b/.github/scripts/fhir-bench/summary_backends.py
@@ -177,7 +177,10 @@ def read_kv(path, sep):
               "`_delete_by_query` with a forced refresh across all of the tenant's indices. Cluster "
               "health `yellow` is expected (1 replica per index, 1 node). Compare search latency with "
               "another leg only if both imported the same entry count and the ES drain above says "
-              "`drained`.")
+              "`drained`. **This worker's rate is expected to make import hit k6's 60-minute cap "
+              "before the corpus finishes, leaving the index incomplete** (see the Import and ES "
+              "drain lines above / `es-drain.txt`; run 36412022609 imported 313/1000 bundles and "
+              "the drain gate reported status=incomplete with 194,664 resources never indexed).")
 if backend.startswith("mongodb"):
     pg_shared_buffers = os.environ.get("IN_PG_SHARED_BUFFERS") or "2GB"
     print(f"\n**How to read this leg.** MongoDB 7.0 single-member replica set (directConnection); "
diff --git a/.github/workflows/fhir-benchmark.yml b/.github/workflows/fhir-benchmark.yml
index 7898c1a39..59d4aec5c 100644
--- a/.github/workflows/fhir-benchmark.yml
+++ b/.github/workflows/fhir-benchmark.yml
@@ -146,13 +146,13 @@ on:
         required: false
         default: "1g"
       es_sync_mode:
-        description: "asynchronous (HFS default): ack on primary commit; ONE worker forwards each resource individually (import/crud throughput = that worker) | synchronous: bundles batched via _bulk + refresh=wait_for, 200ms"
+        description: "synchronous (benchmark default — async import cannot finish the corpus within k6's 60-minute cap, see run 36412022609): bundles batched via _bulk + refresh=wait_for, 200ms | asynchronous (HFS's own out-of-the-box default): ack on primary commit; ONE worker forwards each resource individually (import/crud throughput = that worker)"
         type: choice
         required: false
-        default: asynchronous
+        default: synchronous
         options:
-          - asynchronous
           - synchronous
+          - asynchronous
       mongo_wt_cache_gb:
         description: "mongodb legs: --wiredTigerCacheSizeGB (0.25..6). 2 = parity with Postgres shared_buffers; mongod's own default (~5 GB here) risks a host OOM"
         required: false
@@ -689,14 +689,21 @@ jobs:
               echo "ES container:    $ES_CONTAINER"
               echo "ES volume:       $ES_VOLUME"
               echo "ES index prefix: $ES_PREFIX"
-              # asynchronous = HFS's own defaults, set explicitly so the run
-              # records them: a write acks once the PRIMARY commits and one
-              # background worker forwards it to Elasticsearch; the search
-              # suite's drain gate waits for that backlog before measuring.
-              # synchronous = the conformance workflows' read-your-writes
-              # setting (inferno-us-core.yml — wait_for makes the refresh
-              # interval the price of every write, hence 200ms).
-              case "${IN_ES_SYNC_MODE:-asynchronous}" in
+              # synchronous = this benchmark's default (see the es_sync_mode
+              # input): the conformance workflows' read-your-writes setting
+              # (inferno-us-core.yml — wait_for makes the refresh interval
+              # the price of every write, hence 200ms). Async import cannot
+              # finish the 1000-bundle corpus inside k6's 60-minute cap
+              # (run 36412022609: 313/1000 bundles, drain gate
+              # status=incomplete, 194,664 resources never reached the
+              # index), so synchronous is what a full-corpus leg needs.
+              # asynchronous = HFS's own out-of-the-box defaults, set
+              # explicitly so the run records them: a write acks once the
+              # PRIMARY commits and one background worker forwards it to
+              # Elasticsearch; the search suite's drain gate waits for that
+              # backlog before measuring — this is what run 36412022609
+              # measured.
+              case "${IN_ES_SYNC_MODE:-synchronous}" in
                 synchronous)  SYNC_MODE=synchronous;  WRITE_REFRESH=wait_for; REFRESH_INTERVAL=200ms ;;
                 asynchronous) SYNC_MODE=asynchronous; WRITE_REFRESH=false;    REFRESH_INTERVAL=1s ;;
                 *) echo "::error::Unknown es_sync_mode '$IN_ES_SYNC_MODE'"; exit 1 ;;

From 6d8a707cdcf8a7e8efe2f7ad81db67e10f49d64c Mon Sep 17 00:00:00 2001
From: dougc95 <dougcr95@gmail.com>
Date: Mon, 28 Sep 2026 08:46:10 -0400
Subject: [PATCH 4/4] ci(bench): keep the suites step free of expressions
 (21,000-char cap)

Dispatching cffc076b1 failed before any job ran: "failed to parse
workflow: (Line: 1145, Col: 14): Exceeded max expression length
21000". A run: block that contains any ${{ }} is evaluated as one
expression, and GitHub caps expressions at 21,000 characters. The
"Run benchmark suites" block had grown to ~26,300 characters with 31
substitutions.

Every matrix/inputs/github/needs value that block uses now comes in
through the step's env: (BACKEND, BENCH_PORT, BENCH_RUN_ID,
BENCH_IN_*, BENCH_SHA, BENCH_REF_NAME, BENCH_MAX_PARALLEL,
BENCH_LEG_TIMEOUT_MIN), with the same `||` fallbacks done in bash, so
the block is a plain string with no length cap. A comment above the
env: block records the rule. No behaviour change.

Refs #1475

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013NudzWDu2yTGExYaxdTQYJ
---
 .github/workflows/fhir-benchmark.yml | 100 ++++++++++++++++-----------
 1 file changed, 59 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/fhir-benchmark.yml b/.github/workflows/fhir-benchmark.yml
index 59d4aec5c..f3a644267 100644
--- a/.github/workflows/fhir-benchmark.yml
+++ b/.github/workflows/fhir-benchmark.yml
@@ -1133,16 +1133,38 @@ jobs:
       - name: Run benchmark suites
         working-directory: fhir-benchmark
         env:
-          # Free-text inputs used below reach bash only through env:, never
-          # inlined as ${{ }} inside run: — same reasoning as "Resolve matrix
-          # from backend input" and "Configure backend env". es_sync_mode
-          # needs no entry here: it is already ambient as
-          # $HFS_COMPOSITE_SYNC_MODE, written to $GITHUB_ENV by "Configure
-          # backend env".
+          # This run: block must contain no ${{ }} — GitHub caps a run block
+          # that contains an expression at 21,000 characters (dispatch of
+          # cffc076b1 failed with "Exceeded max expression length 21000" on
+          # this exact step, which was ~26,300 characters with 31 ${{ }}
+          # substitutions). Every matrix/inputs/github/needs value the block
+          # below uses is passed through here instead and read back as a
+          # plain env var in bash. es_sync_mode needs no entry here: it is
+          # already ambient as $HFS_COMPOSITE_SYNC_MODE, written to
+          # $GITHUB_ENV by "Configure backend env".
+          BACKEND: ${{ matrix.backend }}
+          BENCH_PORT: ${{ matrix.port }}
+          BENCH_RUN_ID: ${{ github.run_id }}
+          BENCH_IN_RUN_ID: ${{ inputs.run_id }}
+          BENCH_SHA: ${{ github.sha }}
+          BENCH_REF_NAME: ${{ github.ref_name }}
+          BENCH_IN_PG_MAX_CONNECTIONS: ${{ inputs.hfs_pg_max_connections }}
+          BENCH_MAX_PARALLEL: ${{ needs.setup.outputs.max_parallel }}
+          BENCH_LEG_TIMEOUT_MIN: ${{ needs.setup.outputs.leg_timeout_min }}
+          BENCH_IN_TESTS: ${{ inputs.tests }}
+          BENCH_IN_VUS: ${{ inputs.vus }}
+          BENCH_IN_DURATION: ${{ inputs.duration }}
           IN_ES_HEAP: ${{ inputs.es_heap }}
           IN_MONGO_WT_CACHE_GB: ${{ inputs.mongo_wt_cache_gb }}
           IN_HFS_MONGO_MAX_CONNECTIONS: ${{ inputs.hfs_mongo_max_connections }}
         run: |
+          # This block must contain no inline GitHub Actions expressions
+          # (the double-curly-brace substitution syntax) — GitHub caps a
+          # run: block that contains one at 21,000 characters. Dispatching
+          # commit cffc076b1 failed with "Exceeded max expression length
+          # 21000" on this exact step, which was about 26,300 characters
+          # using 31 such expressions. New values come in through env:
+          # above and are read back here as plain environment variables.
           set -uo pipefail
           # es_drain_gate, capture_backend_stats, write_import_completeness,
           # write_mongo_txn_errors, write_search_counts,
@@ -1154,30 +1176,26 @@ jobs:
           # substitution.
           source "$GITHUB_WORKSPACE/.github/scripts/fhir-bench/suite-lib.sh"
 
-          # Plain (non-exported) shell variables, not `env:` above: this
-          # inline block, unlike the sourced suite-lib.sh functions, still
-          # gets GitHub Actions expression substitution, so there is no need
-          # to export these to the process environment — doing so would leak
-          # them into k6's __ENV and every other child process this step
-          # spawns. Sourcing puts suite-lib.sh's functions in THIS shell, so
-          # they see these like any other shell variable already set by the
-          # time they're called; no export needed for that either.
+          # BACKEND / BENCH_PORT / BENCH_RUN_ID are already in the process
+          # environment from env: above (same matrix.backend / matrix.port /
+          # github.run_id values suite-lib.sh's functions and everything
+          # below need) — no local re-assignment needed. RUN_ID below stays a
+          # plain shell variable rather than moving to env: above because it
+          # is derived (inputs.run_id with a bash-side fallback), not a 1:1
+          # copy of one context value.
           # BENCH_RUN_ID is deliberately NOT named RUN_ID: this step's own
           # $RUN_ID (below) holds the possibly-user-overridden
           # inputs.run_id tag used for k6 --tag runid=, not the actual
           # github.run_id that container/volume names and the drain-probe
           # identifier need.
-          BACKEND="${{ matrix.backend }}"
-          BENCH_PORT="${{ matrix.port }}"
-          BENCH_RUN_ID="${{ github.run_id }}"
-          RUN_ID="${{ inputs.run_id }}"
-          [ -z "$RUN_ID" ] && RUN_ID="${{ github.run_id }}"
+          RUN_ID="$BENCH_IN_RUN_ID"
+          [ -z "$RUN_ID" ] && RUN_ID="$BENCH_RUN_ID"
 
           # Checkout uses clean:false, so wipe any results left by a prior run on
           # this runner (otherwise a stale sibling backend dir rides along in the
           # uploaded artifact).
           rm -rf "$GITHUB_WORKSPACE/bench-results"
-          RESULTS_DIR="$GITHUB_WORKSPACE/bench-results/${{ matrix.backend }}"
+          RESULTS_DIR="$GITHUB_WORKSPACE/bench-results/$BACKEND"
           mkdir -p "$RESULTS_DIR"
 
           # Which machine measured this run, recorded INSIDE the artifact.
@@ -1193,17 +1211,17 @@ jobs:
           # numbers can always be read back against what produced them.
           # Mongo/ES values read n/a off their own legs. CAPACITY_* were
           # written to $GITHUB_ENV by "Docker host capacity gate", above.
-          case "${{ matrix.backend }}" in
+          case "$BACKEND" in
             mongodb*) MONGO_POOL_REC="${IN_HFS_MONGO_MAX_CONNECTIONS:-32}"; MONGO_WT_REC="${IN_MONGO_WT_CACHE_GB:-2}" ;;
             *)        MONGO_POOL_REC=n/a; MONGO_WT_REC=n/a ;;
           esac
-          case "${{ matrix.backend }}" in
+          case "$BACKEND" in
             *-elasticsearch) ES_HEAP_REC="${IN_ES_HEAP:-1g}"
                              ES_WRITE_REFRESH_REC="${HFS_ELASTICSEARCH_WRITE_REFRESH:-unknown}"
                              ES_REFRESH_INTERVAL_REC="${HFS_ELASTICSEARCH_REFRESH_INTERVAL:-unknown}" ;;
             *)               ES_HEAP_REC=n/a; ES_WRITE_REFRESH_REC=n/a; ES_REFRESH_INTERVAL_REC=n/a ;;
           esac
-          case "${{ matrix.backend }}" in
+          case "$BACKEND" in
             mongodb*) MONGO_TXN_LIFETIME_REC="${MONGO_TXN_LIFETIME_S:-unknown}" ;;
             *)        MONGO_TXN_LIFETIME_REC=n/a ;;
           esac
@@ -1214,11 +1232,11 @@ jobs:
             echo "runner_ram:   $(free -g 2>/dev/null | awk '/^Mem:/{print $2"G"}' || echo '?')"
             echo "runner_ip:    ${RUNNER_IP:-unknown}"
             echo "docker_host:  ${DOCKER_HOST_IP:-unknown}"
-            echo "github_run:   ${{ github.run_id }}"
-            echo "git_sha:      ${{ github.sha }}"
-            echo "git_ref:      ${{ github.ref_name }}"
-            echo "backend:      ${{ matrix.backend }}"
-            echo "pg_pool_max:  ${{ inputs.hfs_pg_max_connections || '32' }}"
+            echo "github_run:   $BENCH_RUN_ID"
+            echo "git_sha:      $BENCH_SHA"
+            echo "git_ref:      $BENCH_REF_NAME"
+            echo "backend:      $BACKEND"
+            echo "pg_pool_max:  ${BENCH_IN_PG_MAX_CONNECTIONS:-32}"
             echo "es_heap:            $ES_HEAP_REC"
             echo "es_sync_mode:       ${HFS_COMPOSITE_SYNC_MODE:-n/a}"
             echo "es_write_refresh:   $ES_WRITE_REFRESH_REC"
@@ -1227,8 +1245,8 @@ jobs:
             echo "mongo_pool_max:     $MONGO_POOL_REC"
             echo "mongo_txn_lifetime_s: $MONGO_TXN_LIFETIME_REC"
             echo "hfs_ready_s:        ${HFS_READY_SECONDS:-unknown}"
-            echo "max_parallel:       ${{ needs.setup.outputs.max_parallel }}"
-            echo "leg_timeout_min:    ${{ needs.setup.outputs.leg_timeout_min }}"
+            echo "max_parallel:       $BENCH_MAX_PARALLEL"
+            echo "leg_timeout_min:    $BENCH_LEG_TIMEOUT_MIN"
             echo "capacity_need_mb:   ${CAPACITY_NEED_MB:-unknown}"
             echo "capacity_avail_mb:  ${CAPACITY_AVAIL_MB:-unknown}"
             echo "capacity_wait_s:    ${CAPACITY_WAIT_S:-unknown}"
@@ -1245,7 +1263,7 @@ jobs:
           # Resolve suite list. 'search' needs data, so 'import' must precede it;
           # canonical order keeps that invariant for the 'all' case.
           CANONICAL="prewarm import crud search"
-          INPUT_TESTS="${{ inputs.tests || 'all' }}"
+          INPUT_TESTS="${BENCH_IN_TESTS:-all}"
           if [ "$INPUT_TESTS" = "all" ]; then
             SUITES="$CANONICAL"
           else
@@ -1254,8 +1272,8 @@ jobs:
 
           # Advisory overrides (see header note — ignored when a script pins a scenario).
           EXTRA_ARGS=""
-          [ -n "${{ inputs.vus }}" ]      && EXTRA_ARGS="$EXTRA_ARGS --vus ${{ inputs.vus }}"
-          [ -n "${{ inputs.duration }}" ] && EXTRA_ARGS="$EXTRA_ARGS --duration ${{ inputs.duration }}"
+          [ -n "$BENCH_IN_VUS" ]      && EXTRA_ARGS="$EXTRA_ARGS --vus $BENCH_IN_VUS"
+          [ -n "$BENCH_IN_DURATION" ] && EXTRA_ARGS="$EXTRA_ARGS --duration $BENCH_IN_DURATION"
 
           # Where does a suite's time actually go?
           #
@@ -1280,7 +1298,7 @@ jobs:
           # (main.rs set_search_offloaded), so they would measure an empty
           # table.
           PG_FAMILY=false
-          case "${{ matrix.backend }}" in postgres|postgres-elasticsearch) PG_FAMILY=true ;; esac
+          case "$BACKEND" in postgres|postgres-elasticsearch) PG_FAMILY=true ;; esac
 
           # ES drain gate: ONE deadline shared across the barrier probe and
           # the settle poll (see es_drain_gate in suite-lib.sh) — not 15 min budgeted to
@@ -1328,9 +1346,9 @@ jobs:
             # doesn't), this reading would need the same kind of
             # cross-validation host-mem.sh does for memory — nothing
             # currently checks that assumption for loadavg.
-            local probe_name="hfs-bench-probe-${{ matrix.backend }}-${{ github.run_id }}"
+            local probe_name="hfs-bench-probe-${BACKEND}-${BENCH_RUN_ID}"
             out=$(timeout 60 docker run --rm --name "$probe_name" \
-                --label hfs-bench=1 --label "hfs-bench-run=${{ github.run_id }}" --label "hfs-bench-leg=${{ matrix.backend }}" \
+                --label hfs-bench=1 --label "hfs-bench-run=${BENCH_RUN_ID}" --label "hfs-bench-leg=${BACKEND}" \
                 alpine:3 cat /proc/loadavg 2>/dev/null) || out=""
             # `timeout 60` only kills the local docker CLI; force the
             # container gone so repeated probe_host calls (every suite,
@@ -1395,7 +1413,7 @@ jobs:
             # so this subsumes the plain ANALYZE.
             #
             # Runs once, after the data is in and before search is measured.
-            if [ "$SUITE" = "search" ] && [ "${{ matrix.backend }}" = "postgres" ] \
+            if [ "$SUITE" = "search" ] && [ "$BACKEND" = "postgres" ] \
                && [ -n "${PG_CONTAINER:-}" ]; then
               echo "── VACUUM (ANALYZE) before search (sets the visibility map) ──"
               docker exec "$PG_CONTAINER" psql -U postgres -d postgres -v ON_ERROR_STOP=1 \
@@ -1422,7 +1440,7 @@ jobs:
             fi
 
             echo "=================================================="
-            echo "Suite: $SUITE   backend: ${{ matrix.backend }}   run: $RUN_ID"
+            echo "Suite: $SUITE   backend: $BACKEND   run: $RUN_ID"
             echo "=================================================="
 
             # The search suite tags every request with {name, resourceType,
@@ -1438,7 +1456,7 @@ jobs:
               --no-usage-report \
               --tag runid="$RUN_ID" \
               --tag fhirimpl=hfs \
-              --env BASE_URL="http://localhost:${{ matrix.port }}" \
+              --env BASE_URL="http://localhost:$BENCH_PORT" \
               --env BUNDLE_URL="$BUNDLE_URL" \
               --env FHIRIMPL=hfs \
               --summary-trend-stats "avg,min,med,p(90),p(95),p(99),max" \
@@ -1490,7 +1508,7 @@ jobs:
                 # postgres-elasticsearch writes no search_index (main.rs
                 # set_search_offloaded), so its wall time includes ES sync
                 # backpressure, not just Postgres — say so before the numbers.
-                if [ "${{ matrix.backend }}" = "postgres-elasticsearch" ]; then
+                if [ "$BACKEND" = "postgres-elasticsearch" ]; then
                   echo "note: postgres-elasticsearch — search_index is not written and wall time includes Elasticsearch sync backpressure; a low pg_exec/wall ratio does NOT mean HFS-CPU-bound"
                 fi
                 echo "suite=$SUITE wall_seconds=$SUITE_WALL"
@@ -1547,7 +1565,7 @@ jobs:
           # write still maintained it and it still occupied cache on an 11G host.
           # Evidence for the next round of index pruning; nothing reads it in
           # this run.
-          if [ "${{ matrix.backend }}" = "postgres" ] && [ -n "${PG_CONTAINER:-}" ]; then
+          if [ "$BACKEND" = "postgres" ] && [ -n "${PG_CONTAINER:-}" ]; then
             docker exec "$PG_CONTAINER" psql -U postgres -d postgres \
               -c "SELECT s.indexrelname AS index_name, s.idx_scan AS scans,
                          s.idx_tup_read AS tuples_read,