diff --git a/CHANGELOG.md b/CHANGELOG.md index 7509d4177..c84ac8a02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased +### A Bot's computer is waited for properly on Podman, and the supervisor can reach the engine there + +Two things stopped OpenBot running on Podman, which nothing had tried before. + +The supervisor could not reach the engine at all: `The supervisor could not reach Docker`. The socket +is there and the mount is right, but Podman's virtual machine runs SELinux and labels the socket in a +way a container is not allowed to read. The supervisor now declares `label=disable`, which is what +that needs and which changes nothing on Docker. + +Then every cold start of a computer raced the first request to it. Readiness was read off the image's +`HEALTHCHECK`, and Podman does not report one: its images are OCI-manifest, the OCI image config has +no healthcheck field, and the instruction is dropped both when Podman builds an image and when it +pulls one that has it. With no health to read, the supervisor fell back to accepting a container that +was merely running, and a running container is not a browser that is answering, so the first request +arrived at a port nothing was listening on and came back as a computer that is not running. The +supervisor now states the healthcheck when it creates a computer instead of inheriting it, so +readiness no longer depends on how the image was built. On Docker the behaviour is unchanged. + ## 0.0.7 ### The published service images are zstd rather than gzip diff --git a/docker-compose.yml b/docker-compose.yml index 710ecb6c3..6c8928a94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -223,6 +223,9 @@ services: - /var/run/docker.sock:/var/run/docker.sock:ro # To register an entry per Bot as each computer is created. - spire-server-socket:/tmp/spire-server/private + security_opt: + # The engine socket carries an SELinux label this container is not allowed to read. + - label=disable healthcheck: test: ["CMD-SHELL", "bun -e \"await fetch('http://localhost:4300/health')\""] interval: 10s diff --git a/supervisor/src/docker.ts b/supervisor/src/docker.ts index 6938bb954..ce334cac2 100644 --- a/supervisor/src/docker.ts +++ b/supervisor/src/docker.ts @@ -32,6 +32,37 @@ const docker = new Docker( /** The port the computer listens on inside its own container. */ const COMPUTER_PORT = "4100/tcp"; +/** + * Readiness, stated at create time rather than read off the image. + * + * `agent-computer/Dockerfile` declares the same HEALTHCHECK, and under Docker that was enough: the + * image carried it, the daemon ran it, and {@link waitUntilAnswering} could ask. Podman does not + * report it. Its images are OCI-manifest, the OCI image config has no healthcheck field, and the + * instruction is dropped, both when Podman builds the image and when it pulls one that has it. The + * published `agent-computer` config does carry it; `podman inspect` of that same image reports + * none. + * + * Silently, and into the one branch that cannot tell the difference: with no health to read, + * `waitUntilAnswering` accepts `Running`, and a container that is running is not a Chromium that + * is answering. Every cold start of a computer then raced the first request, which arrived at a + * port nothing was listening on yet and was reported as a computer that is not running. + * + * Passed here, the engine is told what to run instead of asked what it inherited, which is also + * true on Docker and one less thing that depends on how an image was built. Podman honours an + * explicit healthcheck: it is how every service in `docker-compose.yml` reports healthy there. + */ +const COMPUTER_HEALTHCHECK = { + Test: [ + "CMD-SHELL", + `bun -e "const r = await fetch('http://localhost:${COMPUTER_PORT.split("/")[0]}/health'); process.exit(r.ok ? 0 : 1)"`, + ], + // Nanoseconds, which is what the API takes. The same numbers the Dockerfile states. + Interval: 2_000_000_000, + Timeout: 3_000_000_000, + StartPeriod: 2_000_000_000, + Retries: 30, +}; + /** * How many times `ensure` will build a computer before giving up. * @@ -262,8 +293,9 @@ async function waitUntilAnswering( try { const info = await docker.getContainer(container).inspect(); const health = info.State?.Health?.Status; - // An image without a HEALTHCHECK reports nothing. Waiting forever for an answer that will - // never come would be worse than going ahead, so running is accepted as the best available. + // Nothing to read. Every computer this supervisor creates is given a healthcheck, so this is + // an engine that does not report one rather than an image that does not carry one, and + // waiting forever for an answer that will never come would be worse than going ahead. if (!health) { if (info.State?.Running) return; } else if (health === "healthy") { @@ -432,6 +464,7 @@ export async function ensure( Labels: labelsFor(names), Env: options.environment, ExposedPorts: { [COMPUTER_PORT]: {} }, + Healthcheck: COMPUTER_HEALTHCHECK, HostConfig: hostConfig(names, options), }); } catch (error) {