perf: split the Daimon runtime image into cacheable layers

.github/workflows/test.yml

@@ -47,6 +47,9 @@ jobs:
       - name: Verify native helper syscalls
         run: node --test scripts/native-helper-artifacts.test.mjs scripts/native-helper-integration.test.mjs
 
+      - name: Verify Daimon image layering
+        run: node --test scripts/build-local-daimon-runtime.test.mjs
+
       # Preseed reads the candidate volume through its Docker Mountpoint and
       # relies on host rename/fsync/hardlink semantics a container cannot give
       # it. On a rootful Linux daemon /var/lib/docker/volumes/*/_data is

runtime-images/daimon/Dockerfile

@@ -1,5 +1,6 @@
 # syntax=docker/dockerfile:1
 
+ARG DAIMON_DEPENDENCY_MODE=registry
 ARG NODE_BASE_IMAGE=node:24-bookworm-slim@sha256:a9f5f7c91a432850b2a8a7797adf5eadb6c733ceed61167806cee7ea7fbc29df
 FROM daimon_package AS daimon_package
 
@@ -18,9 +19,92 @@ RUN test "$(sha256sum /tmp/daimon.tgz | awk '{print "sha256:" $1}')" = "${DAIMON
   && test -f /probe/node_modules/@noopolis/daimon/dist/runtime/contract-manifest.json \
   && cp /tmp/source-inputs.json /probe/source-inputs.json
 
-FROM ${NODE_BASE_IMAGE} AS build
+FROM ${NODE_BASE_IMAGE} AS base_offline-bundle
+
+FROM ${NODE_BASE_IMAGE} AS base_registry
+RUN apt-get update \
+  && apt-get install --yes --no-install-recommends ca-certificates curl \
+  && rm -rf /var/lib/apt/lists/*
+
+FROM base_${DAIMON_DEPENDENCY_MODE} AS base
+
+FROM base AS grok_source_registry
+ARG GROK_CLI_URL
+RUN curl -fsSL "${GROK_CLI_URL}" -o /tmp/grok
+
+FROM base AS grok_source_offline-bundle
+COPY --from=daimon_package /grok /tmp/grok
+
+FROM grok_source_${DAIMON_DEPENDENCY_MODE} AS grok_cli
+ARG GROK_CLI_SHA256
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+RUN echo "${GROK_CLI_SHA256}  /tmp/grok" | sha256sum -c - \
+  && mkdir -p ${RUNTIME_ROOT}/bin \
+  && install -m 0755 /tmp/grok ${RUNTIME_ROOT}/bin/grok \
+  && test "$(sha256sum ${RUNTIME_ROOT}/bin/grok | awk '{print "sha256:" $1}')" = "sha256:${GROK_CLI_SHA256#sha256:}" \
+  && rm -f /tmp/grok
+
+FROM grok_cli AS agy_source_registry
+ARG AGY_CLI_URL
+RUN curl -fsSL "${AGY_CLI_URL}" -o /tmp/agy.tar.gz
+
+FROM grok_cli AS agy_source_offline-bundle
+COPY --from=daimon_package /agy.tar.gz /tmp/agy.tar.gz
 
+FROM agy_source_${DAIMON_DEPENDENCY_MODE} AS agy_cli
+ARG AGY_CLI_SHA512
+ARG AGY_CLI_SHA256
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+RUN echo "${AGY_CLI_SHA512}  /tmp/agy.tar.gz" | sha512sum -c - \
+  && rm -rf /tmp/agy-extract \
+  && mkdir -p /tmp/agy-extract \
+  && tar -xzf /tmp/agy.tar.gz -C /tmp/agy-extract \
+  && agy_path="$(find /tmp/agy-extract -type f -name antigravity -print -quit)" \
+  && test -n "${agy_path}" \
+  && install -m 0755 "${agy_path}" ${RUNTIME_ROOT}/bin/agy \
+  && test "$(sha256sum ${RUNTIME_ROOT}/bin/agy | awk '{print "sha256:" $1}')" = "sha256:${AGY_CLI_SHA256#sha256:}" \
+  && rm -rf /tmp/agy.tar.gz /tmp/agy-extract
+
+FROM agy_cli AS codex_registry
 ARG CODEX_CLI_VERSION=0.142.3
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  cd ${RUNTIME_ROOT} \
+  && npm install --omit=dev --no-fund --no-audit @openai/codex@${CODEX_CLI_VERSION}
+
+FROM agy_cli AS codex_offline-bundle
+ARG DAIMON_DEPENDENCY_ARCHIVE_SHA256=none
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+COPY --from=daimon_package /dependencies.tar /tmp/dependencies.tar
+RUN test "$(sha256sum /tmp/dependencies.tar | awk '{print "sha256:" $1}')" = "${DAIMON_DEPENDENCY_ARCHIVE_SHA256}" \
+  && mkdir -p ${RUNTIME_ROOT}/node_modules \
+  && tar -xf /tmp/dependencies.tar -C ${RUNTIME_ROOT}/node_modules \
+  && test -x ${RUNTIME_ROOT}/node_modules/@openai/codex/bin/codex.js \
+  && rm -f /tmp/dependencies.tar
+
+FROM codex_registry AS daimon_registry
+ARG DAIMON_PACKAGE_SHA256
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+COPY --from=daimon_package /daimon.tgz /tmp/daimon.tgz
+COPY --from=daimon_package /source-inputs.json /tmp/source-inputs.json
+RUN --mount=type=cache,target=/root/.npm,sharing=locked \
+  test "$(sha256sum /tmp/daimon.tgz | awk '{print "sha256:" $1}')" = "${DAIMON_PACKAGE_SHA256}" \
+  && cd ${RUNTIME_ROOT} \
+  && npm install --omit=dev --no-fund --no-audit /tmp/daimon.tgz \
+  && rm -f /tmp/daimon.tgz
+
+FROM codex_offline-bundle AS daimon_offline-bundle
+ARG DAIMON_PACKAGE_SHA256
+ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
+COPY --from=daimon_package /daimon.tgz /tmp/daimon.tgz
+COPY --from=daimon_package /source-inputs.json /tmp/source-inputs.json
+RUN test "$(sha256sum /tmp/daimon.tgz | awk '{print "sha256:" $1}')" = "${DAIMON_PACKAGE_SHA256}" \
+  && mkdir -p ${RUNTIME_ROOT}/node_modules/@noopolis/daimon \
+  && tar -xzf /tmp/daimon.tgz -C ${RUNTIME_ROOT}/node_modules/@noopolis/daimon --strip-components=1 \
+  && rm -f /tmp/daimon.tgz
+
+FROM daimon_${DAIMON_DEPENDENCY_MODE} AS build
+
 ARG GROK_CLI_VERSION
 ARG GROK_CLI_URL
 ARG GROK_CLI_SHA256
@@ -38,12 +122,6 @@ ARG CODEX_CLI_SHA256
 ARG TARGETARCH
 ARG RUNTIME_ROOT=/opt/spawnfile/runtime-installs/daimon
 
-COPY --from=daimon_package /daimon.tgz /tmp/daimon.tgz
-COPY --from=daimon_package /dependencies.tar /tmp/dependencies.tar
-COPY --from=daimon_package /source-inputs.json /tmp/source-inputs.json
-COPY --from=daimon_package /grok /tmp/offline-grok
-COPY --from=daimon_package /agy.tar.gz /tmp/offline-agy.tar.gz
-
 RUN test -n "${GROK_CLI_VERSION}" \
   && test -n "${GROK_CLI_URL}" \
   && test -n "${GROK_CLI_SHA256}" \
@@ -58,28 +136,6 @@ RUN test -n "${GROK_CLI_VERSION}" \
   && { test "${DAIMON_DEPENDENCY_MODE}" = registry || test "${DAIMON_DEPENDENCY_MODE}" = offline-bundle; } \
   && test -n "${CODEX_CLI_SHA256}" \
   && test -n "${TARGETARCH}" \
-  && test "$(sha256sum /tmp/daimon.tgz | awk '{print "sha256:" $1}')" = "${DAIMON_PACKAGE_SHA256}" \
-  && if test "${DAIMON_DEPENDENCY_MODE}" = registry; then apt-get update && apt-get install --yes --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*; fi \
-  && mkdir -p ${RUNTIME_ROOT}/bin \
-  && cd ${RUNTIME_ROOT} \
-  && if test "${DAIMON_DEPENDENCY_MODE}" = offline-bundle; then \
-       test "$(sha256sum /tmp/dependencies.tar | awk '{print "sha256:" $1}')" = "${DAIMON_DEPENDENCY_ARCHIVE_SHA256}" \
-       && mkdir -p node_modules/@noopolis/daimon \
-       && tar -xf /tmp/dependencies.tar -C node_modules \
-       && tar -xzf /tmp/daimon.tgz -C node_modules/@noopolis/daimon --strip-components=1 \
-       && test -x node_modules/@openai/codex/bin/codex.js; \
-     else npm install --omit=dev --no-fund --no-audit /tmp/daimon.tgz @openai/codex@${CODEX_CLI_VERSION}; fi \
-  && if test "${DAIMON_DEPENDENCY_MODE}" = offline-bundle; then cp /tmp/offline-grok /tmp/grok; else curl -fsSL "${GROK_CLI_URL}" -o /tmp/grok; fi \
-  && echo "${GROK_CLI_SHA256}  /tmp/grok" | sha256sum -c - \
-  && install -m 0755 /tmp/grok ${RUNTIME_ROOT}/bin/grok \
-  && if test "${DAIMON_DEPENDENCY_MODE}" = offline-bundle; then cp /tmp/offline-agy.tar.gz /tmp/agy.tar.gz; else curl -fsSL "${AGY_CLI_URL}" -o /tmp/agy.tar.gz; fi \
-  && echo "${AGY_CLI_SHA512}  /tmp/agy.tar.gz" | sha512sum -c - \
-  && rm -rf /tmp/agy-extract \
-  && mkdir -p /tmp/agy-extract \
-  && tar -xzf /tmp/agy.tar.gz -C /tmp/agy-extract \
-  && agy_path="$(find /tmp/agy-extract -type f -name antigravity -print -quit)" \
-  && test -n "${agy_path}" \
-  && install -m 0755 "${agy_path}" ${RUNTIME_ROOT}/bin/agy \
   && test -x ${RUNTIME_ROOT}/node_modules/@openai/codex/bin/codex.js \
   && test -f ${RUNTIME_ROOT}/node_modules/@noopolis/daimon/dist/runtime/cli.js \
   && case "${TARGETARCH}" in \
@@ -108,8 +164,6 @@ RUN test -n "${GROK_CLI_VERSION}" \
   && test "$(sha256sum ${RUNTIME_ROOT}/bin/daimon-engine-broker | awk '{print $1}')" = "${broker_sha}" \
   && test "$(sha256sum ${RUNTIME_ROOT}/bin/agy | awk '{print "sha256:" $1}')" = "sha256:${AGY_CLI_SHA256#sha256:}" \
   && node -e 'const fs=require("fs"),path=require("path"),root=path.resolve(process.argv[1]);let count=0;const walk=d=>{for(const n of fs.readdirSync(d)){const p=path.join(d,n),s=fs.lstatSync(p);if(s.isDirectory())walk(p);else if(s.isSymbolicLink()){if(++count>4096)throw Error("too many runtime links");const l=fs.readlinkSync(p);if(path.isAbsolute(l))throw Error("absolute runtime link");const r=fs.realpathSync(p);if(!r.startsWith(root+path.sep))throw Error("runtime link escape");const t=fs.statSync(p);if(!t.isFile()||t.dev!==fs.statSync(root).dev)throw Error("unsafe runtime link target");}}};walk(root)' ${RUNTIME_ROOT} \
-  && rm -rf /tmp/agy.tar.gz /tmp/agy-extract /tmp/grok \
-  && npm cache clean --force \
   && test -f ${RUNTIME_ROOT}/node_modules/@noopolis/daimon/package.json
 
 FROM scratch

scripts/build-local-daimon-runtime.test.mjs

@@ -207,18 +207,30 @@ test("dependency lock truth rejects near-empty graphs and a fake Codex version",
 test("Daimon Dockerfile verifies the AGY archive before extracting antigravity and verifies every installed executable", () => {
   const dockerfile = readFileSync(new URL("../runtime-images/daimon/Dockerfile", import.meta.url), "utf8");
   assert.match(dockerfile, /^ARG NODE_BASE_IMAGE=node:24-bookworm-slim@sha256:[a-f0-9]{64}\nFROM daimon_package AS daimon_package/mu);
-  assert.match(dockerfile, /FROM \$\{NODE_BASE_IMAGE\} AS build/u);
+  assert.match(dockerfile, /FROM base_\$\{DAIMON_DEPENDENCY_MODE\} AS base/u);
+  assert.match(dockerfile, /FROM grok_source_\$\{DAIMON_DEPENDENCY_MODE\} AS grok_cli/u);
+  assert.match(dockerfile, /FROM agy_source_\$\{DAIMON_DEPENDENCY_MODE\} AS agy_cli/u);
+  assert.match(dockerfile, /FROM daimon_\$\{DAIMON_DEPENDENCY_MODE\} AS build/u);
   const archiveDownload = dockerfile.indexOf('curl -fsSL "${AGY_CLI_URL}" -o /tmp/agy.tar.gz');
   const archiveVerification = dockerfile.indexOf("sha512sum -c -");
   const archiveExtraction = dockerfile.indexOf("tar -xzf /tmp/agy.tar.gz");
   const executableLookup = dockerfile.indexOf("-name antigravity");
   const executableInstall = dockerfile.indexOf('install -m 0755 "${agy_path}"');
+  const grokInstall = dockerfile.indexOf("install -m 0755 /tmp/grok");
+  const codexInstall = dockerfile.indexOf("npm install --omit=dev --no-fund --no-audit @openai/codex@");
+  const daimonCopy = dockerfile.indexOf("COPY --from=daimon_package /daimon.tgz /tmp/daimon.tgz", dockerfile.indexOf("FROM codex_registry AS daimon_registry"));
+  const offlineDaimonCopy = dockerfile.indexOf("COPY --from=daimon_package /daimon.tgz /tmp/daimon.tgz", dockerfile.indexOf("FROM codex_offline-bundle AS daimon_offline-bundle"));
 
   assert.ok(archiveDownload >= 0);
   assert.ok(archiveDownload < archiveVerification);
   assert.ok(archiveVerification < archiveExtraction);
   assert.ok(archiveExtraction < executableLookup);
   assert.ok(executableLookup < executableInstall);
+  assert.ok(grokInstall < daimonCopy);
+  assert.ok(executableInstall < daimonCopy);
+  assert.ok(codexInstall < daimonCopy);
+  assert.ok(grokInstall < offlineDaimonCopy);
+  assert.ok(executableInstall < offlineDaimonCopy);
   assert.match(dockerfile, /sha256sum \$\{RUNTIME_ROOT\}\/bin\/agy/u);
   assert.match(dockerfile, /sha256sum \$\{RUNTIME_ROOT\}\/bin\/grok/u);
   assert.match(dockerfile, /sha256sum \$\{RUNTIME_ROOT\}\/node_modules\/@openai\/codex\/bin\/codex\.js/u);
@@ -235,5 +247,69 @@ test("Daimon Dockerfile verifies the AGY archive before extracting antigravity a
   assert.match(dockerfile, /DAIMON_DEPENDENCY_MODE.*offline-bundle/su);
   assert.match(dockerfile, /sha256sum \/tmp\/dependencies\.tar/u);
   assert.match(dockerfile, /source_inputs\?\.dependencies\?\.runtime_archive_sha256/u);
-  assert.match(dockerfile, /tar -xf \/tmp\/dependencies\.tar -C node_modules/u);
+  assert.match(dockerfile, /tar -xf \/tmp\/dependencies\.tar -C \$\{RUNTIME_ROOT\}\/node_modules/u);
+  assert.match(dockerfile, /FROM \$\{NODE_BASE_IMAGE\} AS base_offline-bundle/u);
+  assert.match(dockerfile, /FROM \$\{NODE_BASE_IMAGE\} AS base_registry/u);
+  assert.match(dockerfile, /FROM codex_registry AS daimon_registry/u);
+  assert.match(dockerfile, /FROM codex_offline-bundle AS daimon_offline-bundle/u);
+  assert.match(dockerfile, /--mount=type=cache,target=\/root\/\.npm,sharing=locked/u);
+  assert.doesNotMatch(dockerfile, /npm cache clean/u);
+});
+
+test("Daimon Dockerfile stage graph preserves cache and offline-network boundaries", () => {
+  const dockerfile = readFileSync(new URL("../runtime-images/daimon/Dockerfile", import.meta.url), "utf8");
+  const stages = new Map();
+  const fromPattern = /^FROM\s+(\S+)\s+AS\s+(\S+)\s*$/gimu;
+  const declarations = [...dockerfile.matchAll(fromPattern)];
+
+  for (const [index, declaration] of declarations.entries()) {
+    const [, rawParent, name] = declaration;
+    const parent = rawParent.replaceAll("${DAIMON_DEPENDENCY_MODE}", "registry");
+    const bodyStart = declaration.index + declaration[0].length;
+    const bodyEnd = declarations[index + 1]?.index ?? dockerfile.length;
+    stages.set(name, { body: dockerfile.slice(bodyStart, bodyEnd), parent });
+  }
+
+  const ancestry = (graph, target) => {
+    const chain = [];
+    const visited = new Set();
+    let current = target;
+    while (graph.has(current)) {
+      assert.ok(!visited.has(current), `stage ancestry must not contain a cycle at ${current}`);
+      visited.add(current);
+      chain.push(current);
+      current = graph.get(current).parent;
+    }
+    chain.push(current);
+    return chain;
+  };
+
+  assert.deepEqual(ancestry(stages, "build"), [
+    "build", "daimon_registry", "codex_registry", "agy_cli", "agy_source_registry",
+    "grok_cli", "grok_source_registry", "base", "base_registry", "${NODE_BASE_IMAGE}"
+  ]);
+
+  const offlineStages = new Map([...stages].map(([name, stage]) => [
+    name,
+    { ...stage, parent: stage.parent.replaceAll("registry", "offline-bundle") }
+  ]));
+  assert.deepEqual(ancestry(offlineStages, "build"), [
+    "build", "daimon_offline-bundle", "codex_offline-bundle", "agy_cli", "agy_source_offline-bundle",
+    "grok_cli", "grok_source_offline-bundle", "base", "base_offline-bundle", "${NODE_BASE_IMAGE}"
+  ]);
+
+  const assertAncestorsExcludeDaimonInputs = (graph, target) => {
+    for (const ancestor of ancestry(graph, target).slice(1, -1)) {
+      assert.doesNotMatch(graph.get(ancestor).body, /daimon\.tgz|source-inputs\.json/u, `${ancestor} must not depend on Daimon package inputs`);
+    }
+  };
+  assertAncestorsExcludeDaimonInputs(stages, "daimon_registry");
+  assertAncestorsExcludeDaimonInputs(offlineStages, "daimon_offline-bundle");
+
+  const stagesContaining = (pattern) => [...stages]
+    .filter(([, stage]) => pattern.test(stage.body))
+    .map(([name]) => name)
+    .sort();
+  assert.deepEqual(stagesContaining(/\bapt-get\b/u), ["base_registry"]);
+  assert.deepEqual(stagesContaining(/\bcurl\s+-/u), ["agy_source_registry", "grok_source_registry"]);
 });