feat: retry transient Codex capacity failures

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## TBD
 
+- Added automatic bounded retries for native Codex model-capacity failures in local, Docker, and Modal runs, preserving sessions, billing policy, cancellation, deadlines, and per-attempt usage. Set `HEADLESS_CAPACITY_RETRIES=0` to disable.
 - Fixed run coordination to recover stale run and node locks after owner crashes, retain async ownership for the full detached process tree, and handle Windows process probing and state replacement safely (#28).
 
 ## 0.6.1 - 2026-08-12

README.md

@@ -157,7 +157,7 @@ errors, tool output, interruptions, and timeouts do not trigger fallback. Paid
 provider limits still apply; Headless does not impose a local dollar cap.
 
 The policy applies locally, in Docker, and in Modal. Docker keeps an anonymous
-private home across both attempts, removes it after native success, and reports
+private home across attempts, removes it after native success, and reports
 its retained path on failure for recovery. `--session` homes remain durable.
 On Windows, anonymous Docker runs share a Docker-managed volume across attempts;
 success removes it and failure reports its name for recovery. Named durable
@@ -172,6 +172,33 @@ estimates with reported charges. Subscription cost estimates are API list-price
 comparisons, not subscription charges. Auth changes affect child environments
 only; Headless never replaces shared login files.
 
+### Temporary Codex capacity failures
+
+Noninteractive Codex runs automatically retry the native terminal error
+`Selected model is at capacity. Please try a different model.` up to three times.
+Delays are 30, 60, and 120 seconds, each with ±20% jitter. Waiting counts against
+the original command timeout, and cancellation stops the wait immediately.
+
+Set `HEADLESS_CAPACITY_RETRIES=0` to disable these retries, or choose `1`, `2`, or
+`3` to set their limit. Other values are rejected before launching the agent.
+Capacity retries keep the same model, profile, permissions, and billing route.
+They resume the native session with a continuation prompt; without a session,
+Headless retries the original prompt only if no work has been observed.
+
+Retries apply to local, Docker, and Modal runs. Unnamed Docker runs keep a private
+native home across attempts, including subscription-only and API billing; success
+removes it and failure reports its location for recovery. Interactive/tmux launches
+remain under the native CLI's control. Assistant text, tool output, generic errors,
+and recoverable error notices do not trigger capacity retries.
+
+Each wait emits a stderr diagnostic and a `capacity_retry` event in streamed
+JSON/log output, with `retry`, `delayMs`, and `reason: "model-capacity"`.
+`--usage` includes every execution once in `billing.attempts`, with
+`retryReason: "model-capacity"` on executions caused by capacity retries. The
+three capacity retries and the existing single billing fallback have independent
+budgets, allowing at most five executions per invocation. Exhausted capacity
+retries preserve the last failure status and transcript.
+
 ### Empty Pi completions
 
 Pi can finish artifact-producing work with an empty final assistant message.

src/billing-events.ts

@@ -1,4 +1,5 @@
 import { StringDecoder } from "node:string_decoder";
+import { CODEX_CAPACITY_MESSAGE } from "./capacity-retry.js";
 import type { AgentName } from "./types.js";
 
 export type BillingFailureReason = "subscription-quota" | "subscription-model-unsupported";
@@ -55,6 +56,7 @@ export class BillingEventCollector {
   nativeSessionId: string | undefined;
   hasWork = false;
   failed = false;
+  capacityFailure = false;
   private pending = "";
   private pendingBytes = 0;
   private skipping = false;
@@ -106,6 +108,7 @@ export class BillingEventCollector {
   private consumeCodex(event: RecordValue): void {
     if (event.type === "turn.completed") {
       this.failed = false;
+      this.capacityFailure = false;
       this.failureReason = undefined;
     }
     if (event.type === "thread.started") {
@@ -115,6 +118,9 @@ export class BillingEventCollector {
       const item = object(event.item);
       if (typeof item.type === "string" && item.type !== "error") this.hasWork = true;
     }
+    if (event.type === "turn.failed") {
+      this.capacityFailure = object(event.error).message === CODEX_CAPACITY_MESSAGE;
+    }
     if (event.type === "error" || event.type === "turn.failed") {
       this.failureReason ??= codexFailure(event.error) ?? codexFailure(event.message) ??
         codexFailure(object(event.error).message) ?? codexFailure(event);

src/billing-run.ts

@@ -3,9 +3,12 @@ import { BillingEventCollector, type BillingFailureReason } from "./billing-even
 import { aggregateBillingUsage, type BillingUsageReport } from "./billing-usage.js";
 import type { AgentName, BillingMode, BuildOptions, Env } from "./types.js";
 import type { UsageSummary } from "./usage.js";
+import { CAPACITY_CONTINUATION, MAX_EXECUTION_ATTEMPTS, cancellationCode, capacityRetryDelay,
+  resolveCapacityRetries, waitForCapacityRetry, type CapacityRetryEvent } from "./capacity-retry.js";
 
 export interface BillingExecutionResult {
   code: number;
+  terminationSignal?: NodeJS.Signals;
   stdout: string;
   usageTrace?: string;
   finalMessageTrace?: string;
@@ -31,6 +34,10 @@ export interface BillingRunOptions {
   reportUsage?: (trace: string, route: BillingRoute) => Promise<UsageSummary>;
   onTransition?: (event: { type: "billing_transition"; from: BillingRoute; to: BillingRoute; reason: BillingFailureReason }) => void;
   now?: () => number;
+  random?: () => number;
+  sleep?: (delayMs: number, signal?: AbortSignal) => Promise<void>;
+  signal?: AbortSignal;
+  onCapacityRetry?: (event: CapacityRetryEvent) => void;
 }
 
 export interface BillingRunResult {
@@ -40,12 +47,18 @@ export interface BillingRunResult {
   error?: string;
 }
 
-const terminated = new Set([124, 130, 137, 143]);
+const terminated = new Set([124, 129, 130, 131, 137, 143, 149]);
 const continuation = "Continue from where you left off. Your previous turn was interrupted by a subscription usage limit. Preserve completed work and do not repeat completed commands.";
 
-/** One billing transition, sharing the invocation's deadline and native transcript. */
+/** Capacity retries and one billing transition share a deadline and native transcript. */
 export async function runWithBilling(input: BillingRunOptions): Promise<BillingRunResult> {
   const now = input.now ?? Date.now;
+  const capacityLimit = input.agent === "codex" ? resolveCapacityRetries(input.env) : 0;
+  const sleep = input.sleep ?? waitForCapacityRetry;
+  let capacityRetries = 0;
+  let billingTransitioned = false;
+  let hasWork = false;
+  let retryReason: "model-capacity" | undefined;
   const deadline = input.timeoutSeconds === undefined ? undefined : now() + input.timeoutSeconds * 1000;
   let options = input.options;
   let attempt = prepareBillingAttempt(input.agent, options, input.env, input.mode);
@@ -55,7 +68,12 @@ export async function runWithBilling(input: BillingRunOptions): Promise<BillingR
   let error: string | undefined;
   let result: BillingExecutionResult = { code: 124, stdout: "" };
 
-  for (let index = 0; index < 2; index++) {
+  for (let index = 0; index < MAX_EXECUTION_ATTEMPTS; index++) {
+    const cancelled = cancellationCode(input.signal);
+    if (cancelled !== undefined) {
+      result = { ...result, code: cancelled };
+      break;
+    }
     const remaining = deadline === undefined ? undefined : (deadline - now()) / 1000;
     if (remaining !== undefined && remaining <= 0) {
       result = { ...result, code: 124 };
@@ -65,15 +83,57 @@ export async function runWithBilling(input: BillingRunOptions): Promise<BillingR
     result = await input.execute({ ...attempt, options, timeoutSeconds: remaining, observe: (chunk) => events.write(chunk) });
     events.end();
     nativeSessionId = events.nativeSessionId ?? nativeSessionId;
+    hasWork ||= events.hasWork;
     if (input.reportUsage) {
       const trace = result.usageTrace || result.stdout || result.finalMessageTrace || "";
-      reports.push({ route: attempt.route, reason, usage: await input.reportUsage(trace, attempt.route) });
+      reports.push({ route: attempt.route, reason, ...(retryReason ? { retryReason } : {}),
+        usage: await input.reportUsage(trace, attempt.route) });
+    }
+    reason = undefined;
+    retryReason = undefined;
+    const cancelledAfterExecution = cancellationCode(input.signal);
+    if (cancelledAfterExecution !== undefined) {
+      result = { ...result, code: cancelledAfterExecution };
+      break;
     }
-    if (terminated.has(result.code)) break;
+    if (result.terminationSignal || terminated.has(result.code)) break;
     if (events.failed && result.code === 0) result = { ...result, code: 1 };
+    if (events.capacityFailure && !events.failureReason) {
+      if (capacityRetries >= capacityLimit) {
+        error = `model capacity retries exhausted (${capacityRetries} retries)`;
+        break;
+      }
+      const resumeId = nativeSessionId ?? options.sessionId;
+      if (hasWork && !resumeId) {
+        error = "capacity retry cannot safely resume partial work: native session ID unavailable";
+        break;
+      }
+      const remainingMs = deadline === undefined ? Infinity : deadline - now();
+      if (remainingMs <= 0) {
+        result = { ...result, code: 124 };
+        break;
+      }
+      capacityRetries++;
+      const delayMs = Math.min(capacityRetryDelay(capacityRetries, input.random ?? Math.random), remainingMs);
+      input.onCapacityRetry?.({ type: "capacity_retry", retry: capacityRetries, delayMs, reason: "model-capacity" });
+      try {
+        await sleep(delayMs, input.signal);
+      } catch (failure) {
+        const code = cancellationCode(input.signal);
+        if (code === undefined) throw failure;
+        result = { ...result, code };
+        break;
+      }
+      retryReason = "model-capacity";
+      if (resumeId) {
+        options = { ...options, prompt: CAPACITY_CONTINUATION, promptFile: undefined,
+          sessionMode: "resume", sessionId: resumeId };
+      }
+      continue;
+    }
     if (!events.failureReason) break;
     // Explicit subscription-only policy and exhausted paid routes are terminal.
-    if (input.mode !== "auto" || attempt.route !== "subscription" || index !== 0) {
+    if (input.mode !== "auto" || attempt.route !== "subscription" || billingTransitioned) {
       result = { ...result, code: 78 };
       error = `billing unavailable: ${events.failureReason}; no further billing fallback`;
       break;
@@ -83,7 +143,7 @@ export async function runWithBilling(input: BillingRunOptions): Promise<BillingR
       break;
     }
     const resumeId = nativeSessionId ?? options.sessionId;
-    if (events.hasWork && !resumeId) {
+    if (hasWork && !resumeId) {
       result = { ...result, code: 78 };
       error = "billing fallback cannot safely resume partial work: native session ID unavailable";
       break;
@@ -93,6 +153,7 @@ export async function runWithBilling(input: BillingRunOptions): Promise<BillingR
       reason = events.failureReason;
       input.onTransition?.({ type: "billing_transition", from: attempt.route, to: next.route, reason });
       attempt = next;
+      billingTransitioned = true;
       if (resumeId) {
         options = { ...options, prompt: continuation, promptFile: undefined, sessionMode: "resume", sessionId: resumeId };
       }

src/billing-usage.ts

@@ -1,10 +1,12 @@
+import { MAX_EXECUTION_ATTEMPTS } from "./capacity-retry.js";
 import type { BillingFailureReason } from "./billing-events.js";
 import type { UsageCostBreakdown, UsageSummary } from "./usage.js";
 
 export type BillingRoute = "subscription" | "openai-api" | "bedrock" | "native";
 export interface BillingAttempt {
   route: BillingRoute;
   reason?: BillingFailureReason;
+  retryReason?: "model-capacity";
   usage: UsageSummary;
 }
 export interface BillingUsageReport extends UsageSummary {
@@ -13,8 +15,8 @@ export interface BillingUsageReport extends UsageSummary {
 
 /** Keep incompatible cost valuations separate rather than suggesting an actual API bill. */
 export function aggregateBillingUsage(attempts: BillingAttempt[]): BillingUsageReport {
-  if (attempts.length < 1 || attempts.length > 2) {
-    throw new Error("Billing usage requires one or two attempts");
+  if (attempts.length < 1 || attempts.length > MAX_EXECUTION_ATTEMPTS) {
+    throw new Error(`Billing usage requires one to ${MAX_EXECUTION_ATTEMPTS} attempts`);
   }
   const summaries = attempts.map((attempt) => attempt.usage);
   const last = summaries[summaries.length - 1];

src/capacity-retry.ts

@@ -0,0 +1,37 @@
+import { setTimeout } from "node:timers/promises";
+import type { Env } from "./types.js";
+
+export const MAX_CAPACITY_RETRIES = 3;
+export const MAX_EXECUTION_ATTEMPTS = MAX_CAPACITY_RETRIES + 2;
+export const CODEX_CAPACITY_MESSAGE = "Selected model is at capacity. Please try a different model.";
+export const CAPACITY_CONTINUATION = "Continue from where you left off. Your previous turn was interrupted by temporary model capacity exhaustion. Preserve completed work and do not repeat completed commands.";
+
+export interface CapacityRetryEvent {
+  type: "capacity_retry";
+  retry: number;
+  delayMs: number;
+  reason: "model-capacity";
+}
+
+export function resolveCapacityRetries(env: Env): number {
+  const value = env.HEADLESS_CAPACITY_RETRIES;
+  if (value === undefined) return MAX_CAPACITY_RETRIES;
+  if (!/^[0-3]$/.test(value)) {
+    throw new Error("HEADLESS_CAPACITY_RETRIES must be 0, 1, 2, or 3");
+  }
+  return Number(value);
+}
+
+export function capacityRetryDelay(retry: number, random: () => number): number {
+  return Math.round(30_000 * 2 ** (retry - 1) * (0.8 + 0.4 * random()));
+}
+
+export async function waitForCapacityRetry(delayMs: number, signal?: AbortSignal): Promise<void> {
+  await setTimeout(delayMs, undefined, { signal });
+}
+
+export function cancellationCode(signal?: AbortSignal): number | undefined {
+  if (!signal?.aborted) return undefined;
+  return Number.isInteger(signal.reason) && signal.reason > 0 && signal.reason <= 255
+    ? signal.reason as number : 130;
+}