diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index 7516b148..6093935e 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -15,6 +15,9 @@ import { ClaudeSettings, ProviderDriverKind, type ServerProvider } from "@threadlines/contracts"; import * as Cache from "effect/Cache"; import * as DateTime from "effect/DateTime"; +import * as Exit from "effect/Exit"; +import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -58,6 +61,12 @@ const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings); const DRIVER_KIND = ProviderDriverKind.make("claudeAgent"); const SNAPSHOT_REFRESH_INTERVAL = Duration.minutes(5); +/** + * Floor between re-probes triggered by unclassifiable turn errors. One probe + * answers the "was that an auth death?" question; a failing session that + * errors on every send must not turn into a probe stream. + */ +const UNCLASSIFIED_ERROR_PROBE_MIN_INTERVAL_MS = 15_000; const CAPABILITIES_PROBE_TTL = Duration.minutes(5); const CLAUDE_CHAT_AUTH_REQUIRED_MESSAGE = "Claude sign-in expired. Sign in again, then retry."; @@ -369,6 +378,15 @@ export const ClaudeDriver: ProviderDriver = { ), ); + // Repeated turn failures must not stack probes: one re-probe per + // window is enough to catch an auth-shaped death, and the snapshot's + // own refresh semaphore handles the rest. The probes fork into their + // own scope so they outlive the failing turn's fiber but not the + // driver instance. + const lastUnclassifiedErrorProbeMsRef = yield* Ref.make(0); + const errorProbeScope = yield* Scope.make("sequential"); + yield* Effect.addFinalizer(() => Scope.close(errorProbeScope, Exit.void)); + // The adapter is built after the snapshot so mid-turn `rate_limit_event` // messages can be folded straight into the live provider snapshot — // account usage then updates in real time instead of waiting for the @@ -395,6 +413,16 @@ export const ClaudeDriver: ProviderDriver = { }), onChatAuthStateChanged: (state) => snapshot.patchSnapshot((current) => patchClaudeChatAuthState(current, state)), + onUnclassifiedRuntimeError: () => + Effect.gen(function* () { + const nowMs = DateTime.toEpochMillis(yield* DateTime.now); + const lastMs = yield* Ref.get(lastUnclassifiedErrorProbeMsRef); + if (nowMs - lastMs < UNCLASSIFIED_ERROR_PROBE_MIN_INTERVAL_MS) { + return; + } + yield* Ref.set(lastUnclassifiedErrorProbeMsRef, nowMs); + yield* snapshot.refresh.pipe(Effect.forkIn(errorProbeScope), Effect.asVoid); + }), }); return { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 348b86e1..627bb306 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -419,6 +419,14 @@ export interface ClaudeAdapterLiveOptions { * state. */ readonly onChatAuthStateChanged?: (status: "verified" | "unauthenticated") => Effect.Effect; + /** + * Fires when a turn dies with an error the auth patterns cannot classify + * (the SDK's diagnostics for a stream that ended with no stop reason look + * nothing like a 401 even when an expired credential caused them). The + * driver answers by re-probing the instance so an auth-shaped death flips + * the snapshot in seconds instead of after the next scheduled probe. + */ + readonly onUnclassifiedRuntimeError?: () => Effect.Effect; } /** @@ -3112,6 +3120,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( .onChatAuthStateChanged("unauthenticated") .pipe(Effect.ignoreCause({ log: true })); } + if (!isAuthenticationError && options?.onUnclassifiedRuntimeError) { + yield* options.onUnclassifiedRuntimeError().pipe(Effect.ignoreCause({ log: true })); + } const turnState = context.turnState; const stamp = yield* makeEventStamp(); yield* offerRuntimeEvent({ diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 45fbd5a4..68c8cc59 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -1881,6 +1881,45 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { }), ); + it.effect("treats a retryable auth error as terminal instead of a warning", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); + + yield* runtime.emit({ + id: asEventId("evt-retryable-auth-error"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "error", + turnId: asTurnId("turn-1"), + payload: { + threadId: "thread-1", + turnId: "turn-1", + error: { + message: "unexpected status 401 Unauthorized: Missing bearer or basic authentication", + }, + willRetry: true, + }, + } satisfies ProviderEvent); + + const firstEvent = yield* Fiber.join(firstEventFiber); + + assert.equal(firstEvent._tag, "Some"); + if (firstEvent._tag !== "Some") { + return; + } + // Retrying cannot heal an expired credential; the sign-in surface must + // appear on the first attempt, not after the reconnect schedule runs out. + assert.equal(firstEvent.value.type, "runtime.error"); + if (firstEvent.value.type !== "runtime.error") { + return; + } + assert.equal(firstEvent.value.payload.class, "authentication_error"); + }), + ); + it.effect("maps process stderr notifications to runtime.warning", () => Effect.gen(function* () { const { adapter, runtime } = yield* startLifecycleRuntime(); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 7affb743..6748ee1b 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -2400,8 +2400,12 @@ export function mapToRuntimeEvents( if (event.method === "error") { const payload = readPayload(EffectCodexSchema.V2ErrorNotification, event.payload); const message = payload?.error.message ?? event.message ?? "Provider runtime error"; - const willRetry = payload?.willRetry === true; const errorClass = providerErrorClass(message); + // An expired credential cannot heal by retrying: reporting each retry as + // a warning kept the sign-in surface away until the app-server exhausted + // its whole reconnect schedule (20-30s of visible noise). An auth-shaped + // error is terminal on the first attempt. + const willRetry = payload?.willRetry === true && errorClass !== "authentication_error"; return [ { type: willRetry ? "runtime.warning" : "runtime.error", diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 00a32fc2..c91a3938 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -27,6 +27,7 @@ import { import { hideWindowsConsole } from "@threadlines/shared/childProcess"; import { planCliSpawn } from "../../cliSpawn.ts"; import { normalizeModelSlug } from "@threadlines/shared/model"; +import { isProviderAuthErrorMessage } from "@threadlines/shared/providerAuth"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; @@ -1652,7 +1653,12 @@ export const makeCodexSessionRuntime = ( return Effect.void; } const errorMessage = payload.error.message; - const willRetry = payload.willRetry; + // An expired credential cannot heal by retrying: the app-server + // would replay its full reconnect schedule (visible as 20-30s of + // "reconnecting" noise) before failing with the same 401. Treat a + // retrying auth error as terminal so the sign-in surface appears + // on the first attempt instead of the last. + const willRetry = payload.willRetry && !isProviderAuthErrorMessage(errorMessage); return updateSession(sessionRef, { status: willRetry ? "running" : "error", ...(errorMessage ? { lastError: errorMessage } : {}),