From d1b027853b57fdf7d8e3b945249de9b55cfcb38e Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:59:10 -0400 Subject: [PATCH 1/2] Surface auth-shaped provider deaths fast instead of eventually Two hardenings from a real mid-turn token expiry that presented as a silent session crash: - A Claude turn that dies with an error the auth patterns cannot classify (the SDK's stream diagnostics look nothing like a 401 even when an expired credential caused them) now triggers a rate-limited instance re-probe, so the snapshot flips and the sign-in surface appears in seconds instead of after the next scheduled probe, up to five minutes later. - A Codex error notification marked retryable but reading as an auth failure is now terminal at both layers (adapter event class and session status): retrying cannot heal an expired credential, and the old path burned the app-server's whole reconnect schedule before the sign-in surface appeared. --- .../src/provider/Drivers/ClaudeDriver.ts | 28 +++++++++++++ .../src/provider/Layers/ClaudeAdapter.ts | 11 ++++++ .../src/provider/Layers/CodexAdapter.test.ts | 39 +++++++++++++++++++ .../src/provider/Layers/CodexAdapter.ts | 6 ++- .../provider/Layers/CodexSessionRuntime.ts | 8 +++- 5 files changed, 90 insertions(+), 2 deletions(-) diff --git a/apps/server/src/provider/Drivers/ClaudeDriver.ts b/apps/server/src/provider/Drivers/ClaudeDriver.ts index 7516b148..6093935e 100644 --- a/apps/server/src/provider/Drivers/ClaudeDriver.ts +++ b/apps/server/src/provider/Drivers/ClaudeDriver.ts @@ -15,6 +15,9 @@ import { ClaudeSettings, ProviderDriverKind, type ServerProvider } from "@threadlines/contracts"; import * as Cache from "effect/Cache"; import * as DateTime from "effect/DateTime"; +import * as Exit from "effect/Exit"; +import * as Ref from "effect/Ref"; +import * as Scope from "effect/Scope"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; @@ -58,6 +61,12 @@ const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings); const DRIVER_KIND = ProviderDriverKind.make("claudeAgent"); const SNAPSHOT_REFRESH_INTERVAL = Duration.minutes(5); +/** + * Floor between re-probes triggered by unclassifiable turn errors. One probe + * answers the "was that an auth death?" question; a failing session that + * errors on every send must not turn into a probe stream. + */ +const UNCLASSIFIED_ERROR_PROBE_MIN_INTERVAL_MS = 15_000; const CAPABILITIES_PROBE_TTL = Duration.minutes(5); const CLAUDE_CHAT_AUTH_REQUIRED_MESSAGE = "Claude sign-in expired. Sign in again, then retry."; @@ -369,6 +378,15 @@ export const ClaudeDriver: ProviderDriver = { ), ); + // Repeated turn failures must not stack probes: one re-probe per + // window is enough to catch an auth-shaped death, and the snapshot's + // own refresh semaphore handles the rest. The probes fork into their + // own scope so they outlive the failing turn's fiber but not the + // driver instance. + const lastUnclassifiedErrorProbeMsRef = yield* Ref.make(0); + const errorProbeScope = yield* Scope.make("sequential"); + yield* Effect.addFinalizer(() => Scope.close(errorProbeScope, Exit.void)); + // The adapter is built after the snapshot so mid-turn `rate_limit_event` // messages can be folded straight into the live provider snapshot — // account usage then updates in real time instead of waiting for the @@ -395,6 +413,16 @@ export const ClaudeDriver: ProviderDriver = { }), onChatAuthStateChanged: (state) => snapshot.patchSnapshot((current) => patchClaudeChatAuthState(current, state)), + onUnclassifiedRuntimeError: () => + Effect.gen(function* () { + const nowMs = DateTime.toEpochMillis(yield* DateTime.now); + const lastMs = yield* Ref.get(lastUnclassifiedErrorProbeMsRef); + if (nowMs - lastMs < UNCLASSIFIED_ERROR_PROBE_MIN_INTERVAL_MS) { + return; + } + yield* Ref.set(lastUnclassifiedErrorProbeMsRef, nowMs); + yield* snapshot.refresh.pipe(Effect.forkIn(errorProbeScope), Effect.asVoid); + }), }); return { diff --git a/apps/server/src/provider/Layers/ClaudeAdapter.ts b/apps/server/src/provider/Layers/ClaudeAdapter.ts index 348b86e1..627bb306 100644 --- a/apps/server/src/provider/Layers/ClaudeAdapter.ts +++ b/apps/server/src/provider/Layers/ClaudeAdapter.ts @@ -419,6 +419,14 @@ export interface ClaudeAdapterLiveOptions { * state. */ readonly onChatAuthStateChanged?: (status: "verified" | "unauthenticated") => Effect.Effect; + /** + * Fires when a turn dies with an error the auth patterns cannot classify + * (the SDK's diagnostics for a stream that ended with no stop reason look + * nothing like a 401 even when an expired credential caused them). The + * driver answers by re-probing the instance so an auth-shaped death flips + * the snapshot in seconds instead of after the next scheduled probe. + */ + readonly onUnclassifiedRuntimeError?: () => Effect.Effect; } /** @@ -3112,6 +3120,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( .onChatAuthStateChanged("unauthenticated") .pipe(Effect.ignoreCause({ log: true })); } + if (!isAuthenticationError && options?.onUnclassifiedRuntimeError) { + yield* options.onUnclassifiedRuntimeError().pipe(Effect.ignoreCause({ log: true })); + } const turnState = context.turnState; const stamp = yield* makeEventStamp(); yield* offerRuntimeEvent({ diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 45fbd5a4..68c8cc59 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -1881,6 +1881,45 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { }), ); + it.effect("treats a retryable auth error as terminal instead of a warning", () => + Effect.gen(function* () { + const { adapter, runtime } = yield* startLifecycleRuntime(); + const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); + + yield* runtime.emit({ + id: asEventId("evt-retryable-auth-error"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + threadId: asThreadId("thread-1"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "error", + turnId: asTurnId("turn-1"), + payload: { + threadId: "thread-1", + turnId: "turn-1", + error: { + message: "unexpected status 401 Unauthorized: Missing bearer or basic authentication", + }, + willRetry: true, + }, + } satisfies ProviderEvent); + + const firstEvent = yield* Fiber.join(firstEventFiber); + + assert.equal(firstEvent._tag, "Some"); + if (firstEvent._tag !== "Some") { + return; + } + // Retrying cannot heal an expired credential; the sign-in surface must + // appear on the first attempt, not after the reconnect schedule runs out. + assert.equal(firstEvent.value.type, "runtime.error"); + if (firstEvent.value.type !== "runtime.error") { + return; + } + assert.equal(firstEvent.value.payload.class, "authentication_error"); + }), + ); + it.effect("maps process stderr notifications to runtime.warning", () => Effect.gen(function* () { const { adapter, runtime } = yield* startLifecycleRuntime(); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 7affb743..6748ee1b 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -2400,8 +2400,12 @@ export function mapToRuntimeEvents( if (event.method === "error") { const payload = readPayload(EffectCodexSchema.V2ErrorNotification, event.payload); const message = payload?.error.message ?? event.message ?? "Provider runtime error"; - const willRetry = payload?.willRetry === true; const errorClass = providerErrorClass(message); + // An expired credential cannot heal by retrying: reporting each retry as + // a warning kept the sign-in surface away until the app-server exhausted + // its whole reconnect schedule (20-30s of visible noise). An auth-shaped + // error is terminal on the first attempt. + const willRetry = payload?.willRetry === true && errorClass !== "authentication_error"; return [ { type: willRetry ? "runtime.warning" : "runtime.error", diff --git a/apps/server/src/provider/Layers/CodexSessionRuntime.ts b/apps/server/src/provider/Layers/CodexSessionRuntime.ts index 00a32fc2..c91a3938 100644 --- a/apps/server/src/provider/Layers/CodexSessionRuntime.ts +++ b/apps/server/src/provider/Layers/CodexSessionRuntime.ts @@ -27,6 +27,7 @@ import { import { hideWindowsConsole } from "@threadlines/shared/childProcess"; import { planCliSpawn } from "../../cliSpawn.ts"; import { normalizeModelSlug } from "@threadlines/shared/model"; +import { isProviderAuthErrorMessage } from "@threadlines/shared/providerAuth"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; @@ -1652,7 +1653,12 @@ export const makeCodexSessionRuntime = ( return Effect.void; } const errorMessage = payload.error.message; - const willRetry = payload.willRetry; + // An expired credential cannot heal by retrying: the app-server + // would replay its full reconnect schedule (visible as 20-30s of + // "reconnecting" noise) before failing with the same 401. Treat a + // retrying auth error as terminal so the sign-in surface appears + // on the first attempt instead of the last. + const willRetry = payload.willRetry && !isProviderAuthErrorMessage(errorMessage); return updateSession(sessionRef, { status: willRetry ? "running" : "error", ...(errorMessage ? { lastError: errorMessage } : {}), From d9a934527a230da0b8dd5f94af6e60f1116d9d82 Mon Sep 17 00:00:00 2001 From: badcuban <108198679+badcuban@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:07:48 -0400 Subject: [PATCH 2/2] Let a downloaded update be superseded instead of sticking forever Once an update reached the downloaded state, every later check was skipped, including the manual menu action, so the ready pill kept advertising whatever it had downloaded even after a newer release shipped or the release it came from was pulled. Checks now run silently in the downloaded state (no checking flicker on the 4-minute poll): a newer version restarts the cycle, the same version leaves the pill untouched, and a feed that no longer offers the update clears it. --- .../src/updates/DesktopUpdates.test.ts | 71 +++++++++++++++++++ apps/desktop/src/updates/DesktopUpdates.ts | 37 ++++++++-- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src/updates/DesktopUpdates.test.ts b/apps/desktop/src/updates/DesktopUpdates.test.ts index 8a08df11..451697c2 100644 --- a/apps/desktop/src/updates/DesktopUpdates.test.ts +++ b/apps/desktop/src/updates/DesktopUpdates.test.ts @@ -480,6 +480,77 @@ describe("DesktopUpdates", () => { ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); }); + it.effect("replaces a downloaded update when a newer release supersedes it", () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + + harness.emit("update-available", { version: "1.2.4" }); + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + assert.equal((yield* updates.getState).status, "downloaded"); + + // A later poll finds a newer release while the restart is postponed; + // the ready pill must move to it instead of advertising stale bits. + harness.emit("update-available", { version: "1.2.5" }); + yield* flushCallbacks; + + const state = yield* updates.getState; + assert.equal(state.status, "available"); + assert.equal(state.availableVersion, "1.2.5"); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect("keeps the ready pill steady when the downloaded update is still latest", () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + + harness.emit("update-available", { version: "1.2.4" }); + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + + harness.emit("update-available", { version: "1.2.4" }); + yield* flushCallbacks; + + const state = yield* updates.getState; + assert.equal(state.status, "downloaded"); + assert.equal(state.downloadedVersion, "1.2.4"); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + + it.effect("clears a downloaded update the feed no longer offers", () => { + const harness = makeHarness(); + + return Effect.scoped( + Effect.gen(function* () { + const updates = yield* DesktopUpdates.DesktopUpdates; + yield* updates.configure; + + harness.emit("update-available", { version: "1.2.4" }); + harness.emit("update-downloaded", { version: "1.2.4" }); + yield* flushCallbacks; + + // The release was pulled (a failed nightly's cleaned-up draft): the + // stuck-forever pill from that state is the bug this guards against. + harness.emit("update-not-available"); + yield* flushCallbacks; + + const state = yield* updates.getState; + assert.equal(state.status, "up-to-date"); + assert.isNull(state.downloadedVersion); + }), + ).pipe(Effect.provide(Layer.merge(TestClock.layer(), harness.layer))); + }); + it.effect("uses a dev-only preview update state without configuring the updater", () => { const harness = makeHarness({ env: { diff --git a/apps/desktop/src/updates/DesktopUpdates.ts b/apps/desktop/src/updates/DesktopUpdates.ts index be7bf692..aba99ffb 100644 --- a/apps/desktop/src/updates/DesktopUpdates.ts +++ b/apps/desktop/src/updates/DesktopUpdates.ts @@ -623,18 +623,27 @@ const make = Effect.gen(function* () { if (yield* Ref.get(updateCheckInFlightRef)) return false; const state = yield* Ref.get(updateStateRef); - if (state.status === "downloading" || state.status === "downloaded") { - yield* logUpdaterInfo("skipping update check while update is active", { + if (state.status === "downloading") { + yield* logUpdaterInfo("skipping update check while a download is running", { reason, - status: state.status, }); return false; } + // A downloaded update must not block later checks: while the user + // postpones the restart, the release it came from can be superseded by a + // newer one or pulled entirely, and a pill that can never re-check gets + // stuck advertising a version that no longer matters. The check runs + // silently in that state (no transition to "checking") so the ready pill + // doesn't flicker on every poll. + const silentCheck = state.status === "downloaded"; + yield* Ref.set(updateCheckInFlightRef, true); const checkedAt = yield* currentIsoTimestamp; - yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); - yield* logUpdaterInfo("checking for updates", { reason }); + if (!silentCheck) { + yield* setState(reduceDesktopUpdateStateOnCheckStart(state, checkedAt)); + } + yield* logUpdaterInfo("checking for updates", { reason, silent: silentCheck }); return yield* Effect.gen(function* () { if (!(yield* ensurePrivateGitHubUpdateFeed())) { @@ -767,6 +776,16 @@ const make = Effect.gen(function* () { return; } + if (state.status === "downloaded" && state.downloadedVersion === info.version) { + // The downloaded update is still the latest; keep the ready pill + // exactly as it is instead of restarting the cycle for the same + // bits. + yield* logUpdaterInfo("downloaded update is still the latest", { + version: info.version, + }); + return; + } + const checkedAt = yield* currentIsoTimestamp; yield* setState( reduceDesktopUpdateStateOnUpdateAvailable(state, info.version, checkedAt), @@ -786,6 +805,14 @@ const make = Effect.gen(function* () { const handleUpdateNotAvailable = Effect.gen(function* () { const checkedAt = yield* currentIsoTimestamp; const state = yield* Ref.get(updateStateRef); + if (state.status === "downloaded") { + // The release the downloaded update came from is no longer offered + // (pulled or replaced by the running version); clearing to up-to-date + // retires a ready pill that would otherwise advertise it forever. + yield* logUpdaterInfo("downloaded update is no longer offered; clearing it", { + version: state.downloadedVersion, + }); + } yield* setState(reduceDesktopUpdateStateOnNoUpdate(state, checkedAt)); yield* Ref.set(lastLoggedDownloadMilestoneRef, -1); yield* logUpdaterInfo("no updates available");