diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index bb5f88868..870fa2fdd 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -553,7 +553,7 @@ status row (`ok` / `warn` / `fail`) per check. It exits non-zero when any row is - **Wedge history** — the last brain-loop watchdog wedge that was recovered (blocking command and how long it blocked), read from the runtime dir or the brain's reported `lastWedge`. `warn` when the most recent wedge is within the last 24h. - **Sync port** — the sync host port the brain bound. `ok` on the default port, `warn` when bound elsewhere (with the base-port holders it found), `fail` when the brain is up but reported no port. - **Publish health** — account-directory publish state from the brain's sync route health. `ok` when a publish succeeded recently, `fail` when it has been failing for ≥2 min, otherwise `warn`, with the slowest publish leg annotated. -- **Relay** — relay route health as already computed by the brain. `ok` when the relay control is connected, the bridge is validated, and the end-to-end round-trip is verified; `fail` when the route is not fully validated; `warn` when relay is disabled or route health is unavailable. +- **Relay** — relay route health as already computed by the brain. `ok` when the relay control is connected, the bridge is validated, and the end-to-end round-trip is verified; `fail` when the route is not fully validated; `warn` when relay is disabled or route health is unavailable. When another ADE process on this machine has claimed the relay slot, the brain deliberately stops redialing and this row reports that suppression ahead of any lower-level close error, so the detail names the fix (quit the rival process) instead of the symptom. `ade sync status --text` shows the same reason on its `relay` line, plus a `relay failing since` row for how long the current outage has run. - **Account** — whether this machine's brain is signed in to an ADE account (and the credential source), read via the brain's `account.call status`. `warn` when signed out or unavailable. Default doctor does not call provider, GitHub, or Linear networks — it talks only diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index e40fe1c07..8f6803c3a 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -1624,7 +1624,19 @@ export async function createAdeRuntime(args: { ? { userId, expiresAt: refreshed.expiresAt } : null; }, - onPublicationStateChanged: () => resolvedArgs.syncRuntime?.requestAccountMachinePublish?.(), + onPublicationStateChanged: () => { + // Relay state changes are machine-level; without this nudge an idle + // machine emits no sync-status snapshot and the desktop relay banner + // never appears (or never clears). + syncService?.notifyRouteStateChanged(); + resolvedArgs.syncRuntime?.requestAccountMachinePublish?.(); + }, + // The analytics service is machine-scoped and shared, so capturing it in + // this one-per-machine factory closure is safe (unlike the listener + // accessors above, which is why those moved to attachHostListener). + captureAnalytics: (input) => { + productAnalyticsService.captureInternal(input); + }, }); return service; }); @@ -1639,19 +1651,21 @@ export async function createAdeRuntime(args: { if (resolvedArgs.syncRuntime?.sharedSyncListener) { syncTunnelClientService.attachHostListener(resolvedArgs.syncRuntime.sharedSyncListener); } - // Only the runtime that actually hosts phone sync (owns the brain-level - // shared listener) may register the relay tunnel. The relay DO keeps ONE - // host socket per machineKey (last wins), so a headless one-shot CLI - // runtime or embedded fallback starting the tunnel would steal the relay - // from `ade serve` and then fail every phone /connect (no sync port). - const canHostRelayTunnel = resolvedArgs.syncRuntime?.sharedSyncListener != null; - if (canHostRelayTunnel) { - void syncTunnelClientService.start().catch((error) => { - logger.warn("sync.tunnel_start_failed", { - error: error instanceof Error ? error.message : String(error), - }); - }); - } + // Only the runtime that holds the machine-wide sync host lease may register + // the relay tunnel. See relayTunnelAuthorityGate for why the old + // "has a listener" gate let secondary brains evict the real host. + const [{ createRelayTunnelAuthorityGate }, { holdsSyncHostSingleton, onSyncHostSingletonAuthorityChanged }] = + await Promise.all([ + import("./services/sync/relayTunnelAuthorityGate"), + import("./services/sync/syncHostSingleton"), + ]); + const relayTunnelGate = createRelayTunnelAuthorityGate({ + hostListener: resolvedArgs.syncRuntime?.sharedSyncListener ?? null, + tunnel: syncTunnelClientService, + holdsLease: holdsSyncHostSingleton, + subscribe: onSyncHostSingletonAuthorityChanged, + logger, + }); let externalSessionsService: ReturnType | null = null; if (resolvedArgs.syncRuntime?.enabled && agentChatService) { @@ -1882,7 +1896,10 @@ export async function createAdeRuntime(args: { swallow(() => detachPushSources()); // The tunnel client is machine-level and shared across scopes — closing // one project must not sever the relay for the others. The daemon's - // shutdown path (disposeServeResources) stops it. + // shutdown path (disposeServeResources) stops it. Drop only THIS scope's + // lease subscription, or a disposed scope could later stop the shared + // tunnel on a lease transition it no longer has any business observing. + swallow(() => relayTunnelGate.dispose()); swallow(() => automationIngressService?.dispose()); swallow(() => linearIngressService?.stop()); swallow(() => automationService?.dispose()); diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 646a5d1ec..40a7438f0 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -124,6 +124,7 @@ import type { AdeRuntime } from "./bootstrap"; import { reseedBundledAdeSkillsForCli } from "./bootstrap"; import { EncryptedFileCredentialStore } from "./services/credentials/credentialStore"; import type { AccountMachinePublisherService } from "./services/account/accountMachinePublisherService"; +import type { SyncHostSingletonLease } from "./services/sync/syncHostSingleton"; import { shouldRejectDevelopmentEnvCredential, syncAccountAnalyticsIdentity, @@ -13350,6 +13351,23 @@ function createSocketConnection(socketPath: string): net.Socket { return net.createConnection(socketPath); } +/** + * Refuse to run a brain whose RPC socket is already owned. Shared by the + * pre-startup claim and the bind path so the error contract (message, cause, + * `socket_owned_by_other` code that project recovery keys on) lives once. + */ +async function assertBrainSocketUnowned(socketPath: string): Promise { + const liveness = await probeLocalSocketForLiveness(socketPath); + if (liveness !== "live" && liveness !== "unknown") return; + throw Object.assign(new CliExecutionError("ADE brain socket is already in use.", { + socketPath, + cause: liveness === "live" + ? "Another ADE brain is accepting connections on this socket." + : "ADE could not prove the existing socket is stale.", + nextAction: "Stop the existing ADE brain or choose a different --socket path.", + }), { code: "socket_owned_by_other" as const }); +} + async function probeLocalSocketForLiveness(socketPath: string): Promise<"live" | "stale" | "unknown"> { if (socketPath.startsWith("tcp://") || isAdeRuntimeNamedPipePath(socketPath)) { return "unknown"; @@ -14986,6 +15004,22 @@ async function spawnMachineRuntimeDaemon( ): Promise { if (socketPath.startsWith("tcp://")) return false; + // Every failing `ade` command lands here, and the child is detached and + // unref'd — so without this check a burst of failures leaves a burst of + // immortal brains, all of them fighting over the same relay slot. If we + // already spawned one for this socket and it is still alive and recent, let + // it finish coming up instead of adding a rival. + // + // Report success: a spawn for this socket IS in flight, so the caller should + // go on to its connect-with-retry rather than fail immediately. Returning + // false here would turn a slow brain start into a hard error for the second + // of two concurrent commands — and on the build-mismatch path, where the old + // brain has already been shut down, would leave the machine with none. + const [{ hasRecentRuntimeSpawn, recordRuntimeSpawn }, { withSocketSpawnLock }] = await Promise.all([ + import("./services/runtime/runtimeSpawnRecord"), + import("./services/runtime/socketSpawnLock"), + ]); + const { resolveAdeServeCommand } = await import("./serviceManager/common"); const serviceCommand = resolveAdeServeCommand(); const { args, buildHash: runtimeBuildHash } = @@ -15014,14 +15048,23 @@ async function spawnMachineRuntimeDaemon( delete env.ADE_RUNTIME_BUILD_HASH; } - const child = spawn(serviceCommand.command, args, { - detached: true, - stdio: "ignore", - env, + // The record check and the spawn must be ONE atomic claim. Two CLI processes + // reaching an absent record concurrently would otherwise both pass the check + // and both launch a detached brain — the exact burst the record exists to + // prevent, and on named-pipe platforms the loser cannot even use the socket + // liveness abort to exit. + return await withSocketSpawnLock(socketPath, async () => { + if (hasRecentRuntimeSpawn(socketPath)) return true; + const child = spawn(serviceCommand.command, args, { + detached: true, + stdio: "ignore", + env, + }); + child.once("error", () => {}); + if (child.pid != null) recordRuntimeSpawn(socketPath, child.pid); + child.unref(); + return true; }); - child.once("error", () => {}); - child.unref(); - return true; } async function connectMachineRuntimeDaemon( @@ -15087,6 +15130,11 @@ async function connectMachineRuntimeDaemon( throw manualMachineRuntimeSpawnBlockedError(socketPath); } await shutdownMachineRuntimeDaemon(client); + // We just shut a brain down on purpose. If it was one we spawned, its + // record would otherwise suppress the very replacement this path exists + // to start, because the pid can outlive the shutdown by a moment. + const { clearRuntimeSpawnRecord } = await import("./services/runtime/runtimeSpawnRecord"); + clearRuntimeSpawnRecord(socketPath); const repaired = await repairServiceConnection(); if (repaired) return repaired; const spawned = await spawnMachineRuntimeDaemon(socketPath, options); @@ -15986,6 +16034,11 @@ async function runServe( filePath: path.join(layout.secretsDir, "sync-cloud-relay.json"), }); let accountMachinePublisher: AccountMachinePublisherService | null = null; + // Held only while this brain hosts phone sync WITHOUT a project scope (a + // scope's sync service owns its own lease). Machine-exclusive subsystems + // gate on holding one or the other. + let brainSyncHostLease: SyncHostSingletonLease | null = null; + let releaseAccountPublisherAuthoritySubscription: (() => void) | null = null; const getAccountDirectoryHealth = (): SyncAccountDirectoryHealth => accountMachinePublisher?.getPublisherHealth() ?? createSyncAccountDirectoryHealth( "sync_disabled", @@ -16091,7 +16144,22 @@ async function runServe( activeScope = await scopeRegistry.resolveActiveSyncHost(); } if (!activeScope && sharedSyncListener) { - await sharedSyncListener.ensureListening([DEFAULT_SYNC_HOST_PORT]); + // Binding the shared listener IS hosting phone sync, even with no project + // scope to attach to it. Take the machine-wide lease first so this path + // has the same exclusivity as the scoped one — otherwise a projectless + // brain would bind the port, publish itself, and dial the relay while + // another brain legitimately held the lease. Throwing here is correct: + // the startup loop treats a conflict as retryable and waits the other + // brain out. + const { acquireSyncHostSingleton } = await import("./services/sync/syncHostSingleton"); + brainSyncHostLease ??= acquireSyncHostSingleton({ projectRoot: null }); + const listenerPort = await sharedSyncListener.ensureListening([DEFAULT_SYNC_HOST_PORT]); + brainSyncHostLease.updatePort(listenerPort); + } else if (activeScope && brainSyncHostLease) { + // A scope took over hosting and holds its own lease; drop the + // projectless one so the lock file describes the real owner. + brainSyncHostLease.dispose(); + brainSyncHostLease = null; } // A ProjectScope is a complete runtime (DB, search, chat, automation, // polling, PTY, and sync services), not a lightweight metadata cache. @@ -16100,8 +16168,12 @@ async function runServe( return activeScope ?? null; }; const disposeServeResources = async () => { + releaseAccountPublisherAuthoritySubscription?.(); + releaseAccountPublisherAuthoritySubscription = null; accountMachinePublisher?.dispose(); accountMachinePublisher = null; + brainSyncHostLease?.dispose(); + brainSyncHostLease = null; // Before scopes detach (which clears the run map): best-effort Live // Activity `end` so the lock screen doesn't show dead agents until the // stale-date dim. Bounded by the publisher's internal timeout. @@ -16164,6 +16236,21 @@ async function runServe( }); }; + // Claim the RPC socket before entering the sync-host startup loop, not after. + // That loop retries forever by design (so sync recovers when a rival brain + // exits), which meant a brain whose socket was already owned never reached + // the bind check below and simply lived on as a zombie — we found 18 of them + // stacked up on one dev socket, all of them still dialing the relay. A brain + // that cannot own its socket has no reason to exist, so fail fast. + if (!isAdeRuntimeNamedPipePath(socketPath) && fs.existsSync(socketPath)) { + try { + await assertBrainSocketUnowned(socketPath); + } catch (error) { + await disposeServeResources(); + throw error; + } + } + if (syncEnabled) { try { const [{ runSyncHostStartupLoop }, { getRuntimeServiceMainPid }] = await Promise.all([ @@ -16175,12 +16262,33 @@ async function runServe( isDone: () => done, log: (message) => process.stderr.write(`${message}\n`), getServiceMainPid: getRuntimeServiceMainPid, + // The pre-loop claim above only sees a socket that ALREADY existed. Two + // brains started together on a fresh path both pass it, then one wins + // the lease and binds while the loser waits here forever — never + // reaching its own bind check. Re-check while we wait, and only for a + // provably live owner so a probe hiccup can't make a brain quit on + // itself. + abortIf: async () => { + if (isAdeRuntimeNamedPipePath(socketPath) || !fs.existsSync(socketPath)) return false; + return await probeLocalSocketForLiveness(socketPath) === "live"; + }, }); } catch (error: unknown) { // Cross-channel conflict (another build's live brain owns mobile sync): // real builds never run sync-less, so fail before publishing ade.sock. - const { SyncHostSingletonConflictError } = await import("./services/sync/syncHostSingleton"); + const [{ SyncHostSingletonConflictError }, { SyncHostStartupAbortedError }] = await Promise.all([ + import("./services/sync/syncHostSingleton"), + import("./services/sync/syncHostStartupLoop"), + ]); const message = error instanceof Error ? error.message : String(error); + if (error instanceof SyncHostStartupAbortedError) { + await disposeServeResources(); + throw Object.assign(new CliExecutionError("ADE brain socket is already in use.", { + socketPath, + cause: "Another ADE brain took this socket while this one waited for mobile sync.", + nextAction: "Stop the existing ADE brain or choose a different --socket path.", + }), { code: "socket_owned_by_other" as const }); + } if (error instanceof SyncHostSingletonConflictError) { await disposeServeResources(); throw new CliExecutionError("ADE brain refusing to run without mobile sync.", { @@ -16200,16 +16308,7 @@ async function runServe( if (!isAdeRuntimeNamedPipePath(socketPath)) { fs.mkdirSync(path.dirname(socketPath), { recursive: true, mode: 0o700 }); if (fs.existsSync(socketPath)) { - const liveness = await probeLocalSocketForLiveness(socketPath); - if (liveness === "live" || liveness === "unknown") { - throw Object.assign(new CliExecutionError("ADE brain socket is already in use.", { - socketPath, - cause: liveness === "live" - ? "Another ADE brain is accepting connections on this socket." - : "ADE could not prove the existing socket is stale.", - nextAction: "Stop the existing ADE brain or choose a different --socket path.", - }), { code: "socket_owned_by_other" as const }); - } + await assertBrainSocketUnowned(socketPath); try { fs.unlinkSync(socketPath); } catch {} @@ -16245,24 +16344,78 @@ async function runServe( const activeProject = projectRegistry.get(activeProjectId); return activeProject ? [activeProject.rootPath] : []; }; - accountMachinePublisher = createBrainAccountMachinePublisherService({ - secretsDir: layout.secretsDir, - projectRoots: accountProjectRoots, - isSyncEnabled: () => syncEnabled, - logger: headlessProjectLogger, - getSnapshot: async () => { - const activeScope = await scopeRegistry.resolveActiveSyncHost(); - return await activeScope?.runtime.syncService?.getStatus({ - includeTransferReadiness: false, - }) ?? null; - }, - getMachineKey: () => machineCloudRelayStore.getMachineIdentity().machineKey, - directoryBaseUrl: () => process.env.ADE_ACCOUNT_DIRECTORY_URL?.trim() || undefined, - captureAnalytics: (input) => { - brainProductAnalytics.captureInternal(input); - }, + // Publishing this machine to the account directory advertises "reach me + // here", so like the relay tunnel it belongs to whichever brain actually + // holds the machine-wide sync host lease. A second brain publishing its own + // endpoints points phones at a runtime that does not host sync. + const [{ holdsSyncHostSingleton, onSyncHostSingletonAuthorityChanged }, { SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS }] = + await Promise.all([ + import("./services/sync/syncHostSingleton"), + import("./services/sync/relayTunnelAuthorityGate"), + ]); + const startAccountMachinePublisher = (): void => { + if (accountMachinePublisher) return; + accountMachinePublisher = createBrainAccountMachinePublisherService({ + secretsDir: layout.secretsDir, + projectRoots: accountProjectRoots, + isSyncEnabled: () => syncEnabled, + logger: headlessProjectLogger, + getSnapshot: async () => { + const activeScope = await scopeRegistry.resolveActiveSyncHost(); + return await activeScope?.runtime.syncService?.getStatus({ + includeTransferReadiness: false, + }) ?? null; + }, + getMachineKey: () => machineCloudRelayStore.getMachineIdentity().machineKey, + directoryBaseUrl: () => process.env.ADE_ACCOUNT_DIRECTORY_URL?.trim() || undefined, + captureAnalytics: (input) => { + brainProductAnalytics.captureInternal(input); + }, + }); + accountMachinePublisher.start(); + }; + // A project switch deactivates the previous sync host before activating the + // target, so authority momentarily reads false inside one brain. Without the + // grace the publisher would be destroyed and rebuilt on every switch, and + // `ade doctor` would report "Account-directory publishing has not started" + // in the gap. Only a loss that outlives the handoff is a real one. + let accountPublisherReleaseTimer: NodeJS.Timeout | null = null; + const cancelAccountPublisherRelease = (): void => { + if (!accountPublisherReleaseTimer) return; + clearTimeout(accountPublisherReleaseTimer); + accountPublisherReleaseTimer = null; + }; + const unsubscribeAccountPublisherAuthority = onSyncHostSingletonAuthorityChanged((held) => { + if (held) { + cancelAccountPublisherRelease(); + startAccountMachinePublisher(); + return; + } + if (!accountMachinePublisher || accountPublisherReleaseTimer) return; + accountPublisherReleaseTimer = setTimeout(() => { + accountPublisherReleaseTimer = null; + if (holdsSyncHostSingleton()) return; + headlessProjectLogger.info("account_publisher.stopped_without_sync_host_lease", { + reason: "This brain no longer holds the machine-wide sync host lease.", + }); + // The publisher cannot be restarted after dispose, so a later lease + // acquisition rebuilds it above. + accountMachinePublisher?.dispose(); + accountMachinePublisher = null; + }, SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS); + accountPublisherReleaseTimer.unref?.(); }); - accountMachinePublisher.start(); + releaseAccountPublisherAuthoritySubscription = () => { + cancelAccountPublisherRelease(); + unsubscribeAccountPublisherAuthority(); + }; + if (holdsSyncHostSingleton()) { + startAccountMachinePublisher(); + } else { + headlessProjectLogger.info("account_publisher.start_skipped", { + reason: "This brain does not hold the machine-wide sync host lease; another ADE process publishes this machine.", + }); + } } process.stderr.write( @@ -16911,6 +17064,18 @@ function formatSyncStatus(value: unknown): string { relayEndToEndRoundTripMs == null ? "" : ` (${relayEndToEndRoundTripMs}ms)` }` : "not yet verified"; + // Start of the CURRENT uninterrupted control outage. The `relay control + // error` row can't answer "how long has this been broken", because the brain + // restamps its failure on every retry — and that duration is exactly what + // separates an ordinary sleep/wake redial from a route that is really down. + // Null while relay control is connected. + const relayFailingSinceMs = typeof relay.relayControlFailingSinceMs === "number" + && Number.isFinite(relay.relayControlFailingSinceMs) + ? relay.relayControlFailingSinceMs + : null; + const relayFailingSince = relayFailingSinceMs == null + ? null + : relativeTime(new Date(relayFailingSinceMs).toISOString()); const transferReadiness = isRecord(snapshot.transferReadiness) ? snapshot.transferReadiness : null; @@ -16940,6 +17105,7 @@ function formatSyncStatus(value: unknown): string { ["tailscale", tailscaleState], ["relay", relayState], ["relay control error", relay.lastControlError], + ["relay failing since", relayFailingSince], ["relay control opened", relay.lastControlOpenAt], ["relay bridge validated", relay.lastBridgeValidationAt], ["relay end-to-end", relayEndToEndState], diff --git a/apps/ade-cli/src/commands/doctor.test.ts b/apps/ade-cli/src/commands/doctor.test.ts index 12d4d159f..bbec2aee2 100644 --- a/apps/ade-cli/src/commands/doctor.test.ts +++ b/apps/ade-cli/src/commands/doctor.test.ts @@ -234,6 +234,26 @@ describe("doctor row evaluation", () => { expect(rows.filter((row) => row.status === "fail").map((row) => row.key)).toEqual(["brain"]); }); + it("names the rival ADE process when relay control is suppressed", () => { + const input = healthyInput(); + input.relayHealth = { + ...input.relayHealth!, + relayControlConnected: false, + relayControlSuppressed: true, + relayControlSuppressedReason: "Another ADE process owns the relay connection for this machine.", + // A stale bridge/control error must not outrank the actionable reason: + // total relay failure was previously invisible everywhere but here. + lastControlError: "Relay control closed (4505): replaced by newer host", + }; + + const relay = evaluateDoctorRows(input).find((row) => row.key === "relay"); + + expect(relay?.status).toBe("fail"); + expect(relay?.detail).toBe( + "Another ADE process owns the relay connection for this machine.", + ); + }); + it("compares release versions without depending on tag formatting", () => { expect(compareDoctorVersions("v1.2.36", "1.2.35")).toBe(1); expect(compareDoctorVersions("1.2.35", "v1.2.35")).toBe(0); diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts index 252b0a4bd..e96b0bd97 100644 --- a/apps/ade-cli/src/commands/doctor.ts +++ b/apps/ade-cli/src/commands/doctor.ts @@ -636,7 +636,13 @@ function relayRow(relay: DoctorInput["relayHealth"]): DoctorRow { detail: relay.skipReason ?? "disabled", }; } - const failure = relay.relayEndToEndFailure + // Suppression outranks every other failure: while another ADE process owns + // this machine's relay slot, nothing downstream can succeed and no other + // reason tells the user what to do about it. + const failure = (relay.relayControlSuppressed === true + ? relay.relayControlSuppressedReason ?? "Relay control is suppressed." + : null) + ?? relay.relayEndToEndFailure ?? relay.skipReason ?? relay.lastControlError ?? null; diff --git a/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.test.ts b/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.test.ts new file mode 100644 index 000000000..9d0e9fa99 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.test.ts @@ -0,0 +1,85 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + clearRuntimeSpawnRecord, + hasRecentRuntimeSpawn, + readRuntimeSpawnRecord, + recordRuntimeSpawn, + RUNTIME_SPAWN_RECORD_GRACE_MS, + runtimeSpawnRecordPath, +} from "./runtimeSpawnRecord"; + +const sockets = ["/tmp/ade-runtime-spawn-record-test.sock", "/tmp/other/ade.sock"]; +const originalAdeHome = process.env.ADE_HOME; +let tempAdeHome = ""; + +// Records live under the machine ADE home; without an override the suite would +// write into the developer's real ~/.ade. +beforeAll(() => { + tempAdeHome = fs.mkdtempSync(path.join(os.tmpdir(), "ade-spawn-record-home-")); + process.env.ADE_HOME = tempAdeHome; +}); + +afterAll(() => { + if (originalAdeHome === undefined) delete process.env.ADE_HOME; + else process.env.ADE_HOME = originalAdeHome; + if (tempAdeHome) fs.rmSync(tempAdeHome, { recursive: true, force: true }); +}); + +afterEach(() => { + for (const socketPath of sockets) { + fs.rmSync(runtimeSpawnRecordPath(socketPath), { force: true }); + } +}); + +describe("runtimeSpawnRecord", () => { + const socketPath = sockets[0]!; + + it("reports no recent spawn when nothing was recorded", () => { + expect(readRuntimeSpawnRecord(socketPath)).toBe(null); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(false); + }); + + it("suppresses a duplicate spawn while the recorded brain is alive and recent", () => { + recordRuntimeSpawn(socketPath, process.pid); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(true); + }); + + // A wedged brain must not block recovery forever, so the record expires. + it("allows a new spawn once the grace window lapses", () => { + recordRuntimeSpawn(socketPath, process.pid); + const later = Date.now() + RUNTIME_SPAWN_RECORD_GRACE_MS + 1; + expect(hasRecentRuntimeSpawn(socketPath, later)).toBe(false); + }); + + it("allows a new spawn when the recorded brain is gone", () => { + // pid 1 is always alive, so use an implausible-but-valid pid instead. + recordRuntimeSpawn(socketPath, 2_147_483_646); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(false); + }); + + // A deliberate shutdown makes the record stale immediately: the pid may + // outlive the shutdown, and the replacement spawn must not be suppressed. + it("stops suppressing once the record is cleared", () => { + recordRuntimeSpawn(socketPath, process.pid); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(true); + clearRuntimeSpawnRecord(socketPath); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(false); + }); + + it("keys records per socket so one socket's spawn cannot gate another's", () => { + recordRuntimeSpawn(socketPath, process.pid); + expect(hasRecentRuntimeSpawn(sockets[1]!)).toBe(false); + expect(runtimeSpawnRecordPath(socketPath)).not.toBe(runtimeSpawnRecordPath(sockets[1]!)); + }); + + it("treats a corrupt record as absent rather than throwing", () => { + const recordPath = runtimeSpawnRecordPath(socketPath); + fs.mkdirSync(path.dirname(recordPath), { recursive: true }); + fs.writeFileSync(recordPath, "not-json", "utf8"); + expect(readRuntimeSpawnRecord(socketPath)).toBe(null); + expect(hasRecentRuntimeSpawn(socketPath)).toBe(false); + }); +}); diff --git a/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.ts b/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.ts new file mode 100644 index 000000000..379487dc8 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/runtimeSpawnRecord.ts @@ -0,0 +1,116 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { resolveMachineAdeLayout } from "../projects/machineLayout"; + +/** + * Remembers the last brain this machine spawned for a given RPC socket. + * + * Every `ade` command that fails to reach the brain spawns one, detached and + * unref'd, and then forgets it — so N failing commands leak N immortal brains. + * We found 18 stacked on one dev socket, each one signed in and each one + * dialing the relay, which is how a machine ends up evicting its own relay + * connection in a loop. + * + * The record is deliberately advisory: it suppresses a *duplicate* spawn while + * a previously spawned brain is still coming up, and expires so a genuinely + * wedged brain never blocks recovery forever. + */ + +export type RuntimeSpawnRecord = { + pid: number; + socketPath: string; + spawnedAtMs: number; +}; + +/** + * How long a recorded spawn suppresses another one. Sized to a cold brain + * start on a large repo, which is the window during which a second spawn is + * pure harm; past it, a brain that still has not bound its socket is wedged + * and a replacement is the right answer. + */ +export const RUNTIME_SPAWN_RECORD_GRACE_MS = 30_000; + +// Under ~/.ade rather than the system temp dir. On Linux/WSL os.tmpdir() is the +// world-writable /tmp, where another local user could pre-create the record as +// a symlink (writeFileSync follows it) or plant a live pid to suppress this +// user's brain spawns indefinitely. The machine layout dir is ADE-owned and +// created 0700. +function recordDir(): string { + return path.join(resolveMachineAdeLayout().runtimeDir, "spawns"); +} + +export function runtimeSpawnRecordPath(socketPath: string): string { + // Hash rather than sanitize-and-truncate: two socket paths sharing a suffix + // must not share a record and suppress each other's spawns. + const key = createHash("sha256").update(socketPath).digest("hex").slice(0, 32); + return path.join(recordDir(), `${key}.json`); +} + +function processAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM means the pid exists but belongs to another user. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +export function readRuntimeSpawnRecord(socketPath: string): RuntimeSpawnRecord | null { + try { + const parsed = JSON.parse( + fs.readFileSync(runtimeSpawnRecordPath(socketPath), "utf8"), + ) as Partial | null; + if (!parsed || typeof parsed !== "object") return null; + const pid = Number(parsed.pid); + const spawnedAtMs = Number(parsed.spawnedAtMs); + if (!Number.isInteger(pid) || pid <= 0 || !Number.isFinite(spawnedAtMs)) return null; + return { pid, socketPath, spawnedAtMs }; + } catch { + return null; + } +} + +export function recordRuntimeSpawn(socketPath: string, pid: number): void { + if (!Number.isInteger(pid) || pid <= 0) return; + const record: RuntimeSpawnRecord = { pid, socketPath, spawnedAtMs: Date.now() }; + try { + fs.mkdirSync(recordDir(), { recursive: true, mode: 0o700 }); + fs.writeFileSync(runtimeSpawnRecordPath(socketPath), JSON.stringify(record), { + encoding: "utf8", + mode: 0o600, + }); + } catch { + // The record is an optimization; never fail a spawn over it. + } +} + +/** + * Drop the record after deliberately shutting a spawned brain down. Its pid can + * outlive the shutdown by a moment, and without this the stale record would + * suppress the replacement spawn the shutdown exists to make room for. + */ +export function clearRuntimeSpawnRecord(socketPath: string): void { + try { + fs.rmSync(runtimeSpawnRecordPath(socketPath), { force: true }); + } catch { + // Advisory only. + } +} + +/** + * True when a brain we spawned for this socket is still alive and recent + * enough that spawning another would just add a rival, not fix anything. + */ +export function hasRecentRuntimeSpawn( + socketPath: string, + now = Date.now(), + graceMs = RUNTIME_SPAWN_RECORD_GRACE_MS, +): boolean { + const record = readRuntimeSpawnRecord(socketPath); + if (!record) return false; + if (now - record.spawnedAtMs > graceMs) return false; + return processAlive(record.pid); +} diff --git a/apps/ade-cli/src/services/runtime/socketSpawnLock.ts b/apps/ade-cli/src/services/runtime/socketSpawnLock.ts new file mode 100644 index 000000000..443285087 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/socketSpawnLock.ts @@ -0,0 +1,124 @@ +import fs from "node:fs"; +import path from "node:path"; + +/** + * Cross-process mutual exclusion for "spawn a brain for this socket". + * + * Every failing `ade` command and every `ade code` launch can spawn a detached + * brain, so without a lock a burst of commands races: each one checks, each one + * finds nothing, and each one spawns. That is how a machine ends up with a pile + * of immortal brains fighting over the same relay slot. An advisory record + * alone cannot fix it — the check and the spawn must be one atomic claim. + * + * The lock lives next to the socket, is released on completion, and is reaped + * when its owning pid is gone (or, for an unreadable lock, after 30s) so a + * crashed spawner cannot wedge the machine. + */ + +type SocketSpawnLockOwner = { + id: string | null; + pid: number | null; +}; + +function createSocketSpawnLockOwner(): SocketSpawnLockOwner { + return { + id: `${process.pid}:${Date.now()}:${Math.random().toString(36).slice(2)}`, + pid: process.pid, + }; +} + +function serializeSocketSpawnLockOwner(owner: SocketSpawnLockOwner): string { + return JSON.stringify({ + id: owner.id, + pid: owner.pid, + createdAt: new Date().toISOString(), + }); +} + +function readSocketSpawnLockOwner(lockPath: string): SocketSpawnLockOwner { + const raw = fs.readFileSync(lockPath, "utf8"); + try { + const parsed = JSON.parse(raw) as { id?: unknown; pid?: unknown }; + return { + id: typeof parsed.id === "string" ? parsed.id : null, + pid: typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? parsed.pid + : null, + }; + } catch { + const [pidLine] = raw.split(/\r?\n/u); + const pid = Number.parseInt(pidLine ?? "", 10); + return { + id: null, + pid: Number.isInteger(pid) && pid > 0 ? pid : null, + }; + } +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function unlinkSocketSpawnLockIfStale(lockPath: string): boolean { + try { + const stat = fs.statSync(lockPath); + const owner = readSocketSpawnLockOwner(lockPath); + if (owner.pid != null && processExists(owner.pid)) return false; + if (owner.pid == null && Date.now() - stat.mtimeMs <= 30_000) return false; + fs.unlinkSync(lockPath); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; + } +} + +function unlinkSocketSpawnLockIfOwner(lockPath: string, ownerId: string | null): void { + if (!ownerId) return; + try { + const owner = readSocketSpawnLockOwner(lockPath); + if (owner.id !== ownerId) return; + fs.unlinkSync(lockPath); + } catch { + // ignore cleanup races + } +} + +export async function withSocketSpawnLock(socketPath: string, task: () => Promise): Promise { + if (socketPath.startsWith("tcp://")) return await task(); + const lockPath = path.join(path.dirname(socketPath), `${path.basename(socketPath)}.spawn.lock`); + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + const deadline = Date.now() + 10_000; + const owner = createSocketSpawnLockOwner(); + let fd: number | null = null; + while (fd == null) { + try { + fd = fs.openSync(lockPath, "wx", 0o600); + fs.writeFileSync(fd, serializeSocketSpawnLockOwner(owner), "utf8"); + break; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "EEXIST") throw error; + if (unlinkSocketSpawnLockIfStale(lockPath)) continue; + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for ADE socket spawn lock at ${lockPath}.`); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + try { + return await task(); + } finally { + if (fd != null) { + try { + fs.closeSync(fd); + } catch {} + } + unlinkSocketSpawnLockIfOwner(lockPath, owner.id); + } +} diff --git a/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.test.ts b/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.test.ts new file mode 100644 index 000000000..d0f40a7f1 --- /dev/null +++ b/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.test.ts @@ -0,0 +1,268 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { TunnelHostListener } from "./syncTunnelClientService"; +import { + createRelayTunnelAuthorityGate, + SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS, +} from "./relayTunnelAuthorityGate"; + +const listener = { + getPort: () => 8787, + getExpectedLoopbackNonce: () => "n".repeat(32), + getRelayBridgeProof: () => "p".repeat(43), + onLoopbackValidated: () => () => {}, +} satisfies TunnelHostListener; + +function createTunnel() { + return { + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + attachHostListener: vi.fn(), + clearControlSuppression: vi.fn(), + }; +} + +function createLeaseHarness(initiallyHeld: boolean) { + let held = initiallyHeld; + const handlers = new Set<(next: boolean) => void>(); + return { + holdsLease: () => held, + subscribe: (handler: (next: boolean) => void) => { + handlers.add(handler); + return () => handlers.delete(handler); + }, + set(next: boolean) { + held = next; + for (const handler of [...handlers]) handler(next); + }, + subscriberCount: () => handlers.size, + }; +} + +beforeEach(() => { + vi.useFakeTimers(); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("createRelayTunnelAuthorityGate", () => { + it("does not dial the relay while another process holds the sync host lease", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(false); + const info = vi.fn(); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + logger: { info }, + }); + + expect(gate.isRunning()).toBe(false); + expect(tunnel.start).not.toHaveBeenCalled(); + // The whole failure mode was silent, so the decline has to be logged. + expect(info).toHaveBeenCalledWith( + "sync.tunnel_start_skipped", + expect.objectContaining({ hasSyncListener: true, holdsSyncHostLease: false }), + ); + }); + + it("starts once the lease is acquired and clears eviction suppression first", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(false); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + expect(tunnel.start).not.toHaveBeenCalled(); + + lease.set(true); + + expect(gate.isRunning()).toBe(true); + expect(tunnel.clearControlSuppression).toHaveBeenCalledTimes(1); + expect(tunnel.start).toHaveBeenCalledTimes(1); + }); + + it("re-attaches the host listener on every start", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + + createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + expect(tunnel.attachHostListener).toHaveBeenLastCalledWith(listener); + + // stop() drops the listener reference, so a gate-driven restart that did + // not re-attach would come back with a live control socket and no bridge — + // every phone connect rejected with "host sync listener unavailable". + lease.set(false); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS + 1); + expect(tunnel.stop).toHaveBeenCalledTimes(1); + + lease.set(true); + expect(tunnel.attachHostListener).toHaveBeenCalledTimes(2); + expect(tunnel.attachHostListener).toHaveBeenLastCalledWith(listener); + expect(tunnel.start).toHaveBeenCalledTimes(2); + }); + + it("does not re-arm eviction suppression for a gate built while the lease is already held", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + + // Opening a second project constructs a second gate. Clearing suppression + // there would reset the 4505 re-attempt budget and let the eviction war + // restart on every project open. + createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + + expect(tunnel.start).toHaveBeenCalledTimes(1); + expect(tunnel.clearControlSuppression).not.toHaveBeenCalled(); + }); + + it("rides out the authority gap of an in-process sync host switch", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + + // performSyncHostSwitch deactivates the previous host before activating the + // target, so authority blips false mid-switch. Tearing the machine's relay + // down and back up on every project switch would be pure churn. + lease.set(false); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS / 2); + lease.set(true); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS * 2); + + expect(tunnel.stop).not.toHaveBeenCalled(); + expect(tunnel.start).toHaveBeenCalledTimes(1); + expect(gate.isRunning()).toBe(true); + }); + + it("never dials without the shared sync listener, lease or not", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: null, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + + expect(gate.isRunning()).toBe(false); + expect(tunnel.start).not.toHaveBeenCalled(); + }); + + it("stops the tunnel when the lease stays lost past the grace", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + const info = vi.fn(); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + logger: { info }, + }); + expect(gate.isRunning()).toBe(true); + + lease.set(false); + expect(tunnel.stop).not.toHaveBeenCalled(); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS + 1); + + expect(gate.isRunning()).toBe(false); + expect(tunnel.stop).toHaveBeenCalledTimes(1); + expect(info).toHaveBeenCalledWith( + "sync.tunnel_stopped_without_sync_host_lease", + expect.any(Object), + ); + }); + + // stop() and start() are async, so a lease reacquired while a stop is still + // in flight must not be torn down by that stop when it settles. + it("keeps the tunnel running when the lease returns mid-stop", async () => { + const tunnel = createTunnel(); + let releaseStop: (() => void) | undefined; + tunnel.stop.mockImplementation(() => new Promise((resolve) => { + releaseStop = resolve; + })); + const lease = createLeaseHarness(true); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + + lease.set(false); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS + 1); + expect(tunnel.stop).toHaveBeenCalledTimes(1); + + // Lease comes back before the stop settles. + lease.set(true); + expect(gate.isRunning()).toBe(true); + + releaseStop?.(); + await Promise.resolve(); + await Promise.resolve(); + + expect(gate.isRunning()).toBe(true); + // The settling stop re-established the tunnel rather than leaving it down. + expect(tunnel.start).toHaveBeenCalled(); + expect(tunnel.attachHostListener).toHaveBeenLastCalledWith(listener); + }); + + it("does not restart on repeated acquire notifications", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(false); + + createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + lease.set(true); + lease.set(true); + + expect(tunnel.start).toHaveBeenCalledTimes(1); + }); + + it("detaches its subscription and pending timer on dispose without stopping the shared tunnel", () => { + const tunnel = createTunnel(); + const lease = createLeaseHarness(true); + + const gate = createRelayTunnelAuthorityGate({ + hostListener: listener, + tunnel, + holdsLease: lease.holdsLease, + subscribe: lease.subscribe, + }); + lease.set(false); + gate.dispose(); + vi.advanceTimersByTime(SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS * 3); + + expect(lease.subscriberCount()).toBe(0); + // A disposed project scope must not sever relay for the scopes still open. + expect(tunnel.stop).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.ts b/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.ts new file mode 100644 index 000000000..ead8e0622 --- /dev/null +++ b/apps/ade-cli/src/services/sync/relayTunnelAuthorityGate.ts @@ -0,0 +1,166 @@ +import type { TunnelHostListener } from "./syncTunnelClientService"; + +/** + * Decides whether THIS runtime may hold the machine's relay tunnel. + * + * The relay Durable Object keeps one host control socket per machineKey and + * evicts the previous holder (close code 4505). So when two brains on a machine + * both dial it, each eviction triggers the other's reconnect and relay stays + * down for both — silently, since nothing but `ade doctor` reported it. + * + * The gate is the machine-wide sync host LEASE, not the presence of a listener. + * Gating on the listener was the original bug: a dev `serve` or an embedded + * fallback binds an ephemeral listener without ever winning the lease, so it + * passed the check and dialed anyway. The lease is acquired later in the + * process lifetime (when the sync host starts) and can be released again, so + * this subscribes to transitions instead of sampling once. + */ + +export type RelayTunnelGateTunnel = { + start(): Promise; + stop(): Promise; + attachHostListener(listener: TunnelHostListener | null): void; + clearControlSuppression(): void; +}; + +export type RelayTunnelGateLogger = { + info?: (event: string, data?: Record) => void; + warn?: (event: string, data?: Record) => void; +}; + +export type RelayTunnelAuthorityGate = { + /** True when the tunnel is running under this gate. */ + isRunning(): boolean; + /** Detach the lease subscription. Never stops a running tunnel: the client is machine-level and shared across project scopes. */ + dispose(): void; +}; + +/** + * How long authority must stay lost before the tunnel is torn down. + * + * A project switch deactivates the previous sync host BEFORE activating the + * target (ProjectScopeRegistry.performSyncHostSwitch), so within one brain the + * lease legitimately reads "not held" for the width of that handoff. Reacting + * to that transient would stop and restart the machine's relay tunnel — and + * churn the account-directory publisher — on every project switch. A real loss + * of authority outlives this window; a handoff never does. + */ +export const SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS = 5_000; + +export type RelayTunnelAuthorityGateArgs = { + /** + * The brain-level shared listener the relay bridges into, or null for a + * runtime that does not host phone sync at all. + */ + hostListener: TunnelHostListener | null; + tunnel: RelayTunnelGateTunnel; + holdsLease: () => boolean; + subscribe: (handler: (held: boolean) => void) => () => void; + logger?: RelayTunnelGateLogger; + /** Test seam. */ + releaseGraceMs?: number; +}; + +export function createRelayTunnelAuthorityGate( + args: RelayTunnelAuthorityGateArgs, +): RelayTunnelAuthorityGate { + const log = args.logger; + const graceMs = args.releaseGraceMs ?? SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS; + const hasSyncListener = args.hostListener != null; + let running = false; + let releaseTimer: NodeJS.Timeout | null = null; + // Bumped by every start/stop. `stop()` and `start()` are async, so a lease + // reacquired mid-teardown could otherwise be shut down by the stop that was + // already in flight. + let lifecycleGeneration = 0; + // Whether this gate has ever seen authority go away. A gate constructed while + // the lease is already held (opening a second project) must not re-arm the + // eviction suppression: that would reset the 4505 re-attempt budget and let + // the war restart on every project open. + let observedRelease = false; + + const clearReleaseTimer = (): void => { + if (!releaseTimer) return; + clearTimeout(releaseTimer); + releaseTimer = null; + }; + + const startTunnel = (): void => { + running = true; + const generation = ++lifecycleGeneration; + // Re-attach on every start. `stop()` drops the listener reference, and it + // is otherwise attached exactly once per runtime at construction — so a + // gate-driven stop/start cycle would come back with a live control socket + // and no bridge, rejecting every phone connect with "host sync listener + // unavailable" until the brain restarted. + args.tunnel.attachHostListener(args.hostListener); + if (observedRelease) { + // Re-winning the lease makes this runtime the legitimate owner again, so + // an earlier eviction no longer describes reality. + args.tunnel.clearControlSuppression(); + } + void args.tunnel.start().catch((error) => { + if (generation !== lifecycleGeneration) return; + log?.warn?.("sync.tunnel_start_failed", { + error: error instanceof Error ? error.message : String(error), + }); + }); + }; + + const stopTunnel = (): void => { + running = false; + const generation = ++lifecycleGeneration; + log?.info?.("sync.tunnel_stopped_without_sync_host_lease", { + reason: "This runtime no longer holds the machine-wide sync host lease.", + }); + void args.tunnel.stop().then( + () => { + // A start that landed while this stop was in flight owns the tunnel + // now; finishing the stop would tear down the live one. + if (generation !== lifecycleGeneration || !running) return; + args.tunnel.attachHostListener(args.hostListener); + void args.tunnel.start().catch(() => {}); + }, + () => {}, + ); + }; + + const apply = (held: boolean): void => { + const shouldRun = hasSyncListener && held; + if (shouldRun) { + clearReleaseTimer(); + if (running) return; + startTunnel(); + return; + } + if (!held) observedRelease = true; + if (!running || releaseTimer) return; + releaseTimer = setTimeout(() => { + releaseTimer = null; + if (args.holdsLease() && hasSyncListener) return; + stopTunnel(); + }, graceMs); + releaseTimer.unref?.(); + }; + + const release = args.subscribe(apply); + const held = args.holdsLease(); + if (!(hasSyncListener && held)) { + log?.info?.("sync.tunnel_start_skipped", { + reason: hasSyncListener + ? "This runtime does not hold the machine-wide sync host lease; another ADE process owns phone sync and the relay." + : "This runtime does not host the shared sync listener.", + hasSyncListener, + holdsSyncHostLease: held, + }); + } + apply(held); + + return { + isRunning: () => running, + dispose: () => { + clearReleaseTimer(); + release(); + }, + }; +} diff --git a/apps/ade-cli/src/services/sync/syncHostSingleton.test.ts b/apps/ade-cli/src/services/sync/syncHostSingleton.test.ts index 78f061d43..5e8401d49 100644 --- a/apps/ade-cli/src/services/sync/syncHostSingleton.test.ts +++ b/apps/ade-cli/src/services/sync/syncHostSingleton.test.ts @@ -7,7 +7,9 @@ import { buildQuitCommand, detectSyncHostSingletonConflict, formatSyncHostSingletonConflictMessage, + holdsSyncHostSingleton, isSameChannelSyncHostOwner, + onSyncHostSingletonAuthorityChanged, type SyncHostSingletonOwner, } from "./syncHostSingleton"; @@ -168,6 +170,53 @@ describe("sync host singleton", () => { }); }); +describe("sync host authority", () => { + const acquire = (lockPath: string) => + acquireSyncHostSingleton( + { projectRoot: "/Users/admin/Projects/ADE" }, + { lockPath, pidAlive: () => false, scanListeners: () => [] }, + ); + + it("reports authority only while a lease is held", () => { + expect(holdsSyncHostSingleton()).toBe(false); + const lease = acquire(tempLockPath()); + expect(holdsSyncHostSingleton()).toBe(true); + lease.dispose(); + expect(holdsSyncHostSingleton()).toBe(false); + }); + + it("notifies subscribers on authority transitions only", () => { + const seen: boolean[] = []; + const unsubscribe = onSyncHostSingletonAuthorityChanged((held) => seen.push(held)); + + const first = acquire(tempLockPath()); + const second = acquire(tempLockPath()); + // Two leases, one transition: subsystems care about "is it me", not "how + // many scopes". + expect(seen).toEqual([true]); + + first.dispose(); + expect(seen).toEqual([true]); + second.dispose(); + expect(seen).toEqual([true, false]); + + unsubscribe(); + acquire(tempLockPath()).dispose(); + expect(seen).toEqual([true, false]); + }); + + it("survives a throwing subscriber", () => { + const unsubscribe = onSyncHostSingletonAuthorityChanged(() => { + throw new Error("subscriber exploded"); + }); + const lease = acquire(tempLockPath()); + expect(holdsSyncHostSingleton()).toBe(true); + lease.dispose(); + expect(holdsSyncHostSingleton()).toBe(false); + unsubscribe(); + }); +}); + describe("isSameChannelSyncHostOwner", () => { const betaEnv = { ADE_PACKAGE_CHANNEL: "beta", diff --git a/apps/ade-cli/src/services/sync/syncHostSingleton.ts b/apps/ade-cli/src/services/sync/syncHostSingleton.ts index 7fed53171..e0d282beb 100644 --- a/apps/ade-cli/src/services/sync/syncHostSingleton.ts +++ b/apps/ade-cli/src/services/sync/syncHostSingleton.ts @@ -44,6 +44,43 @@ export type SyncHostSingletonDeps = { scanListeners?: () => SyncHostSingletonOwner[]; }; +// Which leases THIS process currently holds. The lock file answers "who owns +// mobile sync on this machine"; this answers "is it me", which is the question +// every other machine-exclusive subsystem (relay tunnel, account-directory +// publisher) actually needs. Without it those subsystems gated on merely +// HAVING a listener, so a secondary brain with an ephemeral fallback listener +// happily dialed the relay and evicted the real host in a ~4s loop. +const heldLeaseIds = new Set(); +const authorityHandlers = new Set<(held: boolean) => void>(); + +function notifyAuthorityChanged(held: boolean): void { + for (const handler of [...authorityHandlers]) { + try { + handler(held); + } catch { + // A subscriber must never break lease bookkeeping. + } + } +} + +/** True when this process holds the machine-wide sync host lease. */ +export function holdsSyncHostSingleton(): boolean { + return heldLeaseIds.size > 0; +} + +/** + * Subscribe to authority transitions (not every acquire/release — only + * none-held → held and held → none-held). Returns an unsubscribe function. + */ +export function onSyncHostSingletonAuthorityChanged( + handler: (held: boolean) => void, +): () => void { + authorityHandlers.add(handler); + return () => { + authorityHandlers.delete(handler); + }; +} + export class SyncHostSingletonConflictError extends Error { readonly conflict: SyncHostSingletonConflict; @@ -434,6 +471,9 @@ export function acquireSyncHostSingleton( if (attempt === 1) writeLock(lockPath, owner, "wx"); } } + const hadAuthority = holdsSyncHostSingleton(); + heldLeaseIds.add(owner.id); + if (!hadAuthority) notifyAuthorityChanged(true); return { owner, updatePort(port: number) { @@ -453,6 +493,9 @@ export function acquireSyncHostSingleton( if (lock?.owner.id === owner.id && lock.owner.pid === process.pid) { unlinkLock(lockPath); } + if (heldLeaseIds.delete(owner.id) && !holdsSyncHostSingleton()) { + notifyAuthorityChanged(false); + } }, }; } diff --git a/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts b/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts index a0bec767c..1c80f2cc5 100644 --- a/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts +++ b/apps/ade-cli/src/services/sync/syncHostStartupLoop.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { runSyncHostStartupLoop } from "./syncHostStartupLoop"; +import { runSyncHostStartupLoop, SyncHostStartupAbortedError } from "./syncHostStartupLoop"; import { SyncHostSingletonConflictError, type SyncHostSingletonOwner, @@ -219,4 +219,40 @@ describe("runSyncHostStartupLoop", () => { }); expect(attempts).toBe(1); }); + + // The loop retries forever on purpose so sync recovers when a rival brain + // exits. That is also how a brain that lost its socket became an immortal + // zombie — it never reached its own bind check. + it("aborts instead of retrying forever once another brain owns the socket", async () => { + let attempts = 0; + let socketTaken = false; + await expect(runSyncHostStartupLoop({ + startSyncHost: () => { + attempts += 1; + if (attempts >= 2) socketTaken = true; + return Promise.reject(new Error("sync host unavailable")); + }, + isDone: () => false, + log: () => {}, + sleep: instantSleep, + abortIf: () => socketTaken, + })).rejects.toBeInstanceOf(SyncHostStartupAbortedError); + expect(attempts).toBe(2); + }); + + it("keeps retrying while the socket is still ours", async () => { + let attempts = 0; + await runSyncHostStartupLoop({ + startSyncHost: () => { + attempts += 1; + if (attempts < 3) return Promise.reject(new Error("not yet")); + return Promise.resolve(null); + }, + isDone: () => false, + log: () => {}, + sleep: instantSleep, + abortIf: () => false, + }); + expect(attempts).toBe(3); + }); }); diff --git a/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts b/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts index 5f753ba05..0ca6f3cff 100644 --- a/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts +++ b/apps/ade-cli/src/services/sync/syncHostStartupLoop.ts @@ -13,6 +13,14 @@ export type SyncHostStartupLoopDeps = { // same-channel sibling; everything else waits, so two recovering brains can // never kill each other in a loop. getServiceMainPid?: () => number | null; + /** + * Checked before every retry. This loop retries forever by design so mobile + * sync recovers when a rival brain exits — but a brain that will never win + * the lease AND has lost its RPC socket to someone else has no reason to + * exist, and staying in the loop is how it becomes an immortal zombie + * (we found 18 stacked on one dev socket). Returning true aborts the loop. + */ + abortIf?: () => boolean | Promise; kill?: (pid: number, signal: NodeJS.Signals | number) => void; pidAlive?: (pid: number) => boolean; sleep?: (ms: number) => Promise; @@ -65,6 +73,14 @@ async function terminatePidAsync( } } +/** Thrown when `abortIf` asks the loop to stop; the caller decides how to exit. */ +export class SyncHostStartupAbortedError extends Error { + constructor() { + super("ADE brain sync host startup was aborted."); + this.name = "SyncHostStartupAbortedError"; + } +} + // Keeps retrying mobile sync host startup until it succeeds or the brain // shuts down. Same-channel conflicts are transient by nature (update races, // restart overlap, a stale sibling about to be evicted), so they retry: @@ -112,6 +128,7 @@ export async function runSyncHostStartupLoop(deps: SyncHostStartupLoopDeps): Pro throw error; } if (deps.maxAttempts != null && attempt >= deps.maxAttempts) return; + if (await deps.abortIf?.()) throw new SyncHostStartupAbortedError(); await sleep(slowRetryDelayMs); continue; } @@ -129,6 +146,7 @@ export async function runSyncHostStartupLoop(deps: SyncHostStartupLoopDeps): Pro } } if (deps.maxAttempts != null && attempt >= deps.maxAttempts) return; + if (await deps.abortIf?.()) throw new SyncHostStartupAbortedError(); await sleep(attempt <= fastRetryCount ? fastRetryDelayMs : slowRetryDelayMs); } } diff --git a/apps/ade-cli/src/services/sync/syncService.ts b/apps/ade-cli/src/services/sync/syncService.ts index d554b32dd..603c9aafe 100644 --- a/apps/ade-cli/src/services/sync/syncService.ts +++ b/apps/ade-cli/src/services/sync/syncService.ts @@ -573,9 +573,20 @@ export function createSyncService(args: SyncServiceArgs) { const status = accountAuthService.getStatus(); return status.signedIn && Boolean(status.userId?.trim()); }; + // Governs whether this runtime ADVERTISES a relay URL to phones (pairing + // connect info, brain_status, hello_ok). A suppressed tunnel has deliberately + // stopped dialing because another ADE process owns this machine's relay slot, + // so continuing to hand out the URL points phones at a route that can only + // fail — and a phone that already saved it never purges the candidate, + // because the purge is driven by an explicitly null cloudRelayWssUrl. + // + // Route health deliberately does NOT use this: `ade doctor` and the desktop + // banner must still report relay as enabled-but-suppressed rather than + // silently "disabled", which is how this whole failure stayed invisible. const isCloudRelayUsable = (): boolean => isRelayAccountSignedIn() - && (args.syncTunnelClientService?.getStatus().accountLeaseValid ?? true); + && (args.syncTunnelClientService?.getStatus().accountLeaseValid ?? true) + && args.syncTunnelClientService?.getStatus().controlSuppressed !== true; const deviceRegistryService = createDeviceRegistryService({ db: args.db, @@ -1362,7 +1373,11 @@ export function createSyncService(args: SyncServiceArgs) { : !tunnelStatus ? "Relay tunnel status is unavailable in this ADE process." : !relayControlConnected - ? tunnelStatus.lastControlError + // A 4505 eviction is a machine-local ownership conflict, not a + // network fault, and its reason is the only one that tells the + // user what to actually do — so it outranks the raw close text. + ? tunnelStatus.controlSuppressedReason + ?? tunnelStatus.lastControlError ?? tunnelStatus.lastError ?? "Relay control is not connected." : !relayBridgeValidated @@ -1393,6 +1408,9 @@ export function createSyncService(args: SyncServiceArgs) { relayEndToEndVerifiedAt: tunnelStatus?.relayEndToEndVerifiedAt ?? null, relayEndToEndFailure: tunnelStatus?.relayEndToEndFailure ?? null, relayEndToEndRoundTripMs: tunnelStatus?.relayEndToEndRoundTripMs ?? null, + relayControlSuppressed: tunnelStatus?.controlSuppressed === true, + relayControlSuppressedReason: tunnelStatus?.controlSuppressedReason ?? null, + relayControlFailingSinceMs: tunnelStatus?.controlFailingSinceMs ?? null, }; const routeHealth: SyncRouteHealth = { listener: { @@ -1740,6 +1758,18 @@ export function createSyncService(args: SyncServiceArgs) { hostService?.broadcastPrsUpdated(); }, + /** + * Push a fresh status snapshot to subscribers now. + * + * The relay tunnel is machine-level and changes state (notably 4505 + * suppression) without any project-scoped activity, so on an idle machine + * nothing would otherwise emit a snapshot and the desktop banner would stay + * hidden until some unrelated sync event happened along. + */ + notifyRouteStateChanged(): void { + void emitStatus(); + }, + getHostService(): SyncHostService | null { return hostService; }, diff --git a/apps/ade-cli/src/services/sync/syncTunnelClientService.test.ts b/apps/ade-cli/src/services/sync/syncTunnelClientService.test.ts index 52b8b0c86..59a72aa39 100644 --- a/apps/ade-cli/src/services/sync/syncTunnelClientService.test.ts +++ b/apps/ade-cli/src/services/sync/syncTunnelClientService.test.ts @@ -17,9 +17,14 @@ import { MAX_PENDING_TUNNEL_BYTES, MAX_RELAY_WEBSOCKET_FRAME_BYTES, parseControlMessage, + CONTROL_REPLACED_REARM_MS, + CONTROL_REPLACED_RETRY_BASE_MS, + MAX_CONTROL_REPLACED_REATTEMPTS, RELAY_CLOSE_BRIDGE_REJECTED, + RELAY_CLOSE_CONTROL_REPLACED, RELAY_CLOSE_FORWARD_FAILED, RELAY_CLOSE_HOST_UNAVAILABLE, + RELAY_CONTROL_REPLACED_MESSAGE, RELAY_READY_VERSION, RELAY_SELF_PROBE_DEBOUNCE_MS, } from "./syncTunnelClientService"; @@ -127,17 +132,32 @@ class StubWebSocket extends EventEmitter { describe("computeBackoffMs", () => { it("grows exponentially and caps at 60s", () => { const max = (attempt: number) => computeBackoffMs(attempt, () => 0.999999); - expect(max(0)).toBeLessThanOrEqual(1_000); - expect(max(1)).toBeLessThanOrEqual(2_000); + expect(max(0)).toBeLessThanOrEqual(3_000); expect(max(2)).toBeLessThanOrEqual(4_000); expect(max(3)).toBeLessThanOrEqual(8_000); expect(max(20)).toBeLessThanOrEqual(60_000); expect(max(20)).toBeGreaterThan(50_000); }); - it("applies full jitter within the ceiling", () => { - expect(computeBackoffMs(5, () => 0)).toBe(0); - expect(computeBackoffMs(5, () => 0.5)).toBe(16_000); + // Full jitter from zero is what let two brains fighting over one relay slot + // resample near-zero delays forever: neither ever stayed connected long + // enough to reset `attempt`, so the war never decayed. + it("never returns a delay below the 1s floor", () => { + for (let attempt = 0; attempt <= 20; attempt += 1) { + expect(computeBackoffMs(attempt, () => 0)).toBeGreaterThanOrEqual(1_000); + expect(computeBackoffMs(attempt, () => 0.0001)).toBeGreaterThanOrEqual(1_000); + } + }); + + it("decorrelates from the previous delay instead of resampling one narrow band", () => { + // With no history the window is the base band... + expect(computeBackoffMs(0, () => 0.999999, 0)).toBeGreaterThan(2_000); + // ...and a long previous delay widens it (prev * 3), so two clients that + // collided once are unlikely to collide again at the same instant. + expect(computeBackoffMs(0, () => 0.999999, 10_000)).toBeGreaterThan(25_000); + expect(computeBackoffMs(0, () => 0.5, 10_000)).toBe(15_500); + // Still capped. + expect(computeBackoffMs(0, () => 0.999999, 50_000)).toBeLessThanOrEqual(60_000); }); }); @@ -1150,6 +1170,9 @@ describe("createSyncTunnelClientService", () => { getRelayBridgeProof: () => null, controlPingIntervalMs: 10, controlPongDeadlineMs: 10, + // This test is about the pong deadline, not the reconnect schedule; the + // real schedule now floors at 1s and would outlast the waitFor window. + reconnectBackoffMs: () => 0, configStore: fakeStore(`http://127.0.0.1:${relayPort}`), }); @@ -1301,7 +1324,9 @@ describe("createSyncTunnelClientService", () => { try { await service.start(); expect(fetchMock).toHaveBeenCalledOnce(); - await vi.advanceTimersByTimeAsync(900); + // First attempt with no history: floor 1s, ceiling 3s, random 0.999 → + // ~2998ms. The 5ms lease poll must not shortcut it. + await vi.advanceTimersByTimeAsync(2_900); expect(fetchMock).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(100); expect(fetchMock).toHaveBeenCalledTimes(2); @@ -1592,6 +1617,8 @@ describe("createSyncTunnelClientService", () => { getExpectedLoopbackNonce: () => "f".repeat(32), getRelayBridgeProof: () => "e".repeat(43), configStore, + // Identity rotation, not backoff, is under test here. + reconnectBackoffMs: () => 0, loopbackProbe: async (port, expectedNonce) => ({ ok: true, port, @@ -2663,3 +2690,235 @@ describe("createSyncTunnelClientService", () => { } }); }); + +describe("relay control eviction (close 4505)", () => { + const REARM_MS = 5 * CONTROL_REPLACED_RETRY_BASE_MS; + const createEvictedService = (sockets: StubWebSocket[]) => + createSyncTunnelClientService({ + getSyncPort: () => null, + getRelayBridgeProof: () => null, + configStore: fakeStore(), + controlReplacedRearmMs: REARM_MS, + createWebSocket: (url) => { + const socket = new StubWebSocket(url); + sockets.push(socket); + return socket as unknown as WebSocket; + }, + }); + + it("does not redial immediately after being replaced by another process", async () => { + vi.useFakeTimers(); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 204 }); + const sockets: StubWebSocket[] = []; + const service = createEvictedService(sockets); + + try { + await service.start(); + sockets[0]!.open(); + sockets[0]!.remoteClose(RELAY_CLOSE_CONTROL_REPLACED, "replaced by newer host"); + await vi.advanceTimersByTimeAsync(0); + + // The old schedule redialed within milliseconds, which evicted the rival + // right back — the ~4s war that left relay dead for both processes. + expect(sockets).toHaveLength(1); + await vi.advanceTimersByTimeAsync(30_000); + expect(sockets).toHaveLength(1); + + expect(service.getStatus()).toMatchObject({ + connected: false, + controlSuppressed: true, + controlSuppressedReason: RELAY_CONTROL_REPLACED_MESSAGE, + }); + } finally { + await service.dispose(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); + + it("retries on the 60s floor and stops entirely after the re-attempt budget", async () => { + vi.useFakeTimers(); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 204 }); + // Zero jitter so the retry lands exactly on the floor. + const random = vi.spyOn(Math, "random").mockReturnValue(0); + const sockets: StubWebSocket[] = []; + const service = createEvictedService(sockets); + + try { + await service.start(); + for (let round = 0; round <= MAX_CONTROL_REPLACED_REATTEMPTS; round += 1) { + const before = sockets.length; + const socket = sockets[before - 1]!; + socket.open(); + socket.remoteClose(RELAY_CLOSE_CONTROL_REPLACED, "replaced by newer host"); + await vi.advanceTimersByTimeAsync(CONTROL_REPLACED_RETRY_BASE_MS - 1_000); + // Nothing before the floor, on every single round. + expect(sockets).toHaveLength(before); + await vi.advanceTimersByTimeAsync(1_001); + if (round < MAX_CONTROL_REPLACED_REATTEMPTS) { + expect(sockets).toHaveLength(before + 1); + } + } + + // 1 initial dial + MAX_CONTROL_REPLACED_REATTEMPTS retries, then silence + // until the re-arm interval. The final round already consumed + // CONTROL_REPLACED_RETRY_BASE_MS of it. + expect(sockets).toHaveLength(MAX_CONTROL_REPLACED_REATTEMPTS + 1); + await vi.advanceTimersByTimeAsync(REARM_MS - CONTROL_REPLACED_RETRY_BASE_MS - 1_000); + expect(sockets).toHaveLength(MAX_CONTROL_REPLACED_REATTEMPTS + 1); + expect(service.getStatus().controlSuppressedReason).toBe(RELAY_CONTROL_REPLACED_MESSAGE); + + // Then it tries again: the rival usually exits, and giving up forever + // would leave relay dead while the UI claims quitting it brings it back. + await vi.advanceTimersByTimeAsync(1_001); + expect(sockets).toHaveLength(MAX_CONTROL_REPLACED_REATTEMPTS + 2); + expect(service.getStatus().controlSuppressed).toBe(false); + } finally { + await service.dispose(); + random.mockRestore(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); + + it("reports one coarse analytics event per suppression episode and no relay identifiers", async () => { + vi.useFakeTimers(); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 204 }); + const random = vi.spyOn(Math, "random").mockReturnValue(0); + const sockets: StubWebSocket[] = []; + const captureAnalytics = vi.fn(); + const service = createSyncTunnelClientService({ + getSyncPort: () => null, + getRelayBridgeProof: () => null, + configStore: fakeStore(), + controlReplacedRearmMs: REARM_MS, + captureAnalytics, + createWebSocket: (url) => { + const socket = new StubWebSocket(url); + sockets.push(socket); + return socket as unknown as WebSocket; + }, + }); + + try { + await service.start(); + for (let round = 0; round <= MAX_CONTROL_REPLACED_REATTEMPTS; round += 1) { + sockets[sockets.length - 1]!.open(); + sockets[sockets.length - 1]!.remoteClose(RELAY_CLOSE_CONTROL_REPLACED, "replaced by newer host"); + await vi.advanceTimersByTimeAsync(CONTROL_REPLACED_RETRY_BASE_MS + 1); + } + + // Four evictions, ONE event: the product fact is the episode, not the + // retry. Raw close reasons, the relay URL and the machineKey stay local. + expect(captureAnalytics).toHaveBeenCalledTimes(1); + const captured = captureAnalytics.mock.calls[0]![0] as Record; + expect(captured).toMatchObject({ + event: "ade_relay_suppressed", + surface: "api", + properties: { attempt: MAX_CONTROL_REPLACED_REATTEMPTS + 1, code: "control_replaced" }, + }); + expect(Object.keys(captured.properties as object).sort()).toEqual(["attempt", "code"]); + expect(JSON.stringify(captured)).not.toContain("replaced by newer host"); + expect(JSON.stringify(captured)).not.toContain(service.getStatus().machineKey); + } finally { + await service.dispose(); + random.mockRestore(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); + + it("re-arms and dials promptly once this process wins the sync host lease", async () => { + vi.useFakeTimers(); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 204 }); + const sockets: StubWebSocket[] = []; + const service = createEvictedService(sockets); + + try { + await service.start(); + sockets[0]!.open(); + sockets[0]!.remoteClose(RELAY_CLOSE_CONTROL_REPLACED, "replaced by newer host"); + await vi.advanceTimersByTimeAsync(0); + expect(service.getStatus().controlSuppressed).toBe(true); + expect(sockets).toHaveLength(1); + + service.clearControlSuppression(); + await vi.advanceTimersByTimeAsync(0); + + expect(sockets).toHaveLength(2); + expect(service.getStatus()).toMatchObject({ + controlSuppressed: false, + controlSuppressedReason: null, + }); + } finally { + await service.dispose(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); + + it("treats an ordinary close as a network drop, not an eviction", async () => { + vi.useFakeTimers(); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 204 }); + const sockets: StubWebSocket[] = []; + const service = createSyncTunnelClientService({ + getSyncPort: () => null, + getRelayBridgeProof: () => null, + configStore: fakeStore(), + reconnectBackoffMs: () => 0, + createWebSocket: (url) => { + const socket = new StubWebSocket(url); + sockets.push(socket); + return socket as unknown as WebSocket; + }, + }); + + try { + await service.start(); + sockets[0]!.open(); + sockets[0]!.remoteClose(1006, "connection reset"); + await vi.advanceTimersByTimeAsync(10); + + expect(sockets).toHaveLength(2); + expect(service.getStatus().controlSuppressed).toBe(false); + } finally { + await service.dispose(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); + + it("reports a control outage start that survives repeated retries", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-29T00:00:00.000Z")); + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(null, { status: 503 }); + const service = createSyncTunnelClientService({ + getSyncPort: () => null, + getRelayBridgeProof: () => null, + configStore: fakeStore(), + reconnectBackoffMs: () => 1_000, + }); + + try { + await service.start(); + const failingSince = service.getStatus().controlFailingSinceMs; + expect(failingSince).toBe(Date.parse("2026-07-29T00:00:00.000Z")); + + // lastFailureAt restamps on every retry; the outage start must not, or + // the desktop banner can never tell a flap from a dead relay. + await vi.advanceTimersByTimeAsync(5_000); + expect(service.getStatus().controlFailingSinceMs).toBe(failingSince); + expect(service.getStatus().lastFailureAt).not.toBe(null); + } finally { + await service.dispose(); + globalThis.fetch = originalFetch; + vi.useRealTimers(); + } + }); +}); diff --git a/apps/ade-cli/src/services/sync/syncTunnelClientService.ts b/apps/ade-cli/src/services/sync/syncTunnelClientService.ts index 55fd3ee88..2f6c0a328 100644 --- a/apps/ade-cli/src/services/sync/syncTunnelClientService.ts +++ b/apps/ade-cli/src/services/sync/syncTunnelClientService.ts @@ -45,6 +45,11 @@ export type SyncTunnelClientStatus = { relayEndToEndRoundTripMs: number | null; relayUrl: string; machineKey: string; + /** True while the client is deliberately not redialing after a 4505 eviction. */ + controlSuppressed?: boolean; + controlSuppressedReason?: string | null; + /** Epoch ms the current uninterrupted control outage began; null when connected. */ + controlFailingSinceMs?: number | null; }; export type SyncTunnelClientService = { @@ -65,6 +70,12 @@ export type SyncTunnelClientService = { validateCurrentBridge(): Promise; /** Dial Relay exactly like a ready-v2 controller and verify bridge readiness. */ runSelfProbe(): Promise<{ ok: boolean; detail: string }>; + /** + * Re-arm after a 4505 eviction suppressed redialing. The caller must have + * established that this process is the legitimate relay owner for the + * machine — in practice, that it just acquired the sync host lease. + */ + clearControlSuppression(): void; getStatus(): SyncTunnelClientStatus; dispose(): Promise; }; @@ -107,6 +118,20 @@ type SyncTunnelClientArgs = { controlJsonPingIntervalMs?: number; controlJsonPongDeadlineMs?: number; controlReadyStableMs?: number; + /** Test seam for the post-eviction re-arm interval. */ + controlReplacedRearmMs?: number; + /** + * Edge-triggered product signal, emitted once per suppression episode when + * this machine gives up on relay because another ADE process owns it. Coarse + * counters only — never a URL, machineKey, or close reason. + */ + captureAnalytics?: (input: { + event: "ade_relay_suppressed"; + surface: "api"; + properties: { attempt: number; code: string }; + dedupeKey: string; + minimumIntervalMs: number; + }) => void; reconnectBackoffMs?: (attempt: number) => number; createWebSocket?: ( url: string, @@ -120,6 +145,13 @@ type SyncTunnelClientArgs = { const BACKOFF_BASE_MS = 1_000; const BACKOFF_CAP_MS = 60_000; +/** + * Never redial faster than this. The old schedule was full jitter from ZERO, + * so two brains fighting over one relay slot both resampled near-zero delays + * and evicted each other forever without either ever reaching the stability + * window that resets `attempt`. + */ +const BACKOFF_MIN_MS = 1_000; const ACCOUNT_STATUS_POLL_MS = 1_000; export const CONTROL_PING_INTERVAL_MS = 30_000; export const CONTROL_PONG_DEADLINE_MS = 10_000; @@ -128,8 +160,34 @@ export const CONTROL_JSON_PONG_DEADLINE_MS = 30_000; export const CONTROL_JSON_INITIAL_PING_DELAY_MS = 1_000; export const RELAY_CLOSE_PARTNER_CLOSED = 4000; export const RELAY_CLOSE_HOST_UNAVAILABLE = 4501; +/** + * The relay Durable Object keeps ONE host control socket per machineKey and + * evicts the previous holder with this code. It therefore never means "the + * network dropped" — it means another process registered the same machine. + */ +export const RELAY_CLOSE_CONTROL_REPLACED = 4505; export const RELAY_CLOSE_BRIDGE_REJECTED = 4507; export const RELAY_CLOSE_FORWARD_FAILED = 4509; +export const RELAY_CONTROL_REPLACED_MESSAGE = + "Another ADE process owns the relay connection for this machine."; +/** Fixed floor for post-eviction redials; jittered up, never down. */ +export const CONTROL_REPLACED_RETRY_BASE_MS = 60_000; +/** + * Evictions tolerated before the client stops dialing. The budget counts only + * evictions that never recovered — a control socket that reaches the stability + * window resets it — so this bounds one continuous war, not a lifetime. + */ +export const MAX_CONTROL_REPLACED_REATTEMPTS = 3; +/** + * How long a stopped client waits before trying once more. + * + * Giving up permanently would be wrong: the rival process usually exits + * (a dev brain is killed, an old app quits) and nothing else would ever redial, + * leaving relay dead until a restart while the UI tells the user that quitting + * the other process fixes it. Re-arming on a long timer makes that advice true + * at a cost of three dials per interval, nowhere near the ~4s war. + */ +export const CONTROL_REPLACED_REARM_MS = 10 * 60_000; export const RELAY_SIGN_IN_REQUIRED_MESSAGE = "Sign in to ADE to use ADE Relay."; export const BRIDGE_VALIDATION_LEASE_MS = 2_000; export const CONTROL_READY_STABLE_MS = 5_000; @@ -181,12 +239,28 @@ type RelaySelfProbeState = { }; /** - * Exponential backoff with full jitter, capped at 60s. Exposed for tests so the - * reconnect schedule is verifiable without waiting on real timers. + * Exponential backoff with DECORRELATED jitter and a 1s floor, capped at 60s. + * Exposed for tests so the reconnect schedule is verifiable without waiting on + * real timers. + * + * `previousDelayMs` is the delay actually used for the last reconnect; passing + * it widens the sampling window (`prev * 3`) instead of resampling the same + * narrow exponential band, so two clients that collide once do not keep + * colliding at the same moment. */ -export function computeBackoffMs(attempt: number, random: () => number = Math.random): number { - const ceiling = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt)); - return Math.floor(random() * ceiling); +export function computeBackoffMs( + attempt: number, + random: () => number = Math.random, + previousDelayMs = 0, +): number { + const exponential = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt)); + const previous = Number.isFinite(previousDelayMs) && previousDelayMs > 0 + ? previousDelayMs + : BACKOFF_MIN_MS; + // `previous` is at least BACKOFF_MIN_MS, so the ceiling is always above the + // floor and no clamp is needed here. + const ceiling = Math.min(BACKOFF_CAP_MS, Math.max(exponential, previous * 3)); + return Math.floor(BACKOFF_MIN_MS + random() * (ceiling - BACKOFF_MIN_MS)); } type ControlOpenMessage = { @@ -292,6 +366,19 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT let accountLeaseUserId: string | null = args.getAccountLease ? null : "legacy"; let accountEligible: boolean | null = null; let attempt = 0; + let lastBackoffDelayMs = 0; + // Relay eviction bookkeeping (close code 4505). Kept separate from `attempt` + // because an eviction is a machine-local ownership conflict, not a network + // failure, and must not be retried on the network schedule. + let controlReplacedAttempts = 0; + let controlReplacedStopped = false; + let controlSuppressedReason: string | null = null; + let controlReplacedRearmTimer: NodeJS.Timeout | null = null; + // Identifies one continuous suppression episode, so the analytics dedupe key + // collapses every eviction in a single war into one accepted event while a + // genuinely new episode (after a recovery) still reports. + let controlSuppressionEpisodeId = 0; + let controlFailingSinceMs: number | null = null; let started = false; let stopped = false; let connected = false; @@ -358,6 +445,11 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT const recordFailure = (reason: string): void => { lastError = reason; lastFailureAt = new Date().toISOString(); + // First failure of an uninterrupted outage. `lastFailureAt` restamps on + // every retry, so it can never answer "how long has relay been down" — + // which is exactly what the desktop banner needs to stay quiet through + // ordinary reconnects. + if (!connected) controlFailingSinceMs ??= Date.now(); }; const requestPublicationStatePublish = ( @@ -526,17 +618,25 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT } }; - const scheduleReconnect = (): void => { + const scheduleReconnect = (options: { delayMs?: number } = {}): void => { if ( stopped + || controlReplacedStopped || reconnectTimer || control || connectingControl || !accountSignedIn() ) return; - const computedDelay = args.reconnectBackoffMs?.(attempt) ?? computeBackoffMs(attempt); - const delay = Number.isFinite(computedDelay) ? Math.max(0, Math.floor(computedDelay)) : BACKOFF_CAP_MS; + // `reconnectBackoffMs` is the test seam for every reconnect delay, + // including the post-eviction one, so it stays outermost. + const computedDelay = args.reconnectBackoffMs?.(attempt) + ?? options.delayMs + ?? computeBackoffMs(attempt, Math.random, lastBackoffDelayMs); + const delay = Number.isFinite(computedDelay) + ? Math.max(0, Math.floor(computedDelay)) + : BACKOFF_CAP_MS; attempt += 1; + lastBackoffDelayMs = delay; reconnectTimer = setTimeout(() => { reconnectTimer = null; void connectControl(); @@ -544,6 +644,106 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT reconnectTimer.unref?.(); }; + /** + * Forget the eviction regime. One definition, used by every reset site, so + * the "which subset does this one clear" divergence cannot creep back in. + */ + const resetControlEvictionState = (): void => { + // Recovery is as invisible as the failure was, so it publishes too. Sits + // here rather than in clearControlSuppression so the markStable path — a + // war that healed on its own — notifies as well. + const wasSuppressed = controlSuppressedReason != null || controlReplacedStopped; + if (controlReplacedAttempts > 0) controlSuppressionEpisodeId += 1; + controlReplacedAttempts = 0; + controlReplacedStopped = false; + controlSuppressedReason = null; + if (controlReplacedRearmTimer) { + clearTimeout(controlReplacedRearmTimer); + controlReplacedRearmTimer = null; + } + if (lastError === RELAY_CONTROL_REPLACED_MESSAGE) lastError = null; + if (wasSuppressed && !stopped) requestPublicationStatePublish("route-state-changed"); + }; + + /** + * Handle a relay eviction. Another ADE process on this machine registered the + * same machineKey, so redialing immediately just evicts it right back — the + * exact loop that made relay permanently unusable while two brains ran. Back + * off hard, give up after a few tries, and say why in the status. + */ + const noteControlReplaced = (): void => { + controlReplacedAttempts += 1; + const wasSuppressed = controlSuppressedReason != null; + controlSuppressedReason = RELAY_CONTROL_REPLACED_MESSAGE; + // Route state just changed in a way the UI must see. Without this the + // desktop banner stays hidden on an idle machine until some unrelated sync + // activity happens to emit a status snapshot — which is precisely the + // silence this whole change exists to end. + if (!wasSuppressed) requestPublicationStatePublish("route-state-changed"); + // Keep the raw `Relay control closed (4505): …` in lastControlError as the + // protocol-level diagnostic; lastError carries the actionable sentence. + recordFailure(RELAY_CONTROL_REPLACED_MESSAGE); + if (controlReplacedAttempts > MAX_CONTROL_REPLACED_REATTEMPTS) { + controlReplacedStopped = true; + clearReconnect(); + log.warn?.("sync_tunnel.control_replaced_stopped", { + attempts: controlReplacedAttempts, + reason: RELAY_CONTROL_REPLACED_MESSAGE, + rearmInMs: args.controlReplacedRearmMs ?? CONTROL_REPLACED_REARM_MS, + }); + // One event per episode, not per eviction: this is the coarse product + // fact "a machine lost relay to a rival ADE process", which is the only + // way to see in the field whether multi-brain hygiene actually holds. + try { + args.captureAnalytics?.({ + event: "ade_relay_suppressed", + surface: "api", + properties: { attempt: controlReplacedAttempts, code: "control_replaced" }, + dedupeKey: `relay-suppressed:${controlSuppressionEpisodeId}`, + minimumIntervalMs: 24 * 60 * 60 * 1_000, + }); + } catch { + // Analytics must never affect relay behavior. + } + controlReplacedRearmTimer ??= (() => { + const timer = setTimeout(() => { + controlReplacedRearmTimer = null; + clearControlSuppression(); + }, args.controlReplacedRearmMs ?? CONTROL_REPLACED_REARM_MS); + timer.unref?.(); + return timer; + })(); + return; + } + // Fixed 60s floor, jittered upward only, so two evicting processes cannot + // land on the same retry instant. + const delayMs = CONTROL_REPLACED_RETRY_BASE_MS + + Math.floor(Math.random() * CONTROL_REPLACED_RETRY_BASE_MS); + log.warn?.("sync_tunnel.control_replaced", { + attempt: controlReplacedAttempts, + retryInMs: delayMs, + }); + scheduleReconnect({ delayMs }); + }; + + /** + * Re-arm after suppression. Called when this process (re)acquires the sync + * host lease — at that point it is the legitimate owner and the rival, if + * any, has been gated out. + */ + const clearControlSuppression = (): void => { + if (controlReplacedAttempts === 0 && !controlSuppressedReason && !controlReplacedStopped) return; + const wasStopped = controlReplacedStopped; + resetControlEvictionState(); + attempt = 0; + lastBackoffDelayMs = 0; + log.info?.("sync_tunnel.control_suppression_cleared", { wasStopped }); + if (!stopped && started && !control && !connectingControl) { + clearReconnect(); + void connectControl(); + } + }; + const claimOnce = async (id: MachineIdentity): Promise => { const relayOrigin = relayHttpUrl().replace(/\/+$/, ""); if ( @@ -608,6 +808,10 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT const connectControl = async (): Promise => { if (stopped || control || connectingControl || reconnectTimer) return; + // Suppression has to hold at the dial itself, not just in the reconnect + // scheduler: the account-lease poll runs every second and calls this + // directly, which would redial straight back into the eviction war. + if (controlReplacedStopped) return; if (!accountSignedIn()) { lastError = RELAY_SIGN_IN_REQUIRED_MESSAGE; return; @@ -710,6 +914,7 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT clearReconnect(); connected = true; lastError = null; + controlFailingSinceMs = null; controlOpenedAtMs = Date.now(); lastControlOpenAt = new Date().toISOString(); socketLivenessStop = armControlLiveness( @@ -802,7 +1007,17 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT || validatedBridgeKey !== bridgeValidationIdentity().key ) return; attempt = 0; + // Decorrelated jitter samples from the PREVIOUS delay, so leaving + // this at a 60s cap would make the first drop after a healthy + // session wait up to a minute instead of a second. + lastBackoffDelayMs = 0; lastControlError = null; + // A control socket that survived the stability window proves the + // eviction war is over. Without this the budget only ever counted + // up, so four self-healed evictions over a long-running brain would + // stop relay for good — and `ade doctor` would keep reporting a + // healthy relay as owned by another process, forever. + resetControlEvictionState(); log.info?.("sync_tunnel.control_ready", { machineKey: id.machineKey, controlEpoch: socketTransportMode === "epoch" ? controlEpoch : null, @@ -943,6 +1158,10 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT socketState.failureReason = closeReason; recordControlFailure(closeReason); } + if (code === RELAY_CLOSE_CONTROL_REPLACED) { + noteControlReplaced(); + return; + } scheduleReconnect(); } }); @@ -1706,6 +1925,10 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT async stop(): Promise { stopped = true; started = false; + // A stop/start cycle is how the lease gate hands relay ownership between + // runtimes, so it must not inherit the previous owner's eviction state. + resetControlEvictionState(); + lastBackoffDelayMs = 0; detachHostListener?.(); detachHostListener = null; hostListener = null; @@ -1720,6 +1943,8 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT validateCurrentBridge, + clearControlSuppression, + async runSelfProbe(): Promise<{ ok: boolean; detail: string }> { if ( stopped @@ -1789,6 +2014,9 @@ export function createSyncTunnelClientService(args: SyncTunnelClientArgs): SyncT relayEndToEndRoundTripMs: eligible ? currentSelfProbeRoundTripMs : null, relayUrl: relayHttpUrl(), machineKey, + controlSuppressed: controlSuppressedReason != null, + controlSuppressedReason, + controlFailingSinceMs: connected ? null : controlFailingSinceMs, }; }, diff --git a/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts b/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts index ad30312b5..916b4d7a7 100644 --- a/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts +++ b/apps/ade-cli/src/tuiClient/__tests__/connection.test.ts @@ -26,7 +26,9 @@ import type { AgentChatEventEnvelope } from "../../../../desktop/src/shared/type import type { ProjectLaunchContext } from "../types"; const childProcess = vi.hoisted(() => { - const child = { unref: vi.fn() }; + // `pid` stays undefined by default so the spawn record is a no-op for the + // tests that only assert argv shape; the duplicate-spawn test sets a live pid. + const child = { unref: vi.fn(), pid: undefined as number | undefined }; return { child, spawn: vi.fn(() => child), @@ -162,6 +164,7 @@ describe("connectToAde embedded mode", () => { embedded.createAdeRpcRequestHandler.mockClear(); childProcess.spawn.mockClear(); childProcess.child.unref.mockClear(); + childProcess.child.pid = undefined; childProcess.spawn.mockImplementation(() => childProcess.child); runtimeService.installRuntimeService.mockClear(); runtimeService.installRuntimeService.mockReturnValue({ @@ -727,6 +730,41 @@ describe("connectToAde embedded mode", () => { expect(fs.existsSync(lockPath)).toBe(false); }); + it("does not spawn a second brain while a recently spawned one is still coming up", async () => { + // The spawn lock only serializes the first attempt. A brain that has not yet + // bound its socket must not attract a rival spawn from the next `ade code`, + // or a burst of launches leaves a burst of immortal brains. + const socketPath = useMissingMachineSocket(); + childProcess.child.pid = process.pid; + mockAttachedClient(); + + const first = await connectToAde({ project }); + await first.close(); + expect(childProcess.spawn).toHaveBeenCalledTimes(1); + + // Socket still absent, so this launch takes the same spawn path. + expect(fs.existsSync(socketPath)).toBe(false); + const second = await connectToAde({ project }); + await second.close(); + + expect(childProcess.spawn).toHaveBeenCalledTimes(1); + }); + + it("spawns again once the recorded brain is gone", async () => { + const socketPath = useMissingMachineSocket(); + // A pid that cannot be alive: the record must not suppress the replacement. + childProcess.child.pid = 0x7fffffff; + mockAttachedClient(); + + const first = await connectToAde({ project }); + await first.close(); + const second = await connectToAde({ project }); + await second.close(); + + expect(fs.existsSync(socketPath)).toBe(false); + expect(childProcess.spawn).toHaveBeenCalledTimes(2); + }); + it("unlinks stale machine socket files before retrying daemon startup", async () => { const socketPath = useMissingMachineSocket(); fs.mkdirSync(path.dirname(socketPath), { recursive: true }); diff --git a/apps/ade-cli/src/tuiClient/connection.ts b/apps/ade-cli/src/tuiClient/connection.ts index 1c739369c..c01e12acc 100644 --- a/apps/ade-cli/src/tuiClient/connection.ts +++ b/apps/ade-cli/src/tuiClient/connection.ts @@ -6,6 +6,12 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { resolveAdeLayout } from "../../../desktop/src/shared/adeLayout"; import { resolveMachineAdeLayout } from "../services/projects/machineLayout"; +import { + clearRuntimeSpawnRecord, + hasRecentRuntimeSpawn, + recordRuntimeSpawn, +} from "../services/runtime/runtimeSpawnRecord"; +import { withSocketSpawnLock } from "../services/runtime/socketSpawnLock"; import { JsonRpcClient } from "./jsonRpcClient"; import type { AdeCodeConnection, ProjectLaunchContext, RuntimeEventGapMetadata } from "./types"; import type { AgentChatEventEnvelope } from "../../../desktop/src/shared/types/chat"; @@ -534,6 +540,14 @@ function attachedRuntimeMismatchReason( } function spawnDaemon(socketPath: string): boolean { + // `ade code` has its own spawn path, so the machine CLI's spawn record is the + // only thing that stops the two from racing each other into duplicate brains. + // The spawn lock below only serializes the *first* attempt: when a cold brain + // takes longer to bind than tryDaemon's ~4.7s retry budget (routine on a large + // repo), the lock is released, the recovery path spawns a second brain, and a + // concurrent `ade code` spawns a third. Report success so the caller still + // runs its connect-with-retry against the brain that is already coming up. + if (hasRecentRuntimeSpawn(socketPath)) return true; const cliEntrypoint = resolveCliEntrypoint(); const buildHash = computeCliEntrypointBuildHash(); const nodeArgs = @@ -559,83 +573,11 @@ function spawnDaemon(socketPath: string): boolean { env, }, ); + if (child.pid != null) recordRuntimeSpawn(socketPath, child.pid); child.unref(); return true; } -type SocketSpawnLockOwner = { - id: string | null; - pid: number | null; -}; - -function createSocketSpawnLockOwner(): SocketSpawnLockOwner { - return { - id: `${process.pid}:${Date.now()}:${Math.random().toString(36).slice(2)}`, - pid: process.pid, - }; -} - -function serializeSocketSpawnLockOwner(owner: SocketSpawnLockOwner): string { - return JSON.stringify({ - id: owner.id, - pid: owner.pid, - createdAt: new Date().toISOString(), - }); -} - -function readSocketSpawnLockOwner(lockPath: string): SocketSpawnLockOwner { - const raw = fs.readFileSync(lockPath, "utf8"); - try { - const parsed = JSON.parse(raw) as { id?: unknown; pid?: unknown }; - return { - id: typeof parsed.id === "string" ? parsed.id : null, - pid: typeof parsed.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 - ? parsed.pid - : null, - }; - } catch { - const [pidLine] = raw.split(/\r?\n/u); - const pid = Number.parseInt(pidLine ?? "", 10); - return { - id: null, - pid: Number.isInteger(pid) && pid > 0 ? pid : null, - }; - } -} - -function processExists(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "EPERM"; - } -} - -function unlinkSocketSpawnLockIfStale(lockPath: string): boolean { - try { - const stat = fs.statSync(lockPath); - const owner = readSocketSpawnLockOwner(lockPath); - if (owner.pid != null && processExists(owner.pid)) return false; - if (owner.pid == null && Date.now() - stat.mtimeMs <= 30_000) return false; - fs.unlinkSync(lockPath); - return true; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ENOENT"; - } -} - -function unlinkSocketSpawnLockIfOwner(lockPath: string, ownerId: string | null): void { - if (!ownerId) return; - try { - const owner = readSocketSpawnLockOwner(lockPath); - if (owner.id !== ownerId) return; - fs.unlinkSync(lockPath); - } catch { - // ignore cleanup races - } -} - async function probeLocalSocketLiveness(socketPath: string): Promise<"live" | "stale" | "unknown"> { if (socketPath.startsWith("tcp://")) return "unknown"; return await new Promise((resolve) => { @@ -675,40 +617,6 @@ async function unlinkStaleLocalSocket(socketPath: string): Promise { } } -async function withSocketSpawnLock(socketPath: string, task: () => Promise): Promise { - if (socketPath.startsWith("tcp://")) return await task(); - const lockPath = path.join(path.dirname(socketPath), `${path.basename(socketPath)}.spawn.lock`); - fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); - const deadline = Date.now() + 10_000; - const owner = createSocketSpawnLockOwner(); - let fd: number | null = null; - while (fd == null) { - try { - fd = fs.openSync(lockPath, "wx", 0o600); - fs.writeFileSync(fd, serializeSocketSpawnLockOwner(owner), "utf8"); - break; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code !== "EEXIST") throw error; - if (unlinkSocketSpawnLockIfStale(lockPath)) continue; - if (Date.now() >= deadline) { - throw new Error(`Timed out waiting for ADE socket spawn lock at ${lockPath}.`); - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - } - - try { - return await task(); - } finally { - if (fd != null) { - try { - fs.closeSync(fd); - } catch {} - } - unlinkSocketSpawnLockIfOwner(lockPath, owner.id); - } -} async function connectAttachedSocket(args: { socketPath: string; @@ -1025,6 +933,11 @@ export async function connectToAde(args: { return await tryDaemon(1); } catch (firstError) { if (firstError instanceof StaleAdeSocketError) { + // tryDaemon ran with shutdownOnStale, so that brain was just asked to + // exit. Its pid can outlive the request by a moment, and if it was one + // we spawned the record would suppress the replacement this path exists + // to start. + clearRuntimeSpawnRecord(machineSocketPath); await new Promise((resolve) => setTimeout(resolve, 200)); } const repaired = await repairService(); diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts index 37ce75578..7f81baa4f 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts @@ -19,7 +19,7 @@ export const INTERNAL_ONLY_EVENTS = new Set([ "ade_update_install_aborted", "ade_update_quit_escalated", "ade_update_install_did_not_land", "ade_update_auto_applied", "ade_update_auto_apply_cancelled", - "ade_brain_recovered", "ade_publish_failing", + "ade_brain_recovered", "ade_publish_failing", "ade_relay_suppressed", ]); export const EVENT_DAILY_BUDGETS: Record = { @@ -42,6 +42,7 @@ export const EVENT_DAILY_BUDGETS: Record = { ade_update_prompted: 10, ade_brain_recovered: 10, ade_publish_failing: 10, + ade_relay_suppressed: 10, }; export const EVENT_MINUTE_BUDGETS: Record = { @@ -64,6 +65,7 @@ export const EVENT_MINUTE_BUDGETS: Record = { ade_update_prompted: 3, ade_brain_recovered: 3, ade_publish_failing: 3, + ade_relay_suppressed: 3, }; const STRING_PROPERTIES = new Set([ @@ -126,6 +128,7 @@ const EVENT_PROPERTY_KEYS: Record ade_update_prompted: new Set(["from_version", "to_version", "user_action"]), ade_brain_recovered: new Set(["blocked_ms", "last_command"]), ade_publish_failing: new Set(["failing_minutes", "leg", "code"]), + ade_relay_suppressed: new Set(["attempt", "code"]), }; const SLUG_VALUE = /^[a-z0-9][a-z0-9._+-]*$/i; diff --git a/apps/desktop/src/renderer/components/app/AppShell.tsx b/apps/desktop/src/renderer/components/app/AppShell.tsx index 1ff4a5991..fabdf47ba 100644 --- a/apps/desktop/src/renderer/components/app/AppShell.tsx +++ b/apps/desktop/src/renderer/components/app/AppShell.tsx @@ -38,6 +38,8 @@ import type { ProjectInfo, OpenProjectBinding, RemoteRuntimeConnectionSnapshot, + SyncRoleSnapshot, + SyncRouteHealth, TerminalSessionSummary, } from "../../../shared/types"; import { @@ -457,6 +459,59 @@ export function AppShell({ children }: { children: React.ReactNode }) { }; }, []); + // Relay leg of THIS machine's sync route health, for the relay-offline banner. + // Push-only: one seed read plus the existing `sync-status` broadcast, which the + // sync host already emits whenever route health changes. No polling — the + // banner host arms a single timer for the outage grace window instead. + const [syncRelayHealth, setSyncRelayHealth] = useState(null); + useEffect(() => { + const syncApi = window.ade.sync; + if (!syncApi) return; + let cancelled = false; + const apply = (snapshot: SyncRoleSnapshot | null | undefined) => { + if (cancelled) return; + const next = snapshot?.routeHealth?.relay ?? null; + // getStatus rebuilds routeHealth.relay as a fresh object every call, so + // committing it unconditionally would re-render the whole shell on every + // peer/status push. Only the fields the banner reads matter here. + setSyncRelayHealth((prev) => ( + prev?.enabled === next?.enabled + && prev?.relayControlConnected === next?.relayControlConnected + && prev?.relayControlSuppressed === next?.relayControlSuppressed + && prev?.relayControlFailingSinceMs === next?.relayControlFailingSinceMs + && prev?.relayControlSuppressedReason === next?.relayControlSuppressedReason + && prev?.skipReason === next?.skipReason + && prev?.lastControlError === next?.lastControlError + ? prev + : next + )); + }; + // Always read the LOCAL snapshot: relay control belongs to the physical + // machine this window runs on, not to whichever runtime a remote-bound + // project routes to. + const readLocal = + typeof syncApi.getLocalStatus === "function" + ? syncApi.getLocalStatus + : syncApi.getStatus; + const refresh = () => { + if (typeof readLocal !== "function") return; + void readLocal.call(syncApi).then(apply).catch(() => {}); + }; + refresh(); + // The event is an INVALIDATION, not the payload. On a remote-bound project + // the preload subscription fans out the remote runtime's snapshot too, and + // applying that directly would let a remote outage raise a warning about + // this machine — or let remote health mask this machine's own outage. + // Same pattern useSyncConnections already uses. + const dispose = syncApi.onEvent?.((event) => { + if (event.type === "sync-status") refresh(); + }); + return () => { + cancelled = true; + dispose?.(); + }; + }, []); + useEffect(() => { const syncApi = window.ade.sync; if (!syncApi?.onEvent || !project?.rootPath || !isLanesRoute) { @@ -1304,6 +1359,7 @@ export function AppShell({ children }: { children: React.ReactNode }) { aiStatusLoaded={aiStatusLoaded && aiStatus !== null} providerMode={providerMode} aiMockProvider={Boolean(aiMockProvider)} + relayHealth={syncRelayHealth} navigate={navigate} /> ) : null} diff --git a/apps/desktop/src/renderer/components/app/IntegrationBannerHost.test.tsx b/apps/desktop/src/renderer/components/app/IntegrationBannerHost.test.tsx index f63e98e43..addf15440 100644 --- a/apps/desktop/src/renderer/components/app/IntegrationBannerHost.test.tsx +++ b/apps/desktop/src/renderer/components/app/IntegrationBannerHost.test.tsx @@ -6,6 +6,7 @@ import type { GitHubAppInstallationStatus, GitHubAppUserAuthStatus, GitHubStatus, + SyncRouteHealth, } from "../../../shared/types"; import { IntegrationBannerHost, type IntegrationBannerHostProps } from "./IntegrationBannerHost"; @@ -85,11 +86,31 @@ function baseProps(overrides: Partial = {}): Integra aiStatusLoaded: true, providerMode: "guest", aiMockProvider: false, + relayHealth: null, navigate: vi.fn() as unknown as IntegrationBannerHostProps["navigate"], ...overrides, }; } +function makeRelayHealth( + overrides: Partial = {}, +): SyncRouteHealth["relay"] { + return { + enabled: true, + relayControlConnected: false, + relayBridgeValidated: false, + lastFailureAt: null, + skipReason: null, + lastControlError: null, + lastControlOpenAt: null, + lastBridgeValidationAt: null, + relayControlSuppressed: false, + relayControlSuppressedReason: null, + relayControlFailingSinceMs: null, + ...overrides, + }; +} + describe("IntegrationBannerHost", () => { beforeEach(() => { cleanup(); @@ -163,3 +184,127 @@ describe("IntegrationBannerHost", () => { expect(stored).toContain("ai-provider:/project/a"); }); }); + +describe("IntegrationBannerHost relay-offline banner", () => { + const NOW = new Date("2026-07-20T12:00:00.000Z").getTime(); + + beforeEach(() => { + cleanup(); + window.localStorage.clear(); + // No GitHub App API → the relay banner is the only candidate in these cases. + setAdeMock(undefined); + vi.useFakeTimers(); + vi.setSystemTime(NOW); + }); + + afterEach(() => { + cleanup(); + vi.useRealTimers(); + }); + + async function renderWithRelay(relay: SyncRouteHealth["relay"] | null): Promise { + await act(async () => { + render(); + }); + } + + it("surfaces the process-conflict case immediately, with no grace period", async () => { + await renderWithRelay( + makeRelayHealth({ + relayControlSuppressed: true, + relayControlSuppressedReason: "Another ADE process owns the relay connection for this machine.", + relayControlFailingSinceMs: NOW - 1_000, + }), + ); + + expect(screen.getByText("Another ADE process owns this machine's relay connection")).toBeTruthy(); + expect( + screen.getByText(/Quit the other ADE app or brain on this machine/), + ).toBeTruthy(); + }); + + it("stays quiet while relay has only been down for 30 seconds", async () => { + await renderWithRelay(makeRelayHealth({ relayControlFailingSinceMs: NOW - 30_000 })); + + expect(screen.queryByRole("status")).toBeNull(); + }); + + it("shows the outage banner once relay has been down for five minutes", async () => { + await renderWithRelay( + makeRelayHealth({ + relayControlFailingSinceMs: NOW - 5 * 60_000, + lastControlError: "relay handshake timed out", + }), + ); + + expect(screen.getByText("ADE Relay is not connected")).toBeTruthy(); + expect(screen.getByText("relay handshake timed out")).toBeTruthy(); + }); + + it("says nothing while the relay control is connected", async () => { + await renderWithRelay( + makeRelayHealth({ + relayControlConnected: true, + relayBridgeValidated: true, + relayControlFailingSinceMs: null, + }), + ); + + expect(screen.queryByRole("status")).toBeNull(); + }); + + it("says nothing when the relay is disabled, however long it has been down", async () => { + await renderWithRelay( + makeRelayHealth({ enabled: false, relayControlFailingSinceMs: NOW - 60 * 60_000 }), + ); + + expect(screen.queryByRole("status")).toBeNull(); + }); + + it("says nothing before the first sync snapshot lands", async () => { + await renderWithRelay(null); + + expect(screen.queryByRole("status")).toBeNull(); + }); + + it("raises the banner on its own once the grace window elapses", async () => { + await renderWithRelay(makeRelayHealth({ relayControlFailingSinceMs: NOW - 30_000 })); + expect(screen.queryByRole("status")).toBeNull(); + + await act(async () => { + await vi.advanceTimersByTimeAsync(95_000); + }); + + expect(screen.getByText("ADE Relay is not connected")).toBeTruthy(); + }); + + it("keeps a dismissed outage from hiding a later process conflict", async () => { + const { rerender } = render( + , + ); + await act(async () => {}); + + await act(async () => { + screen.getByRole("button", { name: "Dismiss: ADE Relay is not connected" }).click(); + }); + expect(screen.queryByRole("status")).toBeNull(); + + await act(async () => { + rerender( + , + ); + }); + + expect(screen.getByText("Another ADE process owns this machine's relay connection")).toBeTruthy(); + }); +}); diff --git a/apps/desktop/src/renderer/components/app/IntegrationBannerHost.tsx b/apps/desktop/src/renderer/components/app/IntegrationBannerHost.tsx index 4637c47c7..3854d3a08 100644 --- a/apps/desktop/src/renderer/components/app/IntegrationBannerHost.tsx +++ b/apps/desktop/src/renderer/components/app/IntegrationBannerHost.tsx @@ -7,7 +7,9 @@ import type { GitHubStatus, PrEventPayload, ProviderMode, + SyncRouteHealth, } from "../../../shared/types"; +import { openConnectionsPanel } from "../../lib/connectionsPanel"; import { deriveGithubAccountAuthState, deriveGithubRealtimeBlock, @@ -34,6 +36,8 @@ import { Banner, type BannerAction, type BannerModel, type BannerSeverity } from * mock-provider banners are migrated from AppShell. */ +export type RelayRouteHealth = SyncRouteHealth["relay"]; + export type IntegrationBannerHostProps = { currentProjectRoot: string | null; githubStatus: GitHubStatus | null; @@ -41,6 +45,12 @@ export type IntegrationBannerHostProps = { aiStatusLoaded: boolean; providerMode: ProviderMode; aiMockProvider: boolean; + /** + * Relay leg of this machine's sync route health, pushed down from AppShell's + * `sync-status` subscription. `null` until the first snapshot lands — the + * relay banner stays silent in that window rather than guessing. + */ + relayHealth: RelayRouteHealth | null; navigate: NavigateFunction; }; @@ -49,6 +59,59 @@ const AI_SETTINGS_ROUTE = "/settings?tab=ai"; const MAX_VISIBLE_BANNERS = 2; const SEVERITY_RANK: Record = { error: 0, warning: 1, info: 2 }; +/** + * How long the relay control has to stay down before we say anything. Relay + * drops and redials constantly (sleep/wake, Wi-Fi hops, worker deploys); a + * banner on every blip would be noise. Two minutes of UNINTERRUPTED outage is + * past every normal reconnect. + */ +const RELAY_OUTAGE_GRACE_MS = 120_000; + +export type RelayOutageState = "suppressed" | "down"; + +/** + * Decide whether the relay leg is in a state worth telling the user about. + * + * `relayControlSuppressed` is immediate: it means this process deliberately + * stopped redialing because another ADE process on this machine claimed the + * same machineKey. Nothing recovers that on its own, so a grace period would + * only delay the fix. + * + * Everything else is measured from `relayControlFailingSinceMs` — the start of + * the CURRENT uninterrupted outage. `lastFailureAt` deliberately isn't used: it + * restamps on every retry, so it can never measure how long relay has been down. + */ +export function deriveRelayOutageState( + relay: RelayRouteHealth | null | undefined, + nowMs: number, +): RelayOutageState | null { + if (!relay) return null; + if (relay.enabled !== true) return null; + if (relay.relayControlConnected === true) return null; + if (relay.relayControlSuppressed === true) return "suppressed"; + const failingSince = relay.relayControlFailingSinceMs; + if (failingSince == null) return null; + return nowMs - failingSince > RELAY_OUTAGE_GRACE_MS ? "down" : null; +} + +/** + * Milliseconds until an ongoing outage becomes reportable, or null when no + * deadline is pending. Shares the predicate above so the timer and the banner + * can never disagree about what counts as an outage. + */ +export function relayGraceRemainingMs( + relay: RelayRouteHealth | null | undefined, + nowMs: number, +): number | null { + if (deriveRelayOutageState(relay, nowMs) != null) return null; + const failingSince = relay?.relayControlFailingSinceMs; + if (relay?.enabled !== true || relay.relayControlConnected === true || failingSince == null) { + return null; + } + const remaining = RELAY_OUTAGE_GRACE_MS - (nowMs - failingSince); + return remaining > 0 ? remaining : null; +} + export function IntegrationBannerHost({ currentProjectRoot, githubStatus, @@ -56,6 +119,7 @@ export function IntegrationBannerHost({ aiStatusLoaded, providerMode, aiMockProvider, + relayHealth, navigate, }: IntegrationBannerHostProps): JSX.Element | null { const dismissals = useBannerDismissals(); @@ -154,6 +218,24 @@ export function IntegrationBannerHost({ }; }, [loadAppStatus]); + // Relay outage crosses its grace threshold on wall-clock time, not on an + // incoming event, so nothing would re-render us at the two-minute mark. Arm a + // SINGLE timer for exactly the remaining grace and let it fire once. No + // polling: while relay is healthy, suppressed, or already past the threshold, + // no timer exists at all. + const [relayGraceTick, setRelayGraceTick] = useState(0); + useEffect(() => { + const remaining = relayGraceRemainingMs(relayHealth, Date.now()); + if (remaining == null) return; + const timer = setTimeout(() => setRelayGraceTick((tick) => tick + 1), remaining + 250); + return () => clearTimeout(timer); + }, [relayHealth, relayGraceTick]); + + // Not a memo: the input is the wall clock. `relayGraceTick` exists only to + // force the re-render when the grace timer fires. The result is a string or + // null, so downstream dep lists stay stable. + const relayOutage = deriveRelayOutageState(relayHealth, Date.now()); + // Clear-on-recovery: when a banner's underlying condition is HEALTHY, drop any // dismissal recorded for it so a later regression to the SAME state resurfaces a // fresh banner instead of staying suppressed under the stale fingerprint for the @@ -175,8 +257,10 @@ export function IntegrationBannerHost({ if (!(providerMode === "subscription" && aiMockProvider)) { clearDismissal(`mock-provider:${currentProjectRoot}`); } + if (relayOutage == null) clearDismissal("relay-offline"); }, [ currentProjectRoot, + relayOutage, appStatusLoaded, appAuth, appInstall, @@ -314,8 +398,44 @@ export function IntegrationBannerHost({ }); } + // 5) ADE Relay control is down (NEW). Total relay failure used to be visible + // only to `ade doctor`: phones and remote clients silently lost their + // off-LAN path while the UI looked fine. Relay identity is machine-wide, so + // the dismiss key is global (like github-app-account) rather than + // project-scoped, and the fingerprint separates the two states so dismissing + // a plain outage can't hide a later process-conflict. + if (relayOutage) { + const reason = + relayHealth?.relayControlSuppressedReason + ?? relayHealth?.skipReason + ?? relayHealth?.lastControlError + ?? null; + const suppressed = relayOutage === "suppressed"; + list.push({ + id: "relay-offline", + severity: "warning", + title: suppressed + ? "Another ADE process owns this machine's relay connection" + : "ADE Relay is not connected", + detail: suppressed + ? "ADE stopped reconnecting so the two processes don't evict each other. Quit the other ADE app or brain on this machine to get the relay back." + : (reason + ?? "Phones and remote machines can't reach this computer over the relay. Local network connections still work."), + actions: [ + { + label: "Open connections", + variant: "primary", + onClick: () => openConnectionsPanel("machines"), + }, + ], + dismiss: { key: "relay-offline", fingerprint: relayOutage }, + }); + } + return list; }, [ + relayOutage, + relayHealth, appStatusLoaded, appAuth, appInstall, diff --git a/apps/desktop/src/shared/types/productAnalytics.ts b/apps/desktop/src/shared/types/productAnalytics.ts index 056bf358e..e02347d9d 100644 --- a/apps/desktop/src/shared/types/productAnalytics.ts +++ b/apps/desktop/src/shared/types/productAnalytics.ts @@ -20,6 +20,7 @@ export const PRODUCT_ANALYTICS_EVENTS = [ "ade_update_prompted", "ade_brain_recovered", "ade_publish_failing", + "ade_relay_suppressed", ] as const; export type ProductAnalyticsEventName = (typeof PRODUCT_ANALYTICS_EVENTS)[number]; diff --git a/apps/desktop/src/shared/types/sync.ts b/apps/desktop/src/shared/types/sync.ts index d61a8a5f9..e2a421590 100644 --- a/apps/desktop/src/shared/types/sync.ts +++ b/apps/desktop/src/shared/types/sync.ts @@ -414,6 +414,21 @@ export type SyncRouteHealth = { lastControlError: string | null; lastControlOpenAt: string | null; lastBridgeValidationAt: string | null; + /** + * The relay control is deliberately NOT redialing. Set when the relay + * evicted this host with close code 4505 ("replaced by newer host"), + * which means another ADE process on this machine claimed the same + * machineKey. Redialing there is self-harm: the two processes evict each + * other in a tight loop and relay stays down for both. + */ + relayControlSuppressed?: boolean; + relayControlSuppressedReason?: string | null; + /** + * Epoch ms when the current uninterrupted control outage began; null while + * connected. Distinct from `lastFailureAt`, which restamps on every failed + * attempt and so can never measure how long relay has been down. + */ + relayControlFailingSinceMs?: number | null; }; accountDirectory: SyncAccountDirectoryHealth; }; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8f4eca6f4..5793745c5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -120,6 +120,8 @@ Product positioning and workflows live in [`docs/PRD.md`](../docs/PRD.md). This - **SSH stdio bridge (`ade rpc --stdio`)** — runs a single-session JSON-RPC runtime over stdin/stdout. This is what desktop's `RemoteConnectionPool` execs over SSH after `bootstrapRemoteRuntime` has uploaded a matching `ade-` binary. Exits when the SSH channel closes. - **Terminal client (`ade code`)** — launches the Ink + React Work chat (`apps/ade-cli/src/tuiClient/`). Defaults to attaching to the machine brain and will start it if the endpoint is missing. `ade --socket /path code` requires a specific endpoint; `ade code --embedded` keeps the in-process runtime fallback explicit. +**Brain startup ordering.** `ade serve` claims its RPC endpoint *before* entering the sync-host startup loop. The order matters: that loop retries forever by design (so mobile sync auto-recovers the moment a rival owner exits), so a brain whose socket was already owned never reached the bind check and simply lived on as a zombie — signed in, dialing the relay, and fighting the legitimate brain for the machine's relay slot. A brain that cannot own its socket has no reason to exist, so both the pre-loop claim and the later bind share one `assertBrainSocketUnowned` contract (message, cause, and the `socket_owned_by_other` code project recovery keys on) and fail fast. Symmetrically, `apps/ade-cli/src/services/runtime/runtimeSpawnRecord.ts` stops the CLI from stacking up detached brains: every `ade` command that cannot reach the brain spawns one detached-and-unref'd and forgets it, so a burst of failures used to leave a burst of immortal brains. The record (under the ADE-owned `runtime/spawns` dir, keyed by a hash of the socket path, `0600`) suppresses a duplicate spawn while a previously spawned brain is still alive and within `RUNTIME_SPAWN_RECORD_GRACE_MS` (30 s), reports success so the caller proceeds to its connect-with-retry, expires so a genuinely wedged brain never blocks recovery, and is cleared explicitly on the deliberate-shutdown path whose whole purpose is to make room for a replacement. + **Machine and multi-project RPC.** The runtime exposes runtime-scoped methods (`projects.list/add/remove/touch`, `sync.*`, `runtime/info`, `machineInfo.get`, `runtimeEvents.subscribe/unsubscribe`) directly. Project-scoped operations dispatch through `ade/actions/call` with a `projectId`. Personal chats use the separate machine methods `personalChats.call` and `personalChats.streamEvents`; they never enter project dispatch and their capability/version is advertised by `runtime/info`. Per-project services are spun up lazily by `ProjectScopeRegistry` (`apps/ade-cli/src/services/projects/projectScope.ts`) which calls `createAdeRuntime({ projectRoot, ... })` the first time a project is touched. `PersonalChatScope` (`apps/ade-cli/src/services/personalChats/personalChatScope.ts`) lazily boots a chat-only runtime under `$ADE_HOME/personal-chats`, with distinct state and scratch roots and no project-registry entry. The project registry (`projectRegistry.ts`) is the durable list of known projects; `machineLayout.ts` resolves machine-wide paths under `$ADE_HOME`. Wire formats live in `apps/ade-cli/src/multiProjectRpcServer.ts`. Runtime-event replay is backed by `apps/ade-cli/src/eventBuffer.ts`, a bounded buffer (10k events, 16 MB total, 1 MB per retained event by default) that returns `eventEpoch`, `gap`, and `oldestCursor` so clients can detect daemon restarts or evicted history. `projects.list` resolves at most 24 host-side project icons within 750 ms, with 128 KiB per-icon and 512 KiB aggregate wire caps; records outside those budgets get a null icon instead of blocking connection setup. **Runtime-side services** (under `apps/ade-cli/src/services/`): @@ -194,7 +196,7 @@ ade brain update status --text Use `ADE_VERSION=vX.Y.Z` for a pinned release or `ADE_INSTALL_DIR` to choose the destination directory. The installer defaults to `$ADE_HOME/bin/ade`; both install and `ade brain update` verify downloaded runtime assets against `SHA256SUMS`. `ade brain update` stages the next release under `$ADE_HOME/runtime/updates/`, verifies the staged binary against the staged native deps, promotes the binary/deps into place, and restarts the per-user brain service. -**Health check (`ade doctor [--online] [--text]`).** `apps/ade-cli/src/commands/doctor.ts` connects to the machine brain over the local socket (bounded ~2 s) and prints one status row (`ok` / `warn` / `fail`) per subsystem: **App** (installed desktop version from the `.app` `Info.plist` vs the latest known version — read from disk, or from GitHub with `--online`), **Brain** (running version/pid/uptime plus any build-hash or role mismatch), **Wedge history** (the most recent recovered event-loop wedge, if any), **Sync port** (whether the shared listener bound the default `8787`, and the holders of the base ports when it drifted — with no visible holder reported as exactly that, since a root-owned holder such as `tailscaled` is invisible to a user-level probe and must be checked with `tailscale serve status` / `netstat -an -p tcp`), **Publish health** (the account-directory publisher's last-leg durations and slowest leg), **Relay** (end-to-end verified vs a classified failure), and **Account** (signed-in state and source). The command exits non-zero when any row is `fail`. The row-evaluation logic (`evaluateDoctorRows`) is pure and dependency-injected so the desktop connection-doctor card and the CLI share one verdict. +**Health check (`ade doctor [--online] [--text]`).** `apps/ade-cli/src/commands/doctor.ts` connects to the machine brain over the local socket (bounded ~2 s) and prints one status row (`ok` / `warn` / `fail`) per subsystem: **App** (installed desktop version from the `.app` `Info.plist` vs the latest known version — read from disk, or from GitHub with `--online`), **Brain** (running version/pid/uptime plus any build-hash or role mismatch), **Wedge history** (the most recent recovered event-loop wedge, if any), **Sync port** (whether the shared listener bound the default `8787`, and the holders of the base ports when it drifted — with no visible holder reported as exactly that, since a root-owned holder such as `tailscaled` is invisible to a user-level probe and must be checked with `tailscale serve status` / `netstat -an -p tcp`), **Publish health** (the account-directory publisher's last-leg durations and slowest leg), **Relay** (end-to-end verified vs a classified failure — with a deliberate suppression, i.e. another ADE process on this machine owning the relay slot, outranking every other reason, since nothing downstream can succeed while it holds and no other reason tells the user what to do), and **Account** (signed-in state and source). The command exits non-zero when any row is `fail`. The row-evaluation logic (`evaluateDoctorRows`) is pure and dependency-injected so the desktop connection-doctor card and the CLI share one verdict. **Install + PATH wiring (when the desktop ships `ade`).** On macOS / Linux the desktop installer drops the launcher at `$HOME/.local/bin/ade`; on Windows it lands at `%LOCALAPPDATA%\ADE\bin\ade.cmd`. After a successful install on Windows, the packaged `.cmd` installer adds the target directory to HKCU `Environment\Path` when needed and broadcasts an environment-change notification. After a successful install on POSIX, `ensureUserBinOnShellPath` appends a marked `export PATH="$HOME/.local/bin:$PATH"` block to the user's shell rc (`.zshrc` for zsh, `.bashrc` for bash, `.profile` otherwise) iff (a) the install dir isn't already on the inherited `PATH` and (b) the file doesn't already contain the marker / line / target dir. The install IPC reply tells the renderer which profile was edited so the Settings/Onboarding UI can prompt the user to open a new terminal or `source` it. @@ -331,7 +333,7 @@ The `/open` route is the HTTPS half of the ADE deeplink scheme (`https://ade-app Four independent Cloudflare Workers, each its own npm package / lockfile / `wrangler.jsonc` with its own trust model. None is a runtime dependency of the desktop app; the brain talks to them over HTTPS/WebSocket. - **`apps/push-relay/`** — merges the bounded ADE Attention snapshots published by every signed-in brain, exposes an incremental account snapshot/ACK/presence/preferences/device API to desktop, hosted web, ADE Code, and iOS, and fans policy-selected events out as APNs alerts plus one prioritized account-wide Live Activity (Worker + one D1 database; free-plan compatible, no Durable Objects). A machine publish requires both its existing HMAC signature and a verified Clerk account token; account routes require a verified Clerk bearer token. Primary and secondary identity domains are complete, distinct issuer/JWKS/OAuth-client triples selected by exact `iss`; OAuth audience metadata must match through `aud` or `azp`, and the D1 user key is namespaced by verified issuer. `npm run deploy` separates schema/trigger health from account-auth health: it requires both binding triples and short-lived issuer-specific smoke tokens, then checks `/health` and calls a real account snapshot with each token after deployment. The relay stores bounded attention previews/destinations/acknowledgments in addition to device tokens and delivery receipts; it does not store chat transcripts or diff contents. APNs auth is an ES256 provider JWT from the `.p8` (wrangler secrets `APNS_KEY` / `APNS_KEY_ID` / `APNS_TEAM_ID`). Brain-side publisher lives at `apps/ade-cli/src/services/push/`; desktop also launches a native AppKit/SwiftUI ADE Notch helper which consumes the renderer's account snapshot through typed IPC instead of polling the relay independently. Physical-notch Macs merge the surface with the hardware cutout; other Macs keep the real ADE icon in the menu bar and open an anchored transient panel instead of a permanent imitation notch. See [features/sync-and-multi-device/push-notifications.md](./features/sync-and-multi-device/push-notifications.md). -- **`apps/tunnel-relay/`** — pipes ADE **sync** WebSocket frames between a controller and a brain when there is no direct LAN/Tailscale path (Worker + Durable Object with SQLite storage, one instance per `machineKey`, WebSocket Hibernation API). The brain holds a persistent HMAC-signed outbound control socket while the machine has a valid ADE account session; a controller dials `/connect/:machineKey`; the DO pairs it with a dedicated brain-side pipe socket and passes bytes through 1:1 with no frame wrapping, so the normal ADE hello / pairing / DPoP handshake is unchanged. Native 30-second ping / 10-second pong transport liveness is the primary keepalive; because a hibernated or wedged DO can leave the edge answering those transport pings after the machine's control registration is dead, the brain adds a low-frequency application-level `{t:"ping"}`/`{t:"pong"}` keepalive (180 s interval, 30 s deadline) to catch such "zombie" controls, and verifies the path end-to-end with a self-probe (`syncRelaySelfProbe`) that dials `/connect/:machineKey?ready=2` like a real controller. The account directory advertises a `relay` endpoint only after that self-probe round-trips (honest relay publication); an at-capacity `4503` close is treated as liveness proof, not failure. Failed bridge opens are rejected explicitly; application close codes and bounded sanitized reasons survive the phone/pipe/local boundaries. Early controller frames are bounded by both 64 frames and 256 KiB, and idle-sweep alarms run only while a client or pipe exists. Brain-side client is `apps/ade-cli/src/services/sync/syncTunnelClientService.ts`, shared one-per-machine and handed the shared sync listener by `attachHostListener()` from whichever runtime actually owns that listener (which is often not the runtime that constructed the client). There is no user relay toggle: sign-in starts and advertises Relay, while sign-out closes it. It remains the lowest-priority `relay` address candidate after LAN and Tailscale. TLS terminates at the Worker, so this is a trusted-operator plaintext path rather than end-to-end encryption; relay payload E2E encryption is planned security work. +- **`apps/tunnel-relay/`** — pipes ADE **sync** WebSocket frames between a controller and a brain when there is no direct LAN/Tailscale path (Worker + Durable Object with SQLite storage, one instance per `machineKey`, WebSocket Hibernation API). The brain holds a persistent HMAC-signed outbound control socket while the machine has a valid ADE account session; a controller dials `/connect/:machineKey`; the DO pairs it with a dedicated brain-side pipe socket and passes bytes through 1:1 with no frame wrapping, so the normal ADE hello / pairing / DPoP handshake is unchanged. Native 30-second ping / 10-second pong transport liveness is the primary keepalive; because a hibernated or wedged DO can leave the edge answering those transport pings after the machine's control registration is dead, the brain adds a low-frequency application-level `{t:"ping"}`/`{t:"pong"}` keepalive (180 s interval, 30 s deadline) to catch such "zombie" controls, and verifies the path end-to-end with a self-probe (`syncRelaySelfProbe`) that dials `/connect/:machineKey?ready=2` like a real controller. The account directory advertises a `relay` endpoint only after that self-probe round-trips (honest relay publication); an at-capacity `4503` close is treated as liveness proof, not failure. Failed bridge opens are rejected explicitly; application close codes and bounded sanitized reasons survive the phone/pipe/local boundaries. Early controller frames are bounded by both 64 frames and 256 KiB, and idle-sweep alarms run only while a client or pipe exists. Brain-side client is `apps/ade-cli/src/services/sync/syncTunnelClientService.ts`, shared one-per-machine and handed the shared sync listener by `attachHostListener()` from whichever runtime actually owns that listener (which is often not the runtime that constructed the client). Because the DO keeps exactly one host control socket per `machineKey` and evicts the previous holder with close code `4505`, dialing it is gated on holding the machine-wide sync host lease (`relayTunnelAuthorityGate` + `syncHostSingleton`, §3.4) — not on merely having a listener. A `4505` close is treated as a machine-local ownership conflict rather than a network fault: the client retries on a 60 s floor at most three times, then stops and reports `routeHealth.relay.relayControlSuppressed` with an actionable reason, re-arming once after 10 minutes or immediately when it (re)acquires the lease. Ordinary reconnect backoff uses decorrelated jitter with a 1 s floor and 60 s cap, so two clients that collide once do not keep colliding. There is no user relay toggle: sign-in starts and advertises Relay, while sign-out closes it. It remains the lowest-priority `relay` address candidate after LAN and Tailscale. TLS terminates at the Worker, so this is a trusted-operator plaintext path rather than end-to-end encryption; relay payload E2E encryption is planned security work. - **`apps/account-directory/`** — Clerk-authenticated machine directory and OAuth device-authorization bridge (Worker + D1). The machine brain publishes a health-filtered registration through `accountMachinePublisherService.ts`: a 30-second heartbeat keeps the row inside the Worker's 90-second online window, while sign-in and publish-relevant relay-route changes trigger coalesced immediate writes and reset the heartbeat deadline. The Worker scopes rows by Clerk `sub`, selects at most the 500 most recently seen machines, then returns online-first order. Machine-list responses expose separate auth and D1 durations through `Server-Timing`, including auth failures. Authentication failures return only fixed classifications such as `token expired`, `invalid issuer`, and `invalid audience`; directory clients consume at most 512 response bytes before exposing the short reason in machine-list results and publisher health. Clients attach `X-ADE-Correlation-ID`; the Worker reflects and CORS-exposes it and logs it with route, method, status, and duration so a connection attempt can be followed without recording account tokens or full endpoint URLs. Desktop, ADE Code, hosted web, and iOS use the compiled HTTPS Worker origin by default. Headless login binds each short-lived device code to a daemon secret, uses Clerk OAuth + PKCE in any browser, and atomically burns the approved token pair on redemption. Each published row also carries the machine's long-lived Ed25519 identity as `pubkey`; a same-account desktop/iOS client verifies that key during the sealed `ade-adopt-v1` handshake to adopt a machine over a direct LAN/Tailscale route (LAN → Tailscale → Relay fallback) without exposing the account bearer in plaintext — see [features/sync-and-multi-device/README.md](./features/sync-and-multi-device/README.md). - **`apps/webhook-relay/`** — the pre-existing GitHub webhook relay (different trust model and lifecycle again). See its own docs. @@ -480,6 +482,8 @@ ADE is a multi-process system on a single machine: the desktop main process, the `ptyService.create()` records `processRegistry.pid` and `processRegistry.startedAt` on the new `terminal_sessions` row's owner columns. `sessionService.reconcileStaleRunningSessions()` accepts both live owners and known local owners: rows with live local owners are left alone, rows with known but no-longer-live local owners can be swept to `detached`, and rows with unknown owner identity are preserved because they may have been synced from another machine. Dispose paths run the same ownership check before tearing down runtimes a sibling still manages. +**Machine-exclusive subsystems** use a separate mechanism, because "which process owns the machine's phone sync" is a single-winner question rather than a per-row one. `apps/ade-cli/src/services/sync/syncHostSingleton.ts` holds an advisory lock file at `$TMPDIR/ade-sync-host-.json` (override `ADE_SYNC_HOST_LOCK_PATH`) naming the owning pid, channel, project root, and bound sync port, and pairs it with a process-wide authority registry: `holdsSyncHostSingleton()` answers "is it me", and `onSyncHostSingletonAuthorityChanged()` publishes none-held → held and held → none-held transitions. The relay tunnel (`relayTunnelAuthorityGate` → `syncTunnelClientService`) and the account-directory publisher both gate on that registry, because relay registration is one control socket per `machineKey` at the Durable Object and directory publication advertises "reach me here". Merely binding a sync listener is not authority — a dev `ade serve`, a headless one-shot, or an embedded fallback can bind one without ever winning the lease. Loss of authority is honored only after `SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS` (5 s), since `ProjectScopeRegistry.performSyncHostSwitch` deactivates the outgoing sync host before activating the target and authority legitimately reads false for the width of a project switch. See [Sync and multi-device](./features/sync-and-multi-device/README.md). + Roles are open-ended strings; today's vocabulary is `desktop-main`, `ade-serve-daemon` for the brain process role, and `tui-runtime`. The desktop main process constructs the registry in `main.ts` and threads it into `ptyService`, `sessionService`, and reconcile callers via the per-project context. The `ade-serve-daemon` literal is retained in live `runtime_processes` rows until the internal role vocabulary is migrated. ### 3.5 Migration strategy @@ -917,7 +921,7 @@ Themes: six shipped themes (`e-paper`, `bloomberg`, `github`, `rainbow`, `sky`, - `renderer/lib/dialogBus.ts` — tiny pub/sub that lets shared UI open/close dialogs by a stable id (`lanes.create`, `settings.ai`, etc.) without prop-drilling. Dialogs subscribe by id; a `subscribeAll` channel exists for devtools. Default singleton export `dialogBus`. - `renderer/components/app/toast/` - shared renderer-only toast primitive. `toastStore.ts` owns stack order, timers, hover pause/resume, sticky toasts, and in-place replacement; `ToastStack.tsx` renders inside AppShell's existing bottom-right notice container. Lane lifecycle and automated rebase terminal events subscribe through `useLaneEventToasts.ts`. -- `renderer/components/shared/Banner.tsx` + `renderer/components/app/IntegrationBannerHost.tsx` — the shared connection/health banner system. `Banner` is the one severity-tinted row every integration banner renders through (error/warning/info accent, normal UI font, never monospace). `IntegrationBannerHost` (mounted once by `AppShell`) computes and renders the whole family — GitHub App account authorization, per-repo App install, gh-CLI/token, missing-AI-provider, and mock-provider — as one severity-ranked list capped at two visible (`MAX_VISIBLE_BANNERS`) with a collapse-the-rest control, in place of the hand-ordered `? :` conditionals that used to live inline in `AppShell` (feature-local one-off banners such as the provider-settings and rebase-tab notices are unaffected). Dismissal is durable and fingerprint-aware via `renderer/lib/bannerDismiss.ts` (localStorage-backed, so a dismissal survives restart and project reopen; a dismissed banner auto-resurfaces after ~2 weeks or the moment its underlying state changes/regresses to a different fingerprint). GitHub App health is derived in `renderer/lib/githubIntegrationStatus.ts` (see [Onboarding and settings](./features/onboarding-and-settings/README.md)), so the banner and Settings never disagree. +- `renderer/components/shared/Banner.tsx` + `renderer/components/app/IntegrationBannerHost.tsx` — the shared connection/health banner system. `Banner` is the one severity-tinted row every integration banner renders through (error/warning/info accent, normal UI font, never monospace). `IntegrationBannerHost` (mounted once by `AppShell`) computes and renders the whole family — GitHub App account authorization, per-repo App install, gh-CLI/token, missing-AI-provider, mock-provider, and ADE Relay outage — as one severity-ranked list capped at two visible (`MAX_VISIBLE_BANNERS`) with a collapse-the-rest control, in place of the hand-ordered `? :` conditionals that used to live inline in `AppShell` (feature-local one-off banners such as the provider-settings and rebase-tab notices are unaffected). Dismissal is durable and fingerprint-aware via `renderer/lib/bannerDismiss.ts` (localStorage-backed, so a dismissal survives restart and project reopen; a dismissed banner auto-resurfaces after ~2 weeks or the moment its underlying state changes/regresses to a different fingerprint). GitHub App health is derived in `renderer/lib/githubIntegrationStatus.ts` (see [Onboarding and settings](./features/onboarding-and-settings/README.md)), so the banner and Settings never disagree. The relay banner reads `routeHealth.relay` from `AppShell`'s `sync-status` subscription seeded by `sync.getLocalStatus` (relay belongs to the physical machine, not to whichever runtime a remote-bound project routes to): a deliberate suppression is reported immediately, while a plain outage waits out `RELAY_OUTAGE_GRACE_MS` (2 minutes) of uninterrupted failure on a single armed timer rather than a poll. See [Sync and multi-device](./features/sync-and-multi-device/README.md). - `renderer/onboarding/docsLinks.ts` — typed registry of internal/public doc URLs (`docs.lanes`, `docs.cto`, …) used by `DidYouKnow`, glossary/help surfaces, and the `HelpMenu`. - `renderer/components/onboarding/LaunchGate.tsx` — fresh-process account-choice gate. New installs see the welcome card first; returning signed-out launches go directly to sign-in or **Continue without an account**. Resolving it is process-local so extra windows and renderer reloads do not repeat it. - `renderer/components/onboarding/WelcomeVideoGate.tsx` — app-level one-time welcome card using the website's canonical desktop/mobile/terminal hero assets, a privacy-enhanced YouTube embed with its real thumbnail/player, and the ADE Mobile TestFlight QR/download/copy panel. Seen/dismissed state is stored in the global app state file, separate from per-project setup onboarding. diff --git a/docs/features/remote-runtime/README.md b/docs/features/remote-runtime/README.md index f8939ab40..ed0847676 100644 --- a/docs/features/remote-runtime/README.md +++ b/docs/features/remote-runtime/README.md @@ -71,11 +71,13 @@ relay payload E2E encryption is planned security work. See the trust boundary in - `apps/ade-cli/src/services/sync/syncTunnelClientService.ts` and `apps/ade-cli/src/bootstrap.ts` — the relay side of a paired route. The tunnel client is shared one-per-machine (`getSharedSyncTunnelClientService`, keyed by - `sync-cloud-relay.json`); bootstrap hands the shared listener to - `attachHostListener()` outside the construction factory so the runtime that - owns the listener supplies the port, loopback nonce, and bridge proof - regardless of which runtime created the instance. See *Relay tunnel and the - sync port* below. + `sync-cloud-relay.json`); bootstrap builds a + `relayTunnelAuthorityGate` per project scope, which starts the tunnel only + while this process holds the machine-wide sync host lease and hands the + shared listener to `attachHostListener()` outside the construction factory, + so the runtime that owns the listener supplies the port, loopback nonce, and + bridge proof regardless of which runtime created the instance. See *Relay + tunnel and the sync port* below. - `apps/ade-cli/src/services/sync/syncHostService.ts` — the host end: paired hello authentication (with the `sync_host.paired_device_rejected` / `sync_host.paired_account_owner_mismatch` rejection logs and the @@ -719,6 +721,22 @@ validates once the listener is bound. Symptom when this is wrong: relay silently never works, and a LAN auth failure becomes a total outage because no fallback route exists. +Whether a runtime may dial the relay at all is a separate question, decided by +`relayTunnelAuthorityGate` on the machine-wide sync host lease +(`syncHostSingleton`), not by owning a listener. The relay Durable Object keeps +one host control socket per `machineKey` and evicts the previous holder with +close code `4505`, so two brains that both dial it evict each other in a tight +loop and relay stays down for both — the failure mode that made this a lease in +the first place. A `4505` close therefore suppresses redialing (bounded +re-attempts on a 60 s floor, then a stop with a 10-minute re-arm) and surfaces +as `routeHealth.relay.relayControlSuppressed` in `ade doctor` and in the +desktop relay banner, rather than being retried as if it were a network fault. +The gate rides out the momentary authority gap of an in-process project switch +with a 5 s grace and re-attaches the host listener on every start, because +`stop()` drops the listener reference and a start without it would leave a live +control socket with no bridge, rejecting every phone connect with "host sync +listener unavailable". + ADE advertises its sync port with `tailscale serve --bg --tcp=`. That outlives the process that registered it, so the served port is reclaimed after each successful publish (`staleAdeTailnetServePorts` + diff --git a/docs/features/sync-and-multi-device/README.md b/docs/features/sync-and-multi-device/README.md index ee8b7a9ba..1713d2808 100644 --- a/docs/features/sync-and-multi-device/README.md +++ b/docs/features/sync-and-multi-device/README.md @@ -68,6 +68,44 @@ diagnostics. It is **disabled by default** and only activates when sessions both leave it off; everything below describes the runtime-hosted path unless explicitly noted. +### The machine-wide sync host lease + +Hosting phone sync is exclusive per machine, and that exclusivity is a real +lease, not a convention. `syncHostSingleton.ts` owns an advisory lock file at +`$TMPDIR/ade-sync-host-.json` (override: `ADE_SYNC_HOST_LOCK_PATH`) +recording the owning pid, channel, project root, and bound port. A project +scope acquires it when its sync service starts; a projectless brain that binds +the shared listener with no active scope acquires a `projectRoot: null` lease +of its own and drops it the moment a scope takes over, so the lock file always +names the real owner. + +The lease is also the answer to *"is it me?"* for every other machine-exclusive +subsystem. Two of them gate on holding it: + +- **the relay tunnel** (`syncTunnelClientService`), because the relay Durable + Object keeps exactly one host control socket per `machineKey` and evicts the + previous holder with close code `4505`; and +- **the account-directory publisher** (`accountMachinePublisherService`), + because publishing endpoints means "reach me here", and a runtime that does + not host sync would be pointing controllers at nothing. + +Holding a *listener* is not sufficient for either — a dev `ade serve`, a +headless one-shot, or an embedded fallback can bind an ephemeral listener +without ever winning the lease. `holdsSyncHostSingleton()` reports current +process authority and `onSyncHostSingletonAuthorityChanged()` publishes +transitions (none-held → held and held → none-held only), because the lease is +acquired well after process start and can be released again. + +Authority transitions are debounced by +`SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS` (5 s) on the *loss* edge. +`ProjectScopeRegistry.performSyncHostSwitch` deactivates the outgoing sync host +before activating the target, so within one brain authority legitimately reads +false for the width of a project switch. A real loss of authority outlives that +window; a handoff never does. Without the grace, every project switch would +stop and restart the machine's relay tunnel and tear down and rebuild the +directory publisher (`ade doctor` would report "Account-directory publishing +has not started" in the gap). + ## Who participates - **Machine runtime** — the per-channel, per-machine `ade serve` runtime. It owns agent @@ -290,7 +328,13 @@ Runtime support files outside `services/sync/`: single machine-brain publisher for the account directory. It derives the stable machine key from the cloud-relay store, publishes only currently validated LAN/Tailscale/relay routes, coalesces overlapping work, and sends - the account bearer only to the trusted HTTPS directory origin. The published + the account bearer only to the trusted HTTPS directory origin. `runServe` + constructs and starts it only while the brain holds the machine-wide sync + host lease, and disposes it after authority has been lost for longer than + `SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS` (a project switch never qualifies); + the publisher cannot be restarted after dispose, so a later lease acquisition + builds a fresh one. A second brain publishing its own endpoints would point + phones at a runtime that does not host sync. The published machine `name` is suffixed by package channel (`publishedMachineName`): a Beta build advertises ` · Beta` and an Alpha build ` · Alpha`, while a stable build (or an already-suffixed name) is left untouched, so the same @@ -420,6 +464,21 @@ Runtime support files outside `services/sync/`: Desktop connection UI: +- `apps/desktop/src/renderer/components/app/IntegrationBannerHost.tsx` — hosts + the `relay-offline` banner alongside the GitHub/AI-provider family. + `AppShell` seeds `routeHealth.relay` from `sync.getLocalStatus` (the physical + machine's relay, not whichever runtime a remote-bound project routes to), + then keeps it current from the existing `sync-status` broadcast, committing a + new object only when a field the banner reads actually changed. + `deriveRelayOutageState` reports `"suppressed"` immediately — nothing + recovers a machine-local ownership conflict on its own, so a grace period + would only delay the fix — and `"down"` only after `RELAY_OUTAGE_GRACE_MS` + (2 minutes) of uninterrupted outage measured from `relayControlFailingSinceMs`, + which is past every ordinary sleep/wake or Wi-Fi-hop redial. The threshold is + crossed on wall-clock time, so `relayGraceRemainingMs` arms exactly one timer + for the remaining grace rather than polling. The dismiss key is global (relay + identity is machine-wide) and its fingerprint is the outage state, so + dismissing a plain outage cannot hide a later process conflict. - `apps/desktop/src/renderer/components/app/ConnectionsPanel.tsx` — the single top-bar Connections surface with Machines, Phone, and Web tabs. The panel owns its header close control and passes the current in-app route to the @@ -706,6 +765,18 @@ Canonical files (`apps/ade-cli/src/services/sync/`): timeout with a vague "took too long" banner. iOS treats that code as transient (retryable and queueable, like a timeout), so queued operations survive host restarts instead of being deleted on replay. +- `syncHostSingleton.ts` — the machine-wide sync host lease. Owns the advisory + lock file (`$TMPDIR/ade-sync-host-.json`, override + `ADE_SYNC_HOST_LOCK_PATH`) that records the owning pid, channel, project + root, and bound port, diagnoses conflicts (`SyncHostSingletonConflictError`) + against live listeners so a stale record cannot strand a new host, and + exposes `updatePort` / `dispose` on the acquired lease. Alongside the file it + keeps a **process-wide authority registry**: `holdsSyncHostSingleton()` + answers whether *this* process currently owns the machine's phone sync, and + `onSyncHostSingletonAuthorityChanged()` notifies subscribers on none-held → + held and held → none-held transitions. That registry is what the relay tunnel + and the account-directory publisher gate on — see *The machine-wide sync host + lease* above. - `syncHostStartupLoop.ts` — retry loop around mobile sync host startup for the brain. Same-channel singleton conflicts (update races, restart overlap, a stale sibling) always retry — the loop may evict a stale @@ -895,11 +966,33 @@ Canonical files (`apps/ade-cli/src/services/sync/`): so the object is created near the machine rather than near whichever request arrives first. Cloudflare honors a hint only at creation, so an existing machineKey keeps its original placement. +- `relayTunnelAuthorityGate.ts` — decides whether a runtime may run the + machine's relay tunnel at all. `createRelayTunnelAuthorityGate` subscribes to + `onSyncHostSingletonAuthorityChanged` and starts the tunnel only while this + process both hosts the brain-level shared listener **and** holds the + machine-wide sync host lease; a loss of authority stops it after + `SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS` (5 s) so an in-process project switch + rides through. Every start re-attaches the host listener, because `stop()` + drops the reference and it is otherwise attached once per runtime — a + gate-driven stop/start would otherwise come back with a live control socket + and no bridge, rejecting phone connects with "host sync listener unavailable" + until the brain restarted. Re-winning the lease also calls + `clearControlSuppression()`, but only for a gate that has actually observed a + release, so opening a second project cannot reset the eviction budget. + `dispose()` detaches the subscription without stopping the tunnel: the client + is machine-level and shared across project scopes, so a closing scope must + not sever relay for the others. `bootstrap.ts` builds one gate per scope. - `syncTunnelClientService.ts` — the brain-side tunnel client. When the machine has a current ADE account lease it keeps an outbound WebSocket registered with the relay worker (HMAC-signed host/pipe upgrades, - exponential backoff with jitter capped at 60 s) so controllers off the - LAN/tailnet can dial the machine through the relay. Connect and reconnect are + exponential backoff with decorrelated jitter, a 1 s floor, and a 60 s cap) + so controllers off the LAN/tailnet can dial the machine through the relay. + `computeBackoffMs` samples from a window widened by the *previous* delay + rather than resampling the same narrow exponential band, so two clients that + collide once do not keep colliding at the same instant; the floor exists + because full jitter from zero let rival processes both resample near-zero + delays forever. The client runs only under `relayTunnelAuthorityGate` — the + runtime holding the machine-wide sync host lease. Connect and reconnect are single-flight: lease reconciliation does not close a still-valid connecting socket, and a transient token-refresh exception retains the current control route through the last known account-lease expiry. Sign-out, an explicit @@ -912,8 +1005,8 @@ Canonical files (`apps/ade-cli/src/services/sync/`): socket opens and whenever the shared listener reports a fresh loopback validation, serializing probes through the same validation queue used by inbound opens. The listener itself arrives through `attachHostListener()`, - which `bootstrap.ts` calls from the runtime that owns the shared listener — - not from the construction factory. The client is shared one-per-machine + which `relayTunnelAuthorityGate` calls on every start from the runtime that + owns the shared listener — not from the construction factory. The client is shared one-per-machine (`getSharedSyncTunnelClientService`, keyed by `sync-cloud-relay.json`) and is built by whichever runtime bootstraps first, which is regularly a scope with no listener at all (a headless one-shot, an embedded fallback), so anything @@ -963,7 +1056,27 @@ Canonical files (`apps/ade-cli/src/services/sync/`): poison a route that already has a ready tunnel. Pipe/local application close codes and sanitized reasons are preserved across the bridge; other closes normalize to `4000`. Account loss clears validation and all sockets, while - account switches force a clean control reconnect. Control observability + account switches force a clean control reconnect. + A close with `RELAY_CLOSE_CONTROL_REPLACED` (`4505`) is handled as its own + regime, not as a network failure: it can only mean another process registered + the same `machineKey`, so redialing on the network schedule just evicts the + rival right back. The client retries on a fixed `CONTROL_REPLACED_RETRY_BASE_MS` + (60 s) floor jittered upward only, at most + `MAX_CONTROL_REPLACED_REATTEMPTS` (3) times, then stops dialing entirely and + reports `controlSuppressed` with the actionable reason "Another ADE process + owns the relay connection for this machine." Suppression is enforced at the + dial itself as well as in the reconnect scheduler, because the once-a-second + account-lease poll calls `connectControl` directly. Because the rival usually + exits on its own, a stopped client re-arms once after + `CONTROL_REPLACED_REARM_MS` (10 minutes); `clearControlSuppression()` also + re-arms immediately when this process (re)acquires the sync host lease, and a + control socket that survives the ready-stability window resets the budget so a + long-lived brain cannot accumulate its way to a permanent stop. Structured + events are `sync_tunnel.control_replaced`, `.control_replaced_stopped`, and + `.control_suppression_cleared`; one edge-triggered `ade_relay_suppressed` + analytics event (coarse attempt count + `control_replaced` code, no URL, + `machineKey`, or close reason) is captured per suppression episode. + Control observability preserves the causal failure rather than replacing it with a generic WebSocket error: upgrade rejection captures the HTTP status and at most 512 sanitized response bytes; close telemetry records code, reason, and whether the socket opened. @@ -973,7 +1086,13 @@ Canonical files (`apps/ade-cli/src/services/sync/`): lifecycle events. `routeHealth.relay` exposes `skipReason` / `lastControlError` plus the end-to-end verdict, while `lastControlOpenAt` and `lastBridgeValidationAt` retain the two independent - success histories. + success histories. It also carries `relayControlSuppressed`, + `relayControlSuppressedReason`, and `relayControlFailingSinceMs` — the last + being the start of the *current uninterrupted* outage, which `lastFailureAt` + can never express because it restamps on every retry. Suppression outranks + every other reason in both the `ade doctor` relay row and the desktop banner, + since nothing downstream can succeed while another process owns the slot and + no other reason tells the user what to do about it. - `syncRelaySelfProbe.ts` — `probeRelayEndToEnd`, the stateless relay round-trip check used by the tunnel client's `runSelfProbe`. It opens `wss:///connect/?ready=2`, requires an `accepted` v2 first @@ -1758,7 +1877,10 @@ feature is merged or because a deliberately isolated-port host is running. payloads. Treat the relay operator as trusted for confidentiality. Adding end-to-end payload encryption to the relay path is planned security work. The host opens and advertises Relay only while its ADE account lease is - current. Every paired Relay hello — including first-time PIN pairing — must + current **and** it holds the machine-wide sync host lease. The relay Durable + Object keeps one host control socket per `machineKey`, so relay ownership is + a machine-level singleton, not a per-process capability; a runtime without + the lease neither dials the relay nor publishes to the directory. Every paired Relay hello — including first-time PIN pairing — must also carry a fresh short-lived Clerk token whose subject matches the account signed in on the host; the proof is never persisted. Direct LAN/Tailscale hellos do not need an account token. Sign-out, account switch, expiry, or a @@ -1845,6 +1967,8 @@ feature is merged or because a deliberately isolated-port host is running. | Device-bound pairing (DPoP, Secure Enclave P-256) | Implemented (host + brain ingress; `requireDpop` / `ADE_SYNC_REQUIRE_DPOP`) | | Cloud tunnel relay (off-LAN transport, `relay` candidate) | Implemented whenever the host is signed in, with no separate toggle and with same-account per-connection proof (`syncTunnelClientService` + `apps/tunnel-relay`) | | Relay end-to-end self-probe + zombie-control detection (honest relay publication) | Implemented (`syncRelaySelfProbe`, JSON control keepalive, `sync.runSelfProbe`, `ade doctor` relay check) | +| Relay tunnel + account-directory publisher gated on the machine sync-host lease | Implemented (`syncHostSingleton` authority registry, `relayTunnelAuthorityGate`, `runServe` publisher gate) | +| Relay eviction (`4505`) suppression + surfaced outage | Implemented (bounded re-attempts, 10-minute re-arm, `routeHealth.relay.relayControlSuppressed*`, `ade doctor` relay row, desktop `relay-offline` banner) | | Sealed account adoption over direct routes (`ade-adopt-v1`, host `pubkey` identity, LAN → tailnet → Relay fallback, negotiated ChaCha20-Poly1305 / AES-256-GCM AEAD) | Implemented (`machineIdentitySigningStore` + `adoptChannelCrypto`; desktop + iOS clients) | | Push notifications + Live Activities (APNs relay) | Implemented (see `push-notifications.md`; on-device E2E needs a physical iPhone) | | Tailscale integration | Implemented (address candidate + mDNS TXT + per-node `tailscale serve` publication on the live sync port) | @@ -1870,6 +1994,16 @@ feature is merged or because a deliberately isolated-port host is running. runtime is. Code that wants the sync service must reach into the runtime IPC bridge, not into the renderer or the Electron main process. +- **Relay ownership is machine-wide, so "has a listener" is never the test.** + Any runtime can bind an ephemeral sync listener — a dev `ade serve`, a + headless one-shot, an embedded fallback. Only one may dial the relay or + publish to the account directory, because the relay Durable Object keeps one + host control socket per `machineKey` and evicts the previous holder with + close code `4505`. New machine-exclusive subsystems must gate on + `holdsSyncHostSingleton()` (through `relayTunnelAuthorityGate` or the same + authority subscription), and must tolerate the momentary `false` that a + project switch produces by riding it out for + `SYNC_HOST_AUTHORITY_RELEASE_GRACE_MS` rather than reacting on the edge. - **`ADE_ENABLE_DESKTOP_SYNC_HOST` is a diagnostics escape hatch.** If you turn it on, both an in-process host and the standing runtime can be alive simultaneously on the same machine — that's intentional for diff --git a/docs/logging.md b/docs/logging.md index fc70fd3a7..1d7c64785 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -92,8 +92,11 @@ The public contract is `apps/desktop/src/shared/types/productAnalytics.ts`. The - `ade_update_prompted` - `ade_brain_recovered` - `ade_publish_failing` +- `ade_relay_suppressed` -The update and reliability events are low-frequency by construction: the five `ade_update_*` events fire at most once per install attempt or idle-apply cycle (daily caps 10–20, minute caps 3–6). `ade_update_install_did_not_land` is emitted once at startup when a requested install relaunched on the old version, so it is bounded by app launches that follow a failed handoff, and carries only a bounded `attempt` counter; `ade_brain_recovered` fires once per wedge recovery at brain startup; `ade_publish_failing` is edge-triggered once per sustained failure episode (first crossing of two minutes), never per attempt. Properties are closed enums and bounded numbers — `reason` is allowlisted to the abort-reason constant, `escalation_reason` to `hard_deadline` / `post_staging`, `last_command` is a closed sync-action slug, and `leg`/`code` are the coarse publish classifications. Worst-case combined volume is a handful of events on a very bad day, inside the shared ceiling. +The update and reliability events are low-frequency by construction: the five `ade_update_*` events fire at most once per install attempt or idle-apply cycle (daily caps 10–20, minute caps 3–6). `ade_update_install_did_not_land` is emitted once at startup when a requested install relaunched on the old version, so it is bounded by app launches that follow a failed handoff, and carries only a bounded `attempt` counter; `ade_brain_recovered` fires once per wedge recovery at brain startup; `ade_publish_failing` is edge-triggered once per sustained failure episode (first crossing of two minutes), never per attempt. + +`ade_relay_suppressed` is the same shape for the relay leg. The relay keeps one host control socket per machine and evicts the previous holder, so two ADE brains on one machine can evict each other in a loop until relay is unusable for both. When the tunnel client exhausts its eviction budget and stops dialing, it emits one event carrying only `attempt` (the bounded eviction count) and a coarse `code` (`control_replaced`). It is keyed to the suppression *episode*, not the eviction, so a whole war collapses into one accepted event, and a 24-hour deduplication window bounds it further; a recovered control socket ends the episode so a genuinely new one still reports. The relay URL, machineKey, and raw WebSocket close reason stay in local logs and never reach the payload. Properties are closed enums and bounded numbers — `reason` is allowlisted to the abort-reason constant, `escalation_reason` to `hard_deadline` / `post_staging`, `last_command` is a closed sync-action slug, and `leg`/`code` are the coarse publish classifications. Worst-case combined volume is a handful of events on a very bad day, inside the shared ceiling. The default machine-wide ceiling is 200 accepted events per UTC day, shared across desktop, runtime, TUI, hosted web, and API-originated aggregates. Each event also has a tighter per-day and per-minute ceiling. Capture ingress is capped, noisy events use persisted deduplication windows, the in-memory transport queue is bounded, and the previous day's accepted/drop totals are summarized in at most two budget events per day. diff --git a/scripts/dev-shared.mjs b/scripts/dev-shared.mjs index 9c96ce21b..3f945301e 100644 --- a/scripts/dev-shared.mjs +++ b/scripts/dev-shared.mjs @@ -473,10 +473,48 @@ export async function ensureRuntime(socketPath, projectRoot = null) { }); child.once("error", () => {}); child.unref(); - await waitForSocket(socketPath); + try { + await waitForSocket(socketPath); + } catch (error) { + // The child is detached and unref'd, so a launcher that gives up here used + // to walk away and leave an immortal brain behind — one per failed dev + // start, each still signed in and still dialing the relay. Reap what we + // spawned before surfacing the failure. + await terminateSpawnedRuntime(child); + throw error; + } return true; } +async function terminateSpawnedRuntime(child) { + const pid = child?.pid; + if (!pid) return; + const alive = () => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } + }; + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + if (!alive()) return; + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, 100)); + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // already gone + } +} + export function devRuntimeEnv(socketPath, projectRoot) { return { ADE_CLI_VERSION: resolveDevAppVersion(), diff --git a/scripts/posthog/dashboard-spec.mjs b/scripts/posthog/dashboard-spec.mjs index 8fa283152..0924b2f10 100644 --- a/scripts/posthog/dashboard-spec.mjs +++ b/scripts/posthog/dashboard-spec.mjs @@ -22,6 +22,7 @@ export const EVENTS = Object.freeze({ UPDATE_PROMPTED: "ade_update_prompted", BRAIN_RECOVERED: "ade_brain_recovered", PUBLISH_FAILING: "ade_publish_failing", + RELAY_SUPPRESSED: "ade_relay_suppressed", MARKETING_APP_OPENED: "ade_marketing_app_opened", MARKETING_SCREEN_VIEWED: "ade_marketing_screen_viewed", MARKETING_CTA_CLICKED: "ade_marketing_cta_clicked", @@ -534,6 +535,7 @@ export const dashboardSpec = Object.freeze({ series: [ eventNode(EVENTS.BRAIN_RECOVERED, "Brain recovered from wedge"), eventNode(EVENTS.PUBLISH_FAILING, "Route publish failing"), + eventNode(EVENTS.RELAY_SUPPRESSED, "Relay suppressed by rival process"), eventNode(EVENTS.UPDATE_INSTALL_ABORTED, "Update install aborted"), eventNode(EVENTS.UPDATE_QUIT_ESCALATED, "Update quit escalated"), eventNode(EVENTS.UPDATE_INSTALL_DID_NOT_LAND, "Update did not land"),