From 7c5adc45ea3ee7456e04339430c2433e3486ffd7 Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Fri, 21 Aug 2026 19:26:26 -0700 Subject: [PATCH] fix(server): wait for launchd shutdown safely --- apps/server/src/cloud/bootService.test.ts | 240 ++++++++++++++++++++-- apps/server/src/cloud/bootService.ts | 192 +++++++++++++---- 2 files changed, 372 insertions(+), 60 deletions(-) diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 1314ccfb9361..6da11f70af7f 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -7,11 +7,14 @@ import { HostProcessUserId, } from "@t3tools/shared/hostProcess"; import * as ConfigProvider from "effect/ConfigProvider"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; +import * as TestClock from "effect/testing/TestClock"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; import * as ProcessRunner from "../processRunner.ts"; @@ -57,6 +60,56 @@ const macPlan = { unitPath: "/Users/theo/Library/LaunchAgents/com.t3tools.t3code.service.plist", }; +const launchdServiceTarget = "gui/501/com.t3tools.t3code.service"; +const launchdNotLoadedMessage = + 'Could not find service "com.t3tools.t3code.service" in domain for user gui: 501'; + +const processResult = (input?: { + readonly code?: number; + readonly stdout?: string; + readonly stderr?: string; +}): ProcessRunner.ProcessRunOutput => ({ + stdout: input?.stdout ?? "", + stderr: input?.stderr ?? "", + code: ChildProcessSpawner.ExitCode(input?.code ?? 0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + stdoutInvalidUtf8: false, + stderrInvalidUtf8: false, +}); + +it("recognizes only launchctl's exact service-not-loaded response", () => { + expect( + BootService.isConfirmedLaunchdNotLoaded( + processResult({ code: 113, stderr: `Bad request.\n${launchdNotLoadedMessage}\n` }), + [launchdNotLoadedMessage], + ), + ).toBe(true); + expect( + BootService.isConfirmedLaunchdNotLoaded( + processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted" }), + [launchdNotLoadedMessage], + ), + ).toBe(false); + expect( + BootService.isConfirmedLaunchdNotLoaded( + processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }), + [launchdNotLoadedMessage], + ), + ).toBe(false); + expect( + BootService.isConfirmedLaunchdBootoutNotLoaded( + processResult({ code: 3, stderr: "Boot-out failed: 3: No such process\n" }), + ), + ).toBe(true); + expect( + BootService.isConfirmedLaunchdBootoutNotLoaded( + processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted\n" }), + ), + ).toBe(false); +}); + it("keeps launchd pinned to the stable launcher rather than a versioned server", () => { const plist = BootService.renderBootServicePlist(macPlan, { homeDir: "/Users/theo" }); @@ -116,23 +169,37 @@ const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( const commands: string[] = []; const timeouts = new Map(); - const control: { failCommand: string | undefined } = { failCommand: undefined }; + const control: { + failCommand: string | undefined; + fixtures: Map ProcessRunner.ProcessRunOutput>; + callCounts: Map; + signals: Map>; + } = { + failCommand: undefined, + fixtures: new Map(), + callCounts: new Map(), + signals: new Map(), + }; const runner = ProcessRunner.ProcessRunner.of({ run: (input) => - Effect.sync(() => { + Effect.gen(function* () { const command = `${input.command} ${input.args.join(" ")}`; commands.push(command); timeouts.set(command, input.timeout); - return { - stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "", - stderr: "", - code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), - timedOut: false, - stdoutTruncated: false, - stderrTruncated: false, - stdoutInvalidUtf8: false, - stderrInvalidUtf8: false, - }; + const call = (control.callCounts.get(command) ?? 0) + 1; + control.callCounts.set(command, call); + const signal = control.signals.get(command); + if (signal !== undefined) yield* Deferred.succeed(signal, undefined); + const fixture = control.fixtures.get(command); + if (fixture) return fixture(call); + if (command === control.failCommand) return processResult({ code: 1 }); + if (command === `launchctl print ${launchdServiceTarget}`) { + return processResult({ + code: 113, + stderr: `Bad request.\n${launchdNotLoadedMessage}\n`, + }); + } + return processResult({ stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "" }); }), }); const service = yield* BootService.make({ @@ -276,9 +343,9 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect((yield* service.status).installed).toBe(false); expect(commands.some((command) => command.startsWith("npm "))).toBe(false); expect(commands.some((command) => command.startsWith("systemctl "))).toBe(false); - // A bootout can block up to the plist's 90s ExitTimeOut; the runner's - // 60s default would cancel it and let bootstrap race a loaded job. - expect(timeouts.get("launchctl bootout --wait gui/501/com.t3tools.t3code.service")).toEqual( + // The bootout command and subsequent bounded print verification both + // allow launchd's 90s ExitTimeOut to elapse. + expect(timeouts.get("launchctl bootout gui/501/com.t3tools.t3code.service")).toEqual( Duration.seconds(120), ); }), @@ -295,7 +362,8 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const error = yield* service.install.pipe(Effect.flip); expect(error._tag).toBe("BootServiceCommandError"); expect(commands.filter((command) => command.startsWith("launchctl "))).toEqual([ - "launchctl bootout --wait gui/501/com.t3tools.t3code.service", + "launchctl bootout gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", "launchctl enable gui/501/com.t3tools.t3code.service", `launchctl bootstrap gui/501 ${plistPath}`, `launchctl bootstrap gui/501 ${plistPath}`, @@ -303,14 +371,145 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); - it.effect("ignores a bootout for an agent that is not loaded", () => + it.effect("removes the launch agent when its GUI domain is absent", () => Effect.gen(function* () { - const { service, control } = yield* makeHarness("darwin"); + const { service, fs, control } = yield* makeHarness("darwin"); + const plan = yield* service.install; + control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () => + processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }), + ); + control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () => + processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }), + ); + + expect(yield* service.uninstall).toBe(true); + expect(yield* fs.exists(plan.unitPath)).toBe(false); + }), + ); + + it.effect("accepts a bootout only when launchd confirms the agent is already absent", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); yield* service.install; - control.failCommand = "launchctl bootout --wait gui/501/com.t3tools.t3code.service"; + commands.length = 0; + control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () => + processResult({ code: 3, stderr: "Boot-out failed: 3: No such process\n" }), + ); yield* service.install; expect((yield* service.status).current).toBe(true); + expect(commands.filter((command) => command.startsWith("launchctl ")).slice(0, 2)).toEqual([ + "launchctl bootout gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", + ]); + }), + ); + + it.effect("waits for a draining launch agent before bootstrap", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + yield* service.install; + commands.length = 0; + control.callCounts.clear(); + const firstPrint = yield* Deferred.make(); + control.signals.set(`launchctl print ${launchdServiceTarget}`, firstPrint); + control.fixtures.set(`launchctl print ${launchdServiceTarget}`, (call) => + call === 1 + ? processResult() + : processResult({ + code: 113, + stderr: `Bad request.\n${launchdNotLoadedMessage}\n`, + }), + ); + + const installFiber = yield* service.install.pipe(Effect.forkChild); + yield* Deferred.await(firstPrint); + yield* Effect.yieldNow; + yield* TestClock.adjust(Duration.millis(100)); + yield* Fiber.join(installFiber); + + expect(commands.filter((command) => command.startsWith("launchctl ")).slice(0, 4)).toEqual([ + "launchctl bootout gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", + "launchctl enable gui/501/com.t3tools.t3code.service", + ]); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("times out instead of bootstrapping while a launch agent remains loaded", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + yield* service.install; + commands.length = 0; + control.callCounts.clear(); + const firstPrint = yield* Deferred.make(); + control.signals.set(`launchctl print ${launchdServiceTarget}`, firstPrint); + control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () => processResult()); + + const installFiber = yield* service.install.pipe(Effect.forkChild); + yield* Deferred.await(firstPrint); + yield* Effect.yieldNow; + yield* TestClock.adjust(Duration.seconds(120)); + const error = yield* Fiber.join(installFiber).pipe(Effect.flip); + + expect(error._tag).toBe("BootServiceCommandError"); + expect(error.message).toContain("timed out while waiting for the launch agent to stop"); + expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service"); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("surfaces bootout permission failures while the launch agent remains loaded", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + yield* service.install; + commands.length = 0; + control.fixtures.set("launchctl bootout gui/501/com.t3tools.t3code.service", () => + processResult({ code: 1, stderr: "Boot-out failed: 1: Operation not permitted\n" }), + ); + + const error = yield* service.install.pipe(Effect.flip); + + expect(error._tag).toBe("BootServiceCommandError"); + expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe( + "stopping the installed launch agent", + ); + expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service"); + }), + ); + + it.effect("surfaces launchd domain failures during stop verification", () => + Effect.gen(function* () { + const { service, commands, control } = yield* makeHarness("darwin"); + yield* service.install; + commands.length = 0; + control.fixtures.set(`launchctl print ${launchdServiceTarget}`, () => + processResult({ code: 125, stderr: "Could not find domain for user gui: 501" }), + ); + + const error = yield* service.install.pipe(Effect.flip); + + expect(error._tag).toBe("BootServiceCommandError"); + expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe( + "checking whether the launch agent stopped", + ); + expect(commands).not.toContain("launchctl enable gui/501/com.t3tools.t3code.service"); + }), + ); + + it.effect("surfaces launchd enable failures", () => + Effect.gen(function* () { + const { service, control } = yield* makeHarness("darwin"); + control.fixtures.set("launchctl enable gui/501/com.t3tools.t3code.service", () => + processResult({ code: 1, stderr: "Enable failed: 1: Operation not permitted\n" }), + ); + + const error = yield* service.install.pipe(Effect.flip); + + expect(error._tag).toBe("BootServiceCommandError"); + expect(error._tag === "BootServiceCommandError" ? error.step : undefined).toBe( + "enabling the launch agent", + ); }), ); @@ -336,7 +535,8 @@ it.layer(NodeServices.layer)("boot service install", (it) => { expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError"); expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true); expect(commands.filter((command) => command.startsWith("launchctl "))).toEqual([ - "launchctl bootout --wait gui/501/com.t3tools.t3code.service", + "launchctl bootout gui/501/com.t3tools.t3code.service", + "launchctl print gui/501/com.t3tools.t3code.service", `launchctl bootstrap gui/501 ${plistPath}`, ]); }), diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 795bf38e979d..9a643d16243b 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -154,27 +154,63 @@ export function renderBootServicePlist( ].join("\n"); } -export interface BootServiceStep { +interface BootServiceStepBase { readonly step: string; readonly command: string; readonly args: ReadonlyArray; - /** - * Non-zero exit is logged and ignored. Reserved for steps whose common - * failures (not loaded, already enabled) leave a state a later strict step - * either tolerates or fails loudly on. - */ - readonly optional?: boolean; /** Override the ProcessRunner default (60s) for steps that block longer. */ readonly timeout?: Duration.Input; } +export type BootServiceStep = BootServiceStepBase & + ( + | { readonly operation?: "command" } + | { + readonly operation: "launchd-bootout"; + /** After launchd bootout, prove the job left the domain before continuing. */ + readonly verifyAbsent: { + readonly serviceTarget: string; + readonly notLoadedMessages: ReadonlyArray; + }; + } + ); + /** - * Stop commands block until the service manager gives up: 90s by default for - * systemd's TimeoutStopSec, and ExitTimeOut=90 in the rendered plist. This - * must stay above both, or the runner cancels the stop mid-shutdown and the - * next step races a still-loaded service. + * A stop can take 90s: systemd blocks for TimeoutStopSec, while launchd's + * supported bootout command returns before the job finishes honoring the + * plist's ExitTimeOut. Keep both the command and launchd verification bounds + * above that window. */ const STOP_STEP_TIMEOUT = Duration.seconds(120); +const LAUNCHD_STOP_POLL_INTERVAL = Duration.millis(100); + +export function isConfirmedLaunchdNotLoaded( + result: ProcessRunner.ProcessRunOutput, + notLoadedMessages: ReadonlyArray, +): boolean { + if (result.code === 0 || result.timedOut) return false; + const lines = `${result.stdout}\n${result.stderr}` + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line !== ""); + return ( + (lines.length === 1 && notLoadedMessages.includes(lines[0] ?? "")) || + (lines.length === 2 && + lines[0] === "Bad request." && + notLoadedMessages.includes(lines[1] ?? "")) + ); +} + +export function isConfirmedLaunchdBootoutNotLoaded( + result: ProcessRunner.ProcessRunOutput, +): boolean { + if (result.code === 0 || result.timedOut) return false; + const lines = `${result.stdout}\n${result.stderr}` + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line !== ""); + return lines.length === 1 && lines[0] === "Boot-out failed: 3: No such process"; +} /** * Platform service-manager integration as data: paths, a pure renderer, and @@ -277,28 +313,28 @@ export function launchdManager(input: { ); const domainTarget = `gui/${input.uid}`; const serviceTarget = `${domainTarget}/${BOOT_SERVICE_LAUNCHD_LABEL}`; - // bootout/enable are optional: they fail on not-loaded states that are fine - // to proceed from. The strict `bootstrap` runs last and is also the start: + const notLoadedMessage = `Could not find service "${BOOT_SERVICE_LAUNCHD_LABEL}" in domain for user gui: ${input.uid}`; + const missingDomainMessage = `Could not find domain for user gui: ${input.uid}`; + // `bootstrap` runs last and is also the start: // loading a RunAtLoad/KeepAlive plist starts the job, so a separate // kickstart would kill and restart a server it just booted. A lingering job - // that survived bootout, or a gui domain with nobody logged in at the - // screen (SSH install), makes bootstrap fail the flow loudly rather than - // silently keeping a stale server. + // that survived bootout, a persisted disable override, or a gui domain with + // nobody logged in at the screen (SSH install) fails the flow loudly. return { kind: "launchd", unitPath, render: (plan) => renderBootServicePlist(plan, { homeDir: input.homeDir }), - // Without --wait, bootout returns in milliseconds while the job drains - // for up to ExitTimeOut, and a bootstrap during the drain fails EIO. - // --wait (present on modern macOS, absent from the man page) blocks until - // the job is removed from the domain; STOP_STEP_TIMEOUT outlives it. + // bootout has no supported wait flag and returns before a draining job is + // absent. Probe the exact service until launchd confirms it left the + // domain, so bootstrap cannot race the prior process. stop: [ { step: "stopping the installed launch agent", command: "launchctl", - args: ["bootout", "--wait", serviceTarget], - optional: true, + args: ["bootout", serviceTarget], timeout: STOP_STEP_TIMEOUT, + operation: "launchd-bootout", + verifyAbsent: { serviceTarget, notLoadedMessages: [notLoadedMessage] }, }, ], activate: [ @@ -307,7 +343,6 @@ export function launchdManager(input: { step: "enabling the launch agent", command: "launchctl", args: ["enable", serviceTarget], - optional: true, }, // Start last. No administrative state write occurs after this succeeds. { @@ -325,15 +360,17 @@ export function launchdManager(input: { ], // No `launchctl disable` here: a persisted override would sabotage a // later reinstall. Removing the plist is what stops the next login load. - // A bootout that fails for a reason other than "not loaded" leaves the - // job running until logout; the failure is in the boot-service log. deactivate: [ { step: "stopping the service", command: "launchctl", - args: ["bootout", "--wait", serviceTarget], - optional: true, + args: ["bootout", serviceTarget], timeout: STOP_STEP_TIMEOUT, + operation: "launchd-bootout", + verifyAbsent: { + serviceTarget, + notLoadedMessages: [notLoadedMessage, missingDomainMessage], + }, }, ], finalize: [], @@ -375,10 +412,14 @@ export class BootServiceCommandError extends Schema.TaggedErrorClass + DateTime.now.pipe( + Effect.flatMap((now) => + fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { + flag: "a", + }), + ), + Effect.ignore, + ); + const runStep = Effect.fn("cloud.boot_service.run_step")(function* ( step: string, command: string, @@ -499,32 +550,93 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { stderrLength: result.stderr.length, }), ), - Effect.tapError((error) => - DateTime.now.pipe( - Effect.flatMap((now) => - fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { - flag: "a", - }), - ), - Effect.ignore, - ), - ), + Effect.tapError(logCommandError), ); }); + const runLaunchdBootout = Effect.fn("cloud.boot_service.run_launchd_bootout")(function* ( + entry: Extract, + ) { + const bootoutResult = yield* runner + .run({ + command: entry.command, + args: entry.args, + timeout: entry.timeout, + }) + .pipe(Effect.mapError((cause) => new BootServiceCommandError({ step: entry.step, cause }))); + const bootoutError = + bootoutResult.code === 0 + ? undefined + : new BootServiceCommandError({ + step: entry.step, + exitCode: bootoutResult.code === null ? undefined : Number(bootoutResult.code), + stdoutLength: bootoutResult.stdout.length, + stderrLength: bootoutResult.stderr.length, + }); + if ( + bootoutError !== undefined && + !isConfirmedLaunchdBootoutNotLoaded(bootoutResult) && + !isConfirmedLaunchdNotLoaded(bootoutResult, entry.verifyAbsent.notLoadedMessages) + ) { + return yield* bootoutError; + } + + yield* Effect.gen(function* () { + while (true) { + const printResult = yield* runner + .run({ + command: "launchctl", + args: ["print", entry.verifyAbsent.serviceTarget], + }) + .pipe( + Effect.mapError( + (cause) => + new BootServiceCommandError({ + step: "checking whether the launch agent stopped", + cause, + }), + ), + ); + if (isConfirmedLaunchdNotLoaded(printResult, entry.verifyAbsent.notLoadedMessages)) { + return; + } + if (bootoutError !== undefined) return yield* bootoutError; + if (printResult.code !== 0) { + return yield* new BootServiceCommandError({ + step: "checking whether the launch agent stopped", + exitCode: printResult.code === null ? undefined : Number(printResult.code), + stdoutLength: printResult.stdout.length, + stderrLength: printResult.stderr.length, + }); + } + yield* Effect.sleep(LAUNCHD_STOP_POLL_INTERVAL); + } + }).pipe( + Effect.timeoutOrElse({ + duration: STOP_STEP_TIMEOUT, + orElse: () => + new BootServiceCommandError({ + step: "waiting for the launch agent to stop", + timedOut: true, + }), + }), + ); + }, Effect.tapError(logCommandError)); + const runSteps = (steps: ReadonlyArray) => Effect.forEach( steps, (entry) => { + if (entry.operation === "launchd-bootout") { + return runLaunchdBootout(entry); + } const run = runStep( entry.step, entry.command, entry.args, entry.timeout === undefined ? undefined : { timeout: entry.timeout }, ); - // runStep's tapError already appends the failure to the log, so an - // ignored optional step still leaves a trace. - return entry.optional === true ? run.pipe(Effect.ignore) : run.pipe(Effect.asVoid); + return run.pipe(Effect.asVoid); }, { discard: true }, );