Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions apps/ade-cli/src/commands/doctor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,35 @@ describe("doctor row evaluation", () => {
status: "fail",
detail: expect.stringMatching(/failing for 5m .* slow leg: http \(9\.2s\)/),
}));
// Only the brain-session states get the restart remedy; a slow HTTP leg is
// not fixed by restarting.
expect(rows.find((row) => row.key === "publish")?.detail).not.toContain("ade brain restart");
});

it("points an unreadable account session at `ade brain restart`", () => {
// Desktop's Connections panel shows a Repair (brain restart) button for
// this state; the CLI has to name the same remedy or an agent is stuck.
const warning = healthyInput();
warning.publishHealth = createSyncAccountDirectoryHealth("token_unreadable", null, {
failingSinceMs: NOW - 30_000,
});
const failing = healthyInput();
failing.publishHealth = createSyncAccountDirectoryHealth("token_unreadable", null, {
failingSinceMs: NOW - 5 * 60_000,
});

expect(evaluateDoctorRows(warning).find((row) => row.key === "publish")).toEqual(
expect.objectContaining({
status: "warn",
detail: expect.stringContaining("ade brain restart"),
}),
);
expect(evaluateDoctorRows(failing).find((row) => row.key === "publish")).toEqual(
expect.objectContaining({
status: "fail",
detail: expect.stringContaining("ade brain restart"),
}),
);
});

it("fails only the brain while dependent checks degrade when the socket is dead", () => {
Expand Down
15 changes: 12 additions & 3 deletions apps/ade-cli/src/commands/doctor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
parseRuntimePublishHealth,
type RuntimePublishHealth,
} from "../../../desktop/src/shared/adeRuntimeProtocol";
import { isBrainAccountSessionFailure } from "../../../desktop/src/shared/types/sync";
import type {
SyncAccountDirectoryHealth,
SyncRouteHealth,
Expand Down Expand Up @@ -647,12 +648,20 @@ function publishRow(
const failingForMs = health.failingSinceMs == null
? null
: Math.max(0, nowMs - health.failingSinceMs);
// The Connections panel offers a "Repair" (brain restart) button for exactly
// the states `isBrainAccountSessionFailure` covers, because a replacement
// brain re-reads the account session from scratch. `ade brain restart` is the
// CLI's equivalent, so name it here rather than leaving an agent holding a
// bare `token_unreadable` with no next step.
const remedy = isBrainAccountSessionFailure(health.state)
? " · run `ade brain restart` so the brain re-reads the account session"
: "";
if (failingForMs != null && failingForMs >= PUBLISH_FAILURE_RED_MS) {
return {
key: "publish",
label: "Publish health",
status: "fail",
detail: `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}`,
detail: `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}${remedy}`,
};
}
if (health.state === "published") {
Expand All @@ -673,8 +682,8 @@ function publishRow(
label: "Publish health",
status: "warn",
detail: failingForMs == null
? `${health.state}${health.skipReason ? ` · ${health.skipReason}` : ""}`
: `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}`,
? `${health.state}${health.skipReason ? ` · ${health.skipReason}` : ""}${remedy}`
: `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}${remedy}`,
};
}

Expand Down
2 changes: 2 additions & 0 deletions apps/ade-cli/src/multiProjectRpcServer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ function makeAccountAuthServiceMock() {
expiresAt: null,
})),
getSessionReadState: vi.fn(() => "missing" as const),
getSessionReadFailureReason: vi.fn(() => null),
getAccessToken: vi.fn(async () => "test-access-token"),
createToken: vi.fn(async () => ({
token: "test-refresh-token",
Expand Down Expand Up @@ -579,6 +580,7 @@ describe("multi-project RPC server", () => {
expiresAt: null,
})),
getSessionReadState: vi.fn(() => "missing" as const),
getSessionReadFailureReason: vi.fn(() => null),
getAccessToken: vi.fn(),
createToken: vi.fn(),
cancelLogin: vi.fn(),
Expand Down
48 changes: 48 additions & 0 deletions apps/ade-cli/src/serviceManager/installWindows.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import {
WINDOWS_REG_COMMAND,
WINDOWS_SCHTASKS_COMMAND,
WINDOWS_TASK_ACTION_FIELD_SEPARATOR,
WINDOWS_TASKKILL_COMMAND,
renderWindowsServiceLauncher,
} from "./installWindows";

Expand Down Expand Up @@ -395,6 +396,53 @@ describe("Windows background service helpers", () => {
]);
});

it.each([
{ label: "an ordinary install", forceEnv: {} },
{ label: "a Repair-forced install", forceEnv: { ADE_FORCE_RUNTIME_SERVICE_RESTART: "1" } },
])("restarts the running supervisor on $label", async ({ forceEnv }) => {
// The desktop Repair button sets ADE_FORCE_RUNTIME_SERVICE_RESTART, and
// ONLY installLaunchd reads it — it exists to defeat launchd's "unchanged
// plist + loaded + responsive => skip" fast path. Windows honours the flag
// by construction rather than by reading it: this install has no skip path,
// so it always taskkills the supervisor tree and starts a fresh one. Assert
// that for BOTH env shapes, so a future "already installed, leave it alone"
// optimisation here cannot silently turn Repair into a no-op on Windows.
const home = makeTempHome("ade-windows-service-force-restart-");
const launcherPath = path.join(home, "brain-service.ps1");
fs.writeFileSync(`${launcherPath}.pid.json`, JSON.stringify(readyPidRecord), "utf8");
const calls: Array<{ command: string; args: string[] }> = [];
const spawnSync = spawnSequence(calls, [
{ status: 3, stdout: "", stderr: "" }, // legacy task: absent
{ status: 3, stdout: "", stderr: "" }, // channel task: absent
{ status: 0, stdout: ` ${taskName} REG_SZ x`, stderr: "" }, // Run entry: installed
{ status: 0, stdout: "", stderr: "" }, // supervisor probe: running
{ status: 0, stdout: "SUCCESS", stderr: "" }, // taskkill /T /F
{ status: 0, stdout: "SUCCESS: deleted", stderr: "" }, // reg delete
{ status: 0, stdout: "SUCCESS: created", stderr: "" }, // reg add
{ status: 0, stdout: "1234", stderr: "" }, // start task
]);

const result = await installWindowsService({
...immediateReadiness,
command: serviceCommand,
env: { USERDOMAIN: "ADEBOX", USERNAME: "arul", ...forceEnv },
launcherPath,
serviceName,
spawnSync,
userName: taskUser,
});

expect(result.ok).toBe(true);
expect(calls).toContainEqual({
command: WINDOWS_TASKKILL_COMMAND,
args: ["/PID", String(readyPidRecord.supervisorPid), "/T", "/F"],
});
expect(calls.at(-1)).toEqual({
command: WINDOWS_POWERSHELL_COMMAND,
args: buildWindowsStartTaskArgs(launcherPath, resolveWindowsStartTaskName(taskName)),
});
});

it("ends and replaces a running channel task before starting the repaired runtime", async () => {
const calls: Array<{ command: string; args: string[] }> = [];
const spawnSync = spawnSequence(calls, [
Expand Down
60 changes: 45 additions & 15 deletions apps/ade-cli/src/services/account/accountAuthService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,10 @@ import {
shouldIgnoreDevelopmentClerkConfiguration,
warnDevelopmentClerkIgnored,
} from "../../../../desktop/src/shared/accountDirectory";
import type { SyncCredentialStore } from "../credentials/credentialStore";
import type {
CredentialStoreReadFailureReason,
SyncCredentialStore,
} from "../credentials/credentialStore";
import { runWithAbortSignal } from "../sync/abortSignal";

export const ACCOUNT_SESSION_CREDENTIAL_KEY = "account.session.v1";
Expand Down Expand Up @@ -105,6 +108,18 @@ export type AccountAuthStatus = {

export type AccountSessionReadState = "available" | "missing" | "unreadable";

/**
* Which read path produced an "unreadable" session. Coarse and closed so it can
* be reported as a product-analytics property.
*/
export type AccountSessionReadFailureReason =
/** Everything the credential store itself can report (decrypt/key-material/format). */
| CredentialStoreReadFailureReason
/** The credential decrypted but the stored session record did not parse. */
| "session_parse"
/** The credential store threw while being read. */
| "read_error";

export type AccountLoginStartResult = {
sessionId: string;
authorizeUrl: string;
Expand Down Expand Up @@ -193,6 +208,8 @@ export type AccountAuthService = {
getStatus(): AccountAuthStatus;
/** Last persisted-session read result, refreshed by getStatus/getAccessToken. */
getSessionReadState(): AccountSessionReadState;
/** Why the last read was unreadable, or null when it was not. */
getSessionReadFailureReason(): AccountSessionReadFailureReason | null;
getAccessToken(options?: AccountAccessTokenOptions): Promise<string>;
createToken(): Promise<AccountTokenCreateResult>;
cancelLogin(sessionId: string): void;
Expand Down Expand Up @@ -793,6 +810,14 @@ export function createAccountAuthService(args: {
let envCredentialEpoch = 0;
let authEpoch = 0;
let sessionReadState: AccountSessionReadState = "missing";
let sessionReadFailureReason: AccountSessionReadFailureReason | null = null;
const setSessionReadState = (
state: AccountSessionReadState,
reason: AccountSessionReadFailureReason | null = null,
): void => {
sessionReadState = state;
sessionReadFailureReason = state === "unreadable" ? reason : null;
};
let lastObservedSignedIn: boolean | null = null;
let locallyRejectedSessionRaw: string | null = null;
const signedInListeners = new Set<() => void>();
Expand Down Expand Up @@ -895,7 +920,7 @@ export function createAccountAuthService(args: {
// rejected on every read instead of being erased.
authEpoch += 1;
lastObservedSignedIn = false;
sessionReadState = "missing";
setSessionReadState("missing");
warnDevelopmentClerkIgnored();
};

Expand All @@ -915,15 +940,19 @@ export function createAccountAuthService(args: {
const session = locallyRejected
? null
: parseStoredSession(stored);
sessionReadState = locallyRejected
? "missing"
: stored == null
? args.credentialStore.getLastReadState?.() === "unreadable"
? "unreadable"
: "missing"
: session
? "available"
: "unreadable";
if (locallyRejected) {
setSessionReadState("missing");
} else if (stored == null) {
const storeUnreadable = args.credentialStore.getLastReadState?.() === "unreadable";
setSessionReadState(
storeUnreadable ? "unreadable" : "missing",
storeUnreadable
? args.credentialStore.getLastReadFailureReason?.() ?? null
: null,
);
} else {
setSessionReadState(session ? "available" : "unreadable", "session_parse");
}
return { raw: stored, session };
};

Expand All @@ -945,12 +974,12 @@ export function createAccountAuthService(args: {
accessToken: retry.session.accessToken,
oauthConfig: retry.session.oauthConfig,
})) {
sessionReadState = "missing";
setSessionReadState("missing");
return { raw: retry.raw, session: null };
}
return retry;
} catch (error) {
sessionReadState = "unreadable";
setSessionReadState("unreadable", "read_error");
logger.warn("account.session_read_failed", {
error: error instanceof Error ? error.message : String(error),
});
Expand All @@ -969,7 +998,7 @@ export function createAccountAuthService(args: {
// peer may be rotating the credential.
authEpoch += 1;
lastObservedSignedIn = false;
sessionReadState = "missing";
setSessionReadState("missing");
return false;
}
let deleted = false;
Expand All @@ -982,7 +1011,7 @@ export function createAccountAuthService(args: {
if (deleted) {
authEpoch += 1;
lastObservedSignedIn = false;
sessionReadState = "missing";
setSessionReadState("missing");
}
return deleted;
};
Expand Down Expand Up @@ -2139,6 +2168,7 @@ export function createAccountAuthService(args: {
pollDeviceLogin,
getStatus,
getSessionReadState: () => sessionReadState,
getSessionReadFailureReason: () => sessionReadFailureReason,
getAccessToken,
createToken,
cancelLogin,
Expand Down
Loading
Loading