Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions apps/ade-cli/src/services/sync/syncHostService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,30 @@ const MOBILE_CHANGESET_EXCLUDED_TABLES = new Set([
// visible-lane refresh upserts here, so scrolling was pushing changesets to
// every phone.
"pull_request_snapshots",
// Event logs the phone stores and never reads. Each exists in
// `DatabaseBootstrap.sql` so an inbound changeset still applies, but there is
// no Swift read path for any of them — verified: zero references outside that
// schema file. Excluding them stops the phone paying apply and storage cost
// for rows it cannot surface.
//
// This list is an OUTBOUND filter only — it drops rows from mobile
// changesets and never touches CRR metadata, so unlike a local-only
// conversion it carries no apply hazard for a peer on an older build.
// Existing rows on a paired device are left alone, and the peer's ack
// watermark still advances through the filtered versions.
//
// `linear_ingress_events` and `worker_agent_runs` are included here but
// deliberately NOT age-pruned on the host: the first is the webhook replay
// guard, the second is a lifecycle table (see RETAINED_EVENT_LOG_TABLES in
// kvDb.ts). Not shipping them to a phone that cannot read them is orthogonal
// to how long the host keeps them.
"linear_ingress_events",
"linear_sync_events",
"linear_workflow_run_events",
"worker_agent_runs",
"worker_agent_cost_events",
"pack_events",
"cto_session_logs",
]);

// Tables the host alone is authoritative for. `sync_cluster_state` is the
Expand Down
10 changes: 10 additions & 0 deletions apps/desktop/src/main/services/ai/aiIntegrationService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1343,6 +1343,16 @@ export function createAiIntegrationService(args: {
return daily;
};

/**
* Successful requests for `feature` so far today.
*
* This reads `ai_usage_log` directly, and that table replicates, so the count
* spans every machine on the account — `dailyLimit` is an account-wide cap,
* not a per-machine one. Anything that stops the table replicating (making it
* local-only, filtering it out of a peer's changesets) turns the cap into a
* per-machine one and multiplies the user's ceiling by their machine count.
* See the note beside `ai_usage_log` in kvDb's LOCAL_ONLY_CRR_EXCLUDED_TABLES.
*/
const countDailyUsage = (feature: AiFeatureKey): number => {
const row = db.get<{ count: number }>(
`
Expand Down
90 changes: 90 additions & 0 deletions apps/desktop/src/main/services/state/dbMaintenanceApi.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import { describe, expect, it, vi } from "vitest";

import {
pruneRowsInBatches,
MAINTENANCE_DELETE_BATCH_ROWS,
MAINTENANCE_DELETE_MAX_BATCHES,
} from "./dbMaintenanceApi";

describe("pruneRowsInBatches", () => {
/** A table of `total` matching rows that deletes at most `limit` per call. */
function fakeTable(total: number) {
let remaining = total;
return {
get remaining() { return remaining; },
deleteRows: vi.fn((sql: string) => {
const limit = Number(/limit (\d+)/.exec(sql)?.[1] ?? 0);
const deleted = Math.min(limit, remaining);
remaining -= deleted;
return deleted;
}),
};
}

it("yields to the event loop between batches", async () => {
const table = fakeTable(MAINTENANCE_DELETE_BATCH_ROWS * 3);
const yields = vi.fn(async () => {});

const removed = await pruneRowsInBatches({
table: "ai_usage_log",
where: "timestamp < ?",
params: ["2026-01-01"],
deleteRows: table.deleteRows,
yieldToEventLoop: yields,
});

expect(removed).toBe(MAINTENANCE_DELETE_BATCH_ROWS * 3);
expect(table.deleteRows).toHaveBeenCalledTimes(4);
// The driver is synchronous, so without these yields the whole delete runs
// as one uninterruptible block — UI, IPC, and the sync pump all stall.
expect(yields).toHaveBeenCalledTimes(3);
});

it("stops on the first short batch without an extra probe", async () => {
const table = fakeTable(10);
const yields = vi.fn(async () => {});

const removed = await pruneRowsInBatches({
table: "cto_session_logs",
where: "created_at < ?",
params: ["2026-01-01"],
deleteRows: table.deleteRows,
yieldToEventLoop: yields,
});

expect(removed).toBe(10);
expect(table.deleteRows).toHaveBeenCalledTimes(1);
expect(yields).not.toHaveBeenCalled();
});

it("bounds one call so a huge backlog cannot run away", async () => {
const table = fakeTable(MAINTENANCE_DELETE_BATCH_ROWS * (MAINTENANCE_DELETE_MAX_BATCHES + 50));

const removed = await pruneRowsInBatches({
table: "ai_usage_log",
where: "timestamp < ?",
params: ["2026-01-01"],
deleteRows: table.deleteRows,
yieldToEventLoop: async () => {},
});

expect(table.deleteRows).toHaveBeenCalledTimes(MAINTENANCE_DELETE_MAX_BATCHES);
expect(removed).toBe(MAINTENANCE_DELETE_BATCH_ROWS * MAINTENANCE_DELETE_MAX_BATCHES);
// The rest is left for the next sweep rather than held in one long pause.
expect(table.remaining).toBe(MAINTENANCE_DELETE_BATCH_ROWS * 50);
});

it("deletes by rowid so each batch costs the batch, not the table", async () => {
const table = fakeTable(1);
await pruneRowsInBatches({
table: "pack_events",
where: "created_at < ?",
params: ["2026-01-01"],
deleteRows: table.deleteRows,
yieldToEventLoop: async () => {},
});
const sql = table.deleteRows.mock.calls[0]?.[0] ?? "";
expect(sql).toContain("where rowid in (");
expect(sql).toContain(`limit ${MAINTENANCE_DELETE_BATCH_ROWS}`);
});
});
70 changes: 70 additions & 0 deletions apps/desktop/src/main/services/state/dbMaintenanceApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,64 @@ export function pruneIngressEventRowsForProject(
return removed;
}

/** Linear/worker/pack/CTO event-log rows older than this are deleted. */
export const EVENT_LOG_RETENTION_DAYS = 30;

/**
* Rows deleted per batch by {@link pruneRowsInBatches}.
*
* ADE's SQLite driver (`node:sqlite` `DatabaseSync`) is fully synchronous, so a
* DELETE runs to completion on the event loop with nothing else able to
* proceed. 2,000 rows is small enough that one batch is imperceptible and
* large enough that a realistic backlog clears in a handful of them.
*/
export const MAINTENANCE_DELETE_BATCH_ROWS = 2_000;

/** Batches per prune, so one call cannot run unbounded on a huge backlog. */
export const MAINTENANCE_DELETE_MAX_BATCHES = 200;

/**
* Delete rows matching `where` in paced batches, yielding to the event loop
* between them.
*
* Every existing prune issues a single unbounded DELETE. That is fine at
* today's row counts and is a latency cliff waiting for the first project that
* accumulates a real backlog: a synchronous driver means the UI, IPC, and sync
* pump all stop for the duration. Batching bounds each pause; the `setImmediate`
* between batches is what actually lets everything else run.
*
* Deleting by `rowid` rather than by the predicate keeps each batch's work
* proportional to the batch, not to the table.
*/
export async function pruneRowsInBatches(args: {
table: string;
where: string;
params: readonly unknown[];
deleteRows: (sql: string, params: readonly unknown[]) => number;
batchRows?: number;
maxBatches?: number;
yieldToEventLoop?: () => Promise<void>;
}): Promise<number> {
const batchRows = args.batchRows ?? MAINTENANCE_DELETE_BATCH_ROWS;
const maxBatches = args.maxBatches ?? MAINTENANCE_DELETE_MAX_BATCHES;
const yieldToEventLoop = args.yieldToEventLoop
?? (() => new Promise<void>((resolve) => { setImmediate(resolve); }));

const sql = `delete from ${args.table} where rowid in (
select rowid from ${args.table} where ${args.where} limit ${batchRows}
)`;
let removed = 0;
for (let batch = 0; batch < maxBatches; batch += 1) {
const changes = args.deleteRows(sql, args.params);
removed += changes;
// A short batch means the predicate is exhausted; stop without paying for
// one more round trip.
if (changes < batchRows) break;
await yieldToEventLoop();
}
return removed;
}

export type DbMaintenanceResult = {
itemsAffected: number;
bytesReclaimed: number;
Expand All @@ -102,6 +160,18 @@ export interface DbMaintenanceApi {
pruneReviewArtifacts(): DbMaintenanceResult;
/** Delete pull_request_snapshots rows not updated in 60 days. */
prunePrSnapshots(): DbMaintenanceResult;
/**
* Delete rows older than 30 days from the append-only event logs that had no
* retention at all: `linear_sync_events`, `linear_workflow_run_events`,
* `worker_agent_cost_events`, and `pack_events`. Every one of them
* replicates.
*
* `linear_ingress_events`, `cto_session_logs`, and `worker_agent_runs` look
* like they belong here and deliberately do not — see
* `RETAINED_EVENT_LOG_TABLES` in `kvDb.ts` for why pruning each would be a
* bug.
*/
pruneEventLogs(): Promise<DbMaintenanceResult>;
/**
* Reclaim cr-sqlite clock/pks bookkeeping. Only safe (and only performed)
* when the project has zero sync peers; otherwise returns skippedReason
Expand Down
103 changes: 103 additions & 0 deletions apps/desktop/src/main/services/state/kvDb.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
type TableRebuildPlan,
} from "./kvDb";
import { isCrsqliteAvailable } from "./crsqliteExtension";
import { EVENT_LOG_RETENTION_DAYS } from "./dbMaintenanceApi";

const testRequire = createRequire(import.meta.url);
const { DatabaseSync } = testRequire("node:sqlite") as {
Expand Down Expand Up @@ -763,3 +764,105 @@ describe("sweepOrphanedRepairStagingTables", () => {
}
});
});


describe("retention maintenance", () => {
async function openScratchDb() {
const root = makeProjectRoot("ade-kvdb-retention-");
const db = await openKvDb(path.join(root, ".ade", "ade.db"), createLogger() as any);
return { db, root };
}

const daysAgo = (days: number): string =>
new Date(Date.now() - days * 24 * 60 * 60 * 1_000).toISOString();

it("prunes event logs on created_at and leaves fresh rows", async () => {
const { db } = await openScratchDb();
try {
db.run(
`insert into pack_events (id, project_id, pack_key, event_type, created_at)
values ('pack-old', 'proj', 'k', 'installed', ?)`,
[daysAgo(EVENT_LOG_RETENTION_DAYS + 5)],
);
db.run(
`insert into pack_events (id, project_id, pack_key, event_type, created_at)
values ('pack-fresh', 'proj', 'k', 'installed', ?)`,
[daysAgo(1)],
);

const result = await db.maintenance!.pruneEventLogs();
expect(result.itemsAffected).toBe(1);
expect(db.all<{ id: string }>("select id from pack_events").map((r) => r.id))
.toEqual(["pack-fresh"]);
} finally {
db.close();
}
});

// The webhook replay guard, the CTO log that a reconcile re-inserts, and the
// worker lifecycle table are all deliberately exempt — pruning any of them
// would be a bug, not a saving.
it("leaves the deliberately-unretained tables alone", async () => {
const { db } = await openScratchDb();
try {
const stale = daysAgo(EVENT_LOG_RETENTION_DAYS + 60);
db.run(
`insert into linear_ingress_events (id, project_id, source, delivery_id, created_at)
values ('ing-1', 'proj', 'linear', 'delivery-1', ?)`,
[stale],
);
db.run(
`insert into cto_session_logs (id, project_id, session_id, created_at)
values ('cto-1', 'proj', 'sess', ?)`,
[stale],
);
db.run(
`insert into worker_agent_runs (id, project_id, agent_id, created_at)
values ('run-1', 'proj', 'agent', ?)`,
[stale],
);

await db.maintenance!.pruneEventLogs();

expect(db.all("select id from linear_ingress_events")).toHaveLength(1);
expect(db.all("select id from cto_session_logs")).toHaveLength(1);
expect(db.all("select id from worker_agent_runs")).toHaveLength(1);
} finally {
db.close();
}
});

// A locked or malformed database must not read as "step not supported" —
// that renders as a tidy completed sweep in the journal, the analytics event,
// and the Settings summary.
it("surfaces a prune failure instead of reporting it as unsupported", async () => {
const { db } = await openScratchDb();
try {
// A malformed table is a real SQL failure, not "this handle does not
// implement the step" — the doctor must record it as an error rather than
// rendering a tidy completed sweep.
db.run("drop table pack_events");
db.run("create table pack_events (id text primary key)");
await expect(db.maintenance!.pruneEventLogs()).rejects.toThrow();
} finally {
db.close();
}
});

// SQLite orders INTEGER before TEXT, so an epoch-number timestamp would match
// every cutoff; CRR repair can also leave an empty-string timestamp.
it("does not treat a non-text or empty timestamp as expired", async () => {
const { db } = await openScratchDb();
try {
db.run("insert into pack_events (id, project_id, pack_key, event_type, created_at) values ('epoch', 'p', 'k', 'e', 1767225600000)");
db.run("insert into pack_events (id, project_id, pack_key, event_type, created_at) values ('empty', 'p', 'k', 'e', '')");

await db.maintenance!.pruneEventLogs();

expect(db.all<{ id: string }>("select id from pack_events order by id").map((r) => r.id))
.toEqual(["empty", "epoch"]);
} finally {
db.close();
}
});
});
Loading
Loading