Skip to content
Closed
11 changes: 11 additions & 0 deletions apps/mobile/src/features/usage/UsageRouteScreen.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -463,10 +463,12 @@ function UsageCoverageNotice(props: {
props.merged.staleEnvironments.includes(environment.environmentId),
);
const duplicateSources = props.merged.duplicateSources;
const incompleteSources = props.merged.incompleteSources;
if (
failed.length === 0 &&
stale.length === 0 &&
duplicateSources.length === 0 &&
incompleteSources.length === 0 &&
!props.isPartial
) {
return null;
Expand All @@ -489,6 +491,15 @@ function UsageCoverageNotice(props: {
{environment.label} runs an older server version and is excluded from totals.
</Text>
))}
{incompleteSources.map((source) => (
<Text
key={`${source.environmentId}:${source.provider}`}
className="text-sm text-foreground-muted"
>
{source.environmentLabel}&apos;s {PROVIDER_LABEL[source.provider]} usage{" "}
{source.status === "failed" ? "could not be read." : "is incomplete."}
</Text>
))}
{duplicateSources.length > 0 ? (
<Text className="text-sm text-foreground-muted">
Counted once across environments sharing a transcript directory:{" "}
Expand Down
7 changes: 5 additions & 2 deletions apps/mobile/src/features/usage/usageProviders.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,24 @@ import { useAppearancePreferences } from "../settings/appearance/AppearancePrefe
* Series and table order. The chart stacks providers from the bottom in this
* order, so it also fixes which band sits on top of the bars.
*/
export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude"];
export const PROVIDER_ORDER: readonly UsageProviderKind[] = ["codex", "claude", "zcode"];

export const PROVIDER_LABEL: Record<UsageProviderKind, string> = {
claude: "Claude Code",
codex: "Codex",
zcode: "ZCode",
};

/**
* Claude's brand orange holds in both themes; Codex is neutral and must flip
* with the theme or its bars vanish against the matching background.
* with the theme or its bars vanish against the matching background. ZCode's
* indigo reads on either theme.
*/
export function useProviderColors(): Record<UsageProviderKind, string> {
const { themeAppearance: scheme } = useAppearancePreferences();
return {
claude: "#d97757",
codex: scheme === "dark" ? "#e6e6e6" : "#3c3c43",
zcode: "#6366f1",
};
}
2 changes: 2 additions & 0 deletions apps/mobile/src/state/usage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ export function useUsage(input: UsageSummaryInput): UsageView {
const windowKey = useMemo(
() =>
JSON.stringify({
contractVersion: input.contractVersion,
sinceDay: input.sinceDay,
untilDay: input.untilDay,
timeZone: input.timeZone,
Expand All @@ -86,6 +87,7 @@ export function useUsage(input: UsageSummaryInput): UsageView {
untilTime: input.untilTime,
}),
[
input.contractVersion,
input.sinceDay,
input.untilDay,
input.timeZone,
Expand Down
35 changes: 35 additions & 0 deletions apps/server/src/usage/UsageService.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { describe, expect, it } from "@effect/vitest";

import { negotiateUsageContractVersion, summarizeSourceReadFailures } from "./UsageService.ts";

describe("negotiateUsageContractVersion", () => {
it("keeps the v4 response shape for clients that do not advertise support", () => {
expect(negotiateUsageContractVersion(undefined)).toBe(4);
expect(negotiateUsageContractVersion(4)).toBe(4);
});

it("serves the current response shape to compatible clients", () => {
expect(negotiateUsageContractVersion(5)).toBe(5);
expect(negotiateUsageContractVersion(6)).toBe(5);
});
});

describe("summarizeSourceReadFailures", () => {
it("reports a healthy source when every file was readable", () => {
expect(summarizeSourceReadFailures(2, 0)).toEqual({ status: "ok", message: null });
});

it("reports partial coverage when only some files failed", () => {
expect(summarizeSourceReadFailures(2, 1)).toEqual({
status: "partial",
message: "1 usage file could not be read.",
});
});

it("reports a failed source when every file failed", () => {
expect(summarizeSourceReadFailures(2, 2)).toEqual({
status: "failed",
message: "2 usage files could not be read.",
});
});
});
124 changes: 100 additions & 24 deletions apps/server/src/usage/UsageService.ts
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@ import {
listTranscriptFiles,
readDirectoryVolumeId,
readTranscriptRecords,
statSqliteUsageStore,
} from "./usageTranscriptReader.ts";
import {
decodeScanCache,
dedupeWithinFile,
encodeScanCache,
isReusableCachedFile,
pruneScanCache,
type ScanCache,
} from "./usageScanCache.ts";
Expand All @@ -66,6 +68,9 @@ const RATES_TTL_MS = 24 * 60 * 60 * 1000;
const MTIME_SLACK_MS = 36 * 60 * 60 * 1000;
const MAX_HOURLY_WINDOW_MS = 24 * 60 * 60 * 1000;

/** Clients predating ZCode omit their supported response contract. */
const PRE_ZCODE_USAGE_CONTRACT_VERSION = 4 as const;

/** Longest window the UI offers, plus slack. Older entries are pruned. */
const CACHE_RETENTION_DAYS = 90;

Expand All @@ -74,6 +79,19 @@ const RatesCacheFile = Schema.Struct({
fetchedAtMs: Schema.Number,
document: Schema.Unknown,
});

/**
* One provider's usage store.
*
* `dir` supplies the source fingerprint's volume id and is walked for
* `*.jsonl` transcripts — unless `file` names a single-file store (ZCode's
* sqlite db), which is checked for existence and read instead.
*/
interface TranscriptSource {
readonly provider: UsageProviderKind;
readonly dir: string;
readonly file?: string;
}
const decodeRatesCache = Schema.decodeUnknownEffect(
Schema.fromJsonString(RatesCacheFile as unknown as Schema.Codec<typeof RatesCacheFile.Type>),
);
Expand All @@ -93,13 +111,32 @@ export class UsageService extends Context.Service<
}
>()("t3/usage/UsageService") {}

export function summarizeSourceReadFailures(
totalFiles: number,
failedFiles: number,
): Pick<UsageSource, "status" | "message"> {
if (failedFiles === 0) return { status: "ok", message: null };
return {
status: failedFiles === totalFiles ? "failed" : "partial",
message: `${failedFiles} usage file${failedFiles === 1 ? "" : "s"} could not be read.`,
};
}

export function negotiateUsageContractVersion(
requestedVersion: number | undefined,
): typeof PRE_ZCODE_USAGE_CONTRACT_VERSION | typeof USAGE_CONTRACT_VERSION {
return requestedVersion !== undefined && requestedVersion >= USAGE_CONTRACT_VERSION
? USAGE_CONTRACT_VERSION
: PRE_ZCODE_USAGE_CONTRACT_VERSION;
}

/** Empty summary, for suites that only need the RPC surface to resolve. */
export const layerTest = Layer.succeed(
UsageService,
UsageService.of({
readSummary: (input) =>
Effect.succeed({
contractVersion: USAGE_CONTRACT_VERSION,
contractVersion: negotiateUsageContractVersion(input.contractVersion),
readAt: "1970-01-01T00:00:00.000Z",
timeZone: input.timeZone,
sinceDay: input.sinceDay,
Expand Down Expand Up @@ -219,10 +256,17 @@ export const make = Effect.gen(function* () {
const claudeDir = yield* resolveClaudeTranscriptDir(claudeHome);
const codexLayout = yield* resolveCodexHomeLayout(settings.providers.codex);

return [
{ provider: "claude" as const, dir: claudeDir },
{ provider: "codex" as const, dir: path.join(codexLayout.sharedHomePath, "sessions") },
// ZCode has no settings-driven home override; its usage store is always
// the app's own sqlite db. A missing directory or db file is reported as a
// missing source.
const zcodeDbDir = path.join(NodeOS.homedir(), ".zcode", "cli", "db");

const sources: readonly TranscriptSource[] = [
{ provider: "claude", dir: claudeDir },
{ provider: "codex", dir: path.join(codexLayout.sharedHomePath, "sessions") },
{ provider: "zcode", dir: zcodeDbDir, file: path.join(zcodeDbDir, "db.sqlite") },
];
return sources;
});

/**
Expand Down Expand Up @@ -263,34 +307,39 @@ export const make = Effect.gen(function* () {
size: number,
mtimeMs: number,
provider: UsageProviderKind,
): Effect.Effect<readonly UsageRecord[]> =>
zcodeSinceMs: number,
): Effect.Effect<readonly UsageRecord[] | null> =>
Effect.gen(function* () {
const cached = fileCache.get(filePath);
// Provider is part of the identity: if both providers were ever pointed
// at one directory, a hit parsed by the other parser must not be reused.
if (
cached &&
cached.size === size &&
cached.mtimeMs === mtimeMs &&
cached.provider === provider
) {
if (cached && isReusableCachedFile(cached, { size, mtimeMs, provider }, zcodeSinceMs)) {
return cached.records;
}

const parsed = yield* Effect.promise(() => readTranscriptRecords(filePath, provider));
const parsed = yield* Effect.promise(() =>
readTranscriptRecords(filePath, provider, zcodeSinceMs),
);
// A read failure is not an empty transcript: caching it under this
// (size, mtime) would silently drop the file's usage until it changes.
if (parsed === null) return [];
if (parsed === null) return null;
// Stored already de-duplicated within the file, which is 99% of all
// duplicates. The aggregator still runs the cross-file dedupe pass.
const records = dedupeWithinFile(parsed);

fileCache.set(filePath, { size, mtimeMs, provider, records });
fileCache.set(filePath, {
size,
mtimeMs,
provider,
completeFromMs: provider === "zcode" ? zcodeSinceMs : null,
records,
});
cacheDirty = true;
return records;
});

const readSummary = Effect.fn("UsageService.readSummary")(function* (input: UsageSummaryInput) {
const contractVersion = negotiateUsageContractVersion(input.contractVersion);
if (input.sinceDay > input.untilDay) {
return yield* new UsageReadError({
reason: "invalidWindow",
Expand Down Expand Up @@ -323,13 +372,20 @@ export const make = Effect.gen(function* () {
}

const startedAtMs = yield* Clock.currentTimeMillis;
const retentionCutoffMs = startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000;
yield* ensureRates();
yield* ensureScanCacheLoaded;

const hostId = NodeOS.hostname();
// The home resolvers ask for `Path` themselves; satisfy them from the
// instance we already hold so `readSummary` stays context-free.
const dirs = yield* resolveTranscriptDirs().pipe(Effect.provideService(Path.Path, path));
const resolvedDirs = yield* resolveTranscriptDirs().pipe(
Effect.provideService(Path.Path, path),
);
const dirs =
contractVersion >= USAGE_CONTRACT_VERSION
? resolvedDirs
: resolvedDirs.filter((source) => source.provider !== "zcode");
const windowStart = DateTime.make(`${input.sinceDay}T00:00:00Z`);
if (Option.isNone(windowStart)) {
return yield* new UsageReadError({
Expand All @@ -353,10 +409,11 @@ export const make = Effect.gen(function* () {
const livePaths = new Set<string>();
const walkedRoots: string[] = [];

for (const { provider, dir } of dirs) {
for (const source of dirs) {
const { provider, dir } = source;
const volumeId = yield* Effect.promise(() => readDirectoryVolumeId(dir));
const exists = yield* fileSystem
.exists(dir)
.exists(source.file ?? dir)
.pipe(Effect.catchCause(() => Effect.succeed(false)));

if (!exists) {
Expand All @@ -367,22 +424,40 @@ export const make = Effect.gen(function* () {
skippedFiles: 0,
malformedRecords: 0,
distinctSessions: 0,
message: "No transcript directory on this environment.",
message:
source.file === undefined
? "No transcript directory on this environment."
: "No usage store on this environment.",
});
continue;
}

walkedRoots.push(dir);
const files = yield* Effect.promise(() => listTranscriptFiles(dir, windowStartMs));
const files = yield* Effect.promise(() =>
source.file === undefined
? listTranscriptFiles(dir, windowStartMs)
: statSqliteUsageStore(source.file, windowStartMs),
);
let scannedFiles = 0;
let skippedFiles = 0;
let failedFiles = 0;
// Distinct per directory. Buckets carry per-cell session counts, but a
// session spans days and models, so clients total this figure instead.
const sessionIds = new Set<string>();

for (const file of files) {
livePaths.add(file.path);
const records = yield* readFileRecords(file.path, file.size, file.mtimeMs, provider);
const records = yield* readFileRecords(
file.path,
file.size,
file.mtimeMs,
provider,
windowStartMs,
);
if (records === null) {
failedFiles += 1;
continue;
}
if (records.length === 0) {
skippedFiles += 1;
continue;
Expand All @@ -397,22 +472,23 @@ export const make = Effect.gen(function* () {
}
}

const readHealth = summarizeSourceReadFailures(files.length, failedFiles);
sources.push({
fingerprint: { hostId, provider, resolvedHomePath: dir, volumeId },
status: "ok",
status: readHealth.status,
scannedFiles,
skippedFiles,
malformedRecords: 0,
distinctSessions: sessionIds.size,
message: null,
message: readHealth.message,
});
}

const pruned = pruneScanCache(fileCache, {
livePaths,
walkedRoots,
windowStartMs,
retentionCutoffMs: startedAtMs - CACHE_RETENTION_DAYS * 24 * 60 * 60 * 1000,
retentionCutoffMs,
});
if (pruned > 0) cacheDirty = true;
yield* persistScanCache();
Expand All @@ -422,7 +498,7 @@ export const make = Effect.gen(function* () {
const finishedAtMs = yield* Clock.currentTimeMillis;

return {
contractVersion: USAGE_CONTRACT_VERSION,
contractVersion,
readAt: DateTime.formatIso(readAt),
timeZone: input.timeZone,
sinceDay: input.sinceDay,
Expand Down
Loading
Loading