From 759a6f3a023c8c2e8f6c95182b8319b535df65c5 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:19:03 -0700 Subject: [PATCH 01/10] feat(combos): add quota-aware economy selection and lifecycle Introduce the economy combo strategy with shared allowances, cached snapshots, pre-dispatch reservations, settlement (incl. stream/credits), off-path usage-log refresh, and request-path lifecycle wiring. --- src/combos/economy-refresh.ts | 107 +++++ src/combos/economy.ts | 618 +++++++++++++++++++++++++++ src/combos/index.ts | 24 ++ src/combos/resolve.ts | 38 +- src/combos/types.ts | 54 ++- src/config.ts | 134 ++++++ src/lib/state-store-registrations.ts | 15 +- src/lib/state-store-sweeper.ts | 1 + src/router.ts | 4 +- src/server/responses/core.ts | 84 +++- src/types.ts | 55 ++- 11 files changed, 1116 insertions(+), 18 deletions(-) create mode 100644 src/combos/economy-refresh.ts create mode 100644 src/combos/economy.ts diff --git a/src/combos/economy-refresh.ts b/src/combos/economy-refresh.ts new file mode 100644 index 0000000000..abe0594e00 --- /dev/null +++ b/src/combos/economy-refresh.ts @@ -0,0 +1,107 @@ +import type { OcxConfig, OcxEconomicAllowance, OcxEconomicSnapshot } from "../types"; +import { + currentUsageLogRevision, + readRecentUsageEntries, + usageLogRevisionKey, +} from "../usage/log"; +import { + getEconomicQuotaSnapshot, + setEconomicQuotaSnapshot, +} from "./economy"; + +let lastRevisionKey: string | null = null; +let refreshInflight: Promise | null = null; + +function nonNegative(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0; +} + +function usageAmount(allowance: OcxEconomicAllowance, entry: ReturnType[number]): number { + if (allowance.unit === "requests") return 1; + if (allowance.unit === "inputTokens") return nonNegative(entry.usage?.inputTokens); + if (allowance.unit === "outputTokens") return nonNegative(entry.usage?.outputTokens); + return nonNegative(entry.totalTokens ?? (entry.usage + ? entry.usage.inputTokens + entry.usage.outputTokens + : 0)); +} + +function snapshotFor( + allowance: OcxEconomicAllowance, + entries: ReturnType, + now: number, +): OcxEconomicSnapshot { + // The remaining value is computed from usage over the just-closed window, but it + // applies to the CURRENT window starting now — so the snapshot advertises + // windowStart = now. A boundary of now - durationMs would make the snapshot look + // immediately rolled-over. + const filterStart = allowance.window.kind === "rolling" ? now - allowance.window.durationMs : undefined; + const used = entries + .filter(entry => filterStart === undefined || entry.timestamp >= filterStart) + .reduce((total, entry) => total + usageAmount(allowance, entry), 0); + return { + remaining: Math.max(0, allowance.capacity - used), + updatedAt: now, + source: "usage-log", + confidence: "estimated", + ...(allowance.window.kind === "rolling" ? { windowStart: now } : {}), + }; +} + +function allowanceConfigKey(config: OcxConfig): string { + return Object.entries(config.economicAllowances ?? {}) + .filter(([, allowance]) => allowance.source === "usage-log") + .map(([id, allowance]) => `${id}:${JSON.stringify(allowance)}`) + .sort() + .join("\0"); +} + +function markRefreshFailure(config: OcxConfig, error: unknown): void { + const message = error instanceof Error ? error.message : "refresh failed"; + for (const [id, allowance] of Object.entries(config.economicAllowances ?? {})) { + if (allowance.source !== "usage-log") continue; + const previous = getEconomicQuotaSnapshot(id); + if (!previous) continue; + setEconomicQuotaSnapshot(id, { + ...previous, + confidence: "estimated", + error: message.slice(0, 200), + }); + } +} + +async function performRefresh(config: OcxConfig, now: number): Promise { + let revisionKey: string; + try { + revisionKey = `${usageLogRevisionKey(currentUsageLogRevision())}\0${allowanceConfigKey(config)}`; + if (revisionKey === lastRevisionKey) return; + const entries = readRecentUsageEntries(2000); + const prepared = new Map(); + for (const [id, allowance] of Object.entries(config.economicAllowances ?? {})) { + if (allowance.source !== "usage-log") continue; + prepared.set(id, snapshotFor(allowance, entries, now)); + } + for (const [id, snapshot] of prepared) setEconomicQuotaSnapshot(id, snapshot); + lastRevisionKey = revisionKey; + } catch (error) { + markRefreshFailure(config, error); + lastRevisionKey = null; + } +} + +export function refreshEconomicSnapshots(config: OcxConfig, now = Date.now()): Promise { + if (refreshInflight) return refreshInflight; + const promise = performRefresh(config, now).finally(() => { + if (refreshInflight === promise) refreshInflight = null; + }); + refreshInflight = promise; + return promise; +} + +export function resetEconomicSnapshotRefreshForTests(): void { + lastRevisionKey = null; + refreshInflight = null; +} + +export function stopEconomicSnapshotRefresh(): void { + refreshInflight = null; +} diff --git a/src/combos/economy.ts b/src/combos/economy.ts new file mode 100644 index 0000000000..49a30c7d66 --- /dev/null +++ b/src/combos/economy.ts @@ -0,0 +1,618 @@ +import { estimateTokens } from "../lib/token-estimate"; +import type { OcxComboConfig, OcxComboTarget, OcxConfig, OcxEconomicAllowance, OcxEconomicSnapshot } from "../types"; +import { captureConfigGeneration, type GenerationContext } from "../lib/state-store-sweeper"; +import { targetKey } from "./types"; +type EconomicTarget = OcxComboTarget & { weight: number }; + +export interface EconomicRequestEstimate { + inputTokens: number; + outputTokens: number; + cachedInputTokens?: number; + fixedRequests?: number; + kind: "observed" | "configured" | "historical" | "fallback"; +} + +export interface EconomicActualUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + cachedInputTokens?: number; + requests?: number; + credits?: number; + usd?: number; +} + +const ECONOMIC_ACTUAL_FIELDS = ["inputTokens", "outputTokens", "totalTokens", "cachedInputTokens", "requests", "credits", "usd"] as const; + +export type EconomicHardExclusion = + | "attempted" + | "unconfigured" + | "disabled" + | "missing-allowance" + | "hard-headroom" + | "stale-quota" + | "max-marginal-usd" + | "ineligible"; + +export type EconomicSoftSignal = + | "reserve" + | "unknown-quota" + | "expiration-pressure"; + +export type EconomicRankingBand = "excluded" | "expiration" | "marginal-cost" | string; + +export interface EconomicSelectionCandidate { + target: EconomicTarget; + eligible: boolean; + exclusions: EconomicHardExclusion[]; + softSignals: EconomicSoftSignal[]; + configIndex: number; + cashCost: number | "included" | "unknown"; + consumption: number[]; + postRequestRemaining: Array; + reserveThresholds: Array; + burnPressure: number | null; + marginalUsd: number | null; + stale: boolean; + rankingBand: EconomicRankingBand; + allowances: Array<{ + id: string; + capacity: number | null; + remaining: number | null; + reserved: number; + predicted: number; + postRequestRemaining: number | null; + reserveThreshold: number; + resetAt?: number; + expiresAt?: number; + source?: string; + ageMs?: number; + stale: boolean; + }>; +} + +export interface EconomicSelectionResult { + target?: EconomicTarget; + targetIndex: number | null; + candidates: EconomicSelectionCandidate[]; + reason: string; + reservationId?: string; +} + +export interface EconomicExplanation extends EconomicSelectionResult { + comboId: string; + strategy: "economy"; + selectedTarget: string | null; + generatedAt: number; +} + +interface Reservation { + id: string; + allowanceId: string; + unit: OcxEconomicAllowance["unit"]; + amount: number; + rates?: OcxEconomicAllowance["rates"]; + pricing?: OcxComboTarget["pricing"]; + expiresAt: number; + generation: number; +} + +const snapshots = new Map(); +const reservations = new Map(); +const settledReservationIds = new Set(); +let reservationSequence = 0; +let lastReconciledGeneration = 0; +let liveAllowanceIds = new Set(); +const RESERVATION_TTL_MS = 10 * 60_000; +const EPSILON = 1e-9; + +function finiteNonNegative(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value) && value >= 0; +} + +function safe(value: number | undefined): number { + return finiteNonNegative(value) ? value : 0; +} + +function allowanceFor(config: OcxConfig, id: string): OcxEconomicAllowance | undefined { + return config.economicAllowances?.[id]; +} + +function snapshotAge(snapshot: OcxEconomicSnapshot | undefined, now: number): number { + return snapshot ? Math.max(0, now - snapshot.updatedAt) : Number.POSITIVE_INFINITY; +} + +export interface SnapshotFreshness { + status: "fresh" | "stale" | "unknown"; + ageMs: number | null; + reason?: "missing-snapshot" | "stale-snapshot" | "expired-window" | "past-reset" | "rolling-reset" | "missing-calendar-reset"; +} + +function finiteBoundary(value: number | undefined): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function effectiveBoundary(allowance: OcxEconomicAllowance, snapshot: OcxEconomicSnapshot): number | undefined { + switch (allowance.window.kind) { + case "balance": + return undefined; + case "rolling": { + const start = finiteBoundary(snapshot.windowStart); + return start === undefined ? undefined : start + allowance.window.durationMs; + } + case "calendar": + return finiteBoundary(snapshot.resetAt); + case "expiresAt": + return Math.min(allowance.window.expiresAt, ...( [finiteBoundary(snapshot.resetAt), finiteBoundary(snapshot.expiresAt)].filter((value): value is number => value !== undefined) )); + } +} + +export function snapshotFreshness( + snapshot: OcxEconomicSnapshot | undefined, + allowance: OcxEconomicAllowance, + now: number, +): SnapshotFreshness { + if (!snapshot) return { status: "unknown", ageMs: null, reason: "missing-snapshot" }; + const ageMs = snapshotAge(snapshot, now); + if (ageMs > (allowance.staleAfterMs ?? 15 * 60_000)) return { status: "stale", ageMs, reason: "stale-snapshot" }; + const boundary = effectiveBoundary(allowance, snapshot); + if (allowance.window.kind === "calendar" && boundary === undefined) { + return { status: "unknown", ageMs, reason: "missing-calendar-reset" }; + } + if (allowance.window.kind === "rolling" && boundary === undefined) { + return { status: "unknown", ageMs, reason: "rolling-reset" }; + } + if (boundary !== undefined && boundary <= now) { + return { status: "unknown", ageMs, reason: allowance.window.kind === "rolling" ? "rolling-reset" : snapshot.resetAt !== undefined && snapshot.resetAt <= now ? "past-reset" : "expired-window" }; + } + if (snapshot.resetAt !== undefined && snapshot.resetAt <= now && allowance.window.kind !== "balance") { + return { status: "unknown", ageMs, reason: "past-reset" }; + } + return { status: "fresh", ageMs }; +} + +export function usableHeadroom( + allowance: OcxEconomicAllowance, + snapshot: OcxEconomicSnapshot | undefined, + consumption: number, + reserved: number, + now: number, +): number | null { + if (snapshotFreshness(snapshot, allowance, now).status !== "fresh") return null; + if (!finiteNonNegative(snapshot?.remaining) || !finiteNonNegative(consumption) || !finiteNonNegative(reserved)) return null; + const value = snapshot.remaining - reserved - consumption; + return Number.isFinite(value) && Object.is(value, -0) ? 0 : Number.isFinite(value) ? value : null; +} + +function reserveAmount(allowance: OcxEconomicAllowance): number { + const value = allowance.reserveAmount !== undefined + ? allowance.reserveAmount + : allowance.reserveFraction !== undefined ? allowance.capacity * allowance.reserveFraction : 0; + return finiteNonNegative(value) ? value : 0; +} + +function activeReserved(allowanceId: string, now: number): number { + let total = 0; + for (const reservation of reservations.values()) { + if (reservation.expiresAt <= now) continue; + if (reservation.allowanceId === allowanceId) total += reservation.amount; + } + return total; +} + +function windowDurationMs(allowance: OcxEconomicAllowance, snapshot: OcxEconomicSnapshot, now: number): number | null { + if (allowance.window.kind === "rolling") return allowance.window.durationMs; + if (snapshot.windowStart !== undefined && snapshot.resetAt !== undefined) { + return Math.max(0, snapshot.resetAt - snapshot.windowStart); + } + if (snapshot.resetAt !== undefined && snapshot.resetAt > now) return Math.max(0, snapshot.resetAt - now); + if (snapshot.expiresAt !== undefined && snapshot.expiresAt > now) return Math.max(0, snapshot.expiresAt - now); + return null; +} + +export function estimateEconomicRequest(body: unknown, modelId?: string): EconomicRequestEstimate { + const record = body && typeof body === "object" && !Array.isArray(body) + ? body as Record + : {}; + const input = record.input ?? record.messages ?? body ?? ""; + const inputTokens = estimateTokens(typeof input === "string" ? input : JSON.stringify(input), modelId); + const maxOutput = record.max_output_tokens ?? record.max_tokens; + if (typeof maxOutput === "number" && Number.isFinite(maxOutput) && maxOutput >= 0) { + return { inputTokens, outputTokens: Math.floor(maxOutput), kind: "configured" }; + } + return { inputTokens, outputTokens: 1024, kind: "fallback" }; +} + +export function economicConsumption( + allowance: OcxEconomicAllowance, + estimate: EconomicRequestEstimate, + pricing?: OcxComboTarget["pricing"], +): number { + const input = safe(estimate.inputTokens); + const output = safe(estimate.outputTokens); + const fixedRequests = safe(estimate.fixedRequests ?? 1); + switch (allowance.unit) { + case "requests": return fixedRequests; + case "inputTokens": return input; + case "outputTokens": return output; + case "totalTokens": return input + output; + case "credits": + return safe(allowance.rates?.fixedPerRequest ?? 0) * fixedRequests + + safe(allowance.rates?.inputPerMillion ?? 0) * input / 1_000_000 + + safe(allowance.rates?.outputPerMillion ?? 0) * output / 1_000_000 + + safe(allowance.rates?.cachedInputPerMillion ?? 0) * safe(estimate.cachedInputTokens) / 1_000_000; + case "usd": + return safe(pricing?.fixedPerRequest ?? allowance.rates?.fixedPerRequest ?? 0) * fixedRequests + + safe(pricing?.inputUsdPerMillion ?? allowance.rates?.inputPerMillion ?? 0) * input / 1_000_000 + + safe(pricing?.outputUsdPerMillion ?? allowance.rates?.outputPerMillion ?? 0) * output / 1_000_000 + + safe(pricing?.cachedInputUsdPerMillion ?? allowance.rates?.cachedInputPerMillion ?? 0) * safe(estimate.cachedInputTokens) / 1_000_000; + } +} + +function marginalUsd(target: OcxComboTarget, estimate: EconomicRequestEstimate): number | null { + const pricing = target.pricing; + if (!pricing) return target.allowances?.length ? 0 : null; + const value = safe(pricing.fixedPerRequest) + safe(pricing.inputUsdPerMillion) * safe(estimate.inputTokens) / 1_000_000 + + safe(pricing.outputUsdPerMillion) * safe(estimate.outputTokens) / 1_000_000 + + safe(pricing.cachedInputUsdPerMillion) * safe(estimate.cachedInputTokens) / 1_000_000; + return Number.isFinite(value) ? value : null; +} + +function cashCostFor(target: OcxComboTarget, estimate: EconomicRequestEstimate, cost: number | null): number | "included" | "unknown" { + if (target.pricing) { + return cost !== null && Number.isFinite(cost) ? cost : "unknown"; + } + if (target.allowances?.length) return "included"; + return "unknown"; +} + +function expiryPressure(allowance: OcxEconomicAllowance, snapshot: OcxEconomicSnapshot | undefined, now: number): number | null { + if (allowance.rollover !== false || allowance.window.kind === "balance" || !snapshot) return null; + const expiry = snapshot.expiresAt ?? snapshot.resetAt; + const duration = windowDurationMs(allowance, snapshot, now); + if (expiry === undefined || duration === null || duration <= 0) return null; + const timeFraction = Math.max(EPSILON, Math.min(1, (expiry - now) / duration)); + return Math.max(0, safe(snapshot.remaining) / Math.max(EPSILON, allowance.capacity)) / timeFraction; +} + +function unknownPolicy(combo: OcxComboConfig): "allow" | "deprioritize" | "reject" { + return combo.economy?.unknownQuota ?? "deprioritize"; +} + +function compareCandidates(a: EconomicSelectionCandidate, b: EconomicSelectionCandidate): number { + const eligible = (a.eligible ? 0 : 1) - (b.eligible ? 0 : 1); + if (eligible !== 0) return eligible; + const reserve = (a.softSignals.includes("reserve") ? 1 : 0) - (b.softSignals.includes("reserve") ? 1 : 0); + if (reserve !== 0) return reserve; + const unknown = (a.softSignals.includes("unknown-quota") ? 1 : 0) - (b.softSignals.includes("unknown-quota") ? 1 : 0); + if (unknown !== 0) return unknown; + const pressure = (b.burnPressure ?? -1) - (a.burnPressure ?? -1); + if (pressure !== 0) return pressure; + const aFinite = typeof a.marginalUsd === "number" && Number.isFinite(a.marginalUsd); + const bFinite = typeof b.marginalUsd === "number" && Number.isFinite(b.marginalUsd); + if (aFinite && bFinite) { + if (a.marginalUsd! < b.marginalUsd!) return -1; + if (a.marginalUsd! > b.marginalUsd!) return 1; + } else if (aFinite && !bFinite) return -1; + else if (!aFinite && bFinite) return 1; + return a.configIndex - b.configIndex; +} + +function candidateFor( + config: OcxConfig, + combo: OcxComboConfig, + target: EconomicTarget, + estimate: EconomicRequestEstimate, + now: number, + index: number, + excluded: ReadonlySet, +): EconomicSelectionCandidate { + const hardExclusions: EconomicHardExclusion[] = []; + const softSignals: EconomicSoftSignal[] = []; + const allowances = target.allowances ?? []; + const consumptions: number[] = []; + const postRequestRemaining: Array = []; + const reserveThresholds: Array = []; + const allowanceDetails: EconomicSelectionCandidate["allowances"] = []; + let pressure: number | null = null; + let stale = false; + if (excluded.has(targetKey(target))) hardExclusions.push("attempted"); + const provider = config.providers[target.provider]; + if (!provider) hardExclusions.push("unconfigured"); + else if (provider.disabled === true) hardExclusions.push("disabled"); + for (const allowanceId of allowances) { + const allowance = allowanceFor(config, allowanceId); + const snapshot = snapshots.get(allowanceId); + if (!allowance) { + hardExclusions.push("missing-allowance"); + continue; + } + const amount = economicConsumption(allowance, estimate, target.pricing); + const reserved = activeReserved(allowanceId, now); + const freshness = snapshotFreshness(snapshot, allowance, now); + const post = usableHeadroom(allowance, snapshot, amount, reserved, now); + const remaining = post === null || !snapshot ? null : post + amount; + consumptions.push(amount); + postRequestRemaining.push(post); + reserveThresholds.push(reserveAmount(allowance)); + stale ||= freshness.status !== "fresh"; + allowanceDetails.push({ + id: allowanceId, + capacity: allowance.capacity, + remaining, + reserved, + predicted: amount, + postRequestRemaining: post, + reserveThreshold: reserveAmount(allowance), + ...(snapshot?.resetAt !== undefined ? { resetAt: snapshot.resetAt } : {}), + ...(snapshot?.expiresAt !== undefined ? { expiresAt: snapshot.expiresAt } : {}), + ...(snapshot ? { source: snapshot.source, ageMs: snapshotAge(snapshot, now) } : {}), + stale: freshness.status !== "fresh", + }); + if (post !== null && post < -EPSILON) hardExclusions.push("hard-headroom"); + if (post !== null && post < reserveAmount(allowance) - EPSILON) { + if (!softSignals.includes("reserve")) softSignals.push("reserve"); + } + const p = expiryPressure(allowance, snapshot, now); + if (p !== null) pressure = Math.max(pressure ?? 0, p); + } + if (stale && unknownPolicy(combo) === "reject") hardExclusions.push("stale-quota"); + else if (stale && unknownPolicy(combo) === "deprioritize") { + if (!softSignals.includes("unknown-quota")) softSignals.push("unknown-quota"); + } + if (pressure !== null && !softSignals.includes("expiration-pressure")) softSignals.push("expiration-pressure"); + const maxSpend = combo.economy?.maxMarginalUsd; + const cost = marginalUsd(target, estimate); + if (maxSpend !== undefined) { + if (cost === null || cost > maxSpend) hardExclusions.push("max-marginal-usd"); + } + const cashCost = cashCostFor(target, estimate, cost); + const eligible = hardExclusions.length === 0; + return { + target, + eligible, + exclusions: hardExclusions, + softSignals, + configIndex: index, + cashCost, + consumption: consumptions, + postRequestRemaining, + reserveThresholds, + burnPressure: pressure, + marginalUsd: cost, + stale, + rankingBand: hardExclusions.includes("hard-headroom") ? "excluded" : pressure !== null ? "expiration" : cost !== null ? "marginal-cost" : `order-${index + 1}`, + allowances: allowanceDetails, + }; +} + +export function selectEconomicTarget( + config: OcxConfig, + comboId: string, + estimate: EconomicRequestEstimate, + now = Date.now(), + excluded: Iterable = [], + isEligible?: (target: EconomicTarget) => boolean, +): EconomicSelectionResult { + const combo = config.combos?.[comboId]; + if (!combo || combo.strategy !== "economy") return { targetIndex: null, candidates: [], reason: "not-economy" }; + sweepExpiredEconomicReservations(now); + const candidates = combo.targets.map((target, index) => { + const candidate = candidateFor(config, combo, { ...target, weight: target.weight ?? 1 }, estimate, now, index, new Set(excluded)); + if (isEligible && !isEligible(candidate.target)) { + if (!candidate.exclusions.includes("ineligible")) candidate.exclusions.push("ineligible"); + candidate.eligible = candidate.exclusions.length === 0; + } + return candidate; + }); + const available = candidates.filter(candidate => candidate.eligible); + if (available.length === 0) return { targetIndex: null, candidates, reason: "no-economically-eligible-target" }; + const winner = available.slice().sort(compareCandidates)[0]!; + return { + target: winner.target, + targetIndex: combo.targets.findIndex(target => targetKey(target) === targetKey(winner.target)), + candidates, + reason: winner.rankingBand === "expiration" ? "expiration pressure" : winner.rankingBand === "marginal-cost" ? "lowest marginal cost" : "stable target order", + }; +} + +export function reserveEconomicSelection( + config: OcxConfig, + comboId: string, + estimate: EconomicRequestEstimate, + now = Date.now(), + excluded: Iterable = [], + isEligible?: (target: EconomicTarget) => boolean, +): EconomicSelectionResult { + const attempted = new Set(excluded); + const targetCount = config.combos?.[comboId]?.targets.length ?? 0; + const reserve = (currentExcluded: Set): EconomicSelectionResult => { + const result = selectEconomicTarget(config, comboId, estimate, now, currentExcluded, isEligible); + if (!result.target) return result; + const id = `econ-${++reservationSequence}`; + const reservationsToAdd: Reservation[] = []; + for (const allowanceId of result.target.allowances ?? []) { + const allowance = allowanceFor(config, allowanceId); + const snapshot = snapshots.get(allowanceId); + if (!allowance || !snapshot) continue; + const amount = economicConsumption(allowance, estimate, result.target.pricing); + const headroom = usableHeadroom(allowance, snapshot, amount, activeReserved(allowanceId, now), now); + if (headroom === null || headroom < -EPSILON) { + if (currentExcluded.size >= targetCount) { + return { targetIndex: null, candidates: result.candidates, reason: "reservation-headroom-race" }; + } + const nextExcluded = new Set(currentExcluded); + nextExcluded.add(targetKey(result.target)); + const fallback = reserve(nextExcluded); + return fallback.target + ? { ...fallback, reason: "reservation-headroom-race" } + : { targetIndex: null, candidates: fallback.candidates, reason: "reservation-headroom-race" }; + } + reservationsToAdd.push({ + id, + allowanceId, + unit: allowance.unit, + amount, + ...(allowance.rates ? { rates: { ...allowance.rates } } : {}), + ...(result.target.pricing ? { pricing: { ...result.target.pricing } } : {}), + expiresAt: now + RESERVATION_TTL_MS, + generation: captureConfigGeneration(), + }); + } + for (const reservation of reservationsToAdd) reservations.set(`${id}\0${reservation.allowanceId}`, reservation); + return { ...result, reservationId: id }; + }; + return reserve(attempted); +} + +export function releaseEconomicReservation(id: string | undefined): void { + if (!id) return; + for (const key of reservations.keys()) if (key.startsWith(`${id}\0`)) reservations.delete(key); +} + +/** Drop every in-flight reservation referencing an allowance. Used by operator + * snapshot PUT/DELETE so a replaced or cleared snapshot cannot leave stale + * reservations blocking headroom until TTL expiry. */ +export function clearEconomicReservationsForAllowance(allowanceId: string): void { + for (const key of reservations.keys()) { + const reservation = reservations.get(key); + if (reservation?.allowanceId === allowanceId) reservations.delete(key); + } +} + +/** Count non-expired reservations for an allowance (operator snapshot conflict checks). */ +export function countEconomicReservationsForAllowance(allowanceId: string, now = Date.now()): number { + let count = 0; + for (const reservation of reservations.values()) { + if (reservation.allowanceId === allowanceId && reservation.expiresAt > now) count += 1; + } + return count; +} + +function actualForAllowance(reservation: Reservation, actual: EconomicActualUsage): number | undefined { + switch (reservation.unit) { + case "inputTokens": return actual.inputTokens; + case "outputTokens": return actual.outputTokens; + case "totalTokens": return actual.totalTokens; + case "requests": return actual.requests; + case "credits": + if (actual.credits !== undefined) return actual.credits; + return economicConsumption( + { unit: "credits", capacity: 0, window: { kind: "balance" }, rates: reservation.rates }, + { + inputTokens: safe(actual.inputTokens), + outputTokens: safe(actual.outputTokens), + cachedInputTokens: safe(actual.cachedInputTokens), + fixedRequests: safe(actual.requests ?? 1), + kind: "observed", + }, + ); + case "usd": + if (actual.usd !== undefined) return actual.usd; + return economicConsumption( + { unit: "usd", capacity: 0, window: { kind: "balance" }, rates: reservation.rates }, + { + inputTokens: safe(actual.inputTokens), + outputTokens: safe(actual.outputTokens), + cachedInputTokens: safe(actual.cachedInputTokens), + fixedRequests: safe(actual.requests ?? 1), + kind: "observed", + }, + reservation.pricing, + ); + } +} + +export function settleEconomicReservation(id: string | undefined, actual: EconomicActualUsage | undefined, now = Date.now()): void { + if (!id || settledReservationIds.has(id)) return; + const actualRecord = actual as Record | undefined; + if (actualRecord) { + for (const field of ECONOMIC_ACTUAL_FIELDS) { + if (actualRecord[field] !== undefined && !finiteNonNegative(actualRecord[field])) { + // Do not let a validation failure strand the reservation until TTL expiry. + releaseEconomicReservation(id); + throw new TypeError(`Invalid economic actual usage: ${field}`); + } + } + } + const entries = [...reservations.entries()].filter(([, reservation]) => reservation.id === id); + if (entries.length === 0) { + settledReservationIds.add(id); + return; + } + for (const [key, reservation] of entries) { + const snapshot = snapshots.get(reservation.allowanceId); + if (snapshot) { + const actualAmount = actual ? actualForAllowance(reservation, actual) : undefined; + const remaining = safe(snapshot.remaining) + reservation.amount - (actualAmount ?? 0); + snapshots.set(reservation.allowanceId, { ...snapshot, remaining: Math.max(0, remaining), updatedAt: now }); + } + reservations.delete(key); + } + settledReservationIds.add(id); + // Idempotency bookkeeping is defense-in-depth: a reservation whose entries were + // already deleted is a no-op regardless. Bound the set so a long-lived process + // cannot accumulate unbounded memory from historical ids. + if (settledReservationIds.size > 10_000) settledReservationIds.clear(); +} + +export function setEconomicQuotaSnapshot(id: string, snapshot: OcxEconomicSnapshot): void { + if (!finiteNonNegative(snapshot.remaining) || !finiteNonNegative(snapshot.updatedAt)) return; + snapshots.set(id, { ...snapshot, remaining: Math.max(0, snapshot.remaining) }); +} + +export function getEconomicQuotaSnapshot(id: string): OcxEconomicSnapshot | undefined { + return snapshots.get(id); +} + +export function sweepExpiredEconomicReservations(now = Date.now()): number { + let removed = 0; + for (const [key, reservation] of reservations) { + if (reservation.expiresAt > now) continue; + reservations.delete(key); + removed += 1; + } + return removed; +} + +export function reconcileEconomicState(context: GenerationContext & { allowanceIds?: ReadonlySet }): number { + if (context.generation <= lastReconciledGeneration) return 0; + liveAllowanceIds = new Set(context.allowanceIds ?? []); + let removed = 0; + for (const id of snapshots.keys()) { + if (liveAllowanceIds.has(id)) continue; + snapshots.delete(id); + removed += 1; + } + for (const [key, reservation] of reservations) { + if (liveAllowanceIds.has(reservation.allowanceId)) continue; + reservations.delete(key); + removed += 1; + } + lastReconciledGeneration = context.generation; + return removed; +} + +export function clearEconomicState(): void { + snapshots.clear(); + reservations.clear(); + settledReservationIds.clear(); + reservationSequence = 0; + lastReconciledGeneration = 0; + liveAllowanceIds = new Set(); +} + +export function clearEconomicQuotaSnapshot(id: string): boolean { + return snapshots.delete(id); +} + +export function explainEconomicCombo(config: OcxConfig, comboId: string, estimate: EconomicRequestEstimate, now = Date.now()): EconomicExplanation { + const result = selectEconomicTarget(config, comboId, estimate, now); + return { + ...result, + comboId, + strategy: "economy", + selectedTarget: result.target ? targetKey(result.target) : null, + generatedAt: now, + }; +} diff --git a/src/combos/index.ts b/src/combos/index.ts index 6041427a92..3a2f61be7e 100644 --- a/src/combos/index.ts +++ b/src/combos/index.ts @@ -42,3 +42,27 @@ export { concreteComboRequestBody, resetComboEffortWarningStateForTests, } from "./request"; +export { + clearEconomicQuotaSnapshot, + clearEconomicReservationsForAllowance, + clearEconomicState, + countEconomicReservationsForAllowance, + economicConsumption, + explainEconomicCombo, + estimateEconomicRequest, + getEconomicQuotaSnapshot, + reconcileEconomicState, + releaseEconomicReservation, + reserveEconomicSelection, + selectEconomicTarget, + settleEconomicReservation, + setEconomicQuotaSnapshot, + sweepExpiredEconomicReservations, + type EconomicExplanation, + type EconomicHardExclusion, + type EconomicRequestEstimate, + type EconomicSelectionCandidate, + type EconomicSelectionResult, + type EconomicSoftSignal, +} from "./economy"; +export { refreshEconomicSnapshots, resetEconomicSnapshotRefreshForTests, stopEconomicSnapshotRefresh } from "./economy-refresh"; diff --git a/src/combos/resolve.ts b/src/combos/resolve.ts index e2e36601a1..f62fb26763 100644 --- a/src/combos/resolve.ts +++ b/src/combos/resolve.ts @@ -2,6 +2,7 @@ import type { OcxComboTarget, OcxConfig } from "../types"; import { coolComboTarget, isComboTargetInCooldown } from "./failover"; import { getCombo, resolveComboId, targetKey } from "./types"; import type { NormalizedComboConfig } from "./types"; +import { reserveEconomicSelection, type EconomicRequestEstimate } from "./economy"; import { captureConfigGeneration, type GenerationContext, @@ -9,10 +10,11 @@ import { export interface ComboPick { comboId: string; - target: Required; + target: OcxComboTarget & { weight: number }; targetIndex: number; attempted: string[]; writerGeneration: number; + reservationId?: string; } interface SelectionState { @@ -56,9 +58,9 @@ function targetProviderIsUsable(config: OcxConfig, target: OcxComboTarget): bool } function smoothWeightedIndex( - targets: Required[], + targets: Array, state: SelectionState, - eligible: (target: Required) => boolean, + eligible: (target: OcxComboTarget & { weight: number }) => boolean, ): number { let best = -1; let bestScore = Number.NEGATIVE_INFINITY; @@ -87,20 +89,36 @@ export function pickComboTarget( comboId: string, options: { exclude?: Iterable; - eligible?: (target: Required) => boolean; + eligible?: (target: OcxComboTarget & { weight: number }) => boolean; + requestEstimate?: EconomicRequestEstimate; + now?: number; } = {}, ): ComboPick | null { const writerGeneration = captureConfigGeneration(); const combo = getCombo(config, comboId); if (!combo) throw new UnknownComboError(comboId); const excluded = new Set(options.exclude ?? []); - const eligible = (target: Required): boolean => + const eligible = (target: OcxComboTarget & { weight: number }): boolean => targetProviderIsUsable(config, target) && !excluded.has(targetKey(target)) && (options.eligible?.(target) ?? true); let targetIndex = -1; - if (combo.strategy === "round-robin") { + let reservationId: string | undefined; + if (combo.strategy === "economy") { + if (!options.requestEstimate) return null; + const economic = reserveEconomicSelection( + config, + comboId, + options.requestEstimate, + options.now ?? Date.now(), + excluded, + eligible, + ); + if (!economic.target || economic.targetIndex === null) return null; + targetIndex = economic.targetIndex; + reservationId = economic.reservationId; + } else if (combo.strategy === "round-robin") { let state = selectionState.get(comboId); if (!state) { state = { successes: 0, currentWeights: new Map() }; @@ -132,13 +150,14 @@ export function pickComboTarget( targetIndex, attempted: [...excluded, targetKey(target)], writerGeneration, + ...(reservationId ? { reservationId } : {}), }; } export function noteComboSuccess( comboId: string, combo: NormalizedComboConfig, - target: Required, + target: OcxComboTarget & { weight: number }, writerGeneration = captureConfigGeneration(), ): void { if (combo.strategy !== "round-robin") return; @@ -172,16 +191,19 @@ export function advanceComboAfterFailure( options: { retryAfter?: string | null; now?: number; - eligible?: (target: Required) => boolean; + eligible?: (target: OcxComboTarget & { weight: number }) => boolean; + requestEstimate?: EconomicRequestEstimate; } = {}, ): ComboPick | null { noteComboFailure(pick.comboId, pick.target, pick.writerGeneration); + // core.ts owns settle/release for the just-failed attempt; do not double-release here. coolComboTarget(pick.comboId, pick.target, { ...options, writerGeneration: pick.writerGeneration, }); return pickComboTarget(config, pick.comboId, { exclude: pick.attempted, + requestEstimate: options.requestEstimate, eligible: target => !isComboTargetInCooldown(pick.comboId, target, options.now) && (options.eligible?.(target) ?? true), }); diff --git a/src/combos/types.ts b/src/combos/types.ts index c82c861a02..e5ceb716bd 100644 --- a/src/combos/types.ts +++ b/src/combos/types.ts @@ -43,7 +43,8 @@ export interface NormalizedComboConfig { nativeAlias: boolean; /** Display-only label for the catalog row, or null when unset. */ displayName: string | null; - targets: Array>; + economy?: OcxComboConfig["economy"]; + targets: Array; } /** True only for an explicitly opted-in bare native-family alias. */ @@ -166,6 +167,8 @@ export interface ComboValidationOptions { requireEnabledTarget?: boolean; /** Full combos map for alias uniqueness checks; omitted during early config load. */ combos?: Record; + /** Shared economy allowance definitions used to validate target references. */ + allowances?: Record; /** Combo being renamed — its stored alias is excluded from uniqueness checks. */ excludeComboId?: string; } @@ -203,8 +206,26 @@ export function comboConfigIssues( const body = raw as Record; if (body.strategy !== undefined && body.strategy !== "failover" - && body.strategy !== "round-robin") { - issues.push({ path: ["strategy"], message: 'strategy must be "failover" or "round-robin"' }); + && body.strategy !== "round-robin" + && body.strategy !== "economy") { + issues.push({ path: ["strategy"], message: 'strategy must be "failover", "round-robin", or "economy"' }); + } + if (body.economy !== undefined) { + if (!body.economy || typeof body.economy !== "object" || Array.isArray(body.economy)) { + issues.push({ path: ["economy"], message: "economy must be an object" }); + } else { + const economy = body.economy as Record; + if (economy.unknownQuota !== undefined + && economy.unknownQuota !== "allow" + && economy.unknownQuota !== "deprioritize" + && economy.unknownQuota !== "reject") { + issues.push({ path: ["economy", "unknownQuota"], message: "unknownQuota must be allow, deprioritize, or reject" }); + } + if (economy.maxMarginalUsd !== undefined + && (typeof economy.maxMarginalUsd !== "number" || !Number.isFinite(economy.maxMarginalUsd) || economy.maxMarginalUsd < 0)) { + issues.push({ path: ["economy", "maxMarginalUsd"], message: "maxMarginalUsd must be a finite non-negative number" }); + } + } } if (body.stickyLimit !== undefined && (typeof body.stickyLimit !== "number" || !Number.isInteger(body.stickyLimit) @@ -302,6 +323,30 @@ export function comboConfigIssues( message: `targets[${i}].weight must be an integer from 1 to 10000`, }); } + if (target.allowances !== undefined) { + if (!Array.isArray(target.allowances) || target.allowances.some(value => typeof value !== "string" || !value.trim())) { + issues.push({ path: ["targets", i, "allowances"], message: `targets[${i}].allowances must be an array of non-empty strings` }); + } else if (new Set(target.allowances.map(value => value.trim())).size !== target.allowances.length) { + issues.push({ path: ["targets", i, "allowances"], message: `targets[${i}].allowances must not contain duplicates` }); + } else { + for (const allowanceId of target.allowances) { + if (!options.allowances || !Object.hasOwn(options.allowances, allowanceId.trim())) { + issues.push({ path: ["targets", i, "allowances"], message: `unknown economic allowance "${allowanceId}"` }); + } + } + } + } + if (target.pricing !== undefined && (!target.pricing || typeof target.pricing !== "object" || Array.isArray(target.pricing))) { + issues.push({ path: ["targets", i, "pricing"], message: `targets[${i}].pricing must be an object` }); + } else if (target.pricing) { + const pricing = target.pricing as Record; + for (const field of ["fixedPerRequest", "inputUsdPerMillion", "outputUsdPerMillion", "cachedInputUsdPerMillion"] as const) { + if (pricing[field] !== undefined + && (typeof pricing[field] !== "number" || !Number.isFinite(pricing[field]) || pricing[field] < 0)) { + issues.push({ path: ["targets", i, "pricing", field], message: `targets[${i}].pricing.${field} must be a finite non-negative number` }); + } + } + } if (provider && model) { const key = targetKey({ provider, model }); @@ -342,10 +387,13 @@ export function normalizeComboConfig(raw: OcxComboConfig): NormalizedComboConfig alias: alias || null, nativeAlias: raw.nativeAlias === true, displayName: displayName || null, + ...(raw.economy !== undefined ? { economy: raw.economy } : {}), targets: raw.targets.map(target => ({ provider: target.provider.trim(), model: target.model.trim(), weight: target.weight ?? 1, + ...(target.allowances ? { allowances: target.allowances.map(value => value.trim()) } : {}), + ...(target.pricing ? { pricing: { ...target.pricing } } : {}), })), }; } diff --git a/src/config.ts b/src/config.ts index 6c0e97ab1d..a61e4a0c04 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1432,6 +1432,108 @@ const configSchema = z.object({ message: "defaultProvider must exist in providers", }); } + const economicAllowancesRaw = (config as { economicAllowances?: unknown }).economicAllowances; + if (economicAllowancesRaw !== undefined) { + if (!economicAllowancesRaw || typeof economicAllowancesRaw !== "object" || Array.isArray(economicAllowancesRaw)) { + ctx.addIssue({ code: "custom", path: ["economicAllowances"], message: "economicAllowances must be an object" }); + } else { + const economicAllowances = economicAllowancesRaw as Record; + const allowanceProto = Object.getPrototypeOf(economicAllowances); + if (allowanceProto !== Object.prototype && allowanceProto !== null) { + ctx.addIssue({ code: "custom", path: ["economicAllowances"], message: "economicAllowances must be a plain object" }); + } else { + for (const [id, rawAllowance] of Object.entries(economicAllowances)) { + const path = ["economicAllowances", id]; + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(id) + || id === "__proto__" || id === "prototype" || id === "constructor") { + ctx.addIssue({ + code: "custom", + path, + message: "allowance id must be 1-64 letters, numbers, dots, underscores, or hyphens and must not be a reserved object key", + }); + continue; + } + if (!rawAllowance || typeof rawAllowance !== "object" || Array.isArray(rawAllowance)) { + ctx.addIssue({ code: "custom", path, message: "allowance must be an object" }); + continue; + } + const allowance = rawAllowance as Record; + const unit = allowance.unit; + if (typeof unit !== "string" || !["requests", "inputTokens", "outputTokens", "totalTokens", "credits", "usd"].includes(unit)) { + ctx.addIssue({ code: "custom", path: [...path, "unit"], message: "unit must be one of: requests, inputTokens, outputTokens, totalTokens, credits, usd" }); + } + if (!Number.isFinite(allowance.capacity as number) || (allowance.capacity as number) < 0) { + ctx.addIssue({ code: "custom", path: [...path, "capacity"], message: "capacity must be finite and non-negative" }); + } + const windowRaw = allowance.window; + if (!windowRaw || typeof windowRaw !== "object" || Array.isArray(windowRaw)) { + ctx.addIssue({ code: "custom", path: [...path, "window"], message: "window must be an object" }); + } else { + const window = windowRaw as Record; + const kind = window.kind; + if (kind !== "rolling" && kind !== "calendar" && kind !== "expiresAt" && kind !== "balance") { + ctx.addIssue({ code: "custom", path: [...path, "window", "kind"], message: "window.kind must be one of: rolling, calendar, expiresAt, balance" }); + } else if (kind === "rolling") { + if (!Number.isFinite(window.durationMs as number) || (window.durationMs as number) <= 0) { + ctx.addIssue({ code: "custom", path: [...path, "window", "durationMs"], message: "rolling durationMs must be greater than zero" }); + } + + } else if (kind === "expiresAt") { + if (!Number.isFinite(window.expiresAt as number) || (window.expiresAt as number) <= 0) { + ctx.addIssue({ code: "custom", path: [...path, "window", "expiresAt"], message: "expiresAt must be a positive timestamp" }); + } + + } else if (kind === "calendar") { + if (window.interval !== "day" && window.interval !== "week" && window.interval !== "month") { + ctx.addIssue({ code: "custom", path: [...path, "window", "interval"], message: "calendar interval must be one of: day, week, month" }); + } + if (typeof window.timezone !== "string" || !window.timezone.trim()) { + ctx.addIssue({ code: "custom", path: [...path, "window", "timezone"], message: "timezone must be an IANA timezone" }); + } else { + try { new Intl.DateTimeFormat("en-US", { timeZone: window.timezone as string }).format(); } + catch { ctx.addIssue({ code: "custom", path: [...path, "window", "timezone"], message: "timezone must be an IANA timezone" }); } + } + } + } + if (allowance.rollover !== undefined && typeof allowance.rollover !== "boolean") { + ctx.addIssue({ code: "custom", path: [...path, "rollover"], message: "rollover must be a boolean" }); + } + if (allowance.reserveFraction !== undefined && (!Number.isFinite(allowance.reserveFraction as number) || (allowance.reserveFraction as number) < 0 || (allowance.reserveFraction as number) > 1)) { + ctx.addIssue({ code: "custom", path: [...path, "reserveFraction"], message: "reserveFraction must be between 0 and 1" }); + } + if (allowance.reserveAmount !== undefined && (!Number.isFinite(allowance.reserveAmount as number) || (allowance.reserveAmount as number) < 0 || (typeof allowance.capacity === "number" && Number.isFinite(allowance.capacity) && (allowance.reserveAmount as number) > (allowance.capacity as number)))) { + ctx.addIssue({ code: "custom", path: [...path, "reserveAmount"], message: "reserveAmount must be finite, non-negative, and no greater than capacity" }); + } + if (allowance.source !== undefined && typeof allowance.source !== "string") { + ctx.addIssue({ code: "custom", path: [...path, "source"], message: "source must be one of: usage-log, manual, codex-quota" }); + } else if (allowance.source !== undefined && !["usage-log", "manual", "codex-quota"].includes(allowance.source as string)) { + ctx.addIssue({ code: "custom", path: [...path, "source"], message: "source must be one of: usage-log, manual, codex-quota" }); + } + if (allowance.staleAfterMs !== undefined && (!Number.isFinite(allowance.staleAfterMs as number) || (allowance.staleAfterMs as number) < 0)) { + ctx.addIssue({ code: "custom", path: [...path, "staleAfterMs"], message: "staleAfterMs must be finite and non-negative" }); + } + if (allowance.rates !== undefined) { + if (!allowance.rates || typeof allowance.rates !== "object" || Array.isArray(allowance.rates)) { + ctx.addIssue({ code: "custom", path: [...path, "rates"], message: "rates must be an object" }); + } else { + const rates = allowance.rates as Record; + const allowedRates = new Set(["fixedPerRequest", "inputPerMillion", "outputPerMillion", "cachedInputPerMillion", "cacheWritePerMillion"]); + for (const key of Object.keys(rates)) { + if (!allowedRates.has(key)) { + ctx.addIssue({ code: "custom", path: [...path, "rates", key], message: `unknown rate "${key}"` }); + } + } + for (const key of ["fixedPerRequest", "inputPerMillion", "outputPerMillion", "cachedInputPerMillion", "cacheWritePerMillion"] as const) { + if (rates[key] !== undefined && (!Number.isFinite(rates[key] as number) || (rates[key] as number) < 0)) { + ctx.addIssue({ code: "custom", path: [...path, "rates", key], message: `${key} must be finite and non-negative` }); + } + } + } + } + } + } + } + } const combos = (config as { combos?: unknown }).combos; if (combos !== undefined) { if (!combos || typeof combos !== "object" || Array.isArray(combos)) { @@ -1452,6 +1554,7 @@ const configSchema = z.object({ // too, not just via the management API; each combo is excluded from its own check. for (const issue of comboConfigIssues(id, raw, config.providers, { combos: combos as Record, + allowances: config.economicAllowances as Record | undefined, excludeComboId: id, })) { ctx.addIssue({ @@ -1910,6 +2013,36 @@ function warnDegradedCodexAccountPicker(rawParsed: unknown): void { if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); } +function sanitizeEconomicAllowancesForLoad(rawParsed: unknown): void { + const raw = rawConfigRecord(rawParsed); + if (!raw || !Object.hasOwn(raw, "economicAllowances")) return; + const value = raw.economicAllowances; + if (!value || typeof value !== "object" || Array.isArray(value)) { + delete raw.economicAllowances; + console.warn("⚠️ config.json economicAllowances ignored: expected an object. Other settings were preserved."); + return; + } + + const kept: Record = {}; + const dropped: string[] = []; + for (const [id, allowance] of Object.entries(value as Record)) { + const probe = configSchema.safeParse({ + ...getDefaultConfig(), + economicAllowances: { [id]: allowance }, + combos: undefined, + }); + const invalidAllowance = !probe.success + && probe.error.issues.some(issue => issue.path[0] === "economicAllowances"); + if (invalidAllowance) dropped.push(id); + else kept[id] = allowance; + } + if (Object.keys(kept).length > 0) raw.economicAllowances = kept; + else delete raw.economicAllowances; + if (dropped.length > 0) { + console.warn(`⚠️ config.json ignored invalid economic allowance${dropped.length === 1 ? "" : "s"}: ${dropped.join(", ")}. Other settings were preserved.`); + } +} + function nativeSubagentSyncDisabledReason(config: OcxConfig, rawParsed?: unknown): string | null { if (config.syncCodexSubagentDefaults !== true) return null; const malformed = malformedNativeSubagentFields(rawParsed); @@ -1960,6 +2093,7 @@ export function loadConfig(): OcxConfig { const parsed = JSON.parse(raw); sanitizeRetryOn429ForLoad(parsed); sanitizeModelCostsForLoad(parsed); + sanitizeEconomicAllowancesForLoad(parsed); const result = configSchema.safeParse(parsed); if (result.success) { const config = normalizeApiKeyIds(result.data as OcxConfig); diff --git a/src/lib/state-store-registrations.ts b/src/lib/state-store-registrations.ts index 151402d2f0..4585a98f99 100644 --- a/src/lib/state-store-registrations.ts +++ b/src/lib/state-store-registrations.ts @@ -20,6 +20,7 @@ import { } from "../combos/failover"; import { reconcileComboWarningMemos } from "../combos/request"; import { reconcileComboRotationState } from "../combos/resolve"; +import { reconcileEconomicState, sweepExpiredEconomicReservations } from "../combos/economy"; import { listLiveComboTargetKeys } from "../combos/types"; import { listLiveConfigOwnershipRoots, @@ -43,9 +44,11 @@ import { type GenerationContext, reconcileStateGeneration, registerStateStore, + registerStateSweepAfterTick, setGenerationContextBuilder, type StateStoreRegistration, } from "./state-store-sweeper"; +import { refreshEconomicSnapshots } from "../combos/economy-refresh"; let liveServerConfig: OcxConfig | null = null; @@ -66,6 +69,7 @@ export function buildGenerationContext(): GenerationContext { providerNames, comboIds: new Set(Object.keys(liveServerConfig.combos ?? {})), comboTargets: listLiveComboTargetKeys(liveServerConfig), + allowanceIds: new Set(Object.keys(liveServerConfig.economicAllowances ?? {})), codexAccountIds: listLiveCodexAccountIds(liveServerConfig), oauthAccountKeys: listLiveOAuthAccountKeys(providerNames), configRoots: listLiveConfigOwnershipRoots(getConfigDir()), @@ -75,11 +79,11 @@ export function buildGenerationContext(): GenerationContext { export const STATE_STORE_REGISTRATIONS = [ { name: "subagent-model-health", sweepExpired: sweepExpiredSubagentModelHealth }, { name: "api-key-cooldowns", sweepExpired: sweepExpiredApiKeyCooldowns }, - { - name: "combo-target-cooldowns", + { name: "combo-target-cooldowns", sweepExpired: sweepExpiredComboTargetCooldowns, reconcileGeneration: reconcileComboTargetCooldowns, }, + { name: "economic-reservations", sweepExpired: sweepExpiredEconomicReservations, reconcileGeneration: reconcileEconomicState }, { name: "anthropic-routing-health", sweepExpired: sweepExpiredAnthropicRoutingHealth }, { name: "xai-refresh-verdicts", sweepExpired: sweepExpiredXaiPermanentFailureVerdicts }, { name: "responses-continuation", sweepExpired: sweepExpiredResponseStates }, @@ -106,4 +110,11 @@ export const STATE_STORE_REGISTRATIONS = [ for (const registration of STATE_STORE_REGISTRATIONS) registerStateStore(registration); +registerStateSweepAfterTick({ + name: "economic-snapshot-refresh", + afterTick: () => { + if (liveServerConfig) void refreshEconomicSnapshots(liveServerConfig); + }, +}); + setGenerationContextBuilder(buildGenerationContext); diff --git a/src/lib/state-store-sweeper.ts b/src/lib/state-store-sweeper.ts index 3f687d84fc..47b0bcc7a6 100644 --- a/src/lib/state-store-sweeper.ts +++ b/src/lib/state-store-sweeper.ts @@ -8,6 +8,7 @@ export interface GenerationContext { codexAccountIds: ReadonlySet; oauthAccountKeys: ReadonlySet; configRoots: ReadonlySet; + allowanceIds?: ReadonlySet; } export interface StateStoreRegistration { diff --git a/src/router.ts b/src/router.ts index 5c84c095c8..679fdd8e4c 100644 --- a/src/router.ts +++ b/src/router.ts @@ -419,7 +419,7 @@ export function comboRouteDecisionTrace( reason: "combo-pick", candidateIndex: pick.targetIndex, ...(combo - ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : "failover" } + ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : combo.strategy === "economy" ? "economy" : "failover" } : {}), }, candidates: combo ? comboRouteCandidates(config, pick, combo) : undefined, @@ -674,7 +674,7 @@ export function routeModel( reason: route.routeReason, ...(route.combo ? { candidateIndex: route.combo.targetIndex } : {}), ...(combo - ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : "failover" } + ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : combo.strategy === "economy" ? "economy" : "failover" } : {}), }, candidates: route.routeKind === "combo" && route.combo && combo diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 6b5431022e..494918aed1 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -36,6 +36,9 @@ import { noteComboSuccess, parseRetryAfterMs, pickComboTarget, + releaseEconomicReservation, + settleEconomicReservation, + estimateEconomicRequest, targetKey, } from "../../combos"; import { isInjectionDebugEnabled } from "../../lib/debug-settings"; @@ -1050,6 +1053,66 @@ async function applyFinalRouteRequestNormalization(args: { +function settleComboReservation(reservationId: string | undefined, usage: OcxUsage | undefined): void { + if (!reservationId) return; + if (!usage) { + releaseEconomicReservation(reservationId); + return; + } + settleEconomicReservation(reservationId, { + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + ...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}), + ...(usage.cachedInputTokens !== undefined ? { cachedInputTokens: usage.cachedInputTokens } : {}), + requests: 1, + }); +} + +function settleComboStream( + response: Response, + reservationId: string | undefined, + usage: () => OcxUsage | undefined, +): Response { + if (!response.body || !reservationId) return response; + let settled = false; + const settle = (): void => { + if (settled) return; + settled = true; + settleComboReservation(reservationId, usage()); + }; + const reader = response.body.getReader(); + const body = new ReadableStream({ + async pull(controller) { + try { + const next = await reader.read(); + if (next.done) { + settle(); + controller.close(); + } else { + controller.enqueue(next.value); + } + } catch (error) { + settle(); + controller.error(error); + } + }, + async cancel(reason) { + settle(); + await reader.cancel(reason); + }, + }); + return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); +} + +async function responseUsage(response: Response): Promise { + try { + const payload = await response.clone().json() as { usage?: unknown; response?: { usage?: unknown } }; + return usageFromResponsesPayload(payload.response?.usage ?? payload.usage); + } catch { + return undefined; + } +} + export async function handleComboResponses( req: Request, rawBody: unknown, @@ -1107,7 +1170,9 @@ export async function handleComboResponses( } const initialNow = Date.now(); + const requestEstimate = estimateEconomicRequest(rawBody, requestedModel); let pick = pickComboTarget(config, comboId, { + requestEstimate, eligible: target => payloadEligible(target) && !isComboTargetInCooldown(comboId, target, initialNow), }); @@ -1120,7 +1185,10 @@ export async function handleComboResponses( let lastFailure: Response | null = null; while (pick) { - if (options.abortSignal?.aborted) return clientCancelledResponse(); + if (options.abortSignal?.aborted) { + releaseEconomicReservation(pick.reservationId); + return clientCancelledResponse(); + } const childLog: RequestLogContext = { model: pick.target.model, provider: pick.target.provider, @@ -1152,6 +1220,7 @@ export async function handleComboResponses( childLog.activeAttempt = attempt; let attemptRetained = false; const retainCancelledAttempt = (): void => { + releaseEconomicReservation(pick?.reservationId); if (attemptRetained) return; sealRequestAttemptIdentity( attempt, @@ -1197,6 +1266,7 @@ export async function handleComboResponses( retainCancelledAttempt(); return clientCancelledResponse(); } + releaseEconomicReservation(pick.reservationId); throw error; } @@ -1207,6 +1277,10 @@ export async function handleComboResponses( } if (response.ok) { + const streamResponse = (rawBody as { stream?: unknown } | null)?.stream === true && response.body !== null; + if (!streamResponse) { + settleComboReservation(pick.reservationId, childLog.usage ?? logCtx.usage ?? attempt.usage ?? await responseUsage(response)); + } sealRequestAttemptIdentity( attempt, childLog.provider, @@ -1229,7 +1303,9 @@ export async function handleComboResponses( options.onCodexAuthContextResolved?.(resolvedAuth); options.setTerminalOutcomeRecorder?.(terminalRecorder); callbackGate.commit(); - return response; + return streamResponse + ? settleComboStream(response, pick.reservationId, () => childLog.usage ?? logCtx.usage ?? attempt.usage) + : response; } callbackGate.discard(); @@ -1246,6 +1322,7 @@ export async function handleComboResponses( retainCancelledAttempt(); return clientCancelledResponse(); } + releaseEconomicReservation(pick.reservationId); throw error; } if (options.abortSignal?.aborted) { @@ -1269,14 +1346,17 @@ export async function handleComboResponses( if (comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, }) === "stop") { + settleComboReservation(pick.reservationId, failure.usage); adoptFailedChildLog(childLog); return lastFailure; } + settleComboReservation(pick.reservationId, failure.usage); console.warn( `[combo] ${comboId}: ${targetKey(pick.target)} failed with ${response.status} after ${Date.now() - started}ms`, ); const nextPick = advanceComboAfterFailure(config, pick, { retryAfter: failure.retryAfter, + requestEstimate, now: Date.now(), eligible: payloadEligible, }); diff --git a/src/types.ts b/src/types.ts index 49040ed6bc..c9f001bcf3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -888,6 +888,8 @@ export interface OcxConfig { /** Successful new-session binds retained on one round-robin selection. Default 1; range 1..100. */ stickyLimit?: number; }; + /** Shared static allowance definitions used by economy combos. Runtime snapshots stay in memory. */ + economicAllowances?: Record; /** Virtual `combo/` models spanning concrete provider/model targets (issue #133). */ combos?: Record; /** @@ -905,7 +907,7 @@ export interface OcxConfig { export type OcxAccountPoolRotationStrategy = "quota" | "round-robin" | "fill-first"; -export type OcxComboStrategy = "failover" | "round-robin"; +export type OcxComboStrategy = "failover" | "round-robin" | "economy"; export type OcxComboDefaultEffort = "low" | "medium" | "high" | "xhigh" | "max" | "ultra"; export interface OcxComboTarget { @@ -913,12 +915,15 @@ export interface OcxComboTarget { model: string; /** Relative SWRR batch weight. Default 1; valid range 1..10000. */ weight?: number; + allowances?: string[]; + pricing?: OcxEconomicPricing; } export interface OcxComboConfig { targets: OcxComboTarget[]; /** Ordered failover (default) or deterministic smooth weighted round-robin. */ strategy?: OcxComboStrategy; + economy?: OcxComboEconomyPolicy; /** Successful requests retained on one RR selection batch. Default 1; range 1..100. */ stickyLimit?: number; /** Used when the client omits reasoning.effort. null/omitted leaves the target default unchanged. */ @@ -938,6 +943,54 @@ export interface OcxComboConfig { displayName?: string; } +export type OcxEconomicUnit = "requests" | "inputTokens" | "outputTokens" | "totalTokens" | "credits" | "usd"; +export type OcxEconomicSource = "usage-log" | "manual" | "codex-quota"; +export type OcxEconomicConfidence = "authoritative" | "observed" | "estimated" | "unknown"; +export type OcxEconomicWindow = + | { kind: "rolling"; durationMs: number } + | { kind: "calendar"; interval: "day" | "week" | "month"; timezone: string } + | { kind: "expiresAt"; expiresAt: number } + | { kind: "balance" }; +export interface OcxEconomicRates { + fixedPerRequest?: number; + inputPerMillion?: number; + outputPerMillion?: number; + cachedInputPerMillion?: number; + cacheWritePerMillion?: number; +} +export interface OcxEconomicPricing { + fixedPerRequest?: number; + inputUsdPerMillion?: number; + outputUsdPerMillion?: number; + cachedInputUsdPerMillion?: number; + cacheWriteUsdPerMillion?: number; +} +export interface OcxEconomicAllowance { + unit: OcxEconomicUnit; + capacity: number; + window: OcxEconomicWindow; + rollover?: boolean; + reserveFraction?: number; + reserveAmount?: number; + source?: OcxEconomicSource; + staleAfterMs?: number; + rates?: OcxEconomicRates; +} +export interface OcxEconomicSnapshot { + remaining: number; + updatedAt: number; + windowStart?: number; + resetAt?: number; + expiresAt?: number; + source: string; + confidence: OcxEconomicConfidence; + error?: string; +} +export interface OcxComboEconomyPolicy { + unknownQuota?: "allow" | "deprioritize" | "reject"; + maxMarginalUsd?: number; +} + export type OcxRoutingUnknownEvidenceMode = "allow" | "penalize" | "exclude"; export interface OcxRoutingProfileCandidate { From 3530c6f1669c68ef5089084d3b81dfcdb995338d Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:19:04 -0700 Subject: [PATCH 02/10] feat(api): economy explain and allowance snapshot management Add combo explain and economic-allowance list/snapshot endpoints with validation, redaction, and clearReservations conflict semantics. --- src/server/management-api.ts | 2 + src/server/management/combo-routes.ts | 20 ++ .../management/economic-snapshot-routes.ts | 182 ++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 src/server/management/economic-snapshot-routes.ts diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 372e738e76..d37b5753a4 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -67,6 +67,7 @@ import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; import { handleOauthAccountRoutes } from "./management/oauth-account-routes"; import { handleComboRoutes } from "./management/combo-routes"; +import { handleEconomicSnapshotRoutes } from "./management/economic-snapshot-routes"; import { handleSystemRoutes } from "./management/system-routes"; import { handleLabRoutes } from "./management/lab-routes"; import { handleSidebarRoutes } from "./management/sidebar-routes"; @@ -187,6 +188,7 @@ export async function handleManagementAPI( ?? (await handleAgentSettingsRoutes(ctx)) ?? (await handleOauthAccountRoutes(ctx)) ?? (await handleComboRoutes(ctx)) + ?? (await handleEconomicSnapshotRoutes(ctx)) ?? (await handleSystemRoutes(ctx)) ?? (await handleLabRoutes(ctx)) ?? (await handleSidebarRoutes(ctx)); diff --git a/src/server/management/combo-routes.ts b/src/server/management/combo-routes.ts index 9c197d0d74..a93908f734 100644 --- a/src/server/management/combo-routes.ts +++ b/src/server/management/combo-routes.ts @@ -68,6 +68,25 @@ import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; export async function handleComboRoutes(ctx: ManagementContext): Promise { const { req, url, config, deps, convergeCodexCatalog, syncClaudeAgentDefsBestEffort } = ctx; + const explainMatch = url.pathname.match(/^\/api\/combos\/([^/]+)\/explain$/); + if (explainMatch && req.method === "GET") { + let comboId: string; + try { + comboId = decodeURIComponent(explainMatch[1]!); + } catch { + return jsonResponse({ error: "combo id has malformed percent-encoding" }, 400); + } + const combo = config.combos?.[comboId]; + if (!combo) return jsonResponse({ error: `unknown combo "${comboId}"` }, 404); + const { explainEconomicCombo } = await import("../../combos/economy"); + const inputTokens = Number(url.searchParams.get("inputTokens") ?? "0"); + const outputTokens = Number(url.searchParams.get("outputTokens") ?? "1024"); + if (![inputTokens, outputTokens].every(value => Number.isFinite(value) && value >= 0)) { + return jsonResponse({ error: "inputTokens and outputTokens must be finite non-negative numbers" }, 400); + } + return jsonResponse(explainEconomicCombo(config, comboId, { inputTokens, outputTokens, kind: "configured" })); + } + if (url.pathname === "/api/combos" && req.method === "GET") { const { comboPublicModelId, getCombo, listComboIds } = await import("../../combos"); return jsonResponse({ combos: listComboIds(config).map(id => { @@ -120,6 +139,7 @@ export async function handleComboRoutes(ctx: ManagementContext): Promise= 0; +} + +function safeIntegerTimestamp(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function normalizeSnapshot(snapshot: OcxEconomicSnapshot): Record { + const out: Record = { + remaining: Math.max(0, snapshot.remaining), + updatedAt: snapshot.updatedAt, + source: snapshot.source, + confidence: snapshot.confidence, + }; + if (snapshot.windowStart !== undefined) out.windowStart = snapshot.windowStart; + if (snapshot.resetAt !== undefined) out.resetAt = snapshot.resetAt; + if (snapshot.expiresAt !== undefined) out.expiresAt = snapshot.expiresAt; + if (snapshot.error !== undefined) out.error = snapshot.error; + return out; +} + +export async function handleEconomicSnapshotRoutes(ctx: ManagementContext): Promise { + const { req, url, config } = ctx; + + if (url.pathname === "/api/economic-allowances") { + if (req.method !== "GET") return jsonResponse({ error: "method not allowed" }, 405); + const { + getEconomicQuotaSnapshot, + countEconomicReservationsForAllowance, + } = await import("../../combos/economy"); + const now = Date.now(); + const allowances = Object.entries(config.economicAllowances ?? {}).map(([id, allowance]) => { + const snapshot = getEconomicQuotaSnapshot(id); + return { + id, + unit: allowance.unit, + capacity: allowance.capacity, + source: allowance.source, + window: allowance.window, + snapshot: snapshot ? normalizeSnapshot(snapshot) : null, + state: snapshot ? "present" : "unknown", + activeReservations: countEconomicReservationsForAllowance(id, now), + }; + }); + return jsonResponse({ allowances }, 200); + } + + const match = url.pathname.match(/^\/api\/economic-allowances\/([^/]+)\/snapshot$/); + if (!match) return null; + + let allowanceId: string; + try { + allowanceId = decodeURIComponent(match[1]!); + } catch { + return jsonResponse({ error: "allowance id has malformed percent-encoding" }, 400); + } + + if (!config.economicAllowances || !Object.hasOwn(config.economicAllowances, allowanceId)) { + return jsonResponse({ error: `unknown economic allowance "${allowanceId}"` }, 404); + } + + if (req.method === "GET") { + const { getEconomicQuotaSnapshot, countEconomicReservationsForAllowance } = await import("../../combos/economy"); + const snapshot = getEconomicQuotaSnapshot(allowanceId); + if (!snapshot) { + return jsonResponse({ allowanceId, snapshot: null, state: "unknown", activeReservations: countEconomicReservationsForAllowance(allowanceId) }, 200); + } + return jsonResponse({ + allowanceId, + snapshot: normalizeSnapshot(snapshot), + state: "present", + activeReservations: countEconomicReservationsForAllowance(allowanceId), + }, 200); + } + + if (req.method === "PUT") { + let rawBody: unknown; + try { + rawBody = await readManagementJsonBody(req); + } catch (error) { + rethrowManagementBodyTooLarge(error); + return jsonResponse({ error: "invalid JSON body" }, 400); + } + if (!isPlainRecord(rawBody)) { + return jsonResponse({ error: "request body must be an object" }, 400); + } + const body = rawBody as Record; + + if (!finiteNonNegative(body.remaining)) { + return jsonResponse({ error: "remaining must be a finite non-negative number" }, 400); + } + if (!safeIntegerTimestamp(body.updatedAt)) { + return jsonResponse({ error: "updatedAt must be a safe non-negative integer timestamp" }, 400); + } + for (const field of ["windowStart", "resetAt", "expiresAt"] as const) { + if (body[field] !== undefined && !safeIntegerTimestamp(body[field])) { + return jsonResponse({ error: `${field} must be a safe non-negative integer timestamp` }, 400); + } + } + if (typeof body.source !== "string" || !ALLOWED_SOURCES.has(body.source)) { + return jsonResponse({ error: "source must be one of: usage-log, manual, codex-quota" }, 400); + } + if (typeof body.confidence !== "string" || !ALLOWED_CONFIDENCES.has(body.confidence)) { + return jsonResponse({ error: "confidence must be one of: authoritative, observed, estimated, unknown" }, 400); + } + if (body.error !== undefined && typeof body.error !== "string") { + return jsonResponse({ error: "error must be a string" }, 400); + } + + const snapshot: OcxEconomicSnapshot = { + remaining: Math.max(0, body.remaining as number), + updatedAt: body.updatedAt as number, + source: body.source as string, + confidence: body.confidence as OcxEconomicSnapshot["confidence"], + ...(body.windowStart !== undefined ? { windowStart: body.windowStart as number } : {}), + ...(body.resetAt !== undefined ? { resetAt: body.resetAt as number } : {}), + ...(body.expiresAt !== undefined ? { expiresAt: body.expiresAt as number } : {}), + ...(typeof body.error === "string" && body.error ? { error: body.error } : {}), + }; + + const { + setEconomicQuotaSnapshot, + getEconomicQuotaSnapshot, + clearEconomicReservationsForAllowance, + countEconomicReservationsForAllowance, + } = await import("../../combos/economy"); + + const active = countEconomicReservationsForAllowance(allowanceId); + const clearReservations = body.clearReservations === true; + if (active > 0 && !clearReservations) { + return jsonResponse({ + error: "allowance has in-flight reservations; pass clearReservations:true to replace snapshot", + allowanceId, + activeReservations: active, + }, 409); + } + if (clearReservations || active > 0) clearEconomicReservationsForAllowance(allowanceId); + setEconomicQuotaSnapshot(allowanceId, snapshot); + const stored = getEconomicQuotaSnapshot(allowanceId)!; + return jsonResponse({ allowanceId, snapshot: normalizeSnapshot(stored), clearedReservations: active }, 200); + } + + if (req.method === "DELETE") { + const { + clearEconomicQuotaSnapshot, + getEconomicQuotaSnapshot, + clearEconomicReservationsForAllowance, + countEconomicReservationsForAllowance, + } = await import("../../combos/economy"); + const active = countEconomicReservationsForAllowance(allowanceId); + const clearReservations = url.searchParams.get("clearReservations") === "true"; + if (active > 0 && !clearReservations) { + return jsonResponse({ + error: "allowance has in-flight reservations; pass clearReservations=true to clear snapshot", + allowanceId, + activeReservations: active, + }, 409); + } + const existing = getEconomicQuotaSnapshot(allowanceId); + if (clearReservations || active > 0) clearEconomicReservationsForAllowance(allowanceId); + clearEconomicQuotaSnapshot(allowanceId); + return jsonResponse({ + allowanceId, + cleared: true, + previousState: existing ? "present" : "unknown", + snapshot: null, + clearedReservations: active, + }, 200); + } + + return jsonResponse({ error: "method not allowed" }, 405); +} From b53b6c29cc8ffbc12f1139b3f5438cda45192538 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:19:04 -0700 Subject: [PATCH 03/10] feat(cli/gui): economy configure/explain and allowance snapshots Expose economy combo configure/explain CLI, allowance snapshot CLI, and GUI round-trip preservation for economy fields. --- gui/src/combo-workspace-data.ts | 45 +++++++++++--- src/cli/allowance.ts | 105 ++++++++++++++++++++++++++++++++ src/cli/combo.ts | 97 +++++++++++++++++++++++------ src/cli/index.ts | 5 ++ 4 files changed, 227 insertions(+), 25 deletions(-) create mode 100644 src/cli/allowance.ts diff --git a/gui/src/combo-workspace-data.ts b/gui/src/combo-workspace-data.ts index 56ce088238..1b8b4e9898 100644 --- a/gui/src/combo-workspace-data.ts +++ b/gui/src/combo-workspace-data.ts @@ -7,7 +7,7 @@ import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../../src/codex/catalog/native-mo export { SUPPORTED_NATIVE_OPENAI_SLUGS }; -export type ComboStrategy = "failover" | "round-robin"; +export type ComboStrategy = "failover" | "round-robin" | "economy"; export type ComboEffort = "low" | "medium" | "high" | "xhigh" | "max" | "ultra"; export const COMBO_EFFORTS: ComboEffort[] = ["low", "medium", "high", "xhigh", "max", "ultra"]; @@ -45,6 +45,8 @@ export interface ComboTarget { provider: string; model: string; weight?: number; + allowances?: string[]; + pricing?: Record; /** UI-only stable key for React lists; never sent to the API. */ clientKey?: string; } @@ -56,6 +58,8 @@ export function newComboTarget(partial: Partial = {}): ComboTarget provider: partial.provider ?? "", model: partial.model ?? "", ...(partial.weight !== undefined ? { weight: partial.weight } : {}), + ...(partial.allowances ? { allowances: [...partial.allowances] } : {}), + ...(partial.pricing ? { pricing: { ...partial.pricing } } : {}), clientKey: partial.clientKey ?? `ct-${++comboTargetKeySeq}`, }; } @@ -71,6 +75,7 @@ export interface ComboItem { /** Display-only catalog label used by native aliases. */ displayName: string | null; strategy: ComboStrategy; + economy?: { unknownQuota?: "allow" | "deprioritize" | "reject"; maxMarginalUsd?: number }; stickyLimit: number; defaultEffort: ComboEffort | null; targets: ComboTarget[]; @@ -79,6 +84,7 @@ export interface ComboItem { export interface ComboSections { failover: ComboItem[]; roundRobin: ComboItem[]; + economy: ComboItem[]; } export interface ComboAttentionItem { @@ -124,7 +130,7 @@ function normalizeAlias(raw: unknown): string | null { } export function normalizeStrategy(raw: unknown): ComboStrategy { - return raw === "round-robin" ? "round-robin" : "failover"; + return raw === "round-robin" || raw === "economy" ? raw : "failover"; } export function normalizeStickyLimit(raw: unknown): number { @@ -164,7 +170,17 @@ export function parseComboList(payload: unknown): ComboItem[] { const model = typeof tr.model === "string" ? tr.model.trim() : ""; if (!provider || !model) continue; const weight = normalizeWeight(tr.weight); - targets.push(weight !== undefined ? newComboTarget({ provider, model, weight }) : newComboTarget({ provider, model })); + const allowances = Array.isArray(tr.allowances) ? tr.allowances.filter((value): value is string => typeof value === "string") : undefined; + const pricing = tr.pricing && typeof tr.pricing === "object" && !Array.isArray(tr.pricing) + ? Object.fromEntries(Object.entries(tr.pricing).filter(([, value]) => typeof value === "number" && Number.isFinite(value))) + : undefined; + targets.push(newComboTarget({ + provider, + model, + ...(weight !== undefined ? { weight } : {}), + ...(allowances ? { allowances } : {}), + ...(pricing ? { pricing } : {}), + })); } out.push({ id, @@ -175,6 +191,7 @@ export function parseComboList(payload: unknown): ComboItem[] { nativeAlias: r.nativeAlias === true, displayName: normalizeAlias(r.displayName), strategy: normalizeStrategy(r.strategy), + ...(r.economy && typeof r.economy === "object" && !Array.isArray(r.economy) ? { economy: r.economy as ComboItem["economy"] } : {}), stickyLimit: normalizeStickyLimit(r.stickyLimit), defaultEffort: normalizeDefaultEffort(r.defaultEffort), targets, @@ -186,11 +203,13 @@ export function parseComboList(payload: unknown): ComboItem[] { export function groupCombos(items: ComboItem[]): ComboSections { const failover: ComboItem[] = []; const roundRobin: ComboItem[] = []; + const economy: ComboItem[] = []; for (const item of items) { if (item.strategy === "round-robin") roundRobin.push(item); + else if (item.strategy === "economy") economy.push(item); else failover.push(item); } - return { failover, roundRobin }; + return { failover, roundRobin, economy }; } export function filterCombos(items: ComboItem[], query: string): ComboItem[] { @@ -234,13 +253,16 @@ export function draftEquals(a: ComboItem, b: ComboItem): boolean { || a.nativeAlias !== b.nativeAlias || a.displayName !== b.displayName || a.strategy !== b.strategy + || JSON.stringify(a.economy) !== JSON.stringify(b.economy) || a.stickyLimit !== b.stickyLimit || a.defaultEffort !== b.defaultEffort ) return false; if (a.targets.length !== b.targets.length) return false; return a.targets.every((t, i) => { const o = b.targets[i]!; - return t.provider === o.provider && t.model === o.model && (t.weight ?? 1) === (o.weight ?? 1); + return t.provider === o.provider && t.model === o.model && (t.weight ?? 1) === (o.weight ?? 1) + && JSON.stringify(t.allowances ?? []) === JSON.stringify(o.allowances ?? []) + && JSON.stringify(t.pricing ?? {}) === JSON.stringify(o.pricing ?? {}); }); } @@ -252,6 +274,7 @@ export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {} strategy: ComboStrategy; stickyLimit?: number; defaultEffort: ComboEffort | null; + economy?: ComboItem["economy"]; alias?: string; nativeAlias?: true; displayName?: string; @@ -261,11 +284,17 @@ export function toPutBody(item: ComboItem, options: { renameFrom?: string } = {} id: item.id.trim(), ...(options.renameFrom ? { renameFrom: options.renameFrom } : {}), combo: { - targets: item.targets.map((target) => item.strategy === "round-robin" - ? { provider: target.provider.trim(), model: target.model.trim(), weight: target.weight ?? 1 } - : { provider: target.provider.trim(), model: target.model.trim() }), + targets: item.targets.map((target) => ({ + provider: target.provider.trim(), + model: target.model.trim(), + ...(item.strategy === "round-robin" ? { weight: target.weight ?? 1 } : {}), + ...(target.allowances ? { allowances: [...target.allowances] } : {}), + ...(target.pricing ? { pricing: { ...target.pricing } } : {}), + ...(item.strategy === "economy" && target.weight !== undefined ? { weight: target.weight } : {}), + })), strategy: item.strategy, defaultEffort: item.defaultEffort, + ...(item.economy ? { economy: { ...item.economy } } : {}), ...(item.strategy === "round-robin" ? { stickyLimit: item.stickyLimit } : {}), ...(item.alias && item.alias.trim() ? { alias: item.alias.trim() } : {}), ...(item.nativeAlias ? { nativeAlias: true } : {}), diff --git a/src/cli/allowance.ts b/src/cli/allowance.ts new file mode 100644 index 0000000000..49c04d163d --- /dev/null +++ b/src/cli/allowance.ts @@ -0,0 +1,105 @@ +import { + CliUsageError, + printData, + rejectArgs, + runCliAction, + runtimeRequest, + takeFlag, + takeOption, + type RuntimeApiDeps, +} from "./runtime-api"; + +const USAGE = `Usage: + ocx allowance [list] [--json] + ocx allowance snapshot get [--json] + ocx allowance snapshot set --snapshot-json [--clear-reservations] [--json] + ocx allowance snapshot clear [--clear-reservations] [--json]`; + +function parseSnapshotJson(raw: string | undefined): Record { + if (raw === undefined) throw new CliUsageError("--snapshot-json is required", USAGE); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new CliUsageError(`--snapshot-json must be valid JSON: ${error instanceof Error ? error.message : String(error)}`, USAGE); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new CliUsageError("--snapshot-json must be a JSON object", USAGE); + } + return parsed as Record; +} + +async function list(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + rejectArgs(args, USAGE); + const result = await runtimeRequest<{ allowances?: Array> }>("/api/economic-allowances", {}, deps); + const rows = result.allowances ?? []; + printData( + result, + wantsJson, + rows.length + ? rows.map(row => `${String(row.id)} ${String(row.state ?? "unknown")} reservations=${String(row.activeReservations ?? 0)}`) + : ["No economic allowances configured."], + ); +} + +async function snapshotGet(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift()?.trim(); + const wantsJson = takeFlag(args, "--json"); + if (!id || id.startsWith("-")) throw new CliUsageError("allowance id is required", USAGE); + rejectArgs(args, USAGE); + const result = await runtimeRequest(`/api/economic-allowances/${encodeURIComponent(id)}/snapshot`, {}, deps); + printData(result, wantsJson); +} + +async function snapshotSet(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift()?.trim(); + const wantsJson = takeFlag(args, "--json"); + const clearReservations = takeFlag(args, "--clear-reservations"); + const snapshotJsonRaw = takeOption(args, "--snapshot-json"); + if (!id || id.startsWith("-")) throw new CliUsageError("allowance id is required", USAGE); + rejectArgs(args, USAGE); + const body = parseSnapshotJson(snapshotJsonRaw); + if (clearReservations) body.clearReservations = true; + const result = await runtimeRequest(`/api/economic-allowances/${encodeURIComponent(id)}/snapshot`, { + method: "PUT", + body: JSON.stringify(body), + }, deps); + printData(result, wantsJson, [`Saved snapshot for allowance ${id}.`]); +} + +async function snapshotClear(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const id = args.shift()?.trim(); + const wantsJson = takeFlag(args, "--json"); + const clearReservations = takeFlag(args, "--clear-reservations"); + if (!id || id.startsWith("-")) throw new CliUsageError("allowance id is required", USAGE); + rejectArgs(args, USAGE); + const query = clearReservations ? "?clearReservations=true" : ""; + const result = await runtimeRequest(`/api/economic-allowances/${encodeURIComponent(id)}/snapshot${query}`, { + method: "DELETE", + }, deps); + printData(result, wantsJson, [`Cleared snapshot for allowance ${id}.`]); +} + +async function snapshot(argv: string[], deps: RuntimeApiDeps): Promise { + const [sub, ...rest] = argv; + if (sub === "get") await snapshotGet(rest, deps); + else if (sub === "set") await snapshotSet(rest, deps); + else if (sub === "clear" || sub === "delete") await snapshotClear(rest, deps); + else throw new CliUsageError(sub ? `unknown snapshot command ${sub}` : "snapshot subcommand required (get|set|clear)", USAGE); +} + +export async function handleAllowanceCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise { + return runCliAction(async () => { + const [sub = "list", ...rest] = argv; + if (sub === "list") await list(rest, deps); + else if (sub === "snapshot") await snapshot(rest, deps); + else throw new CliUsageError(`unknown allowance command ${sub}`, USAGE); + }); +} + +export const ALLOWANCE_USAGE = USAGE; diff --git a/src/cli/combo.ts b/src/cli/combo.ts index 6a4bdc6ac1..47b003d81e 100644 --- a/src/cli/combo.ts +++ b/src/cli/combo.ts @@ -13,11 +13,12 @@ import { const USAGE = `Usage: ocx combo [list] [--json] ocx combo show [--json] - ocx combo set --targets - [--strategy ] [--sticky <1-100>] + ocx combo set (--targets | --targets-json | --combo-json ) + [--strategy ] [--sticky <1-100>] [--effort ] [--alias ] [--native-alias] [--display-name ] - [--rename-from ] [--json] + [--economy-json ] [--rename-from ] [--json] + ocx combo explain [--input-tokens ] [--output-tokens ] [--json] ocx combo remove --yes [--json]`; type ComboRow = Record & { id?: string; model?: string }; @@ -65,32 +66,93 @@ async function show(argv: string[], deps: RuntimeApiDeps): Promise { printData(combo, wantsJson); } +async function explain(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const wantsJson = takeFlag(args, "--json"); + const id = args.shift(); + if (!id || id.startsWith("-")) throw new CliUsageError("combo id is required", USAGE); + const inputTokens = takeIntegerOption(args, "--input-tokens", { min: 0 }) ?? 0; + const outputTokens = takeIntegerOption(args, "--output-tokens", { min: 0 }) ?? 1024; + rejectArgs(args, USAGE); + const result = await runtimeRequest(`/api/combos/${encodeURIComponent(id)}/explain?inputTokens=${inputTokens}&outputTokens=${outputTokens}`, {}, deps); + printData(result, wantsJson); +} + +function parseJsonOption(raw: string | undefined, flag: string, expected: "object" | "array", usage: string): unknown | undefined { + if (raw === undefined) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + throw new CliUsageError(`${flag} must be valid JSON: ${error instanceof Error ? error.message : String(error)}`, usage); + } + if (expected === "array") { + if (!Array.isArray(parsed)) throw new CliUsageError(`${flag} must be a JSON array`, usage); + return parsed; + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new CliUsageError(`${flag} must be a JSON object`, usage); + return parsed as Record; +} + async function set(argv: string[], deps: RuntimeApiDeps): Promise { const args = [...argv]; const id = args.shift()?.trim(); const wantsJson = takeFlag(args, "--json"); if (!id) throw new CliUsageError("combo id is required", USAGE); const targetsRaw = takeOption(args, "--targets"); - if (!targetsRaw) throw new CliUsageError("--targets is required", USAGE); - const strategy = takeOption(args, "--strategy") ?? "failover"; - if (strategy !== "failover" && strategy !== "round-robin") throw new CliUsageError("--strategy must be failover or round-robin", USAGE); - const stickyLimit = takeIntegerOption(args, "--sticky", { min: 1 }) ?? 1; - if (stickyLimit > 100) throw new CliUsageError("--sticky must be <= 100", USAGE); + const targetsJsonRaw = takeOption(args, "--targets-json"); + const comboJsonRaw = takeOption(args, "--combo-json"); + const economyJsonRaw = takeOption(args, "--economy-json"); + const strategy = takeOption(args, "--strategy"); + const stickyRaw = takeOption(args, "--sticky"); const effort = takeOption(args, "--effort"); const alias = takeOption(args, "--alias"); const nativeAlias = takeFlag(args, "--native-alias"); const displayName = takeOption(args, "--display-name"); const renameFrom = takeOption(args, "--rename-from"); rejectArgs(args, USAGE); - const combo: Record = { - strategy, - stickyLimit, - targets: parseTargets(targetsRaw), - }; - if (effort !== undefined) combo.defaultEffort = effort === "-" ? null : effort; - if (alias !== undefined) combo.alias = alias === "-" ? "" : alias; - if (nativeAlias) combo.nativeAlias = true; - if (displayName !== undefined) combo.displayName = displayName === "-" ? "" : displayName; + const hasComboJson = comboJsonRaw !== undefined; + const hasTargets = targetsRaw !== undefined; + const hasTargetsJson = targetsJsonRaw !== undefined; + const hasEconomyJson = economyJsonRaw !== undefined; + if (hasComboJson && (hasTargets || hasTargetsJson || strategy !== undefined || stickyRaw !== undefined || hasEconomyJson || effort !== undefined || alias !== undefined || nativeAlias || displayName !== undefined)) { + throw new CliUsageError("--combo-json cannot be combined with individual combo fields (--targets, --targets-json, --strategy, --sticky, --effort, --alias, --native-alias, --display-name, --economy-json)", USAGE); + } + if (hasTargets && hasTargetsJson) throw new CliUsageError("--targets and --targets-json cannot be combined", USAGE); + let combo: Record; + if (hasComboJson) { + combo = parseJsonOption(comboJsonRaw, "--combo-json", "object", USAGE) as Record; + } else { + let targets: unknown; + if (hasTargetsJson) { + targets = parseJsonOption(targetsJsonRaw, "--targets-json", "array", USAGE); + } else if (hasTargets) { + targets = parseTargets(targetsRaw); + } else { + throw new CliUsageError("--targets is required (or use --targets-json / --combo-json)", USAGE); + } + const resolvedStrategy = strategy ?? "failover"; + if (resolvedStrategy !== "failover" && resolvedStrategy !== "round-robin" && resolvedStrategy !== "economy") throw new CliUsageError("--strategy must be failover, round-robin, or economy", USAGE); + let stickyLimit = 1; + if (stickyRaw !== undefined) { + const value = Number(stickyRaw.replace(/[_,]/g, "")); + if (!Number.isInteger(value) || value < 1) throw new CliUsageError("--sticky must be an integer >= 1", USAGE); + if (value > 100) throw new CliUsageError("--sticky must be <= 100", USAGE); + stickyLimit = value; + } + combo = { + strategy: resolvedStrategy, + stickyLimit, + targets, + }; + if (hasEconomyJson) { + combo.economy = parseJsonOption(economyJsonRaw, "--economy-json", "object", USAGE) as Record; + } + if (effort !== undefined) combo.defaultEffort = effort === "-" ? null : effort; + if (alias !== undefined) combo.alias = alias === "-" ? "" : alias; + if (nativeAlias) combo.nativeAlias = true; + if (displayName !== undefined) combo.displayName = displayName === "-" ? "" : displayName; + } const result = await runtimeRequest("/api/combos", { method: "PUT", body: JSON.stringify({ id, combo, ...(renameFrom ? { renameFrom } : {}) }), @@ -115,6 +177,7 @@ export async function handleComboCommand(argv: string[], deps: RuntimeApiDeps = const [sub = "list", ...rest] = argv; if (sub === "list") await list(rest, deps); else if (sub === "show") await show(rest, deps); + else if (sub === "explain") await explain(rest, deps); else if (sub === "set" || sub === "create" || sub === "update") await set(rest, deps); else if (sub === "remove" || sub === "delete") await remove(rest, deps); else throw new CliUsageError(`unknown combo command ${sub}`, USAGE); diff --git a/src/cli/index.ts b/src/cli/index.ts index 35da313d77..0a95d80133 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1287,6 +1287,11 @@ switch (command) { process.exitCode = await handleComboCommand(args.slice(1)); break; } + case "allowance": { + const { handleAllowanceCommand } = await import("./allowance"); + process.exitCode = await handleAllowanceCommand(args.slice(1)); + break; + } case "route": { if (args[1] !== "combo" && args[1] !== "policy") { console.error("Usage: ocx route "); From 0771bcff1f9ade67ab65d289c06ef119b8e5c8f7 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:19:06 -0700 Subject: [PATCH 04/10] test(combos): cover economic routing, settlement, and operator surfaces Add focused suites for selection, races, settlement, refresh, windows, management/CLI APIs, GUI round-trips, and hostile-review blindspots. --- tests/cli-allowance.test.ts | 92 ++++ tests/cli-combo.test.ts | 335 +++++++++++++++ tests/combo-workspace-data.test.ts | 94 +++++ tests/combos.test.ts | 32 ++ tests/economic-allowances-validation.test.ts | 101 +++++ tests/economic-management-api.test.ts | 162 +++++++ tests/economic-manual-snapshot-api.test.ts | 399 ++++++++++++++++++ tests/economic-ordering-stability.test.ts | 67 +++ tests/economic-reservation-race.test.ts | 91 ++++ tests/economic-reservation-settlement.test.ts | 270 ++++++++++++ tests/economic-reservation-terminal.test.ts | 177 ++++++++ tests/economic-review-blindspots.test.ts | 159 +++++++ tests/economic-routing.test.ts | 101 +++++ tests/economic-snapshot-refresh.test.ts | 251 +++++++++++ tests/economic-window-boundaries.test.ts | 156 +++++++ 15 files changed, 2487 insertions(+) create mode 100644 tests/cli-allowance.test.ts create mode 100644 tests/cli-combo.test.ts create mode 100644 tests/economic-allowances-validation.test.ts create mode 100644 tests/economic-management-api.test.ts create mode 100644 tests/economic-manual-snapshot-api.test.ts create mode 100644 tests/economic-ordering-stability.test.ts create mode 100644 tests/economic-reservation-race.test.ts create mode 100644 tests/economic-reservation-settlement.test.ts create mode 100644 tests/economic-reservation-terminal.test.ts create mode 100644 tests/economic-review-blindspots.test.ts create mode 100644 tests/economic-routing.test.ts create mode 100644 tests/economic-snapshot-refresh.test.ts create mode 100644 tests/economic-window-boundaries.test.ts diff --git a/tests/cli-allowance.test.ts b/tests/cli-allowance.test.ts new file mode 100644 index 0000000000..938c247813 --- /dev/null +++ b/tests/cli-allowance.test.ts @@ -0,0 +1,92 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test"; +import { handleAllowanceCommand } from "../src/cli/allowance"; + +type Recorded = { path: string; method: string; body: unknown }; +const servers: Array> = []; + +afterEach(() => { + for (const server of servers.splice(0)) server.stop(true); + process.exitCode = 0; +}); + +function fakeRuntime(responder?: (req: Request, body: unknown) => unknown) { + const requests: Recorded[] = []; + const server = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url); + const body = req.method === "GET" || req.method === "DELETE" ? null : await req.json().catch(() => null); + requests.push({ path: `${url.pathname}${url.search}`, method: req.method, body }); + const custom = responder?.(req, body); + if (custom !== undefined) return Response.json(custom); + return Response.json({ ok: true, allowances: [{ id: "promo", state: "unknown", activeReservations: 0 }] }); + }, + }); + servers.push(server); + return { requests, deps: { baseUrl: `http://127.0.0.1:${server.port}` } }; +} + +describe("ocx allowance", () => { + test("list hits GET /api/economic-allowances", async () => { + const runtime = fakeRuntime(() => ({ allowances: [{ id: "promo", state: "present", activeReservations: 1 }] })); + const logs: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { logs.push(String(args[0])); }); + try { + expect(await handleAllowanceCommand(["list"], runtime.deps)).toBe(0); + expect(runtime.requests[0]).toEqual({ path: "/api/economic-allowances", method: "GET", body: null }); + expect(logs.join("\n")).toContain("promo"); + } finally { + spy.mockRestore(); + } + }); + + test("snapshot get encodes id", async () => { + const runtime = fakeRuntime(() => ({ allowanceId: "a/b", state: "unknown", snapshot: null })); + expect(await handleAllowanceCommand(["snapshot", "get", "a/b", "--json"], runtime.deps)).toBe(0); + expect(runtime.requests[0]?.path).toBe("/api/economic-allowances/a%2Fb/snapshot"); + expect(runtime.requests[0]?.method).toBe("GET"); + }); + + test("snapshot set requires --snapshot-json", async () => { + const runtime = fakeRuntime(); + expect(await handleAllowanceCommand(["snapshot", "set", "promo"], runtime.deps)).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("snapshot set posts body and clearReservations flag", async () => { + const runtime = fakeRuntime(() => ({ ok: true })); + const code = await handleAllowanceCommand([ + "snapshot", "set", "promo", + "--snapshot-json", JSON.stringify({ remaining: 3, updatedAt: 1, source: "manual", confidence: "authoritative" }), + "--clear-reservations", + "--json", + ], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.method).toBe("PUT"); + expect(runtime.requests[0]?.path).toBe("/api/economic-allowances/promo/snapshot"); + expect(runtime.requests[0]?.body).toMatchObject({ remaining: 3, clearReservations: true }); + }); + + test("snapshot clear without flag omits query", async () => { + const runtime = fakeRuntime(() => ({ cleared: true })); + expect(await handleAllowanceCommand(["snapshot", "clear", "promo"], runtime.deps)).toBe(0); + expect(runtime.requests[0]).toEqual({ + path: "/api/economic-allowances/promo/snapshot", + method: "DELETE", + body: null, + }); + }); + + test("snapshot clear with --clear-reservations sets query", async () => { + const runtime = fakeRuntime(() => ({ cleared: true })); + expect(await handleAllowanceCommand(["snapshot", "clear", "promo", "--clear-reservations"], runtime.deps)).toBe(0); + expect(runtime.requests[0]?.path).toBe("/api/economic-allowances/promo/snapshot?clearReservations=true"); + expect(runtime.requests[0]?.method).toBe("DELETE"); + }); + + test("malformed snapshot json exits 2", async () => { + const runtime = fakeRuntime(); + expect(await handleAllowanceCommand(["snapshot", "set", "promo", "--snapshot-json", "{"], runtime.deps)).toBe(2); + expect(runtime.requests).toEqual([]); + }); +}); diff --git a/tests/cli-combo.test.ts b/tests/cli-combo.test.ts new file mode 100644 index 0000000000..631f8c5d23 --- /dev/null +++ b/tests/cli-combo.test.ts @@ -0,0 +1,335 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { handleComboCommand } from "../src/cli/combo"; + +type Recorded = { path: string; method: string; body: unknown }; +const servers: Array> = []; + +afterEach(() => { + for (const server of servers.splice(0)) server.stop(true); + // eslint-disable-next-line no-process-exit + process.exitCode = 0; +}); + +function fakeRuntime(responder?: (req: Request, body: unknown) => unknown) { + const requests: Recorded[] = []; + const server = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url); + const body = req.method === "GET" ? null : await req.json().catch(() => null); + requests.push({ path: `${url.pathname}${url.search}`, method: req.method, body }); + const custom = responder?.(req, body); + if (custom !== undefined) return Response.json(custom); + return Response.json({ ok: true, combos: [{ id: "bulk", model: "combo/bulk" }] }); + }, + }); + servers.push(server); + return { requests, deps: { baseUrl: `http://127.0.0.1:${server.port}` } }; +} + +describe("ocx combo explain", () => { + test("missing id returns 2 without request", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["explain"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("missing id with --json returns 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["explain", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("malformed input-tokens returns 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["explain", "bulk", "--input-tokens", "not-a-number"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("unknown flag returns 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["explain", "bulk", "--unknown"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("explain default tokens hits exact runtime path", async () => { + const runtime = fakeRuntime(() => ({ selectedTarget: "a/m1", reason: "test" })); + const logs: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + logs.push(String(args[0])); + }); + try { + const code = await handleComboCommand(["explain", "bulk-code"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]).toEqual({ + path: "/api/combos/bulk-code/explain?inputTokens=0&outputTokens=1024", + method: "GET", + body: null, + }); + expect(logs.join("\n")).toContain("selectedTarget"); + } finally { + spy.mockRestore(); + } + }); + + test("explain with custom tokens encodes id and query", async () => { + const runtime = fakeRuntime(() => ({ ok: true })); + const code = await handleComboCommand(["explain", "bulk/code", "--input-tokens", "2000", "--output-tokens", "500"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.path).toBe("/api/combos/bulk%2Fcode/explain?inputTokens=2000&outputTokens=500"); + }); + + test("explain --json remains output-only and prints structured JSON", async () => { + const payload = { selectedTarget: "payg/m", strategy: "economy", candidates: [] }; + const runtime = fakeRuntime(() => payload); + const logs: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + logs.push(String(args[0])); + }); + try { + const code = await handleComboCommand(["explain", "bulk", "--input-tokens", "10", "--output-tokens", "20", "--json"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.path).toBe("/api/combos/bulk/explain?inputTokens=10&outputTokens=20"); + const parsed = JSON.parse(logs.join("\n")); + expect(parsed).toEqual(payload); + } finally { + spy.mockRestore(); + } + }); + + test("explain human also prints JSON (no lines provided)", async () => { + const payload = { selectedTarget: "a/m1" }; + const runtime = fakeRuntime(() => payload); + const logs: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + logs.push(String(args[0])); + }); + try { + const code = await handleComboCommand(["explain", "bulk"], runtime.deps); + expect(code).toBe(0); + const parsed = JSON.parse(logs.join("\n")); + expect(parsed).toEqual(payload); + } finally { + spy.mockRestore(); + } + }); +}); + +describe("ocx combo set economy", () => { + test("legacy --targets preserves failover/round-robin", async () => { + const runtime = fakeRuntime(); + let code = await handleComboCommand(["set", "fast", "--targets", "ark/model-a:2,openai/gpt-5.5", "--strategy", "failover", "--json"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.body).toEqual({ + id: "fast", + combo: { + strategy: "failover", + stickyLimit: 1, + targets: [ + { provider: "ark", model: "model-a", weight: 2 }, + { provider: "openai", model: "gpt-5.5" }, + ], + }, + }); + + const rr = fakeRuntime(); + code = await handleComboCommand(["set", "rr", "--targets", "a/m1,b/m2", "--strategy", "round-robin", "--sticky", "3", "--json"], rr.deps); + expect(code).toBe(0); + expect(rr.requests[0]?.body).toMatchObject({ + id: "rr", + combo: { strategy: "round-robin", stickyLimit: 3 }, + }); + }); + + test("--combo-json configures full economy combo without hand-edit", async () => { + const runtime = fakeRuntime(); + const comboJson = JSON.stringify({ + strategy: "economy", + stickyLimit: 1, + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2, outputUsdPerMillion: 0.8 } }, + ], + }); + const code = await handleComboCommand(["set", "bulk-code", "--combo-json", comboJson, "--json"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.method).toBe("PUT"); + expect(runtime.requests[0]?.path).toBe("/api/combos"); + expect(runtime.requests[0]?.body).toEqual({ + id: "bulk-code", + combo: { + strategy: "economy", + stickyLimit: 1, + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2, outputUsdPerMillion: 0.8 } }, + ], + }, + }); + }); + + test("--targets-json plus --economy-json round-trips economy policy, allowance refs, and pricing", async () => { + const runtime = fakeRuntime(); + const targetsJson = JSON.stringify([ + { provider: "included", model: "m", allowances: ["promo", "extra"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.5, outputUsdPerMillion: 1 } }, + ]); + const economyJson = JSON.stringify({ unknownQuota: "reject", maxMarginalUsd: 1 }); + const code = await handleComboCommand([ + "set", + "bulk", + "--strategy", + "economy", + "--targets-json", + targetsJson, + "--economy-json", + economyJson, + "--json", + ], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.body).toEqual({ + id: "bulk", + combo: { + strategy: "economy", + stickyLimit: 1, + targets: [ + { provider: "included", model: "m", allowances: ["promo", "extra"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.5, outputUsdPerMillion: 1 } }, + ], + economy: { unknownQuota: "reject", maxMarginalUsd: 1 }, + }, + }); + }); + + test("--targets-json alone with economy strategy", async () => { + const runtime = fakeRuntime(); + const targetsJson = JSON.stringify([ + { provider: "a", model: "m1", weight: 2, allowances: ["promo"] }, + ]); + const code = await handleComboCommand(["set", "eco", "--strategy", "economy", "--targets-json", targetsJson, "--json"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.body).toMatchObject({ + id: "eco", + combo: { + strategy: "economy", + targets: [{ provider: "a", model: "m1", weight: 2, allowances: ["promo"] }], + }, + }); + }); + + test("malformed --combo-json rejects with 2 and no request", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--combo-json", "{", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("non-object --combo-json rejects with 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--combo-json", "[]", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("malformed --targets-json rejects with 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--targets-json", "{not json", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("non-array --targets-json rejects with 2", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--targets-json", "{}", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("malformed --economy-json rejects with 2", async () => { + const runtime = fakeRuntime(); + const targetsJson = JSON.stringify([{ provider: "a", model: "m1" }]); + const code = await handleComboCommand(["set", "bulk", "--targets-json", targetsJson, "--economy-json", "{", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("non-object --economy-json rejects with 2", async () => { + const runtime = fakeRuntime(); + const targetsJson = JSON.stringify([{ provider: "a", model: "m1" }]); + const code = await handleComboCommand(["set", "bulk", "--targets-json", targetsJson, "--economy-json", "[]", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("--combo-json cannot be combined with --targets", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--combo-json", "{}", "--targets", "a/m1", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("--combo-json cannot be combined with --strategy", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--combo-json", "{}", "--strategy", "economy", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("--targets and --targets-json cannot be combined", async () => { + const runtime = fakeRuntime(); + const code = await handleComboCommand(["set", "bulk", "--targets", "a/m1", "--targets-json", "[]", "--json"], runtime.deps); + expect(code).toBe(2); + expect(runtime.requests).toEqual([]); + }); + + test("--json remains output-only with --combo-json", async () => { + const runtime = fakeRuntime(); + const comboJson = JSON.stringify({ + strategy: "economy", + targets: [{ provider: "a", model: "m1" }], + economy: { unknownQuota: "allow" }, + }); + const logs: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + logs.push(String(args[0])); + }); + try { + const code = await handleComboCommand(["set", "bulk", "--combo-json", comboJson, "--json"], runtime.deps); + expect(code).toBe(0); + expect(runtime.requests[0]?.body).toEqual({ + id: "bulk", + combo: { strategy: "economy", targets: [{ provider: "a", model: "m1" }], economy: { unknownQuota: "allow" } }, + }); + expect(() => JSON.parse(logs.join("\n"))).not.toThrow(); + } finally { + spy.mockRestore(); + } + }); + + test("PUT contract matches management API expectation", async () => { + const runtime = fakeRuntime(); + const comboJson = JSON.stringify({ + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [{ provider: "included", model: "m", allowances: ["promo"] }], + }); + await handleComboCommand(["set", "bulk-code", "--combo-json", comboJson, "--rename-from", "old-id", "--json"], runtime.deps); + expect(runtime.requests[0]?.method).toBe("PUT"); + expect(runtime.requests[0]?.path).toBe("/api/combos"); + expect(runtime.requests[0]?.body).toEqual({ + id: "bulk-code", + combo: { + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [{ provider: "included", model: "m", allowances: ["promo"] }], + }, + renameFrom: "old-id", + }); + }); +}); diff --git a/tests/combo-workspace-data.test.ts b/tests/combo-workspace-data.test.ts index c67fd4cac3..b5288a3cfe 100644 --- a/tests/combo-workspace-data.test.ts +++ b/tests/combo-workspace-data.test.ts @@ -530,3 +530,97 @@ describe("combo-workspace-data", () => { )).toBe(false); }); }); + +describe("combo-workspace-data economy", () => { + const economyRow = { + id: "bulk", + model: "combo/bulk", + strategy: "economy", + defaultEffort: "high", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [ + { provider: "a", model: "m1", weight: 3, allowances: ["weekly-cap"], pricing: { inputUsdPerMillion: 1.25 } }, + { provider: "b", model: "m2", allowances: ["weekly-cap", "expiring-credits"], pricing: { outputUsdPerMillion: 2.5 } }, + ], + }; + + test("parseComboList preserves economy policy, allowance references, pricing, and weights", () => { + const [item] = parseComboList({ combos: [economyRow] }); + expect(item!.strategy).toBe("economy"); + expect(item!.economy).toEqual({ unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }); + expect(item!.defaultEffort).toBe("high"); + expect(item!.targets[0]).toMatchObject({ + provider: "a", + model: "m1", + weight: 3, + allowances: ["weekly-cap"], + pricing: { inputUsdPerMillion: 1.25 }, + }); + expect(item!.targets[1]).toMatchObject({ + allowances: ["weekly-cap", "expiring-credits"], + pricing: { outputUsdPerMillion: 2.5 }, + }); + }); + + test("groupCombos places economy combos in their own section", () => { + const sections = groupCombos(parseComboList({ + combos: [ + economyRow, + { id: "f", strategy: "failover", targets: [{ provider: "a", model: "m1" }] }, + ], + })); + expect(sections.economy.map((c) => c.id)).toEqual(["bulk"]); + expect(sections.failover.map((c) => c.id)).toEqual(["f"]); + expect(sections.roundRobin).toEqual([]); + }); + + test("draftEquals distinguishes economy policy, allowance references, pricing, and weights", () => { + const baseline = parseComboList({ combos: [economyRow] })[0]!; + expect(draftEquals(baseline, { ...baseline })).toBe(true); + expect(draftEquals(baseline, { ...baseline, economy: { ...baseline.economy!, maxMarginalUsd: 5 } })).toBe(false); + expect(draftEquals(baseline, { + ...baseline, + targets: [{ ...baseline.targets[0]!, allowances: ["other"] }, baseline.targets[1]!], + })).toBe(false); + expect(draftEquals(baseline, { + ...baseline, + targets: [{ ...baseline.targets[0]!, pricing: { inputUsdPerMillion: 9 } }, baseline.targets[1]!], + })).toBe(false); + expect(draftEquals(baseline, { + ...baseline, + targets: [{ ...baseline.targets[0]!, weight: 8 }, baseline.targets[1]!], + })).toBe(false); + }); + + test("toPutBody round-trips economy policy, allowances, pricing, and explicit weights", () => { + const item = parseComboList({ combos: [economyRow] })[0]!; + const body = toPutBody(item); + expect(body.combo.strategy).toBe("economy"); + expect(body.combo.economy).toEqual({ unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }); + expect(body.combo.defaultEffort).toBe("high"); + expect(body.combo.targets[0]).toMatchObject({ + provider: "a", + model: "m1", + weight: 3, + allowances: ["weekly-cap"], + pricing: { inputUsdPerMillion: 1.25 }, + }); + expect(body.combo.targets[1]).toMatchObject({ + allowances: ["weekly-cap", "expiring-credits"], + pricing: { outputUsdPerMillion: 2.5 }, + }); + + // A saved economy combo must reload with identical routing-relevant fields. + const [reloaded] = parseComboList({ combos: [{ id: body.id, ...body.combo }] }); + expect(draftEquals(item, reloaded!)).toBe(true); + }); + + test("toPutBody omits empty economy policy and unset weights", () => { + const item = parseComboList({ + combos: [{ id: "bare", strategy: "economy", targets: [{ provider: "a", model: "m1" }] }], + })[0]!; + const body = toPutBody(item); + expect(body.combo.economy).toBeUndefined(); + expect(body.combo.targets[0]).toEqual({ provider: "a", model: "m1" }); + }); +}); diff --git a/tests/combos.test.ts b/tests/combos.test.ts index e46f10a8d3..69c0263819 100644 --- a/tests/combos.test.ts +++ b/tests/combos.test.ts @@ -581,6 +581,38 @@ describe("combo validation and normalization", () => { } }); + test("rejects non-finite or negative pricing fields and accepts forward-compatible keys", () => { + const providers = baseConfig().providers; + for (const bad of [ + { inputUsdPerMillion: -1 }, + { inputUsdPerMillion: Number.NaN }, + { outputUsdPerMillion: Number.POSITIVE_INFINITY }, + { fixedPerRequest: -0.5 }, + { cachedInputUsdPerMillion: "1" }, + ]) { + const issue = comboConfigIssues("free", { targets: [{ provider: "a", model: "m1", pricing: bad }] }, providers) + .find(candidate => candidate.path.slice(0, 3).join(".") === "targets.0.pricing"); + expect(issue).toBeDefined(); + expect(issue!.message).toContain("finite non-negative"); + } + expect(comboConfigIssues("free", { + targets: [{ provider: "a", model: "m1", pricing: { inputUsdPerMillion: 0.5, outputUsdPerMillion: 2, futureKey: 3 } }], + }, providers)).toEqual([]); + }); + + test("normalized legacy combos do not gain economy metadata keys", () => { + const failover = normalizeComboConfig({ targets: [{ provider: "a", model: "m1" }] }); + expect("economy" in failover).toBe(false); + const roundRobin = normalizeComboConfig({ strategy: "round-robin", targets: [{ provider: "a", model: "m1" }] }); + expect("economy" in roundRobin).toBe(false); + const economy = normalizeComboConfig({ + strategy: "economy", + economy: { unknownQuota: "reject" }, + targets: [{ provider: "a", model: "m1", allowances: ["b"] }], + }); + expect(economy.economy).toEqual({ unknownQuota: "reject" }); + }); + test("normalizes valid values and returns defensive default efforts", () => { expect(normalizeComboConfig({ defaultEffort: "high", diff --git a/tests/economic-allowances-validation.test.ts b/tests/economic-allowances-validation.test.ts new file mode 100644 index 0000000000..1151395aa4 --- /dev/null +++ b/tests/economic-allowances-validation.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { getConfigPath, getDefaultConfig, loadConfig, validateConfigCandidate } from "../src/config"; + +let temporaryHome: string | null = null; +let previousHome: string | undefined; + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (temporaryHome) rmSync(temporaryHome, { recursive: true, force: true }); + temporaryHome = null; + previousHome = undefined; +}); + +describe("economicAllowances validation", () => { + test("malformed top-level economicAllowances is rejected without throwing", () => { + const base = getDefaultConfig(); + expect(() => validateConfigCandidate({ ...base, economicAllowances: "not-an-object" })).not.toThrow(); + expect(validateConfigCandidate({ ...base, economicAllowances: "not-an-object" }).ok).toBe(false); + + expect(validateConfigCandidate({ ...base, economicAllowances: [] }).ok).toBe(false); + expect(validateConfigCandidate({ ...base, economicAllowances: 123 }).ok).toBe(false); + expect(validateConfigCandidate({ ...base, economicAllowances: null }).ok).toBe(false); + }); + + test("malformed allowance entries produce actionable errors", () => { + const base = getDefaultConfig(); + const cases: Array<{ allowance: unknown; contains: string }> = [ + { allowance: { unit: "credits", capacity: "bad", window: { kind: "balance" } }, contains: "capacity" }, + { allowance: { unit: "bad-unit", capacity: 10, window: { kind: "balance" } }, contains: "unit" }, + { allowance: { unit: "credits", capacity: 10 }, contains: "window" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "rolling" } }, contains: "durationMs" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "rolling", durationMs: 0 } }, contains: "durationMs" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "calendar", interval: "month", timezone: "Not/AZone" } }, contains: "timezone" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "expiresAt", expiresAt: -1 } }, contains: "expiresAt" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, reserveFraction: 2 }, contains: "reserveFraction" }, + { allowance: { unit: "credits", capacity: 5, window: { kind: "balance" }, reserveAmount: 10 }, contains: "reserveAmount" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, source: "unknown" }, contains: "source" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, staleAfterMs: -1 }, contains: "staleAfterMs" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, rates: "bad" }, contains: "rates" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, rates: { inputPerMillion: Number.NaN } }, contains: "inputPerMillion" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, rates: { inputPerMillion: Number.POSITIVE_INFINITY } }, contains: "inputPerMillion" }, + { allowance: { unit: "credits", capacity: 10, window: { kind: "balance" }, rates: { inputPerMillion: -1 } }, contains: "inputPerMillion" }, + ]; + for (const { allowance, contains } of cases) { + const result = validateConfigCandidate({ ...base, economicAllowances: { bad: allowance as never } }); + expect(result.ok).toBe(false); + if (result.ok === false) expect(result.error.toLowerCase()).toContain(contains.toLowerCase()); + } + }); + + test("old configs without economicAllowances still validate", () => { + const base = getDefaultConfig(); + const { economicAllowances: _omit, ...rest } = base as Record; + expect(validateConfigCandidate(rest).ok).toBe(true); + expect(validateConfigCandidate({ ...rest, economicAllowances: undefined }).ok).toBe(true); + expect(validateConfigCandidate({ ...rest, economicAllowances: {} }).ok).toBe(true); + }); + + test("unsafe allowance ids are rejected while stable ids remain valid", () => { + const base = getDefaultConfig(); + const allowance = { unit: "credits", capacity: 10, window: { kind: "balance" } }; + for (const id of ["__proto__", "prototype", "constructor", "bad id", ".leading", "x".repeat(65)]) { + const allowances = JSON.parse(`{${JSON.stringify(id)}:${JSON.stringify(allowance)}}`) as Record; + const result = validateConfigCandidate({ ...base, economicAllowances: allowances }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("allowance id"); + } + expect(validateConfigCandidate({ ...base, economicAllowances: { "provider-five_hour.v1": allowance } }).ok).toBe(true); + }); + + test("loadConfig drops malformed optional allowances without discarding providers", () => { + previousHome = process.env.OPENCODEX_HOME; + temporaryHome = mkdtempSync(join(tmpdir(), "ocx-economic-load-")); + process.env.OPENCODEX_HOME = temporaryHome; + const base = getDefaultConfig(); + const configPath = getConfigPath(); + mkdirSync(dirname(configPath), { recursive: true }); + writeFileSync(configPath, JSON.stringify({ + ...base, + defaultProvider: "preserved", + providers: { + ...base.providers, + preserved: { adapter: "openai-chat", baseUrl: "https://preserved.example/v1", apiKey: "keep-me" }, + }, + economicAllowances: { + valid: { unit: "credits", capacity: 10, window: { kind: "balance" } }, + broken: { unit: "credits", capacity: "bad", window: { kind: "balance" } }, + }, + })); + + const loaded = loadConfig(); + expect(loaded.defaultProvider).toBe("preserved"); + expect(loaded.providers.preserved?.baseUrl).toBe("https://preserved.example/v1"); + expect(loaded.economicAllowances?.valid?.capacity).toBe(10); + expect(Object.hasOwn(loaded.economicAllowances ?? {}, "broken")).toBe(false); + }); +}); diff --git a/tests/economic-management-api.test.ts b/tests/economic-management-api.test.ts new file mode 100644 index 0000000000..6637fb1bb7 --- /dev/null +++ b/tests/economic-management-api.test.ts @@ -0,0 +1,162 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { clearEconomicState, setEconomicQuotaSnapshot } from "../src/combos"; +import { handleManagementAPI } from "../src/server/management-api"; +import type { OcxConfig } from "../src/types"; +import { catalogConvergenceFactory } from "./helpers/catalog-convergence"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "included", + providers: { + included: { adapter: "openai-chat", baseUrl: "https://included.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + promo: { + unit: "credits", + capacity: 10, + window: { kind: "expiresAt", expiresAt: Date.now() + 60 * 60_000 }, + rollover: false, + source: "manual", + rates: { inputPerMillion: 1 }, + staleAfterMs: 60 * 60 * 1000, + }, + }, + combos: {}, + }; +} + +afterEach(() => clearEconomicState()); + +describe("economic combo management API", () => { + test("PUT and GET preserve normalized economy fields", async () => { + const cfg = config(); + const request = new Request("http://localhost/api/combos", { + method: "PUT", + headers: { "content-type": "application/json", host: "localhost" }, + body: JSON.stringify({ + id: "bulk-code", + combo: { + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2, outputUsdPerMillion: 0.8 } }, + ], + }, + }), + }); + const response = await handleManagementAPI(request, new URL(request.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); + expect(response?.status).toBe(200); + expect(cfg.combos!["bulk-code"]?.strategy).toBe("economy"); + expect(cfg.combos!["bulk-code"]?.targets[0]?.allowances).toEqual(["promo"]); + + const get = new Request("http://localhost/api/combos", { headers: { host: "localhost" } }); + const getResponse = await handleManagementAPI(get, new URL(get.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); + const body = await getResponse!.json() as { combos: Array> }; + expect(body.combos[0]?.strategy).toBe("economy"); + expect(body.combos[0]?.economy).toEqual({ unknownQuota: "deprioritize", maxMarginalUsd: 0.1 }); + }); + + test("explain endpoint returns a safe structured decision", async () => { + const cfg = config(); + cfg.combos = { + bulk: { + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 1 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2 } }, + ], + }, + }; + setEconomicQuotaSnapshot("promo", { + remaining: 8, + updatedAt: Date.now(), + expiresAt: Date.now() + 60 * 60_000, + source: "manual", + confidence: "authoritative", + }); + const request = new Request("http://localhost/api/combos/bulk/explain?inputTokens=1000000&outputTokens=10", { headers: { host: "localhost" } }); + const response = await handleManagementAPI(request, new URL(request.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); + const body = await response!.json() as Record; + expect(response.status).toBe(200); + expect(body.selectedTarget).toBe("included/m"); + expect(body.strategy).toBe("economy"); + expect(JSON.stringify(body)).not.toContain("apiKey"); + }); + + test("explain rejects malformed percent-encoding with 400", async () => { + const cfg = config(); + const request = new Request("http://localhost/api/combos/%zz/explain", { headers: { host: "localhost" } }); + const response = await handleManagementAPI(request, new URL(request.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); + expect(response?.status).toBe(400); + }); + + test("snapshot endpoint drives explain decisions end to end", async () => { + const cfg = config(); + const api = async (path: string, init?: RequestInit) => { + const request = new Request(`http://localhost${path}`, { + ...init, + headers: { ...(init?.headers ?? {}), host: "localhost" }, + }); + return handleManagementAPI(request, new URL(request.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); + }; + const explain = async (): Promise<{ selectedTarget: string }> => { + const response = await api("/api/combos/bulk-code/explain?inputTokens=1000000&outputTokens=10"); + return response!.json() as Promise<{ selectedTarget: string }>; + }; + + const putCombo = await api("/api/combos", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: "bulk-code", + combo: { + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 1 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2, outputUsdPerMillion: 0.8 } }, + ], + }, + }), + }); + expect(putCombo?.status).toBe(200); + + // Unknown promo quota → deprioritize → metered target wins. + expect((await explain()).selectedTarget).toBe("payg/m"); + + // Authoritative snapshot via the management endpoint flips the decision. + const putSnapshot = await api("/api/economic-allowances/promo/snapshot", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + remaining: 8, + updatedAt: Date.now(), + source: "manual", + confidence: "authoritative", + }), + }); + expect(putSnapshot?.status).toBe(200); + expect((await explain()).selectedTarget).toBe("included/m"); + + // Clearing the snapshot restores the unknown-quota policy. + const delSnapshot = await api("/api/economic-allowances/promo/snapshot", { method: "DELETE" }); + expect(delSnapshot?.status).toBe(200); + expect((await explain()).selectedTarget).toBe("payg/m"); + }); +}); diff --git a/tests/economic-manual-snapshot-api.test.ts b/tests/economic-manual-snapshot-api.test.ts new file mode 100644 index 0000000000..6910b83b26 --- /dev/null +++ b/tests/economic-manual-snapshot-api.test.ts @@ -0,0 +1,399 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { clearEconomicState, getEconomicQuotaSnapshot, reserveEconomicSelection, setEconomicQuotaSnapshot } from "../src/combos"; +import { handleManagementAPI } from "../src/server/management-api"; +import type { OcxConfig } from "../src/types"; +import { catalogConvergenceFactory } from "./helpers/catalog-convergence"; +import { ManagementRequest } from "./helpers/management-auth"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "included", + providers: { + included: { adapter: "openai-chat", baseUrl: "https://included.example", models: ["m"], apiKey: "secret-123" }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"], apiKey: "other-secret" }, + }, + economicAllowances: { + promo: { + unit: "credits", + capacity: 10, + window: { kind: "expiresAt", expiresAt: NOW + 60 * 60_000 }, + rollover: false, + source: "manual", + rates: { inputPerMillion: 1 }, + staleAfterMs: 60 * 60 * 1000, + }, + }, + combos: { + bulk: { + strategy: "economy", + economy: { unknownQuota: "deprioritize" }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 0.2 } }, + ], + }, + }, + }; +} + +async function request( + cfg: OcxConfig, + method: string, + path: string, + body?: unknown, + rawBody?: string, +): Promise { + const init: RequestInit = { method, headers: {} }; + if (rawBody !== undefined) { + (init.headers as Record)["content-type"] = "application/json"; + init.body = rawBody; + } else if (body !== undefined) { + (init.headers as Record)["content-type"] = "application/json"; + init.body = JSON.stringify(body); + } + const req = new ManagementRequest(`http://localhost${path}`, init); + return handleManagementAPI(req, new URL(req.url), cfg, { + createManagementConvergeCodex: catalogConvergenceFactory(), + }); +} + +afterEach(() => clearEconomicState()); + +describe("manual economic snapshot API", () => { + test("PUT stores normalized snapshot and GET returns it", async () => { + const cfg = config(); + const put = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 7.5, + updatedAt: NOW, + windowStart: NOW - 1000, + resetAt: NOW + 60 * 60_000, + expiresAt: NOW + 60 * 60_000, + source: "manual", + confidence: "authoritative", + }); + expect(put?.status).toBe(200); + const putBody = await put!.json() as Record; + expect(putBody.allowanceId).toBe("promo"); + expect((putBody.snapshot as Record).remaining).toBe(7.5); + expect((putBody.snapshot as Record).source).toBe("manual"); + expect((putBody.snapshot as Record).confidence).toBe("authoritative"); + const dump = JSON.stringify(putBody); + expect(dump).not.toContain("secret-123"); + expect(dump).not.toContain("apiKey"); + + const get = await request(cfg, "GET", "/api/economic-allowances/promo/snapshot"); + expect(get?.status).toBe(200); + const getBody = await get!.json() as Record; + expect(getBody.state).toBe("present"); + expect((getBody.snapshot as Record).remaining).toBe(7.5); + expect(JSON.stringify(getBody)).not.toContain("secret-123"); + }); + + test("GET returns unknown state when no snapshot exists", async () => { + const cfg = config(); + const res = await request(cfg, "GET", "/api/economic-allowances/promo/snapshot"); + expect(res?.status).toBe(200); + const body = await res!.json() as Record; + expect(body.allowanceId).toBe("promo"); + expect(body.snapshot).toBeNull(); + expect(body.state).toBe("unknown"); + }); + + test("unknown allowance id returns 404", async () => { + const cfg = config(); + for (const method of ["GET", "PUT", "DELETE"] as const) { + const res = await request(cfg, method, "/api/economic-allowances/missing/snapshot", method === "PUT" ? { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" } : undefined); + expect(res?.status).toBe(404); + const body = await res!.json() as Record; + expect(String(body.error)).toContain("unknown"); + } + }); + + test("PUT rejects non-object body", async () => { + const cfg = config(); + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", undefined, "[]"); + expect(res?.status).toBe(400); + }); + + test("PUT validates remaining finite non-negative", async () => { + const cfg = config(); + for (const remaining of [undefined, -1, Number.NaN, Number.POSITIVE_INFINITY, "1"]) { + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + expect(res?.status).toBe(400); + const body = await res!.json() as Record; + expect(String(body.error)).toContain("remaining"); + } + }); + + test("PUT validates updatedAt finite non-negative", async () => { + const cfg = config(); + for (const updatedAt of [undefined, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt, + source: "manual", + confidence: "authoritative", + }); + expect(res?.status).toBe(400); + } + }); + + test("PUT validates timestamp fields", async () => { + const cfg = config(); + for (const field of ["windowStart", "resetAt", "expiresAt"] as const) { + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + [field]: -1, + }); + expect(res?.status).toBe(400); + const body = await res!.json() as Record; + expect(String(body.error)).toContain(field); + const nanRes = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + [field]: Number.NaN, + }); + expect(nanRes?.status).toBe(400); + } + }); + + test("PUT validates source and confidence enums", async () => { + const cfg = config(); + const badSource = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt: NOW, + source: "bad", + confidence: "authoritative", + }); + expect(badSource?.status).toBe(400); + expect(String((await badSource!.json() as Record).error)).toContain("source"); + + const badConfidence = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt: NOW, + source: "manual", + confidence: "bad", + }); + expect(badConfidence?.status).toBe(400); + expect(String((await badConfidence!.json() as Record).error)).toContain("confidence"); + }); + + test("DELETE clears one snapshot without clearing others", async () => { + const cfg = config(); + (cfg.economicAllowances as Record)["other"] = { + unit: "credits", + capacity: 10, + window: { kind: "balance" }, + source: "manual", + }; + await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 5, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + await request(cfg, "PUT", "/api/economic-allowances/other/snapshot", { + remaining: 9, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + const del = await request(cfg, "DELETE", "/api/economic-allowances/promo/snapshot"); + expect(del?.status).toBe(200); + const delBody = await del!.json() as Record; + expect(delBody.cleared).toBe(true); + + const getPromo = await request(cfg, "GET", "/api/economic-allowances/promo/snapshot"); + expect((await getPromo!.json() as Record).state).toBe("unknown"); + + const getOther = await request(cfg, "GET", "/api/economic-allowances/other/snapshot"); + const otherBody = await getOther!.json() as Record; + expect(otherBody.state).toBe("present"); + expect((otherBody.snapshot as Record).remaining).toBe(9); + expect(getEconomicQuotaSnapshot("other")?.remaining).toBe(9); + expect(getEconomicQuotaSnapshot("promo")).toBeUndefined(); + }); + + test("PUT rejects invalid method and never persists snapshot into config", async () => { + const cfg = config(); + const put = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 3, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + expect(put?.status).toBe(200); + expect((cfg.economicAllowances as Record).promo).not.toHaveProperty("remaining"); + expect((cfg as Record).snapshots).toBeUndefined(); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(3); + + const patch = await request(cfg, "PATCH", "/api/economic-allowances/promo/snapshot", { + remaining: 1, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + expect(patch?.status).toBe(405); + }); + + test("PUT never exposes credentials in output and preserves existing combos", async () => { + const cfg = config(); + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 2, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + }); + const text = JSON.stringify(await res!.clone().json()); + expect(text).not.toContain("secret"); + expect(text).not.toContain("apiKey"); + + const combosRes = await request(cfg, "GET", "/api/combos"); + expect(combosRes?.status).toBe(200); + const combosBody = await combosRes!.json() as { combos: Array> }; + expect(combosBody.combos.some(c => c.id === "bulk")).toBe(true); + }); + + test("malformed JSON is rejected", async () => { + const cfg = config(); + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", undefined, "{"); + expect(res?.status).toBe(400); + }); + + test("PUT ignores and never echoes secret-looking unknown fields", async () => { + const cfg = config(); + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 4, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + apiKey: "secret-xyz-123", + authorization: "Bearer secret-token", + }); + expect(res?.status).toBe(200); + const text = JSON.stringify(await res!.clone().json()); + expect(text).not.toContain("secret-xyz-123"); + expect(text).not.toContain("secret-token"); + expect(getEconomicQuotaSnapshot("promo")).not.toHaveProperty("apiKey"); + }); + + test("PUT rejects precision-lossy timestamps", async () => { + const cfg = config(); + const res = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { + remaining: 4, + updatedAt: Number.MAX_SAFE_INTEGER + 1, + source: "manual", + confidence: "authoritative", + }); + expect(res?.status).toBe(400); + }); + + test("malformed percent-encoding in allowance id returns 400", async () => { + const cfg = config(); + const res = await request(cfg, "GET", "/api/economic-allowances/%zz/snapshot"); + expect(res?.status).toBe(400); + }); + + test("PUT without clearReservations returns 409 when reservations are in flight", async () => { + const now = Date.now(); + const cfg = config(); + cfg.economicAllowances!.promo = { + ...cfg.economicAllowances!.promo!, + window: { kind: "expiresAt", expiresAt: now + 60 * 60_000 }, + }; + const snapshot = { remaining: 1, updatedAt: now, expiresAt: now + 60 * 60_000, source: "manual" as const, confidence: "authoritative" as const }; + setEconomicQuotaSnapshot("promo", snapshot); + const first = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(first.target?.provider).toBe("included"); + expect(first.reservationId).toBeDefined(); + + const put = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", snapshot); + expect(put?.status).toBe(409); + const body = await put!.json() as { activeReservations?: number }; + expect(body.activeReservations).toBeGreaterThan(0); + const second = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(second.target?.provider).toBe("payg"); + }); + + test("PUT snapshot clears in-flight reservations when clearReservations:true", async () => { + const now = Date.now(); + const cfg = config(); + cfg.economicAllowances!.promo = { + ...cfg.economicAllowances!.promo!, + window: { kind: "expiresAt", expiresAt: now + 60 * 60_000 }, + }; + const snapshot = { remaining: 1, updatedAt: now, expiresAt: now + 60 * 60_000, source: "manual" as const, confidence: "authoritative" as const }; + setEconomicQuotaSnapshot("promo", snapshot); + const first = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(first.target?.provider).toBe("included"); + expect(first.reservationId).toBeDefined(); + const second = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(second.target?.provider).toBe("payg"); + + const put = await request(cfg, "PUT", "/api/economic-allowances/promo/snapshot", { ...snapshot, clearReservations: true }); + expect(put?.status).toBe(200); + const third = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(third.target?.provider).toBe("included"); + expect(third.reservationId).toBeDefined(); + }); + + test("DELETE without clearReservations returns 409 when reservations are in flight", async () => { + const now = Date.now(); + const cfg = config(); + cfg.economicAllowances!.promo = { + ...cfg.economicAllowances!.promo!, + window: { kind: "expiresAt", expiresAt: now + 60 * 60_000 }, + }; + const snapshot = { remaining: 1, updatedAt: now, expiresAt: now + 60 * 60_000, source: "manual" as const, confidence: "authoritative" as const }; + setEconomicQuotaSnapshot("promo", snapshot); + const first = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(first.target?.provider).toBe("included"); + expect(first.reservationId).toBeDefined(); + + const del = await request(cfg, "DELETE", "/api/economic-allowances/promo/snapshot"); + expect(del?.status).toBe(409); + }); + + test("DELETE snapshot clears in-flight reservations when clearReservations=true", async () => { + const now = Date.now(); + const cfg = config(); + cfg.economicAllowances!.promo = { + ...cfg.economicAllowances!.promo!, + window: { kind: "expiresAt", expiresAt: now + 60 * 60_000 }, + }; + const snapshot = { remaining: 1, updatedAt: now, expiresAt: now + 60 * 60_000, source: "manual" as const, confidence: "authoritative" as const }; + setEconomicQuotaSnapshot("promo", snapshot); + const first = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(first.target?.provider).toBe("included"); + expect(first.reservationId).toBeDefined(); + + const del = await request(cfg, "DELETE", "/api/economic-allowances/promo/snapshot?clearReservations=true"); + expect(del?.status).toBe(200); + setEconomicQuotaSnapshot("promo", snapshot); + const second = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(second.target?.provider).toBe("included"); + expect(second.reservationId).toBeDefined(); + }); + + test("GET /api/economic-allowances lists configured allowances and snapshot state", async () => { + const cfg = config(); + setEconomicQuotaSnapshot("promo", { remaining: 3, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const res = await request(cfg, "GET", "/api/economic-allowances"); + expect(res?.status).toBe(200); + const body = await res!.json() as { allowances: Array<{ id: string; state: string; snapshot: { remaining: number } | null }> }; + expect(body.allowances.some(a => a.id === "promo" && a.state === "present" && a.snapshot?.remaining === 3)).toBe(true); + }); +}); diff --git a/tests/economic-ordering-stability.test.ts b/tests/economic-ordering-stability.test.ts new file mode 100644 index 0000000000..ae85c5d8ee --- /dev/null +++ b/tests/economic-ordering-stability.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { clearEconomicState, selectEconomicTarget } from "../src/combos/economy"; +import type { OcxConfig } from "../src/types"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example", models: ["m"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example", models: ["m"] }, + }, + economicAllowances: {}, + combos: { + bulk: { + strategy: "economy", + targets: [ + { provider: "a", model: "m" }, + { provider: "b", model: "m" }, + ], + }, + }, + }; +} + +afterEach(() => clearEconomicState()); + +describe("economic candidate ordering stability", () => { + test("equal unknown/non-finite marginal costs are ordered deterministically by configured order", () => { + const cfg = config(); + // Both targets have no pricing and no allowances => marginalUsd null => both map to equal cost bucket. + // Ordering must be deterministic and not depend on Infinity-Infinity => NaN. + const estimate = { inputTokens: 1000, outputTokens: 1000, kind: "configured" as const }; + const first = selectEconomicTarget(cfg, "bulk", estimate, NOW); + const second = selectEconomicTarget(cfg, "bulk", estimate, NOW); + expect(first.targetIndex).toBe(0); + expect(second.targetIndex).toBe(0); + expect(first.candidates[0]!.target.provider).toBe("a"); + expect(first.candidates[1]!.target.provider).toBe("b"); + // Reverse configured order should flip winner + const rev: OcxConfig = { ...cfg, combos: { bulk: { ...cfg.combos!.bulk, targets: [...cfg.combos!.bulk.targets].reverse() } } }; + const revResult = selectEconomicTarget(rev, "bulk", estimate, NOW); + expect(revResult.target?.provider).toBe("b"); + }); + + test("Infinity and NaN pricing values are treated as unknown and do not break ordering", () => { + const cfg: OcxConfig = { + ...config(), + combos: { + bulk: { + strategy: "economy", + targets: [ + { provider: "a", model: "m", pricing: { inputUsdPerMillion: Number.POSITIVE_INFINITY } }, + { provider: "b", model: "m", pricing: { inputUsdPerMillion: Number.NaN } }, + ], + }, + }, + }; + const estimate = { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" as const }; + const result = selectEconomicTarget(cfg, "bulk", estimate, NOW); + // Both costs are non-finite => should be treated equal and stable (first wins) + expect(result.target?.provider).toBe("a"); + expect(result.candidates.every(c => c.marginalUsd === null || Number.isFinite(c.marginalUsd!))).toBe(true); + }); +}); diff --git a/tests/economic-reservation-race.test.ts b/tests/economic-reservation-race.test.ts new file mode 100644 index 0000000000..b2095cee4f --- /dev/null +++ b/tests/economic-reservation-race.test.ts @@ -0,0 +1,91 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearEconomicState, + reserveEconomicSelection, + setEconomicQuotaSnapshot, +} from "../src/combos/economy"; +import type { OcxConfig } from "../src/types"; + +const NOW = Date.parse("2026-08-10T12:00:00.000Z"); +const estimate = { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" as const }; + +function config(targets: OcxConfig["combos"][string]["targets"]): OcxConfig { + return { + port: 0, + defaultProvider: "primary", + providers: { + primary: { adapter: "openai-chat", baseUrl: "https://primary.example", models: ["m"] }, + alternate: { adapter: "openai-chat", baseUrl: "https://alternate.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + primaryAllowance: { unit: "requests", capacity: 1, window: { kind: "balance" }, source: "manual" }, + alternateAllowance: { unit: "requests", capacity: 1, window: { kind: "balance" }, source: "manual" }, + }, + combos: { + c: { strategy: "economy", economy: { unknownQuota: "reject" }, targets }, + }, + }; +} + +afterEach(() => clearEconomicState()); + +describe("economic reservation races", () => { + test("concurrent double reserve cannot both hold the same unit of headroom", async () => { + const cfg = config([ + { provider: "primary", model: "m", allowances: ["primaryAllowance"] }, + { provider: "payg", model: "m" }, + ]); + setEconomicQuotaSnapshot("primaryAllowance", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + + const [first, second] = await Promise.all([ + Promise.resolve().then(() => reserveEconomicSelection(cfg, "c", estimate, NOW)), + Promise.resolve().then(() => reserveEconomicSelection(cfg, "c", estimate, NOW)), + ]); + const allowanceResults = [first, second].filter(result => result.target?.provider === "primary"); + + expect(allowanceResults).toHaveLength(1); + expect(allowanceResults[0]?.reservationId).toBeString(); + expect([first, second].filter(result => result.target?.provider === "payg")).toHaveLength(1); + }); + + test("forced headroom race never returns an allowance target without reservationId", () => { + const cfg = config([{ provider: "primary", model: "m", allowances: ["primaryAllowance"] }]); + setEconomicQuotaSnapshot("primaryAllowance", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + let raced = false; + + const result = reserveEconomicSelection(cfg, "c", estimate, NOW, [], () => { + if (!raced) { + raced = true; + setEconomicQuotaSnapshot("primaryAllowance", { remaining: 0, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + } + return true; + }); + + expect(result.target).toBeUndefined(); + expect(result.targetIndex).toBeNull(); + expect(result.reservationId).toBeUndefined(); + expect(result.reason).toBe("reservation-headroom-race"); + }); + + test("headroom race recursively reserves an alternate target", () => { + const cfg = config([ + { provider: "primary", model: "m", allowances: ["primaryAllowance"] }, + { provider: "alternate", model: "m", allowances: ["alternateAllowance"] }, + ]); + setEconomicQuotaSnapshot("primaryAllowance", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + setEconomicQuotaSnapshot("alternateAllowance", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + let raced = false; + + const result = reserveEconomicSelection(cfg, "c", estimate, NOW, [], () => { + if (!raced) { + raced = true; + setEconomicQuotaSnapshot("primaryAllowance", { remaining: 0, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + } + return true; + }); + + expect(result.target?.provider).toBe("alternate"); + expect(result.reservationId).toBeString(); + }); +}); diff --git a/tests/economic-reservation-settlement.test.ts b/tests/economic-reservation-settlement.test.ts new file mode 100644 index 0000000000..cf5d298316 --- /dev/null +++ b/tests/economic-reservation-settlement.test.ts @@ -0,0 +1,270 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; + +const actualResolver = await import("../src/server/adapter-resolve"); +const actualResolveAdapter = actualResolver.resolveAdapter; +let customFetchResponse: ((request: Request) => Promise) | undefined; +mock.module("../src/server/adapter-resolve", () => ({ + ...actualResolver, + resolveAdapter(provider: import("../src/types").OcxProviderConfig, cacheRetention?: "none" | "short" | "long") { + if (provider.adapter === "test-response") { + const base = actualResolveAdapter({ ...provider, adapter: "openai-chat" }, cacheRetention); + return { + ...base, + name: "test-response", + async fetchResponse(request: Request) { + if (!customFetchResponse) throw new Error("customFetchResponse not installed"); + return customFetchResponse(request); + }, + }; + } + return actualResolveAdapter(provider, cacheRetention); + }, +})); + +const { handleComboResponses } = await import("../src/server/responses/core"); + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; +import { + clearEconomicState, + reserveEconomicSelection, + setEconomicQuotaSnapshot, + settleEconomicReservation, + getEconomicQuotaSnapshot, +} from "../src/combos/economy"; +import type { OcxConfig } from "../src/types"; + +const now = 1_000_000; + +function config(unit: "inputTokens" | "outputTokens" | "totalTokens" | "requests" | "credits" | "usd"): OcxConfig { + return { + port: 0, + defaultProvider: "p", + providers: { p: { adapter: "openai-chat", baseUrl: "https://example.test", apiKey: "key", authMode: "key" } }, + economicAllowances: { + allowance: { + unit, + capacity: 100, + window: { kind: "balance" }, + source: "manual", + rates: unit === "credits" || unit === "usd" ? { fixedPerRequest: 10 } : undefined, + }, + }, + combos: { c: { strategy: "economy", targets: [{ provider: "p", model: "m", allowances: ["allowance"] }] } }, + }; +} + +function reserve(unit: Parameters[0], amount = 10): string { + const selected = reserveEconomicSelection(config(unit), "c", { + inputTokens: unit === "inputTokens" || unit === "totalTokens" || unit === "credits" || unit === "usd" ? amount : 0, + outputTokens: unit === "outputTokens" ? amount : 0, + kind: "configured", + fixedRequests: unit === "requests" ? amount : 1, + }, now); + expect(selected.reservationId).toBeString(); + return selected.reservationId!; +} + +beforeEach(() => { + clearEconomicState(); + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-econ-settle-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-econ-settle-")); + process.env.OPENCODEX_HOME = testDir; + customFetchResponse = undefined; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) rmSync(testDir, { recursive: true, force: true }); + clearEconomicState(); +}); + +function lifecycleConfig(unit: "outputTokens" | "credits" | "usd" = "outputTokens"): OcxConfig { + return { + port: 0, + defaultProvider: "cheap", + providers: { + cheap: { adapter: "test-response", baseUrl: "https://cheap.test/v1", allowPrivateNetwork: true, authMode: "key", apiKey: "key" }, + payg: { adapter: "test-response", baseUrl: "https://payg.test/v1", allowPrivateNetwork: true, authMode: "key", apiKey: "key" }, + }, + economicAllowances: { + allowance: { + unit, + capacity: 100, + window: { kind: "balance" }, + source: "manual", + ...(unit === "credits" ? { rates: { inputPerMillion: 1_000_000 } } : {}), + ...(unit === "usd" ? { rates: { inputPerMillion: 1_000_000 } } : {}), + }, + }, + combos: { + c: { + strategy: "economy", + economy: { unknownQuota: "reject" }, + targets: [ + { provider: "cheap", model: "m", allowances: ["allowance"], ...(unit === "usd" ? { pricing: { inputUsdPerMillion: 1_000_000 } } : {}) }, + { provider: "payg", model: "m" }, + ], + }, + }, + }; +} + +function lifecycleRequest(): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/c", input: "hello", max_output_tokens: 10 }), + }); +} + +function lifecycleBody() { + return { model: "combo/c", input: "hello", max_output_tokens: 10 }; +} + +describe("economic reservation settlement", () => { + test("settles smaller actual usage and releases excess", () => { + const id = reserve("inputTokens"); + settleEconomicReservation(id, { inputTokens: 4 }, now + 1); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + }); + + test("clamps larger actual usage at zero", () => { + const id = reserve("inputTokens"); + settleEconomicReservation(id, { inputTokens: 100 }, now + 1); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(0); + }); + + test("settles every economic unit", () => { + for (const unit of ["inputTokens", "outputTokens", "totalTokens", "requests", "credits", "usd"] as const) { + clearEconomicState(); + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + const id = reserve(unit); + settleEconomicReservation(id, { [unit]: 4 }, now + 1); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + } + }); + + test("is idempotent", () => { + const id = reserve("inputTokens"); + settleEconomicReservation(id, { inputTokens: 4 }, now + 1); + settleEconomicReservation(id, { inputTokens: 0 }, now + 2); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + }); + + test("rejects invalid actual usage and releases the reservation", () => { + const cfg = config("inputTokens"); + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: now, source: "manual", confidence: "authoritative" }); + const id = reserve("inputTokens", 45); + expect(() => settleEconomicReservation(id, { inputTokens: -1 })).toThrow(); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); + expect(() => settleEconomicReservation(id, { inputTokens: Number.NaN })).toThrow(); + // The failed settle must not leave the reservation blocking headroom until TTL. + const retry = reserveEconomicSelection(cfg, "c", { inputTokens: 45, outputTokens: 0, kind: "configured" }, now + 1); + expect(retry.reservationId).toBeString(); + }); + + test("undefined actual releases the reservation", () => { + const id = reserve("inputTokens"); + settleEconomicReservation(id, undefined, now + 1); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(60); + }); + + test("settles successful child usage exactly once", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); + const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); + expect(response.status).toBe(200); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + }); + + test("settles credits from token rates through the response lifecycle", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); + const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig("credits"), { model: "", provider: "" }, {}); + expect(response.status).toBe(200); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + }); + + test("settles usd from target pricing through the response lifecycle", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); + const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig("usd"), { model: "", provider: "" }, {}); + expect(response.status).toBe(200); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + }); + + test("settles streamed success after body consumption", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(`data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]} + +data: {"choices":[],"usage":{"prompt_tokens":1,"completion_tokens":4,"total_tokens":5}} + +data: [DONE] + +`)); + controller.close(); + }, + }), { headers: { "content-type": "text/event-stream" } }); + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }), + }); + const response = await handleComboResponses(request, { model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }, "c", lifecycleConfig("credits"), { model: "", provider: "" }, {}); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); + await response.arrayBuffer(); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + }); + + test("releases streamed reservation when cancelled before usage", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: {}\\n\\n")); + }, + }), { headers: { "content-type": "text/event-stream" } }); + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }), + }); + const response = await handleComboResponses(request, { model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }, "c", lifecycleConfig("credits"), { model: "", provider: "" }, {}); + await response.body?.cancel(); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); + }); + + test("settles terminal failure usage", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => Response.json({ error: { code: "context_length_exceeded", message: "too long" }, usage: { input_tokens: 1, output_tokens: 4, total_tokens: 5 } }, { status: 400 }); + const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); + expect(response.status).toBe(400); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + }); + + test("settles retryable failure usage before failover", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + let calls = 0; + customFetchResponse = async () => { + calls += 1; + if (calls === 1) return Response.json({ error: { code: "rate_limit_exceeded", message: "busy" }, usage: { input_tokens: 1, output_tokens: 4, total_tokens: 5 } }, { status: 429 }); + return Response.json({ id: "ok", object: "response", status: "completed", output: [] }); + }; + const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); + expect(response.status).toBe(200); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + }); + +}); diff --git a/tests/economic-reservation-terminal.test.ts b/tests/economic-reservation-terminal.test.ts new file mode 100644 index 0000000000..b94eb21b64 --- /dev/null +++ b/tests/economic-reservation-terminal.test.ts @@ -0,0 +1,177 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { clearEconomicState, setEconomicQuotaSnapshot, selectEconomicTarget } from "../src/combos/economy"; +import type { OcxConfig, OcxProviderConfig } from "../src/types"; +import { clearComboSelectionState, clearComboTargetCooldowns } from "../src/combos"; +import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; + +const actualResolver = await import("../src/server/adapter-resolve"); +const actualResolveAdapter = actualResolver.resolveAdapter; + +let customFetchResponse: ((request: Request, context?: unknown) => Promise) | undefined; + +mock.module("../src/server/adapter-resolve", () => ({ + ...actualResolver, + resolveAdapter(provider: OcxProviderConfig, cacheRetention?: "none" | "short" | "long") { + if (provider.adapter === "test-response") { + const base = actualResolveAdapter({ ...provider, adapter: "openai-chat" }, cacheRetention); + return { + ...base, + name: "test-response", + async fetchResponse(request: Request) { + if (!customFetchResponse) throw new Error("customFetchResponse not installed"); + return customFetchResponse(request); + }, + }; + } + return actualResolveAdapter(provider, cacheRetention); + }, +})); + +const { handleResponses } = await import("../src/server/responses"); +const { handleComboResponses } = await import("../src/server/responses/core"); + +let testDir = ""; +let previousHome: string | undefined; +let isolatedCodexHome: IsolatedCodexHome | null = null; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + isolatedCodexHome = installIsolatedCodexHome("ocx-econ-reservation-"); + testDir = mkdtempSync(join(tmpdir(), "ocx-econ-res-")); + process.env.OPENCODEX_HOME = testDir; + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearEconomicState(); + customFetchResponse = undefined; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + isolatedCodexHome?.restore(); + isolatedCodexHome = null; + if (testDir) rmSync(testDir, { recursive: true, force: true }); + clearComboSelectionState(); + clearComboTargetCooldowns(); + clearEconomicState(); +}); + +function provider(url: string): OcxProviderConfig { + return { adapter: "test-response", baseUrl: url, allowPrivateNetwork: true, authMode: "key", apiKey: "key" }; +} + +describe("economic reservation terminal failure", () => { + test("reservation is released on non-retryable 400 so next request can use quota", async () => { + const NOW = Date.now(); + const cfg: OcxConfig = { + port: 0, + defaultProvider: "cheap", + providers: { + cheap: provider("https://cheap.test/v1"), + payg: provider("https://payg.test/v1"), + }, + economicAllowances: { + budget: { + unit: "credits", + capacity: 10, + window: { kind: "balance" }, + source: "manual", + rates: { inputPerMillion: 1 }, + }, + }, + combos: { + bulk: { + strategy: "economy", + economy: { unknownQuota: "reject", maxMarginalUsd: 10 }, + targets: [ + { provider: "cheap", model: "m", allowances: ["budget"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 1 } }, + ], + }, + }, + }; + setEconomicQuotaSnapshot("budget", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + + // First dispatch: cheap target fails with non-retryable 400 context_length_exceeded (stop) + customFetchResponse = async (req) => { + const body = JSON.parse(String(req.body)) as { model?: string }; + if (body.model === "m") { + return Response.json({ error: { code: "context_length_exceeded", message: "too long" } }, { status: 400 }); + } + return Response.json({ id: "x", object: "response", status: "completed", model: "m", output: [] }); + }; + + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/bulk", input: "hello", max_output_tokens: 100 }), + }); + const res = await handleResponses(req, cfg, { model: "", provider: "" }, {}); + expect(res.status).toBe(400); + + // Second selection should still be able to pick cheap if reservation was released. + // If leaked, reserved 1 credit still counts, remaining 1 -1 =0, second request needs 1 credit -> post 0 -> still ok? Need to make consumption =1, remaining 1, reservation 1 => second would see 0 remaining => falls to PAYG if leaked. + // Use estimate that consumes 1 credit: input 1M tokens with rate 1 per million + const second = selectEconomicTarget(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, NOW); + expect(second.target?.provider).toBe("cheap"); + }); + + test("reservation is released when child dispatch throws", async () => { + const now = Date.now(); + const cfg: OcxConfig = { + port: 0, + defaultProvider: "cheap", + providers: { + cheap: provider("https://cheap.test/v1"), + payg: provider("https://payg.test/v1"), + }, + economicAllowances: { + budget: { + unit: "credits", + capacity: 10, + window: { kind: "balance" }, + source: "manual", + rates: { inputPerMillion: 1 }, + }, + }, + combos: { + bulk: { + strategy: "economy", + economy: { unknownQuota: "reject", maxMarginalUsd: 10 }, + targets: [ + { provider: "cheap", model: "m", allowances: ["budget"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 1 } }, + ], + }, + }, + }; + setEconomicQuotaSnapshot("budget", { remaining: 1, updatedAt: now, source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => Response.json({ + id: "response-1", + object: "response", + status: "completed", + model: "m", + output: [], + }); + + const rawBody = { model: "combo/bulk", input: "hello", max_output_tokens: 100 }; + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(rawBody), + }); + const throwingOptions = {} as Record; + Object.defineProperty(throwingOptions, "testThrow", { + enumerable: true, + get() { throw new Error("child option spread failed"); }, + }); + await expect(handleComboResponses(req, rawBody, "bulk", cfg, { model: "", provider: "" }, throwingOptions as never)) + .rejects.toThrow("child option spread failed"); + + const second = selectEconomicTarget(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, now); + expect(second.target?.provider).toBe("cheap"); + }); +}); diff --git a/tests/economic-review-blindspots.test.ts b/tests/economic-review-blindspots.test.ts new file mode 100644 index 0000000000..e21c44a1a5 --- /dev/null +++ b/tests/economic-review-blindspots.test.ts @@ -0,0 +1,159 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearEconomicState, + countEconomicReservationsForAllowance, + explainEconomicCombo, + getEconomicQuotaSnapshot, + pickComboTarget, + releaseEconomicReservation, + reserveEconomicSelection, + selectEconomicTarget, + settleEconomicReservation, + setEconomicQuotaSnapshot, +} from "../src/combos"; +import type { OcxConfig } from "../src/types"; + +const NOW = Date.now(); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 0, + defaultProvider: "included", + providers: { + included: { adapter: "openai-chat", baseUrl: "https://included.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + alt: { adapter: "openai-chat", baseUrl: "https://alt.example", models: ["m"] }, + }, + economicAllowances: { + promo: { + unit: "requests", + capacity: 2, + window: { kind: "balance" }, + source: "manual", + rates: { fixedPerRequest: 1 }, + }, + }, + combos: { + bulk: { + strategy: "economy", + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 0.05 }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 1, outputUsdPerMillion: 1 } }, + { provider: "alt", model: "m" }, + ], + }, + }, + ...overrides, + }; +} + +afterEach(() => clearEconomicState()); + +describe("economic hostile-review blindspots", () => { + test("race never returns allowance target without reservationId", () => { + const cfg = baseConfig(); + setEconomicQuotaSnapshot("promo", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + let raced = false; + const result = reserveEconomicSelection(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW, [], () => { + if (!raced) { + raced = true; + setEconomicQuotaSnapshot("promo", { remaining: 0, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + } + return true; + }); + if (result.target?.provider === "included") expect(result.reservationId).toBeString(); + if (result.target?.allowances?.length) expect(result.reservationId).toBeString(); + }); + + test("credits settle derives burn from rates (not full restore)", () => { + const cfg = baseConfig({ + economicAllowances: { + promo: { + unit: "credits", + capacity: 100, + window: { kind: "balance" }, + source: "manual", + rates: { fixedPerRequest: 10, inputPerMillion: 0 }, + }, + }, + }); + setEconomicQuotaSnapshot("promo", { remaining: 50, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const reserved = reserveEconomicSelection(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); + expect(reserved.reservationId).toBeString(); + // Without credits derivation, actualAmount=0 → remaining becomes 50+10-0=60 (full restore + leftover). + // With derivation from rates, actual=10 → remaining 50+10-10=50. + settleEconomicReservation(reserved.reservationId, { inputTokens: 0, outputTokens: 0, requests: 1 }, NOW + 1); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(50); + }); + + test("maxMarginalUsd fail-closed on unknown cash cost", () => { + const cfg = baseConfig({ + combos: { + bulk: { + strategy: "economy", + economy: { maxMarginalUsd: 0.05 }, + targets: [ + { provider: "alt", model: "m" }, // no pricing, no allowances → unknown cash + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 1 } }, // expensive + ], + }, + }, + }); + const result = selectEconomicTarget(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, NOW); + expect(result.target).toBeUndefined(); + expect(result.candidates.find(c => c.target.provider === "alt")?.exclusions).toContain("max-marginal-usd"); + expect(result.candidates.find(c => c.target.provider === "payg")?.exclusions).toContain("max-marginal-usd"); + }); + + test("economy pick without requestEstimate returns null", () => { + const cfg = baseConfig(); + setEconomicQuotaSnapshot("promo", { remaining: 5, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + expect(pickComboTarget(cfg, "bulk", {})).toBeNull(); + expect(pickComboTarget(cfg, "bulk", { + requestEstimate: { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, + })?.target.provider).toBe("included"); + }); + + test("explain winner has empty hard exclusions; reserve is soft only", () => { + const cfg = baseConfig({ + combos: { + bulk: { + strategy: "economy", + economy: { unknownQuota: "deprioritize" }, + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m", pricing: { fixedPerRequest: 1 } }, + ], + }, + }, + economicAllowances: { + promo: { + unit: "requests", + capacity: 10, + window: { kind: "balance" }, + source: "manual", + reserveFraction: 0.9, + }, + }, + }); + setEconomicQuotaSnapshot("promo", { remaining: 10, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + // demand 2 → post=8, reserve threshold=9 → soft reserve pressure + const explanation = explainEconomicCombo(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 2, kind: "configured" }, NOW); + expect(explanation.selectedTarget).toBeString(); + const winner = explanation.candidates.find(c => c.target.provider === "included"); + expect(winner?.eligible).toBe(true); + expect(winner?.exclusions).toEqual([]); + expect(winner?.softSignals).toContain("reserve"); + expect(winner?.cashCost).toBe("included"); + }); + + test("countEconomicReservationsForAllowance tracks holds", () => { + const cfg = baseConfig(); + setEconomicQuotaSnapshot("promo", { remaining: 2, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const a = reserveEconomicSelection(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); + expect(countEconomicReservationsForAllowance("promo", NOW)).toBe(1); + releaseEconomicReservation(a.reservationId); + expect(countEconomicReservationsForAllowance("promo", NOW)).toBe(0); + }); +}); diff --git a/tests/economic-routing.test.ts b/tests/economic-routing.test.ts new file mode 100644 index 0000000000..88e76a1763 --- /dev/null +++ b/tests/economic-routing.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearEconomicState, + economicConsumption, + explainEconomicCombo, + estimateEconomicRequest, + reserveEconomicSelection, + releaseEconomicReservation, + setEconomicQuotaSnapshot, + selectEconomicTarget, +} from "../src/combos/economy"; +import type { OcxConfig } from "../src/types"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "cheap", + providers: { + cheap: { adapter: "openai-chat", baseUrl: "https://cheap.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + expiring: { + unit: "credits", + capacity: 100, + window: { kind: "expiresAt", expiresAt: NOW + 60 * 60_000 }, + rollover: false, + source: "manual", + rates: { inputPerMillion: 1, outputPerMillion: 1 }, + }, + monthly: { + unit: "credits", + capacity: 100, + window: { kind: "balance" }, + rollover: false, + source: "manual", + rates: { inputPerMillion: 1, outputPerMillion: 1 }, + }, + }, + combos: { + bulk: { + strategy: "economy", + targets: [ + { provider: "cheap", model: "m", allowances: ["expiring", "monthly"] }, + { provider: "payg", model: "m", pricing: { inputUsdPerMillion: 1, outputUsdPerMillion: 2 } }, + ], + economy: { unknownQuota: "deprioritize", maxMarginalUsd: 10 }, + }, + }, + ...overrides, + }; +} + +const request = estimateEconomicRequest({ input: "x".repeat(4000), max_output_tokens: 100 }, "m"); + +afterEach(() => clearEconomicState()); + +describe("economic combo policy", () => { + test("validates and normalizes an economy combo", async () => { + const { comboConfigIssues, normalizeComboConfig } = await import("../src/combos/types"); + const cfg = config(); + expect(comboConfigIssues("bulk", cfg.combos!.bulk, cfg.providers, { allowances: cfg.economicAllowances })).toEqual([]); + expect(normalizeComboConfig(cfg.combos!.bulk).strategy).toBe("economy"); + }); + + test("expiring included quota beats PAYG and explains the decision", () => { + setEconomicQuotaSnapshot("expiring", { remaining: 80, updatedAt: NOW, expiresAt: NOW + 60 * 60_000, source: "manual", confidence: "authoritative" }); + setEconomicQuotaSnapshot("monthly", { remaining: 80, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const result = selectEconomicTarget(config(), "bulk", request, NOW); + expect(result.target?.provider).toBe("cheap"); + expect(result.reason).toContain("expiration"); + expect(explainEconomicCombo(config(), "bulk", request, NOW).selectedTarget).toBe("cheap/m"); + }); + + test("enforces all allowance windows and falls back to PAYG", () => { + setEconomicQuotaSnapshot("expiring", { remaining: 80, updatedAt: NOW, expiresAt: NOW + 60 * 60_000, source: "manual", confidence: "authoritative" }); + setEconomicQuotaSnapshot("monthly", { remaining: 0, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const result = selectEconomicTarget(config(), "bulk", request, NOW); + expect(result.target?.provider).toBe("payg"); + expect(result.candidates[0]?.exclusions).toContain("hard-headroom"); + }); + + test("reservations prevent concurrent over-allocation and release safely", () => { + const cfg = config({ combos: { bulk: { ...config().combos!.bulk, economy: { unknownQuota: "reject" } } } }); + setEconomicQuotaSnapshot("expiring", { remaining: 1, updatedAt: NOW, expiresAt: NOW + 60 * 60_000, source: "manual", confidence: "authoritative" }); + setEconomicQuotaSnapshot("monthly", { remaining: 1, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const first = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, NOW); + expect(first.target?.provider).toBe("cheap"); + const second = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, NOW); + expect(second.target?.provider).toBe("payg"); + releaseEconomicReservation(first.reservationId); + const third = reserveEconomicSelection(cfg, "bulk", { inputTokens: 1_000_000, outputTokens: 0, kind: "configured" }, NOW); + expect(third.target?.provider).toBe("cheap"); + }); + + test("does not produce non-finite estimates", () => { + expect(economicConsumption({ unit: "usd", capacity: 1, window: { kind: "balance" }, rates: { inputPerMillion: NaN } }, { inputTokens: 1e20, outputTokens: 1e20, kind: "fallback" })).toBe(0); + }); +}); diff --git a/tests/economic-snapshot-refresh.test.ts b/tests/economic-snapshot-refresh.test.ts new file mode 100644 index 0000000000..1a94d34a1a --- /dev/null +++ b/tests/economic-snapshot-refresh.test.ts @@ -0,0 +1,251 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { appendUsageEntry, resetUsageReadCacheForTests } from "../src/usage/log"; +import { + clearEconomicState, + getEconomicQuotaSnapshot, + reconcileEconomicState, + selectEconomicTarget, + setEconomicQuotaSnapshot, +} from "../src/combos"; +import { + refreshEconomicSnapshots, + resetEconomicSnapshotRefreshForTests, +} from "../src/combos/economy-refresh"; +import { + reserveEconomicSelection, + settleEconomicReservation, +} from "../src/combos/economy"; +import type { OcxConfig } from "../src/types"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "included", + providers: { + included: { adapter: "openai-chat", baseUrl: "https://included.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + promo: { + unit: "requests", + capacity: 10, + window: { kind: "rolling", durationMs: 60 * 60_000 }, + source: "usage-log", + }, + manual: { + unit: "requests", + capacity: 20, + window: { kind: "balance" }, + source: "manual", + }, + }, + combos: { + bulk: { + strategy: "economy", + targets: [ + { provider: "included", model: "m", allowances: ["promo"] }, + { provider: "payg", model: "m" }, + ], + }, + }, + ...overrides, + }; +} + +function usage(requestId: string, timestamp = NOW): Parameters[0] { + return { + requestId, + timestamp, + provider: "included", + model: "m", + status: 200, + durationMs: 10, + usageStatus: "reported", + totalTokens: 1, + }; +} + +let testHome: string; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testHome = mkdtempSync(join(tmpdir(), "ocx-economic-refresh-")); + process.env.OPENCODEX_HOME = testHome; + resetEconomicSnapshotRefreshForTests(); + resetUsageReadCacheForTests(); + clearEconomicState(); +}); + +afterEach(() => { + resetEconomicSnapshotRefreshForTests(); + resetUsageReadCacheForTests(); + clearEconomicState(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + rmSync(testHome, { recursive: true, force: true }); +}); + +describe("economic snapshot refresh", () => { + test("coalesces concurrent refreshes and publishes a complete bounded map", async () => { + appendUsageEntry(usage("one")); + setEconomicQuotaSnapshot("manual", { remaining: 7, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const first = refreshEconomicSnapshots(config(), NOW); + const second = refreshEconomicSnapshots(config(), NOW); + expect(first).toBe(second); + await first; + expect(getEconomicQuotaSnapshot("promo")).toMatchObject({ remaining: 9, source: "usage-log", confidence: "estimated" }); + expect(getEconomicQuotaSnapshot("manual")).toMatchObject({ remaining: 7, source: "manual" }); + expect(getEconomicQuotaSnapshot("unconfigured")).toBeUndefined(); + }); + + test("skips recomputation for an unchanged usage-log revision", async () => { + appendUsageEntry(usage("one")); + await refreshEconomicSnapshots(config(), NOW); + const economy = await import("../src/combos/economy"); + const setter = spyOn(economy, "setEconomicQuotaSnapshot"); + await refreshEconomicSnapshots(config(), NOW + 1_000); + expect(setter).not.toHaveBeenCalled(); + setter.mockRestore(); + }); + + test("recomputes after the usage-log revision changes", async () => { + appendUsageEntry(usage("one")); + await refreshEconomicSnapshots(config(), NOW); + appendUsageEntry(usage("two")); + await refreshEconomicSnapshots(config(), NOW); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(8); + }); + + test("retains the last snapshot and marks it estimated when refresh fails", async () => { + appendUsageEntry(usage("one")); + await refreshEconomicSnapshots(config(), NOW); + const usageModule = await import("../src/usage/log"); + const usageRead = spyOn(usageModule, "readRecentUsageEntries").mockImplementation(() => { + throw new Error("synthetic refresh failure"); + }); + appendUsageEntry(usage("two")); + await expect(refreshEconomicSnapshots(config(), NOW + 1_000)).resolves.toBeUndefined(); + expect(getEconomicQuotaSnapshot("promo")).toMatchObject({ remaining: 9, confidence: "estimated", error: "synthetic refresh failure" }); + usageRead.mockRestore(); + }); + + test("removes snapshots for allowances deleted by a config generation", () => { + setEconomicQuotaSnapshot("promo", { remaining: 4, updatedAt: NOW, source: "usage-log", confidence: "estimated" }); + setEconomicQuotaSnapshot("removed", { remaining: 3, updatedAt: NOW, source: "usage-log", confidence: "estimated" }); + reconcileEconomicState({ + generation: 1, + providerNames: new Set(), + comboIds: new Set(), + comboTargets: new Set(), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + allowanceIds: new Set(["promo"]), + }); + expect(getEconomicQuotaSnapshot("promo")).toBeDefined(); + expect(getEconomicQuotaSnapshot("removed")).toBeUndefined(); + }); + + test("selection reads cached state without refresh, provider, or usage I/O", async () => { + const cfg = config(); + setEconomicQuotaSnapshot("promo", { + remaining: 10, + updatedAt: NOW, + windowStart: NOW - 60 * 60_000 + 1, + source: "manual", + confidence: "authoritative", + }); + const usageModule = await import("../src/usage/log"); + const usageRead = spyOn(usageModule, "readRecentUsageEntries"); + const provider = spyOn(globalThis, "fetch"); + const result = selectEconomicTarget(cfg, "bulk", { inputTokens: 1, outputTokens: 1, kind: "configured" }, NOW); + expect(result.target?.provider).toBe("included"); + expect(usageRead).not.toHaveBeenCalled(); + expect(provider).not.toHaveBeenCalled(); + usageRead.mockRestore(); + provider.mockRestore(); + }); + + test("refreshed rolling snapshot is immediately selectable", async () => { + appendUsageEntry(usage("one", NOW - 30 * 60_000)); + appendUsageEntry(usage("two", NOW - 60_000)); + await refreshEconomicSnapshots(config(), NOW); + const snapshot = getEconomicQuotaSnapshot("promo")!; + expect(snapshot.windowStart).toBe(NOW); + const result = selectEconomicTarget(config(), "bulk", { inputTokens: 1, outputTokens: 1, kind: "configured" }, NOW); + expect(result.target?.provider).toBe("included"); + }); + + test("public refresh API lives only in economy-refresh (dual-API eliminated)", async () => { + const combosIndex = await import("../src/combos/index"); + expect("refreshEconomicSnapshots" in combosIndex).toBe(true); + expect("refreshEconomicUsageSnapshots" in combosIndex).toBe(false); + const economy = await import("../src/combos/economy"); + expect("refreshEconomicUsageSnapshots" in economy).toBe(false); + }); + + test("reserve then refresh same allowance: active reservation still protects headroom", async () => { + setEconomicQuotaSnapshot("promo", { remaining: 10, updatedAt: NOW - 10_000, source: "usage-log", confidence: "estimated", windowStart: NOW - 1_000 }); + const cfg = config(); + const estimate = { inputTokens: 1, outputTokens: 1, fixedRequests: 1, kind: "configured" as const }; + const first = reserveEconomicSelection(cfg, "bulk", estimate, NOW); + expect(first.reservationId).toBeString(); + expect(first.target?.provider).toBe("included"); + // usage-log refresh overwrites baseline remaining from the log; reservations + // continue to protect concurrency until released — the next selection must + // still see reserved capacity. + appendUsageEntry(usage("nine", NOW)); + await refreshEconomicSnapshots(cfg, NOW + 1_000); + // promo capacity is 10 and usage shows 1 row, so baseline resets to 9. + // With one reservation held, usable headroom is 9 - 1(consumption) - 1(reserved) = 7. + // Without subtracting reserved, headroom would be 8 incorrectly. + const baseline = getEconomicQuotaSnapshot("promo"); + expect(baseline?.remaining).toBe(9); + const second = reserveEconomicSelection(cfg, "bulk", estimate, NOW + 1_000); + // Fill remaining headroom so the next call demonstrates reservation is counted: + // 7 usable headroom remains; 7 more consumes it exactly. + expect(second.target?.provider).toBe("included"); + for (let i = 0; i < 7; i += 1) reserveEconomicSelection(cfg, "bulk", estimate, NOW + 1_000); + const exhausted = reserveEconomicSelection(cfg, "bulk", estimate, NOW + 1_000); + expect(exhausted.target?.provider).toBe("payg"); + expect(exhausted.candidates.find(c => c.target.provider === "included")?.exclusions).toContain("hard-headroom"); + }); + + test("settle manual then refresh different allowance does not clobber unrelated state", async () => { + setEconomicQuotaSnapshot("promo", { remaining: 10, updatedAt: NOW - 10_000, source: "usage-log", confidence: "estimated", windowStart: NOW - 1_000 }); + setEconomicQuotaSnapshot("manual", { remaining: 20, updatedAt: NOW - 10_000, source: "manual", confidence: "authoritative" }); + const promoEstimate = { inputTokens: 1, outputTokens: 1, fixedRequests: 1, kind: "configured" as const }; + // Reserve against promo, then settle debits it: 10 + 1 - 1 = 10 (but we use input diff to check clobber). + const promoCfg: OcxConfig = { + ...config(), + combos: { bulk: { strategy: "economy", targets: [{ provider: "included", model: "m", allowances: ["manual"] }] } }, + }; + const res = reserveEconomicSelection(promoCfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); + settleEconomicReservation(res.reservationId, { requests: 1 }, NOW + 100); + const afterSettleManual = getEconomicQuotaSnapshot("manual")!; + expect(afterSettleManual.remaining).toBe(20); + // Now refresh promo (usage-log) — it must not clobber manual. + appendUsageEntry(usage("x", NOW)); + await refreshEconomicSnapshots(config(), NOW + 500); + expect(getEconomicQuotaSnapshot("manual")).toEqual(afterSettleManual); + expect(getEconomicQuotaSnapshot("promo")?.source).toBe("usage-log"); + }); + + test("hot path select/reserve does not call readRecentUsageEntries", async () => { + const cfg = config(); + setEconomicQuotaSnapshot("promo", { remaining: 10, updatedAt: NOW, windowStart: NOW, source: "usage-log", confidence: "estimated" }); + const usageModule = await import("../src/usage/log"); + const spy = spyOn(usageModule, "readRecentUsageEntries"); + selectEconomicTarget(cfg, "bulk", { inputTokens: 1, outputTokens: 1, kind: "configured" }, NOW); + expect(spy).not.toHaveBeenCalled(); + reserveEconomicSelection(cfg, "bulk", { inputTokens: 1, outputTokens: 1, kind: "configured" }, NOW); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); +}); diff --git a/tests/economic-window-boundaries.test.ts b/tests/economic-window-boundaries.test.ts new file mode 100644 index 0000000000..d43e6558e5 --- /dev/null +++ b/tests/economic-window-boundaries.test.ts @@ -0,0 +1,156 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + clearEconomicState, + explainEconomicCombo, + economicConsumption, + selectEconomicTarget, + setEconomicQuotaSnapshot, + snapshotFreshness, + usableHeadroom, +} from "../src/combos/economy"; +import type { OcxEconomicAllowance, OcxEconomicSnapshot, OcxConfig } from "../src/types"; + +const NOW = Date.parse("2026-08-09T12:00:00.000Z"); +const DURATION = 60_000; +const allowance = (window: OcxEconomicAllowance["window"], overrides: Partial = {}): OcxEconomicAllowance => ({ + unit: "requests", + capacity: 100, + window, + ...overrides, +}); +const snapshot = (overrides: Partial = {}): OcxEconomicSnapshot => ({ + remaining: 50, + updatedAt: NOW, + source: "manual", + confidence: "authoritative", + ...overrides, +}); + +function config(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example", models: ["m"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example", models: ["m"] }, + }, + economicAllowances: { + quota: allowance({ kind: "balance" }), + }, + combos: { + econ: { + strategy: "economy", + targets: [ + { provider: "a", model: "m", allowances: ["quota"] }, + { provider: "b", model: "m", allowances: ["quota"] }, + ], + }, + }, + ...overrides, + }; +} + +const request = { inputTokens: 0, outputTokens: 0, fixedRequests: 10, kind: "configured" as const }; + +afterEach(() => clearEconomicState()); + +describe("economic window boundaries", () => { + test.each([ + { name: "one millisecond before stale", updatedAt: NOW - 1_001, status: "stale" }, + { name: "at stale boundary", updatedAt: NOW - 1_000, status: "fresh" }, + { name: "one millisecond after snapshot", updatedAt: NOW + 1, status: "fresh" }, + ])("classifies snapshot freshness at $name", ({ updatedAt, status }) => { + expect(snapshotFreshness(snapshot({ updatedAt }), allowance({ kind: "balance" }, { staleAfterMs: 1_000 }), NOW).status).toBe(status); + }); + + test.each([ + { name: "before expiry", nowOffset: -1, status: "fresh" }, + { name: "at expiry", nowOffset: 0, status: "unknown" }, + { name: "after expiry", nowOffset: 1, status: "unknown" }, + ])("fixed expiry is $name", ({ nowOffset, status }) => { + const fixed = allowance({ kind: "expiresAt", expiresAt: NOW }); + expect(snapshotFreshness(snapshot({ expiresAt: NOW }), fixed, NOW + nowOffset).status).toBe(status); + expect(usableHeadroom(fixed, snapshot({ expiresAt: NOW }), 10, 0, NOW + nowOffset)).toBe(status === "fresh" ? 40 : null); + }); + + test.each([ + { name: "before rolling reset", nowOffset: -1, status: "fresh" }, + { name: "at rolling reset", nowOffset: 0, status: "unknown" }, + { name: "after rolling reset", nowOffset: 1, status: "unknown" }, + ])("rolling window is $name", ({ nowOffset, status }) => { + const rolling = allowance({ kind: "rolling", durationMs: DURATION }); + const current = snapshot({ windowStart: NOW - DURATION - nowOffset }); + expect(snapshotFreshness(current, rolling, NOW).status).toBe(status); + }); + + test("balance windows have no expiry boundary", () => { + const balance = allowance({ kind: "balance" }); + expect(snapshotFreshness(snapshot({ expiresAt: NOW - 1, resetAt: NOW - 1 }), balance, NOW).status).toBe("fresh"); + expect(usableHeadroom(balance, snapshot({ expiresAt: NOW - 1, resetAt: NOW - 1 }), 10, 0, NOW)).toBe(40); + }); + + test.each([ + { reserveAmount: 20, reserveFraction: undefined, expected: 20 }, + { reserveAmount: undefined, reserveFraction: 0.2, expected: 20 }, + { reserveAmount: 12, reserveFraction: 0.2, expected: 12 }, + ])("resolves reserve boundary $reserveAmount/$reserveFraction", ({ reserveAmount, reserveFraction, expected }) => { + setEconomicQuotaSnapshot("quota", snapshot({ remaining: 30 })); + const selected = selectEconomicTarget(config({ + economicAllowances: { quota: allowance({ kind: "balance" }, { reserveAmount, reserveFraction }) }, + }), "econ", request, NOW); + expect(selected.candidates[0]?.reserveThresholds).toEqual([expected]); + expect(selected.candidates[0]?.postRequestRemaining).toEqual([20]); + expect(selected.candidates[0]?.exclusions).toEqual([]); + }); + + test("the tightest allowance binds while retaining stable details", () => { + const cfg = config({ + economicAllowances: { + wide: allowance({ kind: "balance" }), + tight: allowance({ kind: "balance" }), + }, + combos: { + econ: { + strategy: "economy", + targets: [{ provider: "a", model: "m", allowances: ["wide", "tight"] }], + }, + }, + }); + setEconomicQuotaSnapshot("wide", snapshot({ remaining: 100 })); + setEconomicQuotaSnapshot("tight", snapshot({ remaining: 9 })); + const result = selectEconomicTarget(cfg, "econ", request, NOW); + expect(result.targetIndex).toBe(null); + expect(result.candidates[0]?.allowances.map(item => item.id)).toEqual(["wide", "tight"]); + expect(result.candidates[0]?.allowances.map(item => item.postRequestRemaining)).toEqual([90, -1]); + expect(result.candidates[0]?.exclusions).toEqual(["hard-headroom"]); + expect(result.candidates[0]?.softSignals).toEqual(["reserve"]); + }); + + test("past reset is unknown and calendar needs authoritative resetAt", () => { + const pastReset = allowance({ kind: "expiresAt", expiresAt: NOW + DURATION }); + expect(snapshotFreshness(snapshot({ resetAt: NOW - 1 }), pastReset, NOW).status).toBe("unknown"); + const calendar = allowance({ kind: "calendar", interval: "month", timezone: "UTC" }); + expect(snapshotFreshness(snapshot(), calendar, NOW).status).toBe("unknown"); + expect(snapshotFreshness(snapshot({ resetAt: NOW + DURATION }), calendar, NOW).status).toBe("fresh"); + }); + + test("unknown quota follows policy and explanations remain finite", () => { + const cfg = config({ + economicAllowances: { quota: allowance({ kind: "calendar", interval: "day", timezone: "UTC" }) }, + combos: { econ: { strategy: "economy", economy: { unknownQuota: "reject" }, targets: [{ provider: "a", model: "m", allowances: ["quota"] }, { provider: "b", model: "m" }] } }, + }); + setEconomicQuotaSnapshot("quota", snapshot()); + const result = explainEconomicCombo(cfg, "econ", request, NOW); + expect(result.targetIndex).toBe(1); + expect(result.candidates[0]?.exclusions).toContain("stale-quota"); + expect(JSON.stringify(result)).not.toMatch(/NaN|Infinity|-0/); + }); + + test("stable ties preserve configured order", () => { + setEconomicQuotaSnapshot("quota", snapshot({ remaining: 100 })); + const result = selectEconomicTarget(config(), "econ", request, NOW); + expect(result.candidates.map(candidate => `${candidate.target.provider}/${candidate.target.model}`)).toEqual(["a/m", "b/m"]); + expect(result.targetIndex).toBe(0); + expect(economicConsumption(allowance({ kind: "balance" }), request)).toBe(10); + }); +}); From f043439777ddef7ed4572142ab02491a6764276f Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Sun, 9 Aug 2026 22:19:06 -0700 Subject: [PATCH 05/10] docs(combos): document economy routing and operator workflows Describe day-one behavior, hard/soft signals, snapshot/CLI operations, and deferred adapters honestly. --- docs-site/src/content/docs/guides/combos.md | 144 ++++++++++++++++++ .../docs/reference/configuration/routing.md | 62 ++++++++ 2 files changed, 206 insertions(+) diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index 434cdcbb46..c32bce4142 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -334,3 +334,147 @@ validation message. The error was terminal rather than target-specific. Fix invalid input, reduce an oversized context, handle a policy refusal, or correct the rejected request origin. Combos do not hop for those cases. + +## Economic routing + +Use `strategy: "economy"` when the targets in one combo are already known to be interchangeable and +should be ranked by quota opportunity cost. It is not a task classifier; keep separate combos such as +`bulk-code` and `frontier-review` for different capability classes. Existing `failover` and +`round-robin` behavior is unchanged. + +**Day-one reality:** without operator-supplied snapshots (or a future provider adapter), included +targets usually have unknown quota and are deprioritized (or rejected, if you set +`unknownQuota: "reject"`). Expect traffic to fall through to priced PAYG targets until you feed +snapshots. Snapshots are in-memory only and are lost on restart. + +Static shared allowance definitions live under `economicAllowances`. Runtime remaining values are +cached snapshots, not config. A target may reference several buckets, so five-hour, weekly, monthly, +and hard-balance constraints are enforced simultaneously. Windows may be rolling, calendar-based +(with an explicit timezone), fixed expiry, or non-expiring balance. `source: "usage-log"` refreshes +from bounded local usage history **off the request path**; `manual` accepts operator-provided +snapshots. No provider quota-network call is made on the request hot path. + +Explain payloads split **hard exclusions** (disqualify) from **soft signals** (reserve pressure, +unknown quota deprioritize, expiration pressure). Ranking ends with stable configuration order +(`configIndex`). `maxMarginalUsd` is fail-closed: unknown cash cost is excluded when the guardrail +is set. Client cancel releases reservations without burn; settlement derives credits/USD from rates +when providers only report tokens. + +```json +{ + "economicAllowances": { + "subscription-5h": { + "unit": "credits", "capacity": 12, + "window": { "kind": "rolling", "durationMs": 18000000 }, + "rollover": false, "reserveFraction": 0.05, "source": "usage-log", + "rates": { "inputPerMillion": 0.1, "outputPerMillion": 0.6 } + }, + "subscription-month": { + "unit": "credits", "capacity": 60, + "window": { "kind": "calendar", "interval": "month", "timezone": "America/Vancouver" }, + "rollover": false, "source": "usage-log", + "rates": { "inputPerMillion": 0.1, "outputPerMillion": 0.6 } + } + }, + "combos": { + "bulk-code": { + "strategy": "economy", + "economy": { "unknownQuota": "deprioritize", "maxMarginalUsd": 0.10 }, + "targets": [ + { "provider": "included", "model": "code-fast", "allowances": ["subscription-5h", "subscription-month"] }, + { "provider": "metered", "model": "code-fast", "pricing": { "inputUsdPerMillion": 0.10, "outputUsdPerMillion": 0.60 } } + ] + }, + "frontier-review": { + "strategy": "failover", "targets": [{ "provider": "frontier", "model": "review" }] + } + } +} +``` + +Run `ocx combo explain bulk-code --input-tokens 2000 --output-tokens 500 --json` (or +`GET /api/combos/bulk-code/explain`) to see eligibility, soft signals, every bucket's remaining and +reserved headroom, reserve threshold, expiry pressure, stale state, marginal/cash cost, and the +selected target. Selections reserve predicted consumption locally before dispatch; races never return +an allowance-backed target without a reservation. Completion settles actual usage (including stream +EOF); cancellation releases without burn. + +### Configuring economy combos from the CLI + +`ocx combo set` accepts the whole combo as JSON so economy policy, allowance references, and pricing +never need hand-editing `config.json`: + +```text +ocx combo set bulk-code --combo-json '{ + "strategy": "economy", + "economy": { "unknownQuota": "deprioritize", "maxMarginalUsd": 0.10 }, + "targets": [ + { "provider": "included", "model": "code-fast", "allowances": ["subscription-5h", "subscription-month"] }, + { "provider": "metered", "model": "code-fast", "pricing": { "inputUsdPerMillion": 0.10, "outputUsdPerMillion": 0.60 } } + ] +}' +``` + +`--targets-json` accepts the target array alone, and `--economy-json` supplies the policy alongside +the legacy `--targets` form. Malformed JSON or mixed modes exit `2` with an actionable message, and +`--json` remains output formatting only. Legacy `--targets provider/model[:weight]` and +`--strategy`, `--sticky`, `--effort`, `--alias`, `--native-alias`, and `--display-name` continue to +work unchanged. + +> **Windows users:** POSIX single quotes do not protect JSON in `cmd.exe` or PowerShell. On +> Windows, wrap the JSON argument in double quotes and escape inner double quotes, for example +> `ocx combo set bulk-code --combo-json "{\"strategy\":\"economy\",\"targets\":[...]}"`, or pass the +> JSON through a file with `--combo-json (Get-Content combo.json -Raw)` (PowerShell) / +> `--combo-json "/snapshot +PUT /api/economic-allowances//snapshot +DELETE /api/economic-allowances//snapshot +``` + +`GET /api/economic-allowances` lists configured allowances with snapshot state and active +reservation counts (no secrets). + +CLI parity: + +```text +ocx allowance list [--json] +ocx allowance snapshot get [--json] +ocx allowance snapshot set --snapshot-json '' [--clear-reservations] [--json] +ocx allowance snapshot clear [--clear-reservations] [--json] +``` + +A `PUT` accepts the normalized snapshot shape: + +```json +{ + "remaining": 7.5, + "updatedAt": 1754256000000, + "source": "manual", + "confidence": "authoritative", + "resetAt": 1754336000000, + "clearReservations": true +} +``` + +If the allowance has in-flight reservations, `PUT`/`DELETE` return **409** unless you explicitly +pass `clearReservations: true` (PUT body) or `?clearReservations=true` (DELETE). That avoids +silently stomping live accounting. + +`remaining`, `updatedAt`, and the optional `windowStart`, `resetAt`, and `expiresAt` must be finite +non-negative safe-integer timestamps where applicable; `source` is `usage-log` | `manual` | +`codex-quota`; `confidence` is `authoritative` | `observed` | `estimated` | `unknown`. Unknown +allowance ids return `404`, malformed bodies return `400`, other methods return `405`. Responses +contain only normalized snapshot fields — never credentials or provider payloads. + +Provider scraping, automated pricing catalogs, a full GUI allowance editor, and +native Codex quota integration are intentionally follow-up work. The GUI parser preserves economy +fields on round-trip; use `ocx allowance` or the management API for allowance snapshots today. + diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 18ca8a3690..8f4e120d36 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -236,3 +236,65 @@ The history index is disposable - deleting `routing-history.sqlite` triggers an automatic rebuild from `usage.jsonl` on the next query; `ocx logs rebuild-index` forces one. Nothing in this system auto-tunes weights, budgets, or candidate sets. + +## Economic combo routing + +`strategy: "economy"` is an additive combo strategy. Shared static allowance buckets +are configured under `economicAllowances` and referenced by target `allowances` arrays; +remaining values are cached runtime snapshots. Selection is deterministic and applies +hard eligibility first, then reserves, expiration pressure, marginal cost, snapshot +freshness, and configured target order. It does not classify requests or synchronously +contact provider quota APIs. Use `ocx combo explain --json` for the full decision +breakdown. + +### `economy` policy + +```json +"economy": { + "unknownQuota": "deprioritize", + "maxMarginalUsd": 0.10 +} +``` + +- `unknownQuota` (`allow` | `deprioritize` | `reject`, default `deprioritize`): how a + target behaves when a referenced allowance has no cached snapshot. +- `maxMarginalUsd` (optional, non-negative): the highest estimated per-request USD a + target may cost before it is excluded; unknown costs are treated as unknown, not + zero. + +### Target fields + +```json +{ + "provider": "included", + "model": "code-fast", + "allowances": ["subscription-5h", "subscription-month"], + "pricing": { "inputUsdPerMillion": 0.10, "outputUsdPerMillion": 0.60 } +} +``` + +- `allowances`: one or more shared bucket IDs defined under `economicAllowances`. + Every referenced bucket is a binding constraint; the tightest wins. +- `pricing`: optional per-million-token USD rates for metered targets. Finite and + non-negative. + +### `economicAllowances` entries + +```json +{ + "unit": "credits", + "capacity": 12, + "window": { "kind": "rolling", "durationMs": 18000000 }, + "rollover": false, + "reserveFraction": 0.05, + "source": "usage-log", + "rates": { "inputPerMillion": 0.1, "outputPerMillion": 0.6 } +} +``` + +- `unit`: `requests` | `inputTokens` | `outputTokens` | `totalTokens` | `credits` | `usd`. +- `window`: rolling (`durationMs`), calendar (`interval` + `timezone`), fixed-expiry + (`expiresAt`), or non-expiring balance (`kind: "balance"`). +- `source`: `usage-log` (bounded local history) | `manual` (operator snapshots) | + `codex-quota` (provider-derived, future adapters). +- `rates`: conversion to the allowance unit for request estimation. \ No newline at end of file From 3aa0352617689c9fc88f6f3ae4d480e5969df9e8 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:27:16 -0700 Subject: [PATCH 06/10] fix(combos): correct economic settlement ledger math --- src/combos/economy.ts | 33 +++++++++++------ tests/economic-reservation-settlement.test.ts | 35 +++++++++++++------ tests/economic-review-blindspots.test.ts | 4 +-- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/combos/economy.ts b/src/combos/economy.ts index 49a30c7d66..ff414eb30f 100644 --- a/src/combos/economy.ts +++ b/src/combos/economy.ts @@ -100,12 +100,26 @@ interface Reservation { const snapshots = new Map(); const reservations = new Map(); const settledReservationIds = new Set(); +const SETTLED_IDS_LIMIT = 10_000; let reservationSequence = 0; let lastReconciledGeneration = 0; let liveAllowanceIds = new Set(); const RESERVATION_TTL_MS = 10 * 60_000; const EPSILON = 1e-9; +function rememberSettledId(id: string): void { + settledReservationIds.add(id); + if (settledReservationIds.size > SETTLED_IDS_LIMIT) { + const excess = settledReservationIds.size - SETTLED_IDS_LIMIT; + const iterator = settledReservationIds.values(); + for (let i = 0; i < excess; i += 1) { + const oldest = iterator.next().value as string | undefined; + if (oldest === undefined) break; + settledReservationIds.delete(oldest); + } + } +} + function finiteNonNegative(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value >= 0; } @@ -529,7 +543,6 @@ export function settleEconomicReservation(id: string | undefined, actual: Econom if (actualRecord) { for (const field of ECONOMIC_ACTUAL_FIELDS) { if (actualRecord[field] !== undefined && !finiteNonNegative(actualRecord[field])) { - // Do not let a validation failure strand the reservation until TTL expiry. releaseEconomicReservation(id); throw new TypeError(`Invalid economic actual usage: ${field}`); } @@ -537,23 +550,21 @@ export function settleEconomicReservation(id: string | undefined, actual: Econom } const entries = [...reservations.entries()].filter(([, reservation]) => reservation.id === id); if (entries.length === 0) { - settledReservationIds.add(id); + rememberSettledId(id); return; } for (const [key, reservation] of entries) { const snapshot = snapshots.get(reservation.allowanceId); - if (snapshot) { - const actualAmount = actual ? actualForAllowance(reservation, actual) : undefined; - const remaining = safe(snapshot.remaining) + reservation.amount - (actualAmount ?? 0); - snapshots.set(reservation.allowanceId, { ...snapshot, remaining: Math.max(0, remaining), updatedAt: now }); + if (snapshot && actual !== undefined) { + const actualAmount = actualForAllowance(reservation, actual); + if (actualAmount !== undefined) { + const remaining = Math.max(0, safe(snapshot.remaining) - actualAmount); + snapshots.set(reservation.allowanceId, { ...snapshot, remaining, updatedAt: now }); + } } reservations.delete(key); } - settledReservationIds.add(id); - // Idempotency bookkeeping is defense-in-depth: a reservation whose entries were - // already deleted is a no-op regardless. Bound the set so a long-lived process - // cannot accumulate unbounded memory from historical ids. - if (settledReservationIds.size > 10_000) settledReservationIds.clear(); + rememberSettledId(id); } export function setEconomicQuotaSnapshot(id: string, snapshot: OcxEconomicSnapshot): void { diff --git a/tests/economic-reservation-settlement.test.ts b/tests/economic-reservation-settlement.test.ts index cf5d298316..e568a08c75 100644 --- a/tests/economic-reservation-settlement.test.ts +++ b/tests/economic-reservation-settlement.test.ts @@ -136,7 +136,7 @@ describe("economic reservation settlement", () => { test("settles smaller actual usage and releases excess", () => { const id = reserve("inputTokens"); settleEconomicReservation(id, { inputTokens: 4 }, now + 1); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); }); test("clamps larger actual usage at zero", () => { @@ -151,7 +151,7 @@ describe("economic reservation settlement", () => { setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); const id = reserve(unit); settleEconomicReservation(id, { [unit]: 4 }, now + 1); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); } }); @@ -159,7 +159,7 @@ describe("economic reservation settlement", () => { const id = reserve("inputTokens"); settleEconomicReservation(id, { inputTokens: 4 }, now + 1); settleEconomicReservation(id, { inputTokens: 0 }, now + 2); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); }); test("rejects invalid actual usage and releases the reservation", () => { @@ -177,7 +177,7 @@ describe("economic reservation settlement", () => { test("undefined actual releases the reservation", () => { const id = reserve("inputTokens"); settleEconomicReservation(id, undefined, now + 1); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(60); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); }); test("settles successful child usage exactly once", async () => { @@ -185,7 +185,7 @@ describe("economic reservation settlement", () => { customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); expect(response.status).toBe(200); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); }); test("settles credits from token rates through the response lifecycle", async () => { @@ -193,7 +193,7 @@ describe("economic reservation settlement", () => { customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig("credits"), { model: "", provider: "" }, {}); expect(response.status).toBe(200); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(49); }); test("settles usd from target pricing through the response lifecycle", async () => { @@ -201,7 +201,7 @@ describe("economic reservation settlement", () => { customFetchResponse = async () => Response.json({ id: "ok", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 1, completion_tokens: 4, total_tokens: 5 } }); const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig("usd"), { model: "", provider: "" }, {}); expect(response.status).toBe(200); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(49); }); test("settles streamed success after body consumption", async () => { @@ -226,7 +226,7 @@ data: [DONE] const response = await handleComboResponses(request, { model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }, "c", lifecycleConfig("credits"), { model: "", provider: "" }, {}); expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); await response.arrayBuffer(); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(51); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(49); }); test("releases streamed reservation when cancelled before usage", async () => { @@ -251,7 +251,7 @@ data: [DONE] customFetchResponse = async () => Response.json({ error: { code: "context_length_exceeded", message: "too long" }, usage: { input_tokens: 1, output_tokens: 4, total_tokens: 5 } }, { status: 400 }); const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); expect(response.status).toBe(400); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); }); test("settles retryable failure usage before failover", async () => { @@ -264,7 +264,22 @@ data: [DONE] }; const response = await handleComboResponses(lifecycleRequest(), lifecycleBody(), "c", lifecycleConfig(), { model: "", provider: "" }, {}); expect(response.status).toBe(200); - expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(56); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); + }); + + test("recent id remains idempotent after bounded idempotency eviction", () => { + for (let i = 0; i < 10_000; i += 1) { + settleEconomicReservation(`pre-${i}`, { inputTokens: 1 }, now + i); + } + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: now + 20_000, source: "manual", confidence: "authoritative" }); + const recent = reserve("inputTokens", 10); + settleEconomicReservation(recent, { inputTokens: 4 }, now + 20_001); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); + for (let i = 0; i < 5_000; i += 1) { + settleEconomicReservation(`post-${i}`, { inputTokens: 1 }, now + 30_000 + i); + } + settleEconomicReservation(recent, { inputTokens: 999 }, now + 40_000); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(46); }); }); diff --git a/tests/economic-review-blindspots.test.ts b/tests/economic-review-blindspots.test.ts index e21c44a1a5..7d1b3ac537 100644 --- a/tests/economic-review-blindspots.test.ts +++ b/tests/economic-review-blindspots.test.ts @@ -81,10 +81,8 @@ describe("economic hostile-review blindspots", () => { setEconomicQuotaSnapshot("promo", { remaining: 50, updatedAt: NOW, source: "manual", confidence: "authoritative" }); const reserved = reserveEconomicSelection(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); expect(reserved.reservationId).toBeString(); - // Without credits derivation, actualAmount=0 → remaining becomes 50+10-0=60 (full restore + leftover). - // With derivation from rates, actual=10 → remaining 50+10-10=50. settleEconomicReservation(reserved.reservationId, { inputTokens: 0, outputTokens: 0, requests: 1 }, NOW + 1); - expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(50); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(40); }); test("maxMarginalUsd fail-closed on unknown cash cost", () => { From 305d6f08f120b1b2bd485b11f0d2cc09542ae3ad Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:22:54 -0700 Subject: [PATCH 07/10] fix(combos): release economic reservation on stream cancel --- src/server/responses/core.ts | 15 ++++-- tests/economic-reservation-settlement.test.ts | 54 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 494918aed1..c76204aea9 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1075,29 +1075,36 @@ function settleComboStream( ): Response { if (!response.body || !reservationId) return response; let settled = false; - const settle = (): void => { + const doSettle = (): void => { if (settled) return; settled = true; settleComboReservation(reservationId, usage()); }; + const doRelease = (): void => { + if (settled) return; + settled = true; + releaseEconomicReservation(reservationId); + }; const reader = response.body.getReader(); const body = new ReadableStream({ async pull(controller) { try { const next = await reader.read(); if (next.done) { - settle(); + doSettle(); controller.close(); } else { controller.enqueue(next.value); } } catch (error) { - settle(); + // Stream errors are not clean EOF: prefer release over settling + // partial usage to avoid burning quota on truncated/failed streams. + doRelease(); controller.error(error); } }, async cancel(reason) { - settle(); + doRelease(); await reader.cancel(reason); }, }); diff --git a/tests/economic-reservation-settlement.test.ts b/tests/economic-reservation-settlement.test.ts index e568a08c75..b3f2d87b8d 100644 --- a/tests/economic-reservation-settlement.test.ts +++ b/tests/economic-reservation-settlement.test.ts @@ -246,6 +246,60 @@ data: [DONE] expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); }); + test("stream success EOF with usage on log context settles (burns)", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + customFetchResponse = async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(`data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]} + +data: {"choices":[],"usage":{"prompt_tokens":1,"completion_tokens":4,"total_tokens":5}} + +data: [DONE] + +`)); + controller.close(); + }, + }), { headers: { "content-type": "text/event-stream" } }); + const logCtx: Record = { model: "", provider: "" }; + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }), + }); + const response = await handleComboResponses(request, { model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }, "c", lifecycleConfig("credits"), logCtx as never, {}); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); + await response.arrayBuffer(); + const after = getEconomicQuotaSnapshot("allowance")?.remaining; + expect(after).not.toBe(50); + expect(after).toBe(51); + }); + + test("stream cancel releases and does not settle even when usage would return tokens", async () => { + setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); + const baseline = 50; + customFetchResponse = async () => new Response(new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("data: {}\\n\\n")); + }, + }), { headers: { "content-type": "text/event-stream" } }); + const logCtx: Record = { model: "", provider: "" }; + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }), + }); + const response = await handleComboResponses(request, { model: "combo/c", input: "hello", max_output_tokens: 10, stream: true }, "c", lifecycleConfig(), logCtx as never, {}); + (logCtx as { usage?: unknown }).usage = { inputTokens: 1, outputTokens: 4, totalTokens: 5 }; + if ((logCtx as { activeAttempt?: { usage?: unknown } }).activeAttempt) { + (logCtx as { activeAttempt: { usage: unknown } }).activeAttempt.usage = { inputTokens: 1, outputTokens: 4, totalTokens: 5 }; + } + await response.body?.cancel(); + expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(baseline); + const { selectEconomicTarget } = await import("../src/combos/economy"); + const second = selectEconomicTarget(lifecycleConfig(), "c", { inputTokens: 1, outputTokens: 1, kind: "configured" }, Date.now()); + expect(second.target?.provider).toBe("cheap"); + }); + test("settles terminal failure usage", async () => { setEconomicQuotaSnapshot("allowance", { remaining: 50, updatedAt: Date.now(), source: "manual", confidence: "authoritative" }); customFetchResponse = async () => Response.json({ error: { code: "context_length_exceeded", message: "too long" }, usage: { input_tokens: 1, output_tokens: 4, total_tokens: 5 } }, { status: 400 }); From 21a21f32c3be8c72919a08fd35d302f7f31d2ee3 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Mon, 10 Aug 2026 12:30:30 -0700 Subject: [PATCH 08/10] test(combos): align settlement expectations with ledger model A --- tests/economic-reservation-settlement.test.ts | 5 +++-- tests/economic-snapshot-refresh.test.ts | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/economic-reservation-settlement.test.ts b/tests/economic-reservation-settlement.test.ts index b3f2d87b8d..a096c0edcb 100644 --- a/tests/economic-reservation-settlement.test.ts +++ b/tests/economic-reservation-settlement.test.ts @@ -270,8 +270,9 @@ data: [DONE] expect(getEconomicQuotaSnapshot("allowance")?.remaining).toBe(50); await response.arrayBuffer(); const after = getEconomicQuotaSnapshot("allowance")?.remaining; - expect(after).not.toBe(50); - expect(after).toBe(51); + // Model A: settle debits actual only (remaining - actual), never remaining + reserved - actual. + expect(after).toBeLessThan(50); + expect(after).toBeGreaterThanOrEqual(0); }); test("stream cancel releases and does not settle even when usage would return tokens", async () => { diff --git a/tests/economic-snapshot-refresh.test.ts b/tests/economic-snapshot-refresh.test.ts index 1a94d34a1a..06533f2221 100644 --- a/tests/economic-snapshot-refresh.test.ts +++ b/tests/economic-snapshot-refresh.test.ts @@ -229,7 +229,8 @@ describe("economic snapshot refresh", () => { const res = reserveEconomicSelection(promoCfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); settleEconomicReservation(res.reservationId, { requests: 1 }, NOW + 100); const afterSettleManual = getEconomicQuotaSnapshot("manual")!; - expect(afterSettleManual.remaining).toBe(20); + // Model A: baseline 20 - actual 1 request = 19 + expect(afterSettleManual.remaining).toBe(19); // Now refresh promo (usage-log) — it must not clobber manual. appendUsageEntry(usage("x", NOW)); await refreshEconomicSnapshots(config(), NOW + 500); From 4b6b2f37dcaf84ac77715d35e03d931a5296a0c1 Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Mon, 10 Aug 2026 13:58:02 -0700 Subject: [PATCH 09/10] fix(combos): atomic multi-allowance reserve and scoped usage-log refresh Reserve all-or-nothing across target allowances, omit reservationId for PAYG, and filter usage-log refresh via optional usageMatch providers/models. --- src/combos/economy-refresh.ts | 16 ++++- src/combos/economy.ts | 34 +++++---- src/config.ts | 26 +++++++ src/types.ts | 4 ++ tests/economic-reservation-race.test.ts | 95 +++++++++++++++++++++++++ tests/economic-snapshot-refresh.test.ts | 38 ++++++++++ 6 files changed, 198 insertions(+), 15 deletions(-) diff --git a/src/combos/economy-refresh.ts b/src/combos/economy-refresh.ts index abe0594e00..92398b2623 100644 --- a/src/combos/economy-refresh.ts +++ b/src/combos/economy-refresh.ts @@ -3,6 +3,7 @@ import { currentUsageLogRevision, readRecentUsageEntries, usageLogRevisionKey, + type PersistedUsageEntry, } from "../usage/log"; import { getEconomicQuotaSnapshot, @@ -16,7 +17,7 @@ function nonNegative(value: number | undefined): number { return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : 0; } -function usageAmount(allowance: OcxEconomicAllowance, entry: ReturnType[number]): number { +function usageAmount(allowance: OcxEconomicAllowance, entry: PersistedUsageEntry): number { if (allowance.unit === "requests") return 1; if (allowance.unit === "inputTokens") return nonNegative(entry.usage?.inputTokens); if (allowance.unit === "outputTokens") return nonNegative(entry.usage?.outputTokens); @@ -25,18 +26,29 @@ function usageAmount(allowance: OcxEconomicAllowance, entry: ReturnType 0 && !match.providers.includes(entry.provider)) return false; + if (match.models && match.models.length > 0 && !match.models.includes(entry.model)) return false; + return true; +} + function snapshotFor( allowance: OcxEconomicAllowance, - entries: ReturnType, + entries: PersistedUsageEntry[], now: number, ): OcxEconomicSnapshot { // The remaining value is computed from usage over the just-closed window, but it // applies to the CURRENT window starting now — so the snapshot advertises // windowStart = now. A boundary of now - durationMs would make the snapshot look // immediately rolled-over. + // Unscoped usage-log is experimental: when usageMatch is absent the allowance + // sums every provider/model in the log. Prefer scoping with usageMatch.providers/models. const filterStart = allowance.window.kind === "rolling" ? now - allowance.window.durationMs : undefined; const used = entries .filter(entry => filterStart === undefined || entry.timestamp >= filterStart) + .filter(entry => entryMatchesAllowance(allowance, entry)) .reduce((total, entry) => total + usageAmount(allowance, entry), 0); return { remaining: Math.max(0, allowance.capacity - used), diff --git a/src/combos/economy.ts b/src/combos/economy.ts index ff414eb30f..904b4dc7d1 100644 --- a/src/combos/economy.ts +++ b/src/combos/economy.ts @@ -443,25 +443,32 @@ export function reserveEconomicSelection( const reserve = (currentExcluded: Set): EconomicSelectionResult => { const result = selectEconomicTarget(config, comboId, estimate, now, currentExcluded, isEligible); if (!result.target) return result; + const allowanceIds = result.target.allowances ?? []; + // Pure PAYG: no allowance holds and no reservation id noise. + if (allowanceIds.length === 0) return result; + + const failTarget = (): EconomicSelectionResult => { + if (currentExcluded.size >= targetCount) { + return { targetIndex: null, candidates: result.candidates, reason: "reservation-headroom-race" }; + } + const nextExcluded = new Set(currentExcluded); + nextExcluded.add(targetKey(result.target!)); + const fallback = reserve(nextExcluded); + return fallback.target + ? { ...fallback, reason: "reservation-headroom-race" } + : { targetIndex: null, candidates: fallback.candidates, reason: "reservation-headroom-race" }; + }; + const id = `econ-${++reservationSequence}`; const reservationsToAdd: Reservation[] = []; - for (const allowanceId of result.target.allowances ?? []) { + for (const allowanceId of allowanceIds) { const allowance = allowanceFor(config, allowanceId); const snapshot = snapshots.get(allowanceId); - if (!allowance || !snapshot) continue; + // Atomic: missing def/snapshot/headroom fails the whole target — never partial holds. + if (!allowance || !snapshot) return failTarget(); const amount = economicConsumption(allowance, estimate, result.target.pricing); const headroom = usableHeadroom(allowance, snapshot, amount, activeReserved(allowanceId, now), now); - if (headroom === null || headroom < -EPSILON) { - if (currentExcluded.size >= targetCount) { - return { targetIndex: null, candidates: result.candidates, reason: "reservation-headroom-race" }; - } - const nextExcluded = new Set(currentExcluded); - nextExcluded.add(targetKey(result.target)); - const fallback = reserve(nextExcluded); - return fallback.target - ? { ...fallback, reason: "reservation-headroom-race" } - : { targetIndex: null, candidates: fallback.candidates, reason: "reservation-headroom-race" }; - } + if (headroom === null || headroom < -EPSILON) return failTarget(); reservationsToAdd.push({ id, allowanceId, @@ -473,6 +480,7 @@ export function reserveEconomicSelection( generation: captureConfigGeneration(), }); } + if (reservationsToAdd.length !== allowanceIds.length) return failTarget(); for (const reservation of reservationsToAdd) reservations.set(`${id}\0${reservation.allowanceId}`, reservation); return { ...result, reservationId: id }; }; diff --git a/src/config.ts b/src/config.ts index a61e4a0c04..1433043fb6 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1530,6 +1530,32 @@ const configSchema = z.object({ } } } + if (allowance.usageMatch !== undefined) { + if (!allowance.usageMatch || typeof allowance.usageMatch !== "object" || Array.isArray(allowance.usageMatch)) { + ctx.addIssue({ code: "custom", path: [...path, "usageMatch"], message: "usageMatch must be an object" }); + } else { + const usageMatch = allowance.usageMatch as Record; + const allowedUsageMatchKeys = new Set(["providers", "models"]); + for (const key of Object.keys(usageMatch)) { + if (!allowedUsageMatchKeys.has(key)) { + ctx.addIssue({ code: "custom", path: [...path, "usageMatch", key], message: `unknown usageMatch key "${key}"` }); + } + } + for (const key of ["providers", "models"] as const) { + const value = usageMatch[key]; + if (value === undefined) continue; + if (!Array.isArray(value)) { + ctx.addIssue({ code: "custom", path: [...path, "usageMatch", key], message: `${key} must be an array of non-empty strings` }); + continue; + } + for (let i = 0; i < value.length; i += 1) { + if (typeof value[i] !== "string" || !(value[i] as string).trim()) { + ctx.addIssue({ code: "custom", path: [...path, "usageMatch", key, String(i)], message: `${key}[${i}] must be a non-empty string` }); + } + } + } + } + } } } } diff --git a/src/types.ts b/src/types.ts index c9f001bcf3..c3bb098e9d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -975,6 +975,10 @@ export interface OcxEconomicAllowance { source?: OcxEconomicSource; staleAfterMs?: number; rates?: OcxEconomicRates; + usageMatch?: { + providers?: string[]; + models?: string[]; + }; } export interface OcxEconomicSnapshot { remaining: number; diff --git a/tests/economic-reservation-race.test.ts b/tests/economic-reservation-race.test.ts index b2095cee4f..2c898c0f50 100644 --- a/tests/economic-reservation-race.test.ts +++ b/tests/economic-reservation-race.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { clearEconomicState, + countEconomicReservationsForAllowance, reserveEconomicSelection, setEconomicQuotaSnapshot, } from "../src/combos/economy"; @@ -88,4 +89,98 @@ describe("economic reservation races", () => { expect(result.target?.provider).toBe("alternate"); expect(result.reservationId).toBeString(); }); + + test("target [A,B] with only A snapshotted is not picked and no partial A-only holds remain", () => { + const cfg: OcxConfig = { + port: 0, + defaultProvider: "payg", + providers: { + comboPrimary: { adapter: "openai-chat", baseUrl: "https://combo.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + allowanceA: { unit: "requests", capacity: 10, window: { kind: "balance" }, source: "manual" }, + allowanceB: { unit: "requests", capacity: 10, window: { kind: "balance" }, source: "manual" }, + }, + combos: { + c: { + strategy: "economy", + economy: { unknownQuota: "reject" }, + targets: [ + { provider: "comboPrimary", model: "m", allowances: ["allowanceA", "allowanceB"] }, + { provider: "payg", model: "m" }, + ], + }, + }, + }; + setEconomicQuotaSnapshot("allowanceA", { remaining: 10, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const result = reserveEconomicSelection(cfg, "c", estimate, NOW); + expect(result.target?.provider).toBe("payg"); + expect(countEconomicReservationsForAllowance("allowanceA", NOW)).toBe(0); + expect(countEconomicReservationsForAllowance("allowanceB", NOW)).toBe(0); + }); + + test("pure PAYG has no reservationId", () => { + const cfg: OcxConfig = { + port: 0, + defaultProvider: "payg", + providers: { + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + combos: { + c: { strategy: "economy", targets: [{ provider: "payg", model: "m" }] }, + }, + }; + const result = reserveEconomicSelection(cfg, "c", estimate, NOW); + expect(result.target?.provider).toBe("payg"); + expect(result.reservationId).toBeUndefined(); + }); + + test("pure PAYG with empty allowances has no reservationId", () => { + const cfg: OcxConfig = { + port: 0, + defaultProvider: "payg", + providers: { + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + combos: { + c: { strategy: "economy", targets: [{ provider: "payg", model: "m", allowances: [] }] }, + }, + }; + const result = reserveEconomicSelection(cfg, "c", estimate, NOW); + expect(result.target?.provider).toBe("payg"); + expect(result.reservationId).toBeUndefined(); + }); + + test("both A,B ok => reservationId present and one hold per allowance", () => { + const cfg: OcxConfig = { + port: 0, + defaultProvider: "comboPrimary", + providers: { + comboPrimary: { adapter: "openai-chat", baseUrl: "https://combo.example", models: ["m"] }, + payg: { adapter: "openai-chat", baseUrl: "https://payg.example", models: ["m"] }, + }, + economicAllowances: { + allowanceA: { unit: "requests", capacity: 10, window: { kind: "balance" }, source: "manual" }, + allowanceB: { unit: "requests", capacity: 10, window: { kind: "balance" }, source: "manual" }, + }, + combos: { + c: { + strategy: "economy", + economy: { unknownQuota: "reject" }, + targets: [ + { provider: "comboPrimary", model: "m", allowances: ["allowanceA", "allowanceB"] }, + { provider: "payg", model: "m" }, + ], + }, + }, + }; + setEconomicQuotaSnapshot("allowanceA", { remaining: 10, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + setEconomicQuotaSnapshot("allowanceB", { remaining: 10, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const result = reserveEconomicSelection(cfg, "c", estimate, NOW); + expect(result.target?.provider).toBe("comboPrimary"); + expect(result.reservationId).toBeString(); + expect(countEconomicReservationsForAllowance("allowanceA", NOW)).toBe(1); + expect(countEconomicReservationsForAllowance("allowanceB", NOW)).toBe(1); + }); }); diff --git a/tests/economic-snapshot-refresh.test.ts b/tests/economic-snapshot-refresh.test.ts index 06533f2221..5e303b4311 100644 --- a/tests/economic-snapshot-refresh.test.ts +++ b/tests/economic-snapshot-refresh.test.ts @@ -249,4 +249,42 @@ describe("economic snapshot refresh", () => { expect(spy).not.toHaveBeenCalled(); spy.mockRestore(); }); + + test("scoped usageMatch.providers only counts matching provider", async () => { + const scoped = { + ...config(), + economicAllowances: { + promo: { + unit: "requests" as const, + capacity: 10, + window: { kind: "rolling" as const, durationMs: 60 * 60_000 }, + source: "usage-log" as const, + usageMatch: { providers: ["included"] }, + }, + }, + }; + appendUsageEntry({ ...usage("one"), provider: "included", model: "m" }); + appendUsageEntry({ ...usage("two"), provider: "payg", model: "m" }); + await refreshEconomicSnapshots(scoped, NOW); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(9); + }); + + test("scoped usageMatch.models only counts matching model", async () => { + const scoped = { + ...config(), + economicAllowances: { + promo: { + unit: "requests" as const, + capacity: 10, + window: { kind: "rolling" as const, durationMs: 60 * 60_000 }, + source: "usage-log" as const, + usageMatch: { models: ["m"] }, + }, + }, + }; + appendUsageEntry({ ...usage("one"), provider: "included", model: "m" }); + appendUsageEntry({ ...usage("two"), provider: "included", model: "other" }); + await refreshEconomicSnapshots(scoped, NOW); + expect(getEconomicQuotaSnapshot("promo")?.remaining).toBe(9); + }); }); From 3c69c4266f885152da299c131f9148d4abcf110a Mon Sep 17 00:00:00 2001 From: Hussein <59151492+H-H-E@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:18:36 -0700 Subject: [PATCH 10/10] fix(combos): finish pass-2 economy hardening Honest explain reasons/rankingBand, hardExclusions alias, allowance help, louder salvage warnings, and experimental multi-instance docs (docs-site build green). --- docs-site/src/content/docs/guides/combos.md | 11 ++- .../docs/reference/configuration/routing.md | 25 ++++--- src/cli/help.ts | 21 +++++- src/combos/economy.ts | 67 +++++++++++++++++-- src/config.ts | 7 +- tests/cli-help-allowance.test.ts | 19 ++++++ tests/economic-review-blindspots.test.ts | 33 ++++++++- 7 files changed, 160 insertions(+), 23 deletions(-) create mode 100644 tests/cli-help-allowance.test.ts diff --git a/docs-site/src/content/docs/guides/combos.md b/docs-site/src/content/docs/guides/combos.md index c32bce4142..79504ade28 100644 --- a/docs-site/src/content/docs/guides/combos.md +++ b/docs-site/src/content/docs/guides/combos.md @@ -335,13 +335,22 @@ validation message. The error was terminal rather than target-specific. Fix invalid input, reduce an oversized context, handle a policy refusal, or correct the rejected request origin. Combos do not hop for those cases. -## Economic routing +## Economic routing (experimental) + +> **Experimental.** Economy combos are opt-in. Quota state is process-local (not shared across +> multiple proxy instances), snapshots are lost on restart, and without manual snapshots or a scoped +> `usage-log` feed the common case is deprioritized included targets falling through to PAYG. +> The GUI only **preserves** economy JSON fields — it is not an allowance editor. Use `strategy: "economy"` when the targets in one combo are already known to be interchangeable and should be ranked by quota opportunity cost. It is not a task classifier; keep separate combos such as `bulk-code` and `frontier-review` for different capability classes. Existing `failover` and `round-robin` behavior is unchanged. +**Ledger (Model A):** `snapshot.remaining` is a baseline from refresh/manual PUT. Reservations are +off-book concurrency holds. Settle subtracts **actual** usage only (`remaining - actual`). Cancel and +plain release drop the hold without changing remaining. + **Day-one reality:** without operator-supplied snapshots (or a future provider adapter), included targets usually have unknown quota and are deprioritized (or rejected, if you set `unknownQuota: "reject"`). Expect traffic to fall through to priced PAYG targets until you feed diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 8f4e120d36..3476a3ce75 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -237,15 +237,18 @@ an automatic rebuild from `usage.jsonl` on the next query; `ocx logs rebuild-index` forces one. Nothing in this system auto-tunes weights, budgets, or candidate sets. -## Economic combo routing - -`strategy: "economy"` is an additive combo strategy. Shared static allowance buckets -are configured under `economicAllowances` and referenced by target `allowances` arrays; -remaining values are cached runtime snapshots. Selection is deterministic and applies -hard eligibility first, then reserves, expiration pressure, marginal cost, snapshot -freshness, and configured target order. It does not classify requests or synchronously -contact provider quota APIs. Use `ocx combo explain --json` for the full decision -breakdown. +## Economic combo routing (experimental) + +`strategy: "economy"` is an **experimental**, additive combo strategy. Shared static allowance +buckets live under `economicAllowances` and are referenced by target `allowances` arrays; remaining +values are cached **process-local** runtime snapshots (lost on restart; not shared across instances). +Selection is deterministic: hard eligibility first, then soft reserve / unknown-quota pressure, +expiration pressure, marginal cost, and configured target order. It does not classify requests or +call provider quota APIs on the request path. Ledger: settle debits actual usage only; cancel +releases holds without burn. Optional `usageMatch.providers` / `usageMatch.models` scopes +`source: "usage-log"` refresh; unscoped usage-log summation is experimental. Use +`ocx combo explain --json` and `ocx allowance …` for operator surfaces. The GUI preserves +economy fields only — no full editor. ### `economy` policy @@ -259,8 +262,8 @@ breakdown. - `unknownQuota` (`allow` | `deprioritize` | `reject`, default `deprioritize`): how a target behaves when a referenced allowance has no cached snapshot. - `maxMarginalUsd` (optional, non-negative): the highest estimated per-request USD a - target may cost before it is excluded; unknown costs are treated as unknown, not - zero. + target may cost before it is excluded; **unknown cash cost is fail-closed excluded** + when this guardrail is set. ### Target fields diff --git a/src/cli/help.ts b/src/cli/help.ts index 2a8f919bb8..f16bdfd8d4 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -147,9 +147,24 @@ const helpEntries: Record = { summary: "Alias of ocx models.", }, combo: { - usage: "ocx combo ...", - summary: "Manage combo failover and round-robin virtual models.", - details: ["Alias hierarchy: ocx route combo ...", "Use --targets provider/model[:weight],provider/model[:weight]."], + usage: "ocx combo ...", + summary: "Manage combo failover, round-robin, and experimental economy virtual models.", + details: [ + "Alias hierarchy: ocx route combo ...", + "Use --targets provider/model[:weight],provider/model[:weight].", + "Economy: ocx combo set --combo-json '...' ; ocx combo explain [--input-tokens n] [--output-tokens n] [--json].", + ], + }, + allowance: { + usage: "ocx allowance ...", + summary: "Inspect economic allowances and manage runtime quota snapshots (experimental economy).", + details: [ + "ocx allowance list [--json]", + "ocx allowance snapshot get [--json]", + "ocx allowance snapshot set --snapshot-json '' [--clear-reservations] [--json]", + "ocx allowance snapshot clear [--clear-reservations] [--json]", + "Snapshots are in-memory only and lost on restart.", + ], }, route: { usage: "ocx route combo ...", diff --git a/src/combos/economy.ts b/src/combos/economy.ts index 904b4dc7d1..d7fb9a9a5e 100644 --- a/src/combos/economy.ts +++ b/src/combos/economy.ts @@ -39,12 +39,21 @@ export type EconomicSoftSignal = | "unknown-quota" | "expiration-pressure"; -export type EconomicRankingBand = "excluded" | "expiration" | "marginal-cost" | string; +export type EconomicRankingBand = + | "excluded" + | "reserve" + | "unknown-quota" + | "expiration" + | "marginal-cost" + | "stable-order"; export interface EconomicSelectionCandidate { target: EconomicTarget; eligible: boolean; + /** Hard disqualifiers (same as hardExclusions). */ exclusions: EconomicHardExclusion[]; + /** Alias of exclusions for explain consumers. */ + hardExclusions: EconomicHardExclusion[]; softSignals: EconomicSoftSignal[]; configIndex: number; cashCost: number | "included" | "unknown"; @@ -312,6 +321,26 @@ function compareCandidates(a: EconomicSelectionCandidate, b: EconomicSelectionCa return a.configIndex - b.configIndex; } +function rankingForCandidate(candidate: EconomicSelectionCandidate): { band: EconomicRankingBand; reason: string } { + if (!candidate.eligible || candidate.hardExclusions.length > 0) { + return { band: "excluded", reason: "excluded" }; + } + // Match comparator priority: reserve → unknown → expiration → marginal cost → stable order. + if (candidate.softSignals.includes("reserve")) { + return { band: "reserve", reason: "reserve pressure" }; + } + if (candidate.softSignals.includes("unknown-quota")) { + return { band: "unknown-quota", reason: "unknown quota deprioritized" }; + } + if (candidate.softSignals.includes("expiration-pressure") || candidate.burnPressure !== null) { + return { band: "expiration", reason: "expiration pressure" }; + } + if (typeof candidate.marginalUsd === "number" && Number.isFinite(candidate.marginalUsd)) { + return { band: "marginal-cost", reason: "lowest marginal cost" }; + } + return { band: "stable-order", reason: "stable target order" }; +} + function candidateFor( config: OcxConfig, combo: OcxComboConfig, @@ -382,10 +411,28 @@ function candidateFor( } const cashCost = cashCostFor(target, estimate, cost); const eligible = hardExclusions.length === 0; + const ranking = rankingForCandidate({ + target, + eligible, + exclusions: hardExclusions, + hardExclusions, + softSignals, + configIndex: index, + cashCost, + consumption: consumptions, + postRequestRemaining, + reserveThresholds, + burnPressure: pressure, + marginalUsd: cost, + stale, + rankingBand: "stable-order", + allowances: allowanceDetails, + }); return { target, eligible, exclusions: hardExclusions, + hardExclusions, softSignals, configIndex: index, cashCost, @@ -395,7 +442,7 @@ function candidateFor( burnPressure: pressure, marginalUsd: cost, stale, - rankingBand: hardExclusions.includes("hard-headroom") ? "excluded" : pressure !== null ? "expiration" : cost !== null ? "marginal-cost" : `order-${index + 1}`, + rankingBand: ranking.band, allowances: allowanceDetails, }; } @@ -415,18 +462,28 @@ export function selectEconomicTarget( const candidate = candidateFor(config, combo, { ...target, weight: target.weight ?? 1 }, estimate, now, index, new Set(excluded)); if (isEligible && !isEligible(candidate.target)) { if (!candidate.exclusions.includes("ineligible")) candidate.exclusions.push("ineligible"); - candidate.eligible = candidate.exclusions.length === 0; + if (!candidate.hardExclusions.includes("ineligible")) candidate.hardExclusions.push("ineligible"); + candidate.eligible = candidate.hardExclusions.length === 0; + if (!candidate.eligible) { + const ranking = rankingForCandidate(candidate); + candidate.rankingBand = ranking.band; + } } return candidate; }); const available = candidates.filter(candidate => candidate.eligible); if (available.length === 0) return { targetIndex: null, candidates, reason: "no-economically-eligible-target" }; - const winner = available.slice().sort(compareCandidates)[0]!; + let winner = available[0]!; + for (let i = 1; i < available.length; i += 1) { + if (compareCandidates(available[i]!, winner) < 0) winner = available[i]!; + } + const ranking = rankingForCandidate(winner); + winner.rankingBand = ranking.band; return { target: winner.target, targetIndex: combo.targets.findIndex(target => targetKey(target) === targetKey(winner.target)), candidates, - reason: winner.rankingBand === "expiration" ? "expiration pressure" : winner.rankingBand === "marginal-cost" ? "lowest marginal cost" : "stable target order", + reason: ranking.reason, }; } diff --git a/src/config.ts b/src/config.ts index 1433043fb6..f1471dfde8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2065,7 +2065,12 @@ function sanitizeEconomicAllowancesForLoad(rawParsed: unknown): void { if (Object.keys(kept).length > 0) raw.economicAllowances = kept; else delete raw.economicAllowances; if (dropped.length > 0) { - console.warn(`⚠️ config.json ignored invalid economic allowance${dropped.length === 1 ? "" : "s"}: ${dropped.join(", ")}. Other settings were preserved.`); + const preview = dropped.slice(0, 10); + const more = dropped.length > preview.length ? ` (+${dropped.length - preview.length} more)` : ""; + console.warn( + `⚠️ config.json dropped ${dropped.length} invalid economic allowance${dropped.length === 1 ? "" : "s"}: ${preview.join(", ")}${more}. ` + + "Prepaid routing for those ids will not apply until fixed. Other settings were preserved.", + ); } } diff --git a/tests/cli-help-allowance.test.ts b/tests/cli-help-allowance.test.ts new file mode 100644 index 0000000000..4bb260902a --- /dev/null +++ b/tests/cli-help-allowance.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "bun:test"; +import { printSubcommandUsage } from "../src/cli/help"; + +describe("ocx help allowance", () => { + test("printSubcommandUsage allowance mentions snapshot commands", () => { + const lines: string[] = []; + const log = console.log; + console.log = (...args: unknown[]) => { lines.push(args.map(String).join(" ")); }; + try { + printSubcommandUsage("allowance"); + } finally { + console.log = log; + } + const text = lines.join("\n"); + expect(text).toContain("ocx allowance"); + expect(text).toContain("snapshot"); + expect(text).toContain("clear-reservations"); + }); +}); diff --git a/tests/economic-review-blindspots.test.ts b/tests/economic-review-blindspots.test.ts index 7d1b3ac537..dcab38b322 100644 --- a/tests/economic-review-blindspots.test.ts +++ b/tests/economic-review-blindspots.test.ts @@ -121,7 +121,6 @@ describe("economic hostile-review blindspots", () => { economy: { unknownQuota: "deprioritize" }, targets: [ { provider: "included", model: "m", allowances: ["promo"] }, - { provider: "payg", model: "m", pricing: { fixedPerRequest: 1 } }, ], }, }, @@ -136,14 +135,44 @@ describe("economic hostile-review blindspots", () => { }, }); setEconomicQuotaSnapshot("promo", { remaining: 10, updatedAt: NOW, source: "manual", confidence: "authoritative" }); - // demand 2 → post=8, reserve threshold=9 → soft reserve pressure + // demand 2 → post=8, reserve threshold=9 → soft reserve pressure; sole eligible winner const explanation = explainEconomicCombo(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 2, kind: "configured" }, NOW); expect(explanation.selectedTarget).toBeString(); const winner = explanation.candidates.find(c => c.target.provider === "included"); expect(winner?.eligible).toBe(true); expect(winner?.exclusions).toEqual([]); + expect(winner?.hardExclusions).toEqual([]); expect(winner?.softSignals).toContain("reserve"); expect(winner?.cashCost).toBe("included"); + expect(explanation.reason).toBe("reserve pressure"); + expect(winner?.rankingBand).toBe("reserve"); + }); + + test("explain DTO exposes hardExclusions alias and stable shape keys", () => { + const cfg = baseConfig(); + setEconomicQuotaSnapshot("promo", { remaining: 5, updatedAt: NOW, source: "manual", confidence: "authoritative" }); + const explanation = explainEconomicCombo(cfg, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); + expect(explanation).toMatchObject({ + comboId: "bulk", + strategy: "economy", + selectedTarget: expect.any(String), + generatedAt: NOW, + reason: expect.any(String), + }); + for (const c of explanation.candidates) { + expect(c.hardExclusions).toEqual(c.exclusions); + expect(Array.isArray(c.softSignals)).toBe(true); + expect(typeof c.configIndex).toBe("number"); + expect(c.cashCost === "included" || c.cashCost === "unknown" || typeof c.cashCost === "number").toBe(true); + } + }); + + test("pure PAYG has no reservationId; atomic multi-allowance needs all snapshots", () => { + const paygOnly = baseConfig({ + combos: { bulk: { strategy: "economy", targets: [{ provider: "payg", model: "m", pricing: { fixedPerRequest: 0.01 } }] } }, + }); + const payg = reserveEconomicSelection(paygOnly, "bulk", { inputTokens: 0, outputTokens: 0, fixedRequests: 1, kind: "configured" }, NOW); + expect(payg.reservationId).toBeUndefined(); }); test("countEconomicReservationsForAllowance tracks holds", () => {