From 0a67ec6b3c125d72ea660ffad01edadf8316c721 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 19:25:25 +0000 Subject: [PATCH 1/4] refactor: single-responsibility modules; store classes over factories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boyscout pass over the recent bus work and its relatives: - bus.ts owns only the bus and the AgentBusStore seam. The store implementations move to their own modules as real classes: MemoryBusStore (memory-store.ts, the default) and FileBusStore (file-store.ts, owning all file IO, serialization, and recovery). The createFileBusStore/createMemoryBusStore closure factories and the implementation-specific FileBusStore interface are gone — abstractions stay where multiple implementations justify them (AgentBusStore), and implementations are classes. - The harness package entry no longer carries the run loop: the implementation lives in harness.ts and index.ts is a pure re-export barrel, matching the training and rewrite packages. - windowedContext moves out of the harness-loop adapter into its own provider module (src/providers/context.ts) — context management and loop adaptation are separate concerns. - Training's createMemoryTrainingStore closure factory becomes the MemoryTrainingStore class. - The harness README example still showed the removed { file } bus settings; corrected to the store API. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NG6fSKqgt8nD7JnEY9q245 --- packages/harness/README.md | 11 +- packages/harness/src/bus.ts | 101 +--------- packages/harness/src/file-store.ts | 68 +++++++ packages/harness/src/harness.ts | 230 +++++++++++++++++++++++ packages/harness/src/index.ts | 254 +++----------------------- packages/harness/src/memory-store.ts | 15 ++ packages/harness/test/harness.test.ts | 6 +- packages/training/src/index.ts | 2 +- packages/training/src/records.ts | 25 +-- packages/training/src/training.ts | 4 +- src/index.ts | 5 +- src/providers/context.ts | 17 ++ src/providers/harness.ts | 22 +-- 13 files changed, 391 insertions(+), 369 deletions(-) create mode 100644 packages/harness/src/file-store.ts create mode 100644 packages/harness/src/harness.ts create mode 100644 packages/harness/src/memory-store.ts create mode 100644 src/providers/context.ts diff --git a/packages/harness/README.md b/packages/harness/README.md index 6c74772..d306f63 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -24,7 +24,7 @@ npm install ts-autocode-harness ```ts import { join } from "node:path"; -import { defineTrainingHarness, WriteAheadAgentBus } from "ts-autocode-harness"; +import { defineTrainingHarness, FileBusStore, WriteAheadAgentBus } from "ts-autocode-harness"; const harness = defineTrainingHarness({ maxRounds: 3, @@ -32,7 +32,7 @@ const harness = defineTrainingHarness({ }); const result = await harness.run({ - bus: new WriteAheadAgentBus({ file: join(root, "actions.jsonl") }), + bus: new WriteAheadAgentBus({ store: new FileBusStore(join(root, "actions.jsonl")) }), task: { objective, target }, rubric: "The candidate must pass AgentV and preserve its public contract.", student: myStudent, @@ -59,9 +59,10 @@ Configure `redact` when payloads may contain sensitive application data. Storage is pluggable through `AgentBusStore` — anything with `append(entry)` and `load()` works, so entries can live in memory, on disk, or behind a remote -service. `createMemoryBusStore()` is the default; `createFileBusStore(path)` -is the durable JSONL implementation (fsynced per append, resilient to an -incomplete trailing line). Messages and entries are parsed at the boundary +service. Each implementation is its own class in its own module: +`MemoryBusStore` is the default, and `FileBusStore` is the durable JSONL +implementation (fsynced per append, resilient to an incomplete trailing +line). Messages and entries are parsed at the boundary with zod schemas (`agentMessage`, `agentBusEntry`), so malformed values never enter the log. diff --git a/packages/harness/src/bus.ts b/packages/harness/src/bus.ts index 637ad3f..f4d5f02 100644 --- a/packages/harness/src/bus.ts +++ b/packages/harness/src/bus.ts @@ -1,16 +1,7 @@ import { randomUUID } from "node:crypto"; -import { mkdir, open, readFile } from "node:fs/promises"; -import { dirname } from "node:path"; -import { - absolutePath, - agentBusEntry, - agentMessage, - messageId, - type AbsolutePath, - type AgentBusEntry, - type AgentMessage, -} from "./schema.js"; +import { MemoryBusStore } from "./memory-store.js"; +import { agentBusEntry, agentMessage, messageId, type AgentBusEntry, type AgentMessage } from "./schema.js"; /** One requested bus operation, offered to the `allow` hook. */ export type AgentBusAccess = @@ -18,11 +9,10 @@ export type AgentBusAccess = | Readonly<{ operation: "read"; actor?: string }>; /** Ordered storage for bus entries. Implementations may keep entries in - * memory, on disk, or behind a remote service — the bus does not care, and - * any object with these two methods plugs in. A store belongs to one writing - * bus at a time: the bus resumes sequence numbering from the store's tail and - * then owns it, so concurrent writers need a store with its own reservation - * semantics behind this interface. */ + * memory, on disk, or behind a remote service — the bus does not care. A + * store belongs to one writing bus at a time: the bus resumes sequence + * numbering from the store's tail and then owns it, so concurrent writers + * need a store with its own reservation semantics behind this interface. */ export interface AgentBusStore { /** Appends one entry, preserving sequence order. */ append(entry: AgentBusEntry): Promise; @@ -56,7 +46,7 @@ export class WriteAheadAgentBus { #pending: Promise = Promise.resolve(); constructor(settings: AgentBusSettings = {}) { - this.#store = settings.store ?? createMemoryBusStore(); + this.#store = settings.store ?? new MemoryBusStore(); this.#idFactory = settings.idFactory ?? randomUUID; this.#now = settings.now ?? (() => new Date()); this.#redact = settings.redact ?? ((value) => value); @@ -101,80 +91,3 @@ export class WriteAheadAgentBus { return Object.freeze(actor === undefined ? [...entries] : entries.filter((entry) => entry.actor === actor)); } } - -/** Volatile in-memory storage: the default when no store is configured. */ -export function createMemoryBusStore(): AgentBusStore { - const entries: AgentBusEntry[] = []; - return { - append: async (entry) => { - entries.push(entry); - }, - load: async () => [...entries], - }; -} - -export interface FileBusStore extends AgentBusStore { - /** Resolved absolute path of the JSONL log. */ - readonly file: AbsolutePath; -} - -/** Durable JSONL storage, fsynced per append. On load, an incomplete trailing - * line (a crashed writer) is ignored; anything else that fails to parse as an - * entry is an error rather than silently accepted. */ -export function createFileBusStore(file: string): FileBusStore { - const path = absolutePath.parse(file); - return { - file: path, - append: async (entry) => { - await mkdir(dirname(path), { recursive: true }); - const handle = await open(path, "a"); - try { - await handle.appendFile(`${safeJson(entry)}\n`, "utf8"); - await handle.sync(); - } finally { - await handle.close(); - } - }, - load: async () => { - let content: string; - try { - content = await readFile(path, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } - return parseEntries(content); - }, - }; -} - -function parseEntries(content: string): AgentBusEntry[] { - const lines = content.split("\n"); - const entries: AgentBusEntry[] = []; - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]?.trim(); - if (!line) continue; - let value: unknown; - try { - value = JSON.parse(line); - } catch (error) { - // Only an incomplete trailing fragment (a crashed writer) is - // recoverable; a complete record that fails the schema is not. - if (index === lines.length - 1) continue; - throw error; - } - entries.push(agentBusEntry.parse(value)); - } - return entries; -} - -function safeJson(value: unknown): string { - const seen = new WeakSet(); - return JSON.stringify(value, (_key, item: unknown) => { - if (typeof item === "bigint") return item.toString(); - if (!item || typeof item !== "object") return item; - if (seen.has(item)) return "[Circular]"; - seen.add(item); - return item; - }); -} diff --git a/packages/harness/src/file-store.ts b/packages/harness/src/file-store.ts new file mode 100644 index 0000000..d9eef9f --- /dev/null +++ b/packages/harness/src/file-store.ts @@ -0,0 +1,68 @@ +import { mkdir, open, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import type { AgentBusStore } from "./bus.js"; +import { absolutePath, agentBusEntry, type AbsolutePath, type AgentBusEntry } from "./schema.js"; + +/** Durable JSONL storage, fsynced per append. On load, an incomplete trailing + * JSON fragment (a crashed writer) is ignored; a complete record that fails + * the entry schema is an error rather than silently accepted. */ +export class FileBusStore implements AgentBusStore { + /** Resolved absolute path of the JSONL log. */ + readonly file: AbsolutePath; + + constructor(file: string) { + this.file = absolutePath.parse(file); + } + + async append(entry: AgentBusEntry): Promise { + await mkdir(dirname(this.file), { recursive: true }); + const handle = await open(this.file, "a"); + try { + await handle.appendFile(`${serialize(entry)}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + } + + async load(): Promise { + let content: string; + try { + content = await readFile(this.file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + return parse(content); + } +} + +function parse(content: string): AgentBusEntry[] { + const lines = content.split("\n"); + const entries: AgentBusEntry[] = []; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]?.trim(); + if (!line) continue; + let value: unknown; + try { + value = JSON.parse(line); + } catch (error) { + if (index === lines.length - 1) continue; + throw error; + } + entries.push(agentBusEntry.parse(value)); + } + return entries; +} + +function serialize(value: unknown): string { + const seen = new WeakSet(); + return JSON.stringify(value, (_key, item: unknown) => { + if (typeof item === "bigint") return item.toString(); + if (!item || typeof item !== "object") return item; + if (seen.has(item)) return "[Circular]"; + seen.add(item); + return item; + }); +} diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts new file mode 100644 index 0000000..8b99c9d --- /dev/null +++ b/packages/harness/src/harness.ts @@ -0,0 +1,230 @@ +import { type WriteAheadAgentBus } from "./bus.js"; +import { dispatchAction, decisionKind, type ActionGate } from "./dispatch.js"; +import { candidateKey, judgeDecision, roundLimit, rubricText, type AgentBusEntry, type JudgeDecision } from "./schema.js"; + +/** Shapes the bus history handed to actors and the judge. The bus itself does + * no context management: windowing, rolling summaries, or any other + * optimization belong to the consumer's provider (in the spirit of Semantic + * Kernel's chat-history reducers). The full history is passed when unset. */ +export type ContextProvider = ( + entries: readonly AgentBusEntry[], +) => readonly AgentBusEntry[] | Promise; + +export interface StudentTurn { + readonly round: number; + readonly task: unknown; + readonly rubric: string; + readonly feedback: readonly TFeedback[]; + readonly context: readonly AgentBusEntry[]; + readonly signal?: AbortSignal; +} + +export interface TeacherResult { + readonly assessment: TAssessment; + readonly feedback: readonly TFeedback[]; +} + +export interface RubricRevision { + readonly rubric: string; + readonly feedback: readonly TFeedback[]; +} + +export type JudgeRequest = + | Readonly<{ subject: "action"; action: AgentBusEntry; context: readonly AgentBusEntry[] }> + | Readonly<{ subject: "candidate"; task: unknown; candidate: TCandidate; assessment: TAssessment; rubric: string; context: readonly AgentBusEntry[] }> + | Readonly<{ subject: "adversary"; task: unknown; candidate: TCandidate; challenge: TChallenge; rubric: string; context: readonly AgentBusEntry[] }>; + +export interface HarnessRound { + readonly round: number; + readonly candidate: TCandidate; + readonly assessment: TAssessment; + readonly judgeDecision: JudgeDecision; + readonly adversary?: Readonly<{ challenge: TChallenge; decision: JudgeDecision }>; + readonly rubric: string; +} + +export interface HarnessRun { + readonly outcome: "accepted" | "stalled" | "exhausted"; + readonly rounds: readonly HarnessRound[]; + readonly final: HarnessRound; + readonly rubric: string; +} + +export interface HarnessInput { + readonly bus: WriteAheadAgentBus; + readonly task: unknown; + readonly rubric: string; + readonly student: (turn: StudentTurn) => TCandidate | Promise; + readonly teacher: ( + candidate: TCandidate, + turn: StudentTurn, + ) => TeacherResult | Promise>; + readonly judge: (input: unknown) => JudgeDecision | Promise; + readonly adversary: ( + candidate: TCandidate, + turn: Readonly<{ task: unknown; context: readonly AgentBusEntry[]; signal?: AbortSignal }>, + ) => TChallenge | Promise; + readonly reviseRubric: ( + challenge: TChallenge, + turn: Readonly<{ + task: unknown; + candidate: TCandidate; + assessment: TAssessment; + rubric: string; + context: readonly AgentBusEntry[]; + signal?: AbortSignal; + }>, + ) => RubricRevision | Promise>; + /** Shapes bus history into actor context; full history when unset. */ + readonly contextProvider?: ContextProvider; + readonly signal?: AbortSignal; +} + +export interface HarnessSettings { + readonly maxRounds?: number; + readonly candidateId: (candidate: TCandidate) => string; +} + +/** How many student rounds a harness runs when `maxRounds` is unset. */ +export const defaultMaxRounds = 3; + +export interface TrainingHarness { + run(input: HarnessInput): Promise>; +} + +export function defineTrainingHarness( + settings: HarnessSettings, +): TrainingHarness { + const maxRounds = roundLimit.parse(settings.maxRounds ?? defaultMaxRounds); + + return Object.freeze({ + async run(input: HarnessInput) { + let rubric: string = rubricText.parse(input.rubric); + const rounds: HarnessRound[] = []; + let feedback: readonly TFeedback[] = []; + let previousCandidate: string | undefined; + const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); + + // Every actor invocation is written ahead and gated through the judge + // callback; the judge itself is just one more actor whose verdicts + // land on the bus as ordinary messages. + const gate: ActionGate = async (action, context) => + input.judge(Object.freeze({ subject: "action", action, context: await provide(context) })); + const dispatch = (actor: string, kind: string, payload: unknown, execute: () => Promise | T) => + dispatchAction(input.bus, actor, kind, payload, gate, execute); + + for (let round = 1; round <= maxRounds; round += 1) { + input.signal?.throwIfAborted(); + const turn = await studentTurn(input, provide, round, rubric, feedback); + const candidate = await dispatch("student", "agent.propose", { round, task: input.task, rubric, feedback }, + () => input.student(turn)); + input.signal?.throwIfAborted(); + const candidateId: string = candidateKey.parse(settings.candidateId(candidate)); + if (candidateId === previousCandidate) return result("stalled", rounds, rubric); + previousCandidate = candidateId; + + const assessment = await dispatch("teacher", "agent.assess", { round, candidateId }, + () => input.teacher(candidate, turn)); + input.signal?.throwIfAborted(); + const candidateDecision = await decide(input, { candidateId }, { + subject: "candidate", + task: input.task, + candidate, + assessment: assessment.assessment, + rubric, + context: [], + }); + + if (candidateDecision === "fail") { + rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, rubric })); + feedback = Object.freeze([...assessment.feedback]); + continue; + } + + const adversary = await dispatch("adversary", "agent.challenge", { candidateId }, async () => + input.adversary(candidate, { + task: input.task, + context: await provide(await input.bus.read("adversary")), + ...(input.signal === undefined ? {} : { signal: input.signal }), + })); + input.signal?.throwIfAborted(); + const adversaryDecision = await decide(input, { candidateId }, { + subject: "adversary", + task: input.task, + candidate, + challenge: adversary, + rubric, + context: [], + }); + + if (adversaryDecision === "fail") { + rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, + adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); + return result("accepted", rounds, rubric); + } + + const revision = await dispatch("teacher", "agent.revise-rubric", { round, candidateId }, async () => + input.reviseRubric(adversary, { + task: input.task, + candidate, + assessment: assessment.assessment, + rubric, + context: await provide(await input.bus.read()), + ...(input.signal === undefined ? {} : { signal: input.signal }), + })); + const revised: string = rubricText.parse(revision.rubric); + if (revised === rubric) throw new Error("teacher must improve the rubric after an approved adversarial challenge"); + rubric = revised; + feedback = Object.freeze([...revision.feedback]); + rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, + adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); + } + + return result("exhausted", rounds, rubric); + }, + }); +} + +async function studentTurn( + input: HarnessInput, + provide: ContextProvider, + round: number, + rubric: string, + feedback: readonly TFeedback[], +): Promise> { + return Object.freeze({ + round, + task: input.task, + rubric, + feedback, + context: await provide(await input.bus.read()), + ...(input.signal === undefined ? {} : { signal: input.signal }), + }); +} + +/** Asks the judge callback and records its verdict on the bus as the judge's + * own message — the same way any actor communicates. */ +async function decide( + input: HarnessInput, + payload: Readonly>, + request: JudgeRequest, +): Promise { + const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); + const context = request.subject === "action" ? request.context : await provide(await input.bus.read()); + const decision = judgeDecision.parse(await input.judge(Object.freeze({ ...request, context }))); + await input.bus.append({ + actor: "judge", + kind: decisionKind, + payload: { subject: request.subject, ...payload, decision }, + }); + return decision; +} + +function result( + outcome: HarnessRun["outcome"], + rounds: HarnessRound[], + rubric: string, +): HarnessRun { + return Object.freeze({ outcome, rounds: Object.freeze([...rounds]), + final: rounds.at(-1) as HarnessRound, rubric }); +} diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 829bccf..bbb8425 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -1,241 +1,31 @@ -import { type WriteAheadAgentBus } from "./bus.js"; -import { dispatchAction, decisionKind, type ActionGate } from "./dispatch.js"; -import { candidateKey, judgeDecision, roundLimit, rubricText, type AgentBusEntry, type JudgeDecision } from "./schema.js"; +export { WriteAheadAgentBus } from "./bus.js"; +export type { AgentBusAccess, AgentBusSettings, AgentBusStore } from "./bus.js"; + +export { MemoryBusStore } from "./memory-store.js"; +export { FileBusStore } from "./file-store.js"; -export { createFileBusStore, createMemoryBusStore, WriteAheadAgentBus } from "./bus.js"; -export type { AgentBusAccess, AgentBusSettings, AgentBusStore, FileBusStore } from "./bus.js"; export { AgentActionDeniedError, decisionKind, dispatchAction } from "./dispatch.js"; export type { ActionGate } from "./dispatch.js"; + export { agentBusEntry, agentMessage, judgeDecision } from "./schema.js"; export type { AbsolutePath, AgentBusEntry, AgentMessage, JudgeDecision } from "./schema.js"; + +export { defaultMaxRounds, defineTrainingHarness } from "./harness.js"; +export type { + ContextProvider, + HarnessInput, + HarnessRound, + HarnessRun, + HarnessSettings, + JudgeRequest, + RubricRevision, + StudentTurn, + TeacherResult, + TrainingHarness, +} from "./harness.js"; + export { createHarnessPolicy, sandboxPolicyVersion } from "./policy.js"; export type { HarnessPolicySettings } from "./policy.js"; + export { MxcSandbox } from "./sandbox.js"; export type { MxcSandboxSettings } from "./sandbox.js"; - -/** Shapes the bus history handed to actors and the judge. The bus itself does - * no context management: windowing, rolling summaries, or any other - * optimization belong to the consumer's provider (in the spirit of Semantic - * Kernel's chat-history reducers). The full history is passed when unset. */ -export type ContextProvider = ( - entries: readonly AgentBusEntry[], -) => readonly AgentBusEntry[] | Promise; - -export interface StudentTurn { - readonly round: number; - readonly task: unknown; - readonly rubric: string; - readonly feedback: readonly TFeedback[]; - readonly context: readonly AgentBusEntry[]; - readonly signal?: AbortSignal; -} - -export interface TeacherResult { - readonly assessment: TAssessment; - readonly feedback: readonly TFeedback[]; -} - -export interface RubricRevision { - readonly rubric: string; - readonly feedback: readonly TFeedback[]; -} - -export type JudgeRequest = - | Readonly<{ subject: "action"; action: AgentBusEntry; context: readonly AgentBusEntry[] }> - | Readonly<{ subject: "candidate"; task: unknown; candidate: TCandidate; assessment: TAssessment; rubric: string; context: readonly AgentBusEntry[] }> - | Readonly<{ subject: "adversary"; task: unknown; candidate: TCandidate; challenge: TChallenge; rubric: string; context: readonly AgentBusEntry[] }>; - -export interface HarnessRound { - readonly round: number; - readonly candidate: TCandidate; - readonly assessment: TAssessment; - readonly judgeDecision: JudgeDecision; - readonly adversary?: Readonly<{ challenge: TChallenge; decision: JudgeDecision }>; - readonly rubric: string; -} - -export interface HarnessRun { - readonly outcome: "accepted" | "stalled" | "exhausted"; - readonly rounds: readonly HarnessRound[]; - readonly final: HarnessRound; - readonly rubric: string; -} - -export interface HarnessInput { - readonly bus: WriteAheadAgentBus; - readonly task: unknown; - readonly rubric: string; - readonly student: (turn: StudentTurn) => TCandidate | Promise; - readonly teacher: ( - candidate: TCandidate, - turn: StudentTurn, - ) => TeacherResult | Promise>; - readonly judge: (input: unknown) => JudgeDecision | Promise; - readonly adversary: ( - candidate: TCandidate, - turn: Readonly<{ task: unknown; context: readonly AgentBusEntry[]; signal?: AbortSignal }>, - ) => TChallenge | Promise; - readonly reviseRubric: ( - challenge: TChallenge, - turn: Readonly<{ - task: unknown; - candidate: TCandidate; - assessment: TAssessment; - rubric: string; - context: readonly AgentBusEntry[]; - signal?: AbortSignal; - }>, - ) => RubricRevision | Promise>; - /** Shapes bus history into actor context; full history when unset. */ - readonly contextProvider?: ContextProvider; - readonly signal?: AbortSignal; -} - -export interface HarnessSettings { - readonly maxRounds?: number; - readonly candidateId: (candidate: TCandidate) => string; -} - -/** How many student rounds a harness runs when `maxRounds` is unset. */ -export const defaultMaxRounds = 3; - -export interface TrainingHarness { - run(input: HarnessInput): Promise>; -} - -export function defineTrainingHarness( - settings: HarnessSettings, -): TrainingHarness { - const maxRounds = roundLimit.parse(settings.maxRounds ?? defaultMaxRounds); - - return Object.freeze({ - async run(input: HarnessInput) { - let rubric: string = rubricText.parse(input.rubric); - const rounds: HarnessRound[] = []; - let feedback: readonly TFeedback[] = []; - let previousCandidate: string | undefined; - const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); - - // Every actor invocation is written ahead and gated through the judge - // callback; the judge itself is just one more actor whose verdicts - // land on the bus as ordinary messages. - const gate: ActionGate = async (action, context) => - input.judge(Object.freeze({ subject: "action", action, context: await provide(context) })); - const dispatch = (actor: string, kind: string, payload: unknown, execute: () => Promise | T) => - dispatchAction(input.bus, actor, kind, payload, gate, execute); - - for (let round = 1; round <= maxRounds; round += 1) { - input.signal?.throwIfAborted(); - const turn = await studentTurn(input, provide, round, rubric, feedback); - const candidate = await dispatch("student", "agent.propose", { round, task: input.task, rubric, feedback }, - () => input.student(turn)); - input.signal?.throwIfAborted(); - const candidateId: string = candidateKey.parse(settings.candidateId(candidate)); - if (candidateId === previousCandidate) return result("stalled", rounds, rubric); - previousCandidate = candidateId; - - const assessment = await dispatch("teacher", "agent.assess", { round, candidateId }, - () => input.teacher(candidate, turn)); - input.signal?.throwIfAborted(); - const candidateDecision = await decide(input, { candidateId }, { - subject: "candidate", - task: input.task, - candidate, - assessment: assessment.assessment, - rubric, - context: [], - }); - - if (candidateDecision === "fail") { - rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, rubric })); - feedback = Object.freeze([...assessment.feedback]); - continue; - } - - const adversary = await dispatch("adversary", "agent.challenge", { candidateId }, async () => - input.adversary(candidate, { - task: input.task, - context: await provide(await input.bus.read("adversary")), - ...(input.signal === undefined ? {} : { signal: input.signal }), - })); - input.signal?.throwIfAborted(); - const adversaryDecision = await decide(input, { candidateId }, { - subject: "adversary", - task: input.task, - candidate, - challenge: adversary, - rubric, - context: [], - }); - - if (adversaryDecision === "fail") { - rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, - adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); - return result("accepted", rounds, rubric); - } - - const revision = await dispatch("teacher", "agent.revise-rubric", { round, candidateId }, async () => - input.reviseRubric(adversary, { - task: input.task, - candidate, - assessment: assessment.assessment, - rubric, - context: await provide(await input.bus.read()), - ...(input.signal === undefined ? {} : { signal: input.signal }), - })); - const revised: string = rubricText.parse(revision.rubric); - if (revised === rubric) throw new Error("teacher must improve the rubric after an approved adversarial challenge"); - rubric = revised; - feedback = Object.freeze([...revision.feedback]); - rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, - adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); - } - - return result("exhausted", rounds, rubric); - }, - }); -} - -async function studentTurn( - input: HarnessInput, - provide: ContextProvider, - round: number, - rubric: string, - feedback: readonly TFeedback[], -): Promise> { - return Object.freeze({ - round, - task: input.task, - rubric, - feedback, - context: await provide(await input.bus.read()), - ...(input.signal === undefined ? {} : { signal: input.signal }), - }); -} - -/** Asks the judge callback and records its verdict on the bus as the judge's - * own message — the same way any actor communicates. */ -async function decide( - input: HarnessInput, - payload: Readonly>, - request: JudgeRequest, -): Promise { - const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); - const context = request.subject === "action" ? request.context : await provide(await input.bus.read()); - const decision = judgeDecision.parse(await input.judge(Object.freeze({ ...request, context }))); - await input.bus.append({ - actor: "judge", - kind: decisionKind, - payload: { subject: request.subject, ...payload, decision }, - }); - return decision; -} - -function result( - outcome: HarnessRun["outcome"], - rounds: HarnessRound[], - rubric: string, -): HarnessRun { - return Object.freeze({ outcome, rounds: Object.freeze([...rounds]), - final: rounds.at(-1) as HarnessRound, rubric }); -} diff --git a/packages/harness/src/memory-store.ts b/packages/harness/src/memory-store.ts new file mode 100644 index 0000000..f02df44 --- /dev/null +++ b/packages/harness/src/memory-store.ts @@ -0,0 +1,15 @@ +import type { AgentBusStore } from "./bus.js"; +import type { AgentBusEntry } from "./schema.js"; + +/** Volatile in-memory storage: the default when a bus gets no store. */ +export class MemoryBusStore implements AgentBusStore { + readonly #entries: AgentBusEntry[] = []; + + async append(entry: AgentBusEntry): Promise { + this.#entries.push(entry); + } + + async load(): Promise { + return [...this.#entries]; + } +} diff --git a/packages/harness/test/harness.test.ts b/packages/harness/test/harness.test.ts index f16fece..b1527df 100644 --- a/packages/harness/test/harness.test.ts +++ b/packages/harness/test/harness.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import { AgentActionDeniedError, - createFileBusStore, + FileBusStore, createHarnessPolicy, defineTrainingHarness, dispatchAction, @@ -166,9 +166,9 @@ describe("training harness", () => { it("continues sequence numbers and recovers an incomplete trailing entry", async () => { const directory = await mkdtemp(join(tmpdir(), "ts-autocode-bus-recovery-")); const file = join(directory, "actions.jsonl"); - const first = new WriteAheadAgentBus({ store: createFileBusStore(file) }); + const first = new WriteAheadAgentBus({ store: new FileBusStore(file) }); await dispatchAction(first, "student", "first", {}, () => "pass", async () => "one"); - const second = new WriteAheadAgentBus({ store: createFileBusStore(file) }); + const second = new WriteAheadAgentBus({ store: new FileBusStore(file) }); await dispatchAction(second, "teacher", "second", {}, () => "pass", async () => "two"); await appendFile(file, "{\"incomplete\"", "utf8"); expect((await second.read()).map(({ sequence }) => sequence)).toEqual([1, 2, 3, 4, 5, 6]); diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 2de4801..b01500d 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -57,5 +57,5 @@ export type { TrainableEvalRun } from "./evaluation.js"; export { defaultPromotionGates, evaluatePromotionGate } from "./promotion.js"; export type { PromotionDecision, PromotionGate, PromotionGateContext, PromotionGateInput } from "./promotion.js"; -export { createMemoryTrainingStore } from "./records.js"; +export { MemoryTrainingStore } from "./records.js"; export type { TrainingRecord, TrainingStore } from "./records.js"; diff --git a/packages/training/src/records.ts b/packages/training/src/records.ts index f928bc0..610d16a 100644 --- a/packages/training/src/records.ts +++ b/packages/training/src/records.ts @@ -18,16 +18,17 @@ export interface TrainingStore { list(trainableId?: TrainableId): Promise; } -export function createMemoryTrainingStore(): TrainingStore { - const records: TrainingRecord[] = []; - return { - async append(record) { - records.push(structuredClone(record)); - }, - async list(trainableId) { - return structuredClone( - trainableId === undefined ? records : records.filter((record) => record.trainableId === trainableId), - ); - }, - }; +/** Volatile in-memory storage: the default when a runtime gets no store. */ +export class MemoryTrainingStore implements TrainingStore { + readonly #records: TrainingRecord[] = []; + + async append(record: TrainingRecord): Promise { + this.#records.push(structuredClone(record)); + } + + async list(trainableId?: TrainableId): Promise { + return structuredClone( + trainableId === undefined ? this.#records : this.#records.filter((record) => record.trainableId === trainableId), + ); + } } diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index c4c39b2..3d9df47 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -16,7 +16,7 @@ import { import { evaluateTrainable, type TrainableEvalRun } from "./evaluation.js"; import { sequentialLoop, type TrainingLoop, type TrainingRound } from "./loop.js"; import { evaluatePromotionGate, type PromotionDecision, type PromotionGate } from "./promotion.js"; -import { createMemoryTrainingStore, type TrainingRecord, type TrainingStore } from "./records.js"; +import { MemoryTrainingStore, type TrainingRecord, type TrainingStore } from "./records.js"; import { findTrainable, type SourceSettings, @@ -163,7 +163,7 @@ class TrainingRuntime implements Training { constructor(settings: TrainingSettings) { this.#settings = settings; this.#variables = Object.freeze({ ...settings.variables }); - this.#store = settings.store ?? createMemoryTrainingStore(); + this.#store = settings.store ?? new MemoryTrainingStore(); this.#tracer = settings.tracing?.tracer ?? trace.getTracer(tracerName); } diff --git a/src/index.ts b/src/index.ts index dd508d4..2d1f796 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,8 +18,9 @@ provideTrainingDefaults({ }); configureRewriteCapture(); -export { createHarnessLoop, defaultActionLogFile, defaultContextWindow, windowedContext } from "./providers/harness.js"; +export { createHarnessLoop, defaultActionLogFile } from "./providers/harness.js"; export type { HarnessLoopOptions } from "./providers/harness.js"; +export { defaultContextWindow, windowedContext } from "./providers/context.js"; export { configureRewriteCapture, rewritePromotion } from "./providers/rewrite.js"; export { instrumentTrainable, trainable, wrapTrainable } from "./instrumentation.js"; export type { TrainableDecorator } from "./instrumentation.js"; @@ -27,7 +28,7 @@ export type { TrainableDecorator } from "./instrumentation.js"; export { captureTrainable, configureTraining, - createMemoryTrainingStore, + MemoryTrainingStore, defaultEvolution, defaultObjective, defaultOutputDir, diff --git a/src/providers/context.ts b/src/providers/context.ts new file mode 100644 index 0000000..609a746 --- /dev/null +++ b/src/providers/context.ts @@ -0,0 +1,17 @@ +import type { ContextProvider } from "ts-autocode-harness"; +import { z } from "zod"; + +/** How many trailing bus entries the default context provider keeps. */ +export const defaultContextWindow = 100; + +const contextWindow = z.number().int().min(0, "context window must be a non-negative integer"); + +/** Rolling-window context: actors see the trailing `limit` bus entries (zero + * means none). The bus does no context management, so optimization lives here + * — a consumer needing more than a window (rolling summaries in the style of + * Semantic Kernel's chat-history reduction, relevance filtering, ...) + * substitutes its own ContextProvider. */ +export function windowedContext(limit = defaultContextWindow): ContextProvider { + const window = contextWindow.parse(limit); + return (entries) => entries.slice(Math.max(entries.length - window, 0)); +} diff --git a/src/providers/harness.ts b/src/providers/harness.ts index ff23e76..1904cc9 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -1,36 +1,22 @@ import { resolve } from "node:path"; import { - createFileBusStore, defineTrainingHarness, + FileBusStore, WriteAheadAgentBus, type ContextProvider, type JudgeRequest, } from "ts-autocode-harness"; -import { z } from "zod"; import type { CandidatePatch, CandidateReview, TrainingLoop } from "ts-autocode-training"; +import { windowedContext } from "./context.js"; + type Request = JudgeRequest; /** Where the write-ahead action log lands inside the run's output directory * when `createHarnessLoop` is not given a filename. */ export const defaultActionLogFile = "harness-actions.jsonl"; -/** How many trailing bus entries the default context provider keeps. */ -export const defaultContextWindow = 100; - -const contextWindow = z.number().int().min(0, "context window must be a non-negative integer"); - -/** Rolling-window context: actors see the trailing `limit` bus entries (zero - * means none). The bus does no context management, so optimization lives here - * — a consumer needing more than a window (rolling summaries in the style of - * Semantic Kernel's chat-history reduction, relevance filtering, ...) - * substitutes its own ContextProvider. */ -export function windowedContext(limit = defaultContextWindow): ContextProvider { - const window = contextWindow.parse(limit); - return (entries) => entries.slice(Math.max(entries.length - window, 0)); -} - export interface HarnessLoopOptions { readonly actionLogFile?: string; /** Context management for harness actors; a rolling window when unset. */ @@ -49,7 +35,7 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo candidateId: (candidate) => candidate.id, ...(input.maxRounds === undefined ? {} : { maxRounds: input.maxRounds }), }); - const bus = new WriteAheadAgentBus({ store: createFileBusStore(resolve(input.outputDir, actionLogFile)) }); + const bus = new WriteAheadAgentBus({ store: new FileBusStore(resolve(input.outputDir, actionLogFile)) }); const result = await harness.run({ bus, contextProvider, From f2af9fed6f2bbfb71606064bb4fdf796f57021ea Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 19:52:03 +0000 Subject: [PATCH 2/4] Harness role defaults and fully injectable harness loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only student and teacher are required to run the harness. Every other role defaults under one evidence convention (feedback is the verdict): the judge passes a candidate when the teacher reports no feedback and lets a challenge stand when the adversary reports evidence; a missing adversary accepts passing candidates; rubric revision appends the challenge evidence as new criteria; the bus defaults to memory and is returned on the run result for auditing. The adversary now reports AdversaryResult { challenge, feedback }, mirroring TeacherResult. createHarnessLoop no longer hardcodes any collaborator: the bus is built by an injectable factory (file-backed write-ahead log only as the default), the judge is injectable, and the training-specific judge/reviseRubric/candidateId boilerplate is deleted outright — training promotes exactly when a review reports no gate failures, so the harness defaults are equivalent by construction. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NG6fSKqgt8nD7JnEY9q245 --- README.md | 24 ++-- packages/harness/README.md | 38 +++++- packages/harness/src/harness.ts | 177 +++++++++++++++----------- packages/harness/src/index.ts | 1 + packages/harness/test/harness.test.ts | 54 ++++++-- src/providers/harness.ts | 59 ++++----- 6 files changed, 222 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index 90eaa51..ea4d863 100644 --- a/README.md +++ b/README.md @@ -203,13 +203,13 @@ configured promotion policy. Training rounds run through the provider-neutral `TrainingLoop` contract. This package registers `createHarnessLoop()` as the default, so `ts-autocode-harness` owns bounded rounds, feedback, cancellation, and stall -detection: the same callback path accepts arbitrary judge inputs, requires an -exact pass/fail decision, tests -approved candidates with an isolated adversary, and makes the teacher revise -the rubric when the adversary exposes an accepted gap. Baseline results are -never treated as proof that a rewrite passes. Set `TrainingSettings.loop` to -substitute your own orchestration; the lower-level `evaluate` and -`ts-autocode-rewrite` promotion primitives also remain available. +detection. Training reviews serve as the harness's evidence: a candidate +passes exactly when its review reports no gate failures, accepted candidates +are re-reviewed by an isolated adversary, and a standing challenge tightens +the rubric before the next round. Baseline results are never treated as proof +that a rewrite passes. Set `TrainingSettings.loop` to substitute your own +orchestration; the lower-level `evaluate` and `ts-autocode-rewrite` +promotion primitives also remain available. The built-in loop is an observable round sequence (`trainingRounds()` pushes each reviewed round to a subscriber; `sequentialLoop` collects the @@ -235,10 +235,12 @@ Runtime dependencies enter through `TrainingSettings`: is not the desired project. - `outputDir` relocates run artifacts and eval output (default `.agentv`, exported as `defaultOutputDir`); a run's `EvalConfig.outputDir` still - overrides it. `createHarnessLoop({ actionLogFile, contextProvider })` - renames the write-ahead action log inside that directory and replaces the - default rolling-window context management (`windowedContext`) with your own - provider — for example a rolling-summary reducer. + overrides it. Every `createHarnessLoop` collaborator is injectable: + `bus` builds the run's message bus (replacing the default file-backed + write-ahead log named by `actionLogFile` inside that directory), `judge` + gates every harness action and verdict, and `contextProvider` replaces the + default rolling-window context management (`windowedContext`) — for example + with a rolling-summary reducer. AgentV's `workers` option parallelizes live-trace and candidate evals. Independent trainables can be trained concurrently by the application, while the configured diff --git a/packages/harness/README.md b/packages/harness/README.md index d306f63..d6cd69b 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -3,12 +3,19 @@ A policy-enforced code-training harness: a bounded callback loop, a durable agent message bus, and an MXC-sandboxed execution backend. -The harness coordinates four callbacks a consumer supplies: +The harness coordinates callbacks the consumer supplies. Only two are +required: - **student** proposes a candidate from the rubric, teacher feedback, and recent bus history; -- **teacher** assesses objective evidence and revises the rubric when an adversarial challenge exposes a gap; -- **judge** accepts any input, returns only `pass` or `fail`, and never supplies rejection feedback; -- **adversary** receives only the artifact under test and its own prior messages, so it has no knowledge of the training loop. +- **teacher** assesses objective evidence and reports feedback against the candidate. + +Every other role has a default, and all defaults follow one evidence +convention — feedback is the verdict: + +- **judge** accepts any input and returns only `pass` or `fail`. Unset, a candidate passes when the teacher reports no feedback, a challenge stands when the adversary reports evidence, and actions are logged ungated. +- **adversary** receives only the artifact under test and its own prior messages, and reports `{ challenge, feedback }`. Unset, a passing candidate is accepted without adversarial review. +- **reviseRubric** tightens the rubric after a standing challenge. Unset, the challenge evidence is appended as new criteria. +- **bus** defaults to an in-memory write-ahead bus, returned on the run result for auditing. The harness does not create, configure, or select agents — no models, prompts, or agent frameworks appear in its API. Callbacks are the whole contract: bring @@ -22,6 +29,22 @@ npm install ts-autocode-harness ## Run the loop +The minimal loop is two callbacks: + +```ts +import { defineTrainingHarness } from "ts-autocode-harness"; + +const result = await defineTrainingHarness().run({ + task: { objective, target }, + rubric: "The candidate must pass AgentV and preserve its public contract.", + student: myStudent, + teacher: myTeacher, +}); +``` + +Every default is replaceable — a durable bus, a gating judge, an adversary, +and a bespoke rubric revision: + ```ts import { join } from "node:path"; import { defineTrainingHarness, FileBusStore, WriteAheadAgentBus } from "ts-autocode-harness"; @@ -46,8 +69,8 @@ const result = await harness.run({ The judge first evaluates the candidate. A `fail` carries no judge feedback; the next student turn receives only teacher feedback. A passing candidate is challenged by the adversary. The candidate is accepted only when that -challenge fails. If the challenge passes, the teacher must revise the rubric -before the next round. +challenge fails. If the challenge stands, the rubric must be revised before +the next round. ## The message bus @@ -84,7 +107,8 @@ kind, payload, gate, execute)`: `defineTrainingHarness` dispatches every student, teacher, and adversary invocation through this convention with the run's judge callback as the gate. -Without a gate, `dispatchAction` still records intent and outcome. +Without a configured judge, `dispatchAction` still records intent and outcome, +and the evidence convention's verdicts are appended the same way. ## Sandboxed execution diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts index 8b99c9d..e056bd9 100644 --- a/packages/harness/src/harness.ts +++ b/packages/harness/src/harness.ts @@ -1,4 +1,4 @@ -import { type WriteAheadAgentBus } from "./bus.js"; +import { WriteAheadAgentBus } from "./bus.js"; import { dispatchAction, decisionKind, type ActionGate } from "./dispatch.js"; import { candidateKey, judgeDecision, roundLimit, rubricText, type AgentBusEntry, type JudgeDecision } from "./schema.js"; @@ -24,6 +24,14 @@ export interface TeacherResult { readonly feedback: readonly TFeedback[]; } +/** What the adversary reports back: the challenge artifact plus the evidence + * it gathered against the candidate. Mirrors `TeacherResult`, and the feedback + * is what the default judge weighs — a challenge without evidence fails. */ +export interface AdversaryResult { + readonly challenge: TChallenge; + readonly feedback: readonly TFeedback[]; +} + export interface RubricRevision { readonly rubric: string; readonly feedback: readonly TFeedback[]; @@ -48,10 +56,15 @@ export interface HarnessRun { readonly rounds: readonly HarnessRound[]; readonly final: HarnessRound; readonly rubric: string; + /** The run's message bus — the full audit log, even when defaulted. */ + readonly bus: WriteAheadAgentBus; } +/** A run needs only a student and a teacher; every other role has a default. + * The defaults follow one evidence convention: feedback is the verdict. A + * candidate passes when the teacher reports no feedback, and an adversarial + * challenge stands when the adversary reports evidence against the candidate. */ export interface HarnessInput { - readonly bus: WriteAheadAgentBus; readonly task: unknown; readonly rubric: string; readonly student: (turn: StudentTurn) => TCandidate | Promise; @@ -59,13 +72,21 @@ export interface HarnessInput { candidate: TCandidate, turn: StudentTurn, ) => TeacherResult | Promise>; - readonly judge: (input: unknown) => JudgeDecision | Promise; - readonly adversary: ( + /** Message log for the run; an in-memory write-ahead bus when unset. */ + readonly bus?: WriteAheadAgentBus; + /** Gates every action and verdict. When unset, actions are logged ungated + * and verdicts follow the evidence convention above. */ + readonly judge?: (input: unknown) => JudgeDecision | Promise; + /** Challenges candidates the judge accepted. When unset, a passing + * candidate is accepted without adversarial review. */ + readonly adversary?: ( candidate: TCandidate, turn: Readonly<{ task: unknown; context: readonly AgentBusEntry[]; signal?: AbortSignal }>, - ) => TChallenge | Promise; - readonly reviseRubric: ( - challenge: TChallenge, + ) => AdversaryResult | Promise>; + /** Revises the rubric after a standing challenge; when unset the challenge + * evidence is appended to the rubric as new criteria. */ + readonly reviseRubric?: ( + challenge: AdversaryResult, turn: Readonly<{ task: unknown; candidate: TCandidate; @@ -82,7 +103,9 @@ export interface HarnessInput { export interface HarnessSettings { readonly maxRounds?: number; - readonly candidateId: (candidate: TCandidate) => string; + /** Candidate identity for stall detection; the candidate's own string form + * when unset, so identical proposals stall without configuration. */ + readonly candidateId?: (candidate: TCandidate) => string; } /** How many student rounds a harness runs when `maxRounds` is unset. */ @@ -93,47 +116,72 @@ export interface TrainingHarness { } export function defineTrainingHarness( - settings: HarnessSettings, + settings: HarnessSettings = {}, ): TrainingHarness { const maxRounds = roundLimit.parse(settings.maxRounds ?? defaultMaxRounds); + const identify = settings.candidateId ?? stringifyCandidate; return Object.freeze({ async run(input: HarnessInput) { + const bus = input.bus ?? new WriteAheadAgentBus(); + const provide = input.contextProvider ?? fullHistory; + const judge = input.judge; + const revise = input.reviseRubric ?? appendCriteria; let rubric: string = rubricText.parse(input.rubric); const rounds: HarnessRound[] = []; let feedback: readonly TFeedback[] = []; let previousCandidate: string | undefined; - const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); - // Every actor invocation is written ahead and gated through the judge - // callback; the judge itself is just one more actor whose verdicts - // land on the bus as ordinary messages. - const gate: ActionGate = async (action, context) => - input.judge(Object.freeze({ subject: "action", action, context: await provide(context) })); + // Every actor invocation is written ahead; a configured judge also + // gates it, and every verdict — the judge's or the evidence + // convention's — lands on the bus as an ordinary judge message. + const gate: ActionGate | undefined = judge === undefined ? undefined : async (action, context) => + judge(Object.freeze({ subject: "action", action, context: await provide(context) })); const dispatch = (actor: string, kind: string, payload: unknown, execute: () => Promise | T) => - dispatchAction(input.bus, actor, kind, payload, gate, execute); + dispatchAction(bus, actor, kind, payload, gate, execute); + const decide = async ( + payload: Readonly>, + request: JudgeRequest, + fallback: () => JudgeDecision, + ): Promise => { + const decision = judge === undefined + ? fallback() + : judgeDecision.parse(await judge(Object.freeze({ ...request, context: await provide(await bus.read()) }))); + await bus.append({ actor: "judge", kind: decisionKind, payload: { subject: request.subject, ...payload, decision } }); + return decision; + }; + const result = (outcome: HarnessRun["outcome"]) => + Object.freeze({ outcome, rounds: Object.freeze([...rounds]), + final: rounds.at(-1) as HarnessRound, rubric, bus }); for (let round = 1; round <= maxRounds; round += 1) { input.signal?.throwIfAborted(); - const turn = await studentTurn(input, provide, round, rubric, feedback); + const turn: StudentTurn = Object.freeze({ + round, + task: input.task, + rubric, + feedback, + context: await provide(await bus.read()), + ...(input.signal === undefined ? {} : { signal: input.signal }), + }); const candidate = await dispatch("student", "agent.propose", { round, task: input.task, rubric, feedback }, () => input.student(turn)); input.signal?.throwIfAborted(); - const candidateId: string = candidateKey.parse(settings.candidateId(candidate)); - if (candidateId === previousCandidate) return result("stalled", rounds, rubric); + const candidateId: string = candidateKey.parse(identify(candidate)); + if (candidateId === previousCandidate) return result("stalled"); previousCandidate = candidateId; const assessment = await dispatch("teacher", "agent.assess", { round, candidateId }, () => input.teacher(candidate, turn)); input.signal?.throwIfAborted(); - const candidateDecision = await decide(input, { candidateId }, { + const candidateDecision = await decide({ candidateId }, { subject: "candidate", task: input.task, candidate, assessment: assessment.assessment, rubric, context: [], - }); + }, () => assessment.feedback.length === 0 ? "pass" : "fail"); if (candidateDecision === "fail") { rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, rubric })); @@ -141,35 +189,41 @@ export function defineTrainingHarness( continue; } - const adversary = await dispatch("adversary", "agent.challenge", { candidateId }, async () => - input.adversary(candidate, { + const adversary = input.adversary; + if (adversary === undefined) { + rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, rubric })); + return result("accepted"); + } + + const challenge = await dispatch("adversary", "agent.challenge", { candidateId }, async () => + adversary(candidate, { task: input.task, - context: await provide(await input.bus.read("adversary")), + context: await provide(await bus.read("adversary")), ...(input.signal === undefined ? {} : { signal: input.signal }), })); input.signal?.throwIfAborted(); - const adversaryDecision = await decide(input, { candidateId }, { + const challengeDecision = await decide({ candidateId }, { subject: "adversary", task: input.task, candidate, - challenge: adversary, + challenge: challenge.challenge, rubric, context: [], - }); + }, () => challenge.feedback.length > 0 ? "pass" : "fail"); - if (adversaryDecision === "fail") { + if (challengeDecision === "fail") { rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, - adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); - return result("accepted", rounds, rubric); + adversary: Object.freeze({ challenge: challenge.challenge, decision: challengeDecision }), rubric })); + return result("accepted"); } const revision = await dispatch("teacher", "agent.revise-rubric", { round, candidateId }, async () => - input.reviseRubric(adversary, { + revise(challenge, { task: input.task, candidate, assessment: assessment.assessment, rubric, - context: await provide(await input.bus.read()), + context: await provide(await bus.read()), ...(input.signal === undefined ? {} : { signal: input.signal }), })); const revised: string = rubricText.parse(revision.rubric); @@ -177,54 +231,31 @@ export function defineTrainingHarness( rubric = revised; feedback = Object.freeze([...revision.feedback]); rounds.push(Object.freeze({ round, candidate, assessment: assessment.assessment, judgeDecision: candidateDecision, - adversary: Object.freeze({ challenge: adversary, decision: adversaryDecision }), rubric })); + adversary: Object.freeze({ challenge: challenge.challenge, decision: challengeDecision }), rubric })); } - return result("exhausted", rounds, rubric); + return result("exhausted"); }, }); } -async function studentTurn( - input: HarnessInput, - provide: ContextProvider, - round: number, - rubric: string, - feedback: readonly TFeedback[], -): Promise> { - return Object.freeze({ - round, - task: input.task, - rubric, - feedback, - context: await provide(await input.bus.read()), - ...(input.signal === undefined ? {} : { signal: input.signal }), - }); -} +const fullHistory: ContextProvider = (entries) => entries; -/** Asks the judge callback and records its verdict on the bus as the judge's - * own message — the same way any actor communicates. */ -async function decide( - input: HarnessInput, - payload: Readonly>, - request: JudgeRequest, -): Promise { - const provide = input.contextProvider ?? ((entries: readonly AgentBusEntry[]) => entries); - const context = request.subject === "action" ? request.context : await provide(await input.bus.read()); - const decision = judgeDecision.parse(await input.judge(Object.freeze({ ...request, context }))); - await input.bus.append({ - actor: "judge", - kind: decisionKind, - payload: { subject: request.subject, ...payload, decision }, - }); - return decision; +/** Default candidate identity: the candidate's own string form, so identical + * proposals are detected as a stall without configuration. */ +function stringifyCandidate(candidate: unknown): string { + return typeof candidate === "string" ? candidate : JSON.stringify(candidate) ?? String(candidate); } -function result( - outcome: HarnessRun["outcome"], - rounds: HarnessRound[], - rubric: string, -): HarnessRun { - return Object.freeze({ outcome, rounds: Object.freeze([...rounds]), - final: rounds.at(-1) as HarnessRound, rubric }); +/** Default rubric revision: a standing challenge's evidence becomes new + * criteria, so the rubric always tightens and the loop cannot re-accept the + * same gap. */ +function appendCriteria( + challenge: AdversaryResult, + turn: Readonly<{ rubric: string }>, +): RubricRevision { + const criteria = challenge.feedback + .map((item) => typeof item === "string" ? item : JSON.stringify(item)) + .join("; "); + return { rubric: `${turn.rubric}\nAdversarial criteria: ${criteria}`, feedback: challenge.feedback }; } diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index bbb8425..805748c 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -12,6 +12,7 @@ export type { AbsolutePath, AgentBusEntry, AgentMessage, JudgeDecision } from ". export { defaultMaxRounds, defineTrainingHarness } from "./harness.js"; export type { + AdversaryResult, ContextProvider, HarnessInput, HarnessRound, diff --git a/packages/harness/test/harness.test.ts b/packages/harness/test/harness.test.ts index b1527df..823fb21 100644 --- a/packages/harness/test/harness.test.ts +++ b/packages/harness/test/harness.test.ts @@ -16,11 +16,48 @@ import { } from "../src/index.js"; describe("training harness", () => { + it("runs with only a student and a teacher; defaults supply the rest", async () => { + const harness = defineTrainingHarness(); + const result = await harness.run({ + task: "task", + rubric: "rubric", + student: ({ round }) => `candidate-${round}`, + teacher: (candidate) => ({ assessment: "evidence", feedback: candidate === "candidate-1" ? ["needs work"] : [] }), + }); + + expect(result.outcome).toBe("accepted"); + expect(result.final.round).toBe(2); + expect(result.final.adversary).toBeUndefined(); + // The defaulted in-memory bus still carries the full audit log, + // including evidence-convention verdicts as ordinary judge messages. + const verdicts = await result.bus.read("judge"); + expect(verdicts).toHaveLength(2); + expect(verdicts.every(({ kind }) => kind === "agent.decision")).toBe(true); + }); + + it("applies the evidence convention when no judge is configured", async () => { + const evidence = [["breaks on unicode"], []]; + const harness = defineTrainingHarness(); + const result = await harness.run({ + task: "task", + rubric: "Initial rubric", + student: ({ round }) => `candidate-${round}`, + teacher: () => ({ assessment: "evidence", feedback: [] }), + adversary: () => ({ challenge: "challenge", feedback: evidence.shift() ?? [] }), + }); + + // Round one's challenge stood and tightened the rubric by default; + // round two's challenge found nothing, so the candidate was accepted. + expect(result.outcome).toBe("accepted"); + expect(result.final.round).toBe(2); + expect(result.rounds[0]?.rubric).toBe("Initial rubric\nAdversarial criteria: breaks on unicode"); + }); + it("uses one callback loop for teacher feedback, judge decisions, and adversarial review", async () => { const callbacks = await loopCallbacks(["fail", "pass", "fail"]); const student = vi.fn(({ round }) => `candidate-${round}`); - const adversary = vi.fn(() => "counterexample"); - const harness = defineTrainingHarness({ maxRounds: 2, candidateId: (candidate) => candidate }); + const adversary = vi.fn(() => ({ challenge: "counterexample", feedback: [] })); + const harness = defineTrainingHarness({ maxRounds: 2 }); const result = await harness.run({ ...callbacks, @@ -46,7 +83,7 @@ describe("training harness", () => { it("does not accept when cancellation occurs during the teacher turn", async () => { const callbacks = await loopCallbacks(["pass"]); const controller = new AbortController(); - const harness = defineTrainingHarness({ candidateId: (candidate) => candidate }); + const harness = defineTrainingHarness(); await expect(harness.run({ ...callbacks, @@ -58,8 +95,7 @@ describe("training harness", () => { controller.abort(); return { assessment: null, feedback: [] }; }, - adversary: () => "challenge", - reviseRubric: () => ({ rubric: "revised", feedback: [] }), + adversary: () => ({ challenge: "challenge", feedback: [] }), })).rejects.toThrow(); }); @@ -67,7 +103,7 @@ describe("training harness", () => { const callbacks = await loopCallbacks(["pass", "pass"]); const teacher = vi.fn(() => ({ assessment: "passes", feedback: [] as string[] })); const reviseRubric = vi.fn(() => ({ rubric: "Check tests and adversarial edge cases", feedback: ["handle edge case"] })); - const harness = defineTrainingHarness({ maxRounds: 1, candidateId: (candidate) => candidate }); + const harness = defineTrainingHarness({ maxRounds: 1 }); const result = await harness.run({ ...callbacks, @@ -77,7 +113,7 @@ describe("training harness", () => { teacher, adversary: (_candidate, turn) => { expect(JSON.stringify(turn)).not.toMatch(/teacher|rubric|student/i); - return "edge-case failure"; + return { challenge: "edge-case failure", feedback: ["handle edge case"] }; }, reviseRubric, }); @@ -142,7 +178,7 @@ describe("training harness", () => { it("hands actors provider-shaped context instead of the raw log", async () => { const callbacks = await loopCallbacks(["fail", "pass", "fail"]); const contexts: number[] = []; - const harness = defineTrainingHarness({ maxRounds: 2, candidateId: (candidate) => candidate }); + const harness = defineTrainingHarness({ maxRounds: 2 }); await harness.run({ ...callbacks, @@ -154,7 +190,7 @@ describe("training harness", () => { return `candidate-${round}`; }, teacher: () => ({ assessment: "evidence", feedback: [] }), - adversary: () => "challenge", + adversary: () => ({ challenge: "challenge", feedback: [] }), reviseRubric: () => ({ rubric: "revised", feedback: [] }), }); diff --git a/src/providers/harness.ts b/src/providers/harness.ts index 1904cc9..f360ba7 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -5,40 +5,47 @@ import { FileBusStore, WriteAheadAgentBus, type ContextProvider, - type JudgeRequest, + type JudgeDecision, } from "ts-autocode-harness"; -import type { CandidatePatch, CandidateReview, TrainingLoop } from "ts-autocode-training"; +import type { CandidatePatch, CandidateReview, TrainingLoop, TrainingLoopInput } from "ts-autocode-training"; import { windowedContext } from "./context.js"; -type Request = JudgeRequest; - -/** Where the write-ahead action log lands inside the run's output directory - * when `createHarnessLoop` is not given a filename. */ +/** Where the default file-backed bus lands inside the run's output directory. */ export const defaultActionLogFile = "harness-actions.jsonl"; +/** Every collaborator is injectable; the options only choose defaults. */ export interface HarnessLoopOptions { + /** Builds the message bus for a run. Unset, each run gets a write-ahead bus + * over a JSONL `FileBusStore` in its output directory — swap in any + * `AgentBusStore`-backed bus (memory, remote, ...) here. */ + readonly bus?: (input: TrainingLoopInput) => WriteAheadAgentBus; + /** File name for the default file-backed bus; ignored when `bus` is set. */ readonly actionLogFile?: string; /** Context management for harness actors; a rolling window when unset. */ readonly contextProvider?: ContextProvider; + /** Gates every harness action and verdict. Unset, the harness's evidence + * convention decides — equivalent here, because training promotes a + * candidate exactly when its review reports no gate failures. */ + readonly judge?: (input: unknown) => JudgeDecision | Promise; } /** Adapts the governed ts-autocode-harness loop to the provider-neutral - * TrainingLoop contract: a write-ahead action bus, an exact pass/fail judge on - * the promotion decision, adversarial re-verification of accepted candidates, - * and rubric revision when a challenge exposes a gap. */ + * TrainingLoop contract. Training reviews serve as every role's evidence: + * the teacher assesses the candidate, the adversary re-reviews an accepted + * one, and the review's gate failures are the feedback the harness weighs. */ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoop { - const actionLogFile = options.actionLogFile ?? defaultActionLogFile; + const createBus = options.bus ?? ((input: TrainingLoopInput) => + new WriteAheadAgentBus({ store: new FileBusStore(resolve(input.outputDir, options.actionLogFile ?? defaultActionLogFile)) })); const contextProvider = options.contextProvider ?? windowedContext(); return async (input) => { - const harness = defineTrainingHarness({ - candidateId: (candidate) => candidate.id, - ...(input.maxRounds === undefined ? {} : { maxRounds: input.maxRounds }), - }); - const bus = new WriteAheadAgentBus({ store: new FileBusStore(resolve(input.outputDir, actionLogFile)) }); + const harness = defineTrainingHarness( + input.maxRounds === undefined ? {} : { maxRounds: input.maxRounds }, + ); const result = await harness.run({ - bus, + bus: createBus(input), contextProvider, + ...(options.judge === undefined ? {} : { judge: options.judge }), task: { trainable: input.trainableId, objective: input.objective }, rubric: input.rubric, ...(input.signal === undefined ? {} : { signal: input.signal }), @@ -52,23 +59,13 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo }); return { assessment: review, feedback: review.decision.failures }; }, - judge: (request) => { - const typed = request as Request; - if (typed.subject === "action") return "pass"; - if (typed.subject === "candidate") return typed.assessment.decision.promote ? "pass" : "fail"; - // The adversary re-verified an accepted candidate; a failed gate means - // the challenge stands and the rubric must be revised. - return typed.challenge.decision.promote ? "fail" : "pass"; - }, - adversary: (candidate, { signal }) => - input.review(candidate, { + adversary: async (candidate, { signal }) => { + const challenge = await input.review(candidate, { label: `adversary-${candidate.id}`, ...(signal === undefined ? {} : { signal }), - }), - reviseRubric: (challenge, { rubric }) => ({ - rubric: `${rubric}\nAdversarial criteria: ${challenge.decision.failures.join("; ")}`, - feedback: challenge.decision.failures, - }), + }); + return { challenge, feedback: challenge.decision.failures }; + }, }); return { outcome: result.outcome === "accepted" ? "ready" : result.outcome, From ef5c844cc47e9ddbe42cf9e56637c0be17de0d71 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 20:05:15 +0000 Subject: [PATCH 3/4] Replace bespoke bus stores with one JSONL store over the standard fs seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MemoryBusStore and FileBusStore reinvented what the ecosystem already standardizes. JsonlBusStore is now the only shipped store: an append-only JSONL log written through BusFileSystem — the slice of the node:fs/promises API it uses, the TypeScript equivalent of C#'s IFileProvider or Python's fsspec. Inject node:fs/promises for disk (the default), a memfs volume's promises for memory (JsonlBusStore.inMemory(), the bus's default store), or any fs-compatible implementation for remote storage. The AgentBusStore seam stays for services that store entries natively. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NG6fSKqgt8nD7JnEY9q245 --- package-lock.json | 497 ++++++++++++++++++++++++++ packages/harness/README.md | 25 +- packages/harness/package.json | 1 + packages/harness/src/bus.ts | 6 +- packages/harness/src/file-store.ts | 68 ---- packages/harness/src/index.ts | 4 +- packages/harness/src/jsonl-store.ts | 99 +++++ packages/harness/src/memory-store.ts | 15 - packages/harness/test/harness.test.ts | 18 +- src/providers/harness.ts | 9 +- 10 files changed, 637 insertions(+), 105 deletions(-) delete mode 100644 packages/harness/src/file-store.ts create mode 100644 packages/harness/src/jsonl-store.ts delete mode 100644 packages/harness/src/memory-store.ts diff --git a/package-lock.json b/package-lock.json index d70ec96..55759cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -756,6 +756,416 @@ "dev": true, "license": "MIT" }, + "node_modules/@jsonjoy.com/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/buffers": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-17.67.0.tgz", + "integrity": "sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/codegen": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-1.0.0.tgz", + "integrity": "sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-core": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-core/-/fs-core-4.64.0.tgz", + "integrity": "sha512-zs2TAq7Six5jgMuoMNjpspAvOP3mhtgq/k1UyQodEzCtQi/N83y2/y+zcvnZSGp/Rxq96DBN+bValOBQAyn/ew==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-fsa/-/fs-fsa-4.64.0.tgz", + "integrity": "sha512-nMWOVbkLFyEgmXZih3wyvxA9XpgyyqyfrINMHvEFqhi7uqfRl7c9ERJt6yX7vgMPrB9Uo+OJO+Spa0cFzPD01w==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "thingies": "^2.5.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-node": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node/-/fs-node-4.64.0.tgz", + "integrity": "sha512-dO+NNkODbUli4uV42bcNrrLvq5rE7SNpdZ5TNd0dtbLsAaNK3MDiIC9lUi+brboGoIjW6vd2fB1qao60nrk5xA==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "glob-to-regex.js": "^1.0.0", + "thingies": "^2.5.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-node-builtins": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-builtins/-/fs-node-builtins-4.64.0.tgz", + "integrity": "sha512-/o7WRFhUWaM/fOrslwLZGnzn4RmRILykn+lAL+mNObqqRNw+CQSiij6hpCeZ+C7buhdoVo7go/OYqzaSUfDYmA==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-node-to-fsa": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-to-fsa/-/fs-node-to-fsa-4.64.0.tgz", + "integrity": "sha512-WDD9WVs0hb7UAEKTgZW2f66WDrbj7gIIWwpP3spbLyXa0rghtUaFTB8L4gdR3ZCWwiKIsj38/CNijpVmpnuPUw==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-node-utils": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-node-utils/-/fs-node-utils-4.64.0.tgz", + "integrity": "sha512-k5Indsx9hWW9xSF7Y6oSKKwtCUNhzZxadub3owhIlitc+iMRVlPPdX2duTKQWBL3qNWpXya8jykgaaWpheeS4w==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "glob-to-regex.js": "^1.0.1" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-print": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-print/-/fs-print-4.64.0.tgz", + "integrity": "sha512-PHZFccchvkhWrwPWHjmVAhbC3vSHCtyZvlZfJJ3ho2bnzl450hXri6/8e6pbkWdH+SkmLXNml0sV8e5HDAfxKw==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-node-utils": "4.64.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/fs-snapshot/-/fs-snapshot-4.64.0.tgz", + "integrity": "sha512-oM7UDeL83q6NBzzsfKAsYKXKVXlykKFqqOLh4xZZKAzzROTlInkPbc6LTDGThEOnPiFiUzA7tYziHG9xavd76Q==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/buffers": "^17.65.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/json-pack": "^17.65.0", + "@jsonjoy.com/util": "^17.65.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/base64": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/base64/-/base64-17.67.0.tgz", + "integrity": "sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/codegen": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/codegen/-/codegen-17.67.0.tgz", + "integrity": "sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/json-pack": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-17.67.0.tgz", + "integrity": "sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/base64": "17.67.0", + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0", + "@jsonjoy.com/json-pointer": "17.67.0", + "@jsonjoy.com/util": "17.67.0", + "hyperdyperid": "^1.2.0", + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/json-pointer": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-17.67.0.tgz", + "integrity": "sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/util": "17.67.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/fs-snapshot/node_modules/@jsonjoy.com/util": { + "version": "17.67.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-17.67.0.tgz", + "integrity": "sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/buffers": "17.67.0", + "@jsonjoy.com/codegen": "17.67.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/json-pack": { + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pack/-/json-pack-1.21.0.tgz", + "integrity": "sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/base64": "^1.1.2", + "@jsonjoy.com/buffers": "^1.2.0", + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/json-pointer": "^1.0.2", + "@jsonjoy.com/util": "^1.9.0", + "hyperdyperid": "^1.2.0", + "thingies": "^2.5.0", + "tree-dump": "^1.1.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/json-pack/node_modules/@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/json-pointer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/json-pointer/-/json-pointer-1.0.2.tgz", + "integrity": "sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/codegen": "^1.0.0", + "@jsonjoy.com/util": "^1.9.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/util": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/util/-/util-1.9.0.tgz", + "integrity": "sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/buffers": "^1.0.0", + "@jsonjoy.com/codegen": "^1.0.0" + }, + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, + "node_modules/@jsonjoy.com/util/node_modules/@jsonjoy.com/buffers": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsonjoy.com/buffers/-/buffers-1.2.1.tgz", + "integrity": "sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, "node_modules/@langchain/core": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.2.tgz", @@ -2276,6 +2686,22 @@ "node": ">= 6" } }, + "node_modules/glob-to-regex.js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/glob-to-regex.js/-/glob-to-regex.js-1.2.0.tgz", + "integrity": "sha512-QMwlOQKU/IzqMUOAZWubUOT8Qft+Y0KQWnX9nK3ch0CJg0tTp4TvGZsTfudYKv2NzoQSyPcnA6TYeIQ3jGichQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, "node_modules/google-auth-library": { "version": "10.9.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", @@ -2328,6 +2754,15 @@ "node": ">= 14" } }, + "node_modules/hyperdyperid": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hyperdyperid/-/hyperdyperid-1.2.0.tgz", + "integrity": "sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==", + "license": "MIT", + "engines": { + "node": ">=10.18" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2766,6 +3201,35 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/memfs": { + "version": "4.64.0", + "resolved": "https://registry.npmjs.org/memfs/-/memfs-4.64.0.tgz", + "integrity": "sha512-Kw72fgY7Wn+sD8KmtNWSafl1dz0UvAsE/PHs3YVfLiaZuA3HxNm9sRLqAu0ATiBGJvME1PxZXbBZPv5GycDeAw==", + "license": "Apache-2.0", + "dependencies": { + "@jsonjoy.com/fs-core": "4.64.0", + "@jsonjoy.com/fs-fsa": "4.64.0", + "@jsonjoy.com/fs-node": "4.64.0", + "@jsonjoy.com/fs-node-builtins": "4.64.0", + "@jsonjoy.com/fs-node-to-fsa": "4.64.0", + "@jsonjoy.com/fs-node-utils": "4.64.0", + "@jsonjoy.com/fs-print": "4.64.0", + "@jsonjoy.com/fs-snapshot": "4.64.0", + "@jsonjoy.com/json-pack": "^1.11.0", + "@jsonjoy.com/util": "^1.9.0", + "glob-to-regex.js": "^1.0.1", + "thingies": "^2.5.0", + "tree-dump": "^1.0.3", + "tslib": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -3228,6 +3692,22 @@ "dev": true, "license": "MIT" }, + "node_modules/thingies": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/thingies/-/thingies-2.6.0.tgz", + "integrity": "sha512-rMHRjmlFLM1R96UYPvpmnc3LYtdFrT33JIB7L9hetGue1qAPfn1N2LJeEjxUSidu1Iku+haLZXDuEXUHNGO/lg==", + "license": "MIT", + "engines": { + "node": ">=10.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "^2" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -3284,6 +3764,22 @@ "node": ">=8.0" } }, + "node_modules/tree-dump": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/tree-dump/-/tree-dump-1.1.0.tgz", + "integrity": "sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/streamich" + }, + "peerDependencies": { + "tslib": "2" + } + }, "node_modules/ts-algebra": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", @@ -3597,6 +4093,7 @@ "dependencies": { "@microsoft/mxc-sdk": "^0.7.0", "deepagents": "^1.10.7", + "memfs": "^4.64.0", "zod": "^4.4.3" }, "engines": { diff --git a/packages/harness/README.md b/packages/harness/README.md index d6cd69b..b68e3a0 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -47,7 +47,7 @@ and a bespoke rubric revision: ```ts import { join } from "node:path"; -import { defineTrainingHarness, FileBusStore, WriteAheadAgentBus } from "ts-autocode-harness"; +import { defineTrainingHarness, JsonlBusStore, WriteAheadAgentBus } from "ts-autocode-harness"; const harness = defineTrainingHarness({ maxRounds: 3, @@ -55,7 +55,7 @@ const harness = defineTrainingHarness({ }); const result = await harness.run({ - bus: new WriteAheadAgentBus({ store: new FileBusStore(join(root, "actions.jsonl")) }), + bus: new WriteAheadAgentBus({ store: new JsonlBusStore(join(root, "actions.jsonl")) }), task: { objective, target }, rubric: "The candidate must pass AgentV and preserve its public contract.", student: myStudent, @@ -80,14 +80,19 @@ identity, ordering, and time, and `read(actor?)` returns the full history. An optional `allow` hook decides whether a given append or read may proceed. Configure `redact` when payloads may contain sensitive application data. -Storage is pluggable through `AgentBusStore` — anything with `append(entry)` -and `load()` works, so entries can live in memory, on disk, or behind a remote -service. Each implementation is its own class in its own module: -`MemoryBusStore` is the default, and `FileBusStore` is the durable JSONL -implementation (fsynced per append, resilient to an incomplete trailing -line). Messages and entries are parsed at the boundary -with zod schemas (`agentMessage`, `agentBusEntry`), so malformed values never -enter the log. +Storage is pluggable at two standard seams. `AgentBusStore` — anything with +`append(entry)` and `load()` — is the entry-level seam for services that store +entries natively (a database, a queue). The shipped implementation is +`JsonlBusStore`: an append-only JSONL log (synced per append, resilient to an +incomplete trailing line) written through `BusFileSystem`, the slice of the +standard `node:fs/promises` API it uses. That filesystem seam is the +TypeScript ecosystem's equivalent of C#'s `IFileProvider` or Python's fsspec: +inject `node:fs/promises` for disk (the default), a [memfs](https://www.npmjs.com/package/memfs) +volume's `.promises` for memory (`JsonlBusStore.inMemory()` does exactly +this, and is what a bus uses when given no store), or any compatible +implementation for remote storage. Messages and entries are parsed at the +boundary with zod schemas (`agentMessage`, `agentBusEntry`), so malformed +values never enter the log. The bus does **no context management** — no trailing windows, no truncation. Shaping history into actor context is the consumer's job through diff --git a/packages/harness/package.json b/packages/harness/package.json index 92ee9e4..c4a28bb 100644 --- a/packages/harness/package.json +++ b/packages/harness/package.json @@ -35,6 +35,7 @@ "dependencies": { "@microsoft/mxc-sdk": "^0.7.0", "deepagents": "^1.10.7", + "memfs": "^4.64.0", "zod": "^4.4.3" }, "keywords": [ diff --git a/packages/harness/src/bus.ts b/packages/harness/src/bus.ts index f4d5f02..d51a298 100644 --- a/packages/harness/src/bus.ts +++ b/packages/harness/src/bus.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; -import { MemoryBusStore } from "./memory-store.js"; +import { JsonlBusStore } from "./jsonl-store.js"; import { agentBusEntry, agentMessage, messageId, type AgentBusEntry, type AgentMessage } from "./schema.js"; /** One requested bus operation, offered to the `allow` hook. */ @@ -21,7 +21,7 @@ export interface AgentBusStore { } export interface AgentBusSettings { - /** Where entries live; volatile in-memory storage when unset. */ + /** Where entries live; a JSONL log on an in-memory filesystem when unset. */ readonly store?: AgentBusStore; readonly idFactory?: () => string; readonly now?: () => Date; @@ -46,7 +46,7 @@ export class WriteAheadAgentBus { #pending: Promise = Promise.resolve(); constructor(settings: AgentBusSettings = {}) { - this.#store = settings.store ?? new MemoryBusStore(); + this.#store = settings.store ?? JsonlBusStore.inMemory(); this.#idFactory = settings.idFactory ?? randomUUID; this.#now = settings.now ?? (() => new Date()); this.#redact = settings.redact ?? ((value) => value); diff --git a/packages/harness/src/file-store.ts b/packages/harness/src/file-store.ts deleted file mode 100644 index d9eef9f..0000000 --- a/packages/harness/src/file-store.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { mkdir, open, readFile } from "node:fs/promises"; -import { dirname } from "node:path"; - -import type { AgentBusStore } from "./bus.js"; -import { absolutePath, agentBusEntry, type AbsolutePath, type AgentBusEntry } from "./schema.js"; - -/** Durable JSONL storage, fsynced per append. On load, an incomplete trailing - * JSON fragment (a crashed writer) is ignored; a complete record that fails - * the entry schema is an error rather than silently accepted. */ -export class FileBusStore implements AgentBusStore { - /** Resolved absolute path of the JSONL log. */ - readonly file: AbsolutePath; - - constructor(file: string) { - this.file = absolutePath.parse(file); - } - - async append(entry: AgentBusEntry): Promise { - await mkdir(dirname(this.file), { recursive: true }); - const handle = await open(this.file, "a"); - try { - await handle.appendFile(`${serialize(entry)}\n`, "utf8"); - await handle.sync(); - } finally { - await handle.close(); - } - } - - async load(): Promise { - let content: string; - try { - content = await readFile(this.file, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; - throw error; - } - return parse(content); - } -} - -function parse(content: string): AgentBusEntry[] { - const lines = content.split("\n"); - const entries: AgentBusEntry[] = []; - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]?.trim(); - if (!line) continue; - let value: unknown; - try { - value = JSON.parse(line); - } catch (error) { - if (index === lines.length - 1) continue; - throw error; - } - entries.push(agentBusEntry.parse(value)); - } - return entries; -} - -function serialize(value: unknown): string { - const seen = new WeakSet(); - return JSON.stringify(value, (_key, item: unknown) => { - if (typeof item === "bigint") return item.toString(); - if (!item || typeof item !== "object") return item; - if (seen.has(item)) return "[Circular]"; - seen.add(item); - return item; - }); -} diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 805748c..ec35b38 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -1,8 +1,8 @@ export { WriteAheadAgentBus } from "./bus.js"; export type { AgentBusAccess, AgentBusSettings, AgentBusStore } from "./bus.js"; -export { MemoryBusStore } from "./memory-store.js"; -export { FileBusStore } from "./file-store.js"; +export { JsonlBusStore } from "./jsonl-store.js"; +export type { BusFileSystem } from "./jsonl-store.js"; export { AgentActionDeniedError, decisionKind, dispatchAction } from "./dispatch.js"; export type { ActionGate } from "./dispatch.js"; diff --git a/packages/harness/src/jsonl-store.ts b/packages/harness/src/jsonl-store.ts new file mode 100644 index 0000000..ac711fc --- /dev/null +++ b/packages/harness/src/jsonl-store.ts @@ -0,0 +1,99 @@ +import { mkdir, open, readFile } from "node:fs/promises"; +import { dirname } from "node:path"; + +import { Volume } from "memfs"; + +import type { AgentBusStore } from "./bus.js"; +import { absolutePath, agentBusEntry, type AbsolutePath, type AgentBusEntry } from "./schema.js"; + +/** The slice of `node:fs/promises` the store writes through — the TypeScript + * ecosystem's standard filesystem seam, in the spirit of C#'s `IFileProvider` + * or Python's fsspec. `node:fs/promises` itself, a memfs volume's `.promises`, + * or any compatible implementation (ZenFS and friends for remote backends) + * plugs in unchanged. */ +export interface BusFileSystem { + mkdir(path: string, options: { readonly recursive: true }): Promise; + readFile(path: string, encoding: "utf8"): Promise; + open(path: string, flags: "a"): Promise<{ + appendFile(data: string, encoding: "utf8"): Promise; + /** Flush to durable storage; optional because purely virtual + * filesystems (memfs types, notably) have nothing to flush. */ + sync?(): Promise; + close(): Promise; + }>; +} + +const localFileSystem: BusFileSystem = { mkdir, open, readFile }; + +/** Append-only JSONL storage over an injected filesystem — as durable as that + * filesystem is, since each append goes through a synced handle. On load, an + * incomplete trailing JSON fragment (a crashed writer) is ignored; a complete + * record that fails the entry schema is an error rather than silently + * accepted. */ +export class JsonlBusStore implements AgentBusStore { + /** Resolved absolute path of the JSONL log. */ + readonly file: AbsolutePath; + readonly #filesystem: BusFileSystem; + + constructor(file: string, filesystem: BusFileSystem = localFileSystem) { + this.file = absolutePath.parse(file); + this.#filesystem = filesystem; + } + + /** A store over a fresh in-memory filesystem (a memfs volume): what a bus + * uses when it is given no store. */ + static inMemory(file = "/agent-bus.jsonl"): JsonlBusStore { + return new JsonlBusStore(file, new Volume().promises); + } + + async append(entry: AgentBusEntry): Promise { + await this.#filesystem.mkdir(dirname(this.file), { recursive: true }); + const handle = await this.#filesystem.open(this.file, "a"); + try { + await handle.appendFile(`${serialize(entry)}\n`, "utf8"); + await handle.sync?.(); + } finally { + await handle.close(); + } + } + + async load(): Promise { + let content: string | Uint8Array; + try { + content = await this.#filesystem.readFile(this.file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + return parse(typeof content === "string" ? content : new TextDecoder().decode(content)); + } +} + +function parse(content: string): AgentBusEntry[] { + const lines = content.split("\n"); + const entries: AgentBusEntry[] = []; + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]?.trim(); + if (!line) continue; + let value: unknown; + try { + value = JSON.parse(line); + } catch (error) { + if (index === lines.length - 1) continue; + throw error; + } + entries.push(agentBusEntry.parse(value)); + } + return entries; +} + +function serialize(value: unknown): string { + const seen = new WeakSet(); + return JSON.stringify(value, (_key, item: unknown) => { + if (typeof item === "bigint") return item.toString(); + if (!item || typeof item !== "object") return item; + if (seen.has(item)) return "[Circular]"; + seen.add(item); + return item; + }); +} diff --git a/packages/harness/src/memory-store.ts b/packages/harness/src/memory-store.ts deleted file mode 100644 index f02df44..0000000 --- a/packages/harness/src/memory-store.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { AgentBusStore } from "./bus.js"; -import type { AgentBusEntry } from "./schema.js"; - -/** Volatile in-memory storage: the default when a bus gets no store. */ -export class MemoryBusStore implements AgentBusStore { - readonly #entries: AgentBusEntry[] = []; - - async append(entry: AgentBusEntry): Promise { - this.#entries.push(entry); - } - - async load(): Promise { - return [...this.#entries]; - } -} diff --git a/packages/harness/test/harness.test.ts b/packages/harness/test/harness.test.ts index 823fb21..bd460d0 100644 --- a/packages/harness/test/harness.test.ts +++ b/packages/harness/test/harness.test.ts @@ -4,12 +4,13 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { Volume } from "memfs"; import { AgentActionDeniedError, - FileBusStore, createHarnessPolicy, defineTrainingHarness, dispatchAction, + JsonlBusStore, MxcSandbox, WriteAheadAgentBus, type AgentBusEntry, @@ -202,14 +203,25 @@ describe("training harness", () => { it("continues sequence numbers and recovers an incomplete trailing entry", async () => { const directory = await mkdtemp(join(tmpdir(), "ts-autocode-bus-recovery-")); const file = join(directory, "actions.jsonl"); - const first = new WriteAheadAgentBus({ store: new FileBusStore(file) }); + const first = new WriteAheadAgentBus({ store: new JsonlBusStore(file) }); await dispatchAction(first, "student", "first", {}, () => "pass", async () => "one"); - const second = new WriteAheadAgentBus({ store: new FileBusStore(file) }); + const second = new WriteAheadAgentBus({ store: new JsonlBusStore(file) }); await dispatchAction(second, "teacher", "second", {}, () => "pass", async () => "two"); await appendFile(file, "{\"incomplete\"", "utf8"); expect((await second.read()).map(({ sequence }) => sequence)).toEqual([1, 2, 3, 4, 5, 6]); }); + it("runs the same store over any filesystem — a memfs volume standing in for disk", async () => { + const volume = new Volume(); + const first = new WriteAheadAgentBus({ store: new JsonlBusStore("/bus/actions.jsonl", volume.promises) }); + await first.append({ actor: "student", kind: "test.first" }); + // A second bus over the same volume resumes where the first left off. + const second = new WriteAheadAgentBus({ store: new JsonlBusStore("/bus/actions.jsonl", volume.promises) }); + const appended = await second.append({ actor: "teacher", kind: "test.second" }); + expect(appended.sequence).toBe(2); + expect(volume.toJSON()["/bus/actions.jsonl"]).toContain("test.first"); + }); + it("gates sandbox file actions and keeps the bus outside writable workspaces", async () => { const workspace = await mkdtemp(join(tmpdir(), "ts-autocode-sandbox-")); const { bus } = await newBus(); diff --git a/src/providers/harness.ts b/src/providers/harness.ts index f360ba7..07e5323 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -2,7 +2,7 @@ import { resolve } from "node:path"; import { defineTrainingHarness, - FileBusStore, + JsonlBusStore, WriteAheadAgentBus, type ContextProvider, type JudgeDecision, @@ -17,8 +17,9 @@ export const defaultActionLogFile = "harness-actions.jsonl"; /** Every collaborator is injectable; the options only choose defaults. */ export interface HarnessLoopOptions { /** Builds the message bus for a run. Unset, each run gets a write-ahead bus - * over a JSONL `FileBusStore` in its output directory — swap in any - * `AgentBusStore`-backed bus (memory, remote, ...) here. */ + * over a `JsonlBusStore` in its output directory on the local filesystem — + * swap in any `AgentBusStore`-backed bus, or the same store over another + * `BusFileSystem` (memfs, a remote filesystem, ...), here. */ readonly bus?: (input: TrainingLoopInput) => WriteAheadAgentBus; /** File name for the default file-backed bus; ignored when `bus` is set. */ readonly actionLogFile?: string; @@ -36,7 +37,7 @@ export interface HarnessLoopOptions { * one, and the review's gate failures are the feedback the harness weighs. */ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoop { const createBus = options.bus ?? ((input: TrainingLoopInput) => - new WriteAheadAgentBus({ store: new FileBusStore(resolve(input.outputDir, options.actionLogFile ?? defaultActionLogFile)) })); + new WriteAheadAgentBus({ store: new JsonlBusStore(resolve(input.outputDir, options.actionLogFile ?? defaultActionLogFile)) })); const contextProvider = options.contextProvider ?? windowedContext(); return async (input) => { const harness = defineTrainingHarness( From be2340357b7f659fd6310def9eeb8ea8ad6ede20 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 12 Jul 2026 20:18:07 +0000 Subject: [PATCH 4/4] Address review: evidence in judge requests, cancellation after revision Candidate and adversary judge requests now carry the same feedback the default verdicts weigh, and the judge callback is typed with the JudgeRequest union instead of unknown, so a configured judge can apply the evidence convention even with windowed or redacted bus context. The rubric-revision path checks the abort signal after its dispatch like every other actor call, and the root README qualifies the evidence policy as the default a configured judge may override. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NG6fSKqgt8nD7JnEY9q245 --- README.md | 3 ++- packages/harness/src/harness.ts | 18 +++++++++++++----- src/providers/harness.ts | 5 ++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ea4d863..802eeb5 100644 --- a/README.md +++ b/README.md @@ -203,7 +203,8 @@ configured promotion policy. Training rounds run through the provider-neutral `TrainingLoop` contract. This package registers `createHarnessLoop()` as the default, so `ts-autocode-harness` owns bounded rounds, feedback, cancellation, and stall -detection. Training reviews serve as the harness's evidence: a candidate +detection. By default, training reviews serve as the harness's evidence — a +configured judge may decide differently: a candidate passes exactly when its review reports no gate failures, accepted candidates are re-reviewed by an isolated adversary, and a standing challenge tightens the rubric before the next round. Baseline results are never treated as proof diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts index e056bd9..e2a7362 100644 --- a/packages/harness/src/harness.ts +++ b/packages/harness/src/harness.ts @@ -37,10 +37,13 @@ export interface RubricRevision { readonly feedback: readonly TFeedback[]; } -export type JudgeRequest = +/** Every request carries the evidence the default verdicts weigh, so a + * configured judge can apply the same convention even when its bus context is + * windowed or redacted. */ +export type JudgeRequest = | Readonly<{ subject: "action"; action: AgentBusEntry; context: readonly AgentBusEntry[] }> - | Readonly<{ subject: "candidate"; task: unknown; candidate: TCandidate; assessment: TAssessment; rubric: string; context: readonly AgentBusEntry[] }> - | Readonly<{ subject: "adversary"; task: unknown; candidate: TCandidate; challenge: TChallenge; rubric: string; context: readonly AgentBusEntry[] }>; + | Readonly<{ subject: "candidate"; task: unknown; candidate: TCandidate; assessment: TAssessment; feedback: readonly TFeedback[]; rubric: string; context: readonly AgentBusEntry[] }> + | Readonly<{ subject: "adversary"; task: unknown; candidate: TCandidate; challenge: TChallenge; feedback: readonly TFeedback[]; rubric: string; context: readonly AgentBusEntry[] }>; export interface HarnessRound { readonly round: number; @@ -76,7 +79,9 @@ export interface HarnessInput { readonly bus?: WriteAheadAgentBus; /** Gates every action and verdict. When unset, actions are logged ungated * and verdicts follow the evidence convention above. */ - readonly judge?: (input: unknown) => JudgeDecision | Promise; + readonly judge?: ( + request: JudgeRequest, + ) => JudgeDecision | Promise; /** Challenges candidates the judge accepted. When unset, a passing * candidate is accepted without adversarial review. */ readonly adversary?: ( @@ -141,7 +146,7 @@ export function defineTrainingHarness( dispatchAction(bus, actor, kind, payload, gate, execute); const decide = async ( payload: Readonly>, - request: JudgeRequest, + request: JudgeRequest, fallback: () => JudgeDecision, ): Promise => { const decision = judge === undefined @@ -179,6 +184,7 @@ export function defineTrainingHarness( task: input.task, candidate, assessment: assessment.assessment, + feedback: assessment.feedback, rubric, context: [], }, () => assessment.feedback.length === 0 ? "pass" : "fail"); @@ -207,6 +213,7 @@ export function defineTrainingHarness( task: input.task, candidate, challenge: challenge.challenge, + feedback: challenge.feedback, rubric, context: [], }, () => challenge.feedback.length > 0 ? "pass" : "fail"); @@ -226,6 +233,7 @@ export function defineTrainingHarness( context: await provide(await bus.read()), ...(input.signal === undefined ? {} : { signal: input.signal }), })); + input.signal?.throwIfAborted(); const revised: string = rubricText.parse(revision.rubric); if (revised === rubric) throw new Error("teacher must improve the rubric after an approved adversarial challenge"); rubric = revised; diff --git a/src/providers/harness.ts b/src/providers/harness.ts index 07e5323..f43ca5c 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -6,6 +6,7 @@ import { WriteAheadAgentBus, type ContextProvider, type JudgeDecision, + type JudgeRequest, } from "ts-autocode-harness"; import type { CandidatePatch, CandidateReview, TrainingLoop, TrainingLoopInput } from "ts-autocode-training"; @@ -28,7 +29,9 @@ export interface HarnessLoopOptions { /** Gates every harness action and verdict. Unset, the harness's evidence * convention decides — equivalent here, because training promotes a * candidate exactly when its review reports no gate failures. */ - readonly judge?: (input: unknown) => JudgeDecision | Promise; + readonly judge?: ( + request: JudgeRequest, + ) => JudgeDecision | Promise; } /** Adapts the governed ts-autocode-harness loop to the provider-neutral