From 1d1badabc8609d2161ba9d74383f439e48c40793 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 29 Jul 2026 15:02:35 -0600 Subject: [PATCH 1/2] feat(intelligence): accept directly authored profiles --- docs/api/intelligence.md | 15 +- docs/api/primitive-catalog.md | 2 +- ...ate-agent-improvement-proposal-fixtures.ts | 16 +- src/intelligence/improvement-cycle.ts | 5 +- src/intelligence/improvement-surfaces.ts | 249 +++++++++++++++-- src/intelligence/profile-activation.ts | 52 ++-- .../agent-profile-improvement-proposal.json | 148 +++++----- tests/helpers/profile-improvement-fixture.ts | 118 ++------ tests/improvement-cycle.test.ts | 256 +++++++++++++++++- tests/improvement-surfaces.test.ts | 232 +++++++++++++++- 10 files changed, 847 insertions(+), 246 deletions(-) diff --git a/docs/api/intelligence.md b/docs/api/intelligence.md index 15467a3b..a354ee62 100644 --- a/docs/api/intelligence.md +++ b/docs/api/intelligence.md @@ -3881,7 +3881,7 @@ Usage class for billing. Base-stream tokens bill `'inference'`; every ##### surface -> **surface**: [`AgentImprovementProfileSurface`](#agentimprovementprofilesurface) +> **surface**: [`AgentProfileMeasuredSurface`](#agentprofilemeasuredsurface) *** @@ -3893,7 +3893,7 @@ Usage class for billing. Base-stream tokens bill `'inference'`; every ##### surface -> **surface**: [`AgentImprovementProfileSurface`](#agentimprovementprofilesurface) +> **surface**: [`AgentProfileMeasuredSurface`](#agentprofilemeasuredsurface) *** @@ -4072,10 +4072,11 @@ Agent improvement surfaces delivered as exact `AgentProfileDiff` replacements. ### AGENT\_PROFILE\_MEASURED\_SURFACES -> `const` **AGENT\_PROFILE\_MEASURED\_SURFACES**: readonly \[`"prompt"`, `"skills"`\] +> `const` **AGENT\_PROFILE\_MEASURED\_SURFACES**: readonly \[`"prompt"`, `"skills"`, `"tools"`, `"mcp"`, `"hooks"`, `"subagents"`, `"agent-profile"`\] -Portable profile surfaces eligible for shared measured comparisons. -Other profile settings can contain credentials or executable configuration. +Profile changes eligible for the product-owned measured comparison path. +The six directly deliverable profile surfaces retain their granular labels; +any residual profile axis also adds the complete `agent-profile` surface. ## Functions @@ -4760,7 +4761,7 @@ surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" ### isAgentProfileMeasuredSurface() -> **isAgentProfileMeasuredSurface**(`surface`): surface is "prompt" \| "skills" +> **isAgentProfileMeasuredSurface**(`surface`): surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile" Return whether a surface is eligible for shared profile measurement. @@ -4772,7 +4773,7 @@ Return whether a surface is eligible for shared profile measurement. #### Returns -surface is "prompt" \| "skills" +surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile" *** diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 4fd1de05..5abf0309 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -409,7 +409,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. | `verifyCandidateExecutionEvidence` | function | Recheck one Runtime receipt against its exact signed experiment cell. | | `withIntelligence` | function | Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt | | `AGENT_IMPROVEMENT_PROFILE_SURFACES` | const | Agent improvement surfaces delivered as exact `AgentProfileDiff` replacements. | -| `AGENT_PROFILE_MEASURED_SURFACES` | const | Portable profile surfaces eligible for shared measured comparisons. | +| `AGENT_PROFILE_MEASURED_SURFACES` | const | Profile changes eligible for the product-owned measured comparison path. | | `defaultEffortTier` | const | The default tier when a client declares no effort. `'standard'` turns | | `exactProcessCandidateExperimentExecutionSupport` | const | Candidate surfaces implemented by the neutral exact-process executor. | | `AgentCandidateExperimentCellExecutionError` | class | A failed baseline or candidate cell with its complete Runtime failure result. | diff --git a/scripts/generate-agent-improvement-proposal-fixtures.ts b/scripts/generate-agent-improvement-proposal-fixtures.ts index 4f0a78ab..bd1b82eb 100644 --- a/scripts/generate-agent-improvement-proposal-fixtures.ts +++ b/scripts/generate-agent-improvement-proposal-fixtures.ts @@ -150,18 +150,16 @@ try { fixturePath, `${JSON.stringify(runtime.verifyAgentImprovementProposal(proposal), null, 2)}\n`, ) - const profileFixture = profileFixtures.createProfileImprovementFixture() + const profileFixture = profileFixtures.createProfileImprovementFixture({ + metadata: { + fixture: 'agent-profile-improvement-proposal', + runtimeVersion: packageJson.version, + }, + }) const profileProposal = runtime.createAgentImprovementProposal({ runId: profileFixture.evaluation.provenance.runId, findings: [], - evaluation: { - ...profileFixture.evaluation, - metadata: { - ...(profileFixture.evaluation.metadata ?? {}), - fixture: 'agent-profile-improvement-proposal', - runtimeVersion: packageJson.version, - }, - }, + evaluation: profileFixture.evaluation, now: () => new Date('2026-07-10T01:30:00.000Z'), }) const serializedProfileProposalFixture = formatFixture( diff --git a/src/intelligence/improvement-cycle.ts b/src/intelligence/improvement-cycle.ts index f804e771..fc4c3f80 100644 --- a/src/intelligence/improvement-cycle.ts +++ b/src/intelligence/improvement-cycle.ts @@ -62,7 +62,6 @@ import { agentImprovementSourceSchema, agentProfileImprovementArmSchema, agentProfileImprovementExecutionRefSchema, - agentProfileImprovementMeasuredComparisonSchema, candidateExecutionEvidenceSchema, numbersApproximatelyEqual, } from '@tangle-network/agent-interface' @@ -765,7 +764,7 @@ export async function proposeAgentProfileImprovement +const profileDifferIsExhaustive: MissingProfileDifferField extends never ? true : never = true +void profileDifferIsExhaustive + +type ProfileResourceDifferHandledField = + | 'files' + | 'tools' + | 'skills' + | 'agents' + | 'commands' + | 'instructions' + | 'failOnError' +type MissingProfileResourceDifferField = Exclude< + keyof NonNullable, + ProfileResourceDifferHandledField +> +const profileResourceDifferIsExhaustive: MissingProfileResourceDifferField extends never + ? true + : never = true +void profileResourceDifferIsExhaustive + export interface AgentImprovementTargetProfileDiffOptions { id: string source?: AgentProfileDiff['source'] @@ -355,31 +396,189 @@ export function agentImprovementProfileDiffs( omitUndefinedObjectFields(candidateInput, 'profile improvement candidate'), 'profile improvement candidate', ) - const surfaces = AGENT_PROFILE_MEASURED_SURFACES.filter( - (surface) => - agentImprovementProfileSurfaceDigest(baseline, surface) !== - agentImprovementProfileSurfaceDigest(candidate, surface), - ) - if (surfaces.length === 0) { - throw new Error('profile improvement candidate does not change a deliverable profile surface') - } - const changes = surfaces.flatMap((surface) => - agentImprovementTargetProfileDiffs( - { surface, desiredInput: agentImprovementProfileSurfaceInput(candidate, surface) }, - options, - ), - ) as [AgentProfileDiff, ...AgentProfileDiff[]] - const applied = changes.reduce( + if (canonicalCandidateDigest(baseline) === canonicalCandidateDigest(candidate)) { + throw new Error('profile improvement candidate does not change the profile') + } + const completeChanges = completeAgentProfileReplacementDiffs(baseline, candidate, options) + const applied = completeChanges.reduce( (profile, change) => - applyExactAgentProfileDiff(profile, change, 'profile improvement candidate change'), + applyExactAgentProfileDiff(profile, change, 'complete profile improvement candidate change'), baseline, ) if (canonicalCandidateDigest(applied) !== canonicalCandidateDigest(candidate)) { - throw new Error( - 'profile improvement candidate changes fields that the measured profile contract cannot apply', - ) + throw new Error('complete profile improvement change did not reproduce the candidate') + } + return completeChanges +} + +function completeAgentProfileReplacementDiffs( + baseline: AgentProfile, + candidate: AgentProfile, + options: AgentImprovementTargetProfileDiffOptions, +): [AgentProfileDiff, ...AgentProfileDiff[]] { + const remove: AgentProfileDiffRemoval = {} + const set: AgentProfile = {} + if ( + profileValuesDiffer(baseline.name, candidate.name) || + profileValuesDiffer(baseline.description, candidate.description) || + profileValuesDiffer(baseline.version, candidate.version) + ) { + remove.identity = true + if (candidate.name !== undefined) set.name = candidate.name + if (candidate.description !== undefined) set.description = candidate.description + if (candidate.version !== undefined) set.version = candidate.version } - return changes + if (profileValuesDiffer(baseline.tags, candidate.tags)) { + remove.tags = true + if (candidate.tags !== undefined) set.tags = candidate.tags + } + if (profileValuesDiffer(baseline.prompt, candidate.prompt)) { + remove.prompt = true + if (candidate.prompt !== undefined) set.prompt = candidate.prompt + } + if (profileValuesDiffer(baseline.model, candidate.model)) { + remove.model = true + if (candidate.model !== undefined) set.model = candidate.model + } + if (profileValuesDiffer(baseline.harness, candidate.harness)) { + remove.harness = true + if (candidate.harness !== undefined) set.harness = candidate.harness + } + if (profileValuesDiffer(baseline.permissions, candidate.permissions)) { + remove.permissions = true + if (candidate.permissions !== undefined) set.permissions = candidate.permissions + } + if (profileValuesDiffer(baseline.tools, candidate.tools)) { + remove.tools = true + if (candidate.tools !== undefined) set.tools = candidate.tools + } + if (profileValuesDiffer(baseline.mcp, candidate.mcp)) { + remove.mcp = true + if (candidate.mcp !== undefined) set.mcp = candidate.mcp + } + if (profileValuesDiffer(baseline.connections, candidate.connections)) { + remove.connections = true + if (candidate.connections !== undefined) set.connections = candidate.connections + } + if (profileValuesDiffer(baseline.subagents, candidate.subagents)) { + remove.subagents = true + if (candidate.subagents !== undefined) set.subagents = candidate.subagents + } + replaceChangedProfileResources(baseline.resources, candidate.resources, remove, set) + if (profileValuesDiffer(baseline.hooks, candidate.hooks)) { + remove.hooks = true + if (candidate.hooks !== undefined) set.hooks = candidate.hooks + } + if (profileValuesDiffer(baseline.modes, candidate.modes)) { + remove.modes = true + if (candidate.modes !== undefined) set.modes = candidate.modes + } + if (profileValuesDiffer(baseline.confidential, candidate.confidential)) { + remove.confidential = true + if (candidate.confidential !== undefined) set.confidential = candidate.confidential + } + if (profileValuesDiffer(baseline.metadata, candidate.metadata)) { + remove.metadata = true + if (candidate.metadata !== undefined) set.metadata = candidate.metadata + } + if (profileValuesDiffer(baseline.extensions, candidate.extensions)) { + remove.extensions = true + if (candidate.extensions !== undefined) set.extensions = candidate.extensions + } + if (Object.keys(remove).length === 0) { + throw new Error('complete profile differ found no changed fields') + } + const common = { + kind: 'agent-profile-diff' as const, + ...(options.source ? { source: options.source } : {}), + metadata: { + ...(options.metadata ?? {}), + surface: 'agent-profile', + }, + } + const reset = agentProfileDiffSchema.parse( + defineAgentProfileDiff({ + ...common, + id: `${options.id}:agent-profile:reset`, + title: 'Reset changed agent profile fields', + remove, + }), + ) as AgentProfileDiff + if (Object.keys(set).length === 0) return [reset] + const replacement = agentProfileDiffSchema.parse( + defineAgentProfileDiff({ + ...common, + id: `${options.id}:agent-profile:set`, + title: 'Set changed agent profile fields', + set, + }), + ) as AgentProfileDiff + return [reset, replacement] +} + +function replaceChangedProfileResources( + baseline: AgentProfile['resources'], + candidate: AgentProfile['resources'], + remove: AgentProfileDiffRemoval, + set: AgentProfile, +): void { + if (!profileValuesDiffer(baseline, candidate)) return + const resourceRemoval: Exclude = {} + const resourceSet: NonNullable = {} + let changedSubfields = 0 + + if (profileValuesDiffer(baseline?.files, candidate?.files)) { + changedSubfields += 1 + resourceRemoval.files = true + if (candidate?.files !== undefined) resourceSet.files = candidate.files + } + if (profileValuesDiffer(baseline?.tools, candidate?.tools)) { + changedSubfields += 1 + resourceRemoval.tools = true + if (candidate?.tools !== undefined) resourceSet.tools = candidate.tools + } + if (profileValuesDiffer(baseline?.skills, candidate?.skills)) { + changedSubfields += 1 + resourceRemoval.skills = true + if (candidate?.skills !== undefined) resourceSet.skills = candidate.skills + } + if (profileValuesDiffer(baseline?.agents, candidate?.agents)) { + changedSubfields += 1 + resourceRemoval.agents = true + if (candidate?.agents !== undefined) resourceSet.agents = candidate.agents + } + if (profileValuesDiffer(baseline?.commands, candidate?.commands)) { + changedSubfields += 1 + resourceRemoval.commands = true + if (candidate?.commands !== undefined) resourceSet.commands = candidate.commands + } + if (profileValuesDiffer(baseline?.instructions, candidate?.instructions)) { + changedSubfields += 1 + resourceRemoval.instructions = true + if (candidate?.instructions !== undefined) resourceSet.instructions = candidate.instructions + } + if (profileValuesDiffer(baseline?.failOnError, candidate?.failOnError)) { + changedSubfields += 1 + resourceRemoval.failOnError = true + if (candidate?.failOnError !== undefined) resourceSet.failOnError = candidate.failOnError + } + + if (changedSubfields === 0) { + remove.resources = true + if (candidate !== undefined) set.resources = candidate + return + } + remove.resources = resourceRemoval + if (Object.keys(resourceSet).length > 0) set.resources = resourceSet +} + +function profileValuesDiffer(baseline: unknown, candidate: unknown): boolean { + const comparable = (value: unknown) => + value === undefined ? { present: false } : { present: true, value } + return ( + canonicalCandidateDigest(comparable(baseline)) !== + canonicalCandidateDigest(comparable(candidate)) + ) } function improvementSurfaceReplacement(target: { diff --git a/src/intelligence/profile-activation.ts b/src/intelligence/profile-activation.ts index 6d73b585..f340ab17 100644 --- a/src/intelligence/profile-activation.ts +++ b/src/intelligence/profile-activation.ts @@ -18,10 +18,11 @@ import type { } from './activation' import { type AgentImprovementProfileSurface, + type AgentProfileMeasuredSurface, agentImprovementProfileSurfaceDigest, agentImprovementTargetProfileDiffs, assertProfileImprovementTargetsShareIdentity, - isAgentImprovementProfileSurface, + isAgentProfileMeasuredSurface, } from './improvement-surfaces' export type AgentImprovementProfileActivationTarget = Omit< @@ -31,15 +32,22 @@ export type AgentImprovementProfileActivationTarget = Omit< surface: AgentImprovementProfileSurface } +type AgentProfileImprovementTransitionTarget = Omit< + AgentImprovementActivationTargetPlan, + 'surface' +> & { + surface: AgentProfileMeasuredSurface +} + export type AgentImprovementProfileTargetState = Omit< AgentImprovementActivationTargetState, 'surface' -> & { surface: AgentImprovementProfileSurface } +> & { surface: AgentProfileMeasuredSurface } export type AgentImprovementProfileTargetTransition = Omit< AgentImprovementActivationTargetTransition, 'surface' -> & { surface: AgentImprovementProfileSurface } +> & { surface: AgentProfileMeasuredSurface } export interface AgentImprovementProfileReplacement { identity: string @@ -118,15 +126,17 @@ interface CurrentProfiles { export function prepareAgentImprovementProfileActivation( input: AgentImprovementProfileActivationInput, ): AgentImprovementProfileActivationPreparation { - const targets = - 'profileTransition' in input ? profileTargets(input.profileTransition.targets) : input.targets - assertUniqueProfileTargets(targets) - const current = readCurrentProfiles(input.currentByIdentity, targets) - if ('status' in current) return immutableCandidateValue(current) - if ('profileTransition' in input) { + const targets = profileTargets(input.profileTransition.targets) + assertUniqueProfileTargets(targets) + const current = readCurrentProfiles(input.currentByIdentity, targets) + if ('status' in current) return immutableCandidateValue(current) return prepareProfileImprovementActivation(input, targets, current) } + const targets = input.targets + assertUniqueProfileTargets(targets) + const current = readCurrentProfiles(input.currentByIdentity, targets) + if ('status' in current) return immutableCandidateValue(current) return prepareSurfaceReplacementActivation(targets, current) } @@ -191,8 +201,8 @@ function prepareSurfaceReplacementActivation( function prepareProfileImprovementActivation( input: Extract, targets: readonly [ - AgentImprovementProfileActivationTarget, - ...AgentImprovementProfileActivationTarget[], + AgentProfileImprovementTransitionTarget, + ...AgentProfileImprovementTransitionTarget[], ], currentProfiles: CurrentProfiles, ): AgentImprovementProfileActivationPreparation { @@ -258,7 +268,7 @@ function restoreProfileState( function assertProfileTransitionTargets( transition: ProfileImprovementActivationTransitionInput, - targets: readonly AgentImprovementProfileActivationTarget[], + targets: readonly AgentProfileImprovementTransitionTarget[], ): void { assertProfileImprovementTargetsShareIdentity(targets) const operationDigest = canonicalCandidateDigest(transition.operation) @@ -278,8 +288,8 @@ function appliedProfileReplacement( identities: readonly string[], profiles: ReadonlyMap, targets: readonly [ - AgentImprovementProfileActivationTarget, - ...AgentImprovementProfileActivationTarget[], + AgentProfileImprovementTransitionTarget, + ...AgentProfileImprovementTransitionTarget[], ], ): AgentImprovementProfileActivationPreparation { return immutableCandidateValue({ @@ -300,7 +310,7 @@ function appliedProfileReplacement( function targetStates( current: readonly { - target: AgentImprovementProfileActivationTarget + target: AgentProfileImprovementTransitionTarget currentDigest: Sha256Digest }[], ): [AgentImprovementProfileTargetState, ...AgentImprovementProfileTargetState[]] { @@ -324,7 +334,7 @@ function profileStateDigest( function readCurrentProfiles( currentByIdentity: ReadonlyMap, - targets: readonly AgentImprovementProfileActivationTarget[], + targets: readonly AgentProfileImprovementTransitionTarget[], ): CurrentProfiles | { status: 'missing'; identities: string[] } { const identities = [...new Set(targets.map((target) => target.identity))].sort() const missing = identities.filter((identity) => !currentByIdentity.has(identity)) @@ -344,20 +354,20 @@ function readCurrentProfiles( function profileTargets( targets: readonly AgentImprovementActivationTargetPlan[], -): [AgentImprovementProfileActivationTarget, ...AgentImprovementProfileActivationTarget[]] { - if (!targets.every((target) => isAgentImprovementProfileSurface(target.surface))) { +): [AgentProfileImprovementTransitionTarget, ...AgentProfileImprovementTransitionTarget[]] { + if (!targets.every((target) => isAgentProfileMeasuredSurface(target.surface))) { throw new Error('agent profile activation contains an unsupported surface') } const first = targets[0] if (!first) throw new Error('agent profile activation requires a target') return targets as [ - AgentImprovementProfileActivationTarget, - ...AgentImprovementProfileActivationTarget[], + AgentProfileImprovementTransitionTarget, + ...AgentProfileImprovementTransitionTarget[], ] } function assertUniqueProfileTargets( - targets: readonly AgentImprovementProfileActivationTarget[], + targets: readonly AgentProfileImprovementTransitionTarget[], ): void { const targetKeys = targets.map((target) => `${target.identity}\u0000${target.surface}`) if (new Set(targetKeys).size !== targetKeys.length) { diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 61f9ef3d..76e2d4dc 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,18 +1,38 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:5a6888221983315e661a139d32348538dc054443e03ce2ef7f0cdf6380fb3558", + "digest": "sha256:2d5c13dbf6aa5b331fd581294429e93e4313c41f196aab5bd9e58c0771592006", "evaluation": { "decision": { "contributingChecks": [ { "name": "paired-significance", "passed": true + }, + { + "name": "paired-precision", + "passed": true + }, + { + "name": "all-runs-completed", + "passed": true + }, + { + "name": "candidate-task-pass", + "passed": true + }, + { + "name": "critical-dimensions", + "passed": true + }, + { + "name": "budget", + "passed": true } ], "outcome": "ship", - "reasons": ["paired comparison passed"] + "reasons": ["all measured checks passed"] }, - "diff": "prompt: add source and uncertainty instructions", + "diff": "[{\"id\":\"add-source-and-uncertainty\",\"kind\":\"agent-profile-diff\",\"set\":{\"prompt\":{\"systemPrompt\":\"Answer directly, cite the source, and state uncertainty.\"},\"resources\":{\"skills\":[{\"content\":\"Cite the evidence you use.\",\"kind\":\"inline\",\"name\":\"sources.SKILL.md\"}]}},\"source\":{\"artifacts\":[\"traces://run/profile-improvement-1\"],\"kind\":\"optimizer\"}}]", "evaluation": { "generationsExplored": 1, "measurement": { @@ -165,7 +185,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:53e9a7340ff97286ab016388519ce56ea2f8ba89fd9d02f19b62ccd559c2a0e6", + "digest": "sha256:9fb7c227fa5da24956b7ea5b59dfe7cd33e4cd3aa21c81e1506191da9dc23843", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-0", "executionRef": { @@ -177,7 +197,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -201,7 +221,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 110, @@ -292,7 +312,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:bb83b019f5a68bb0b0ea3dcc4d334aac6cea8f46309957b7d8ec71d6c2aaeb6f", + "digest": "sha256:6caab48474a18ccb40129f298bc498434937ffd5274f98c4f82f5b615087a13c", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-0", "executionRef": { @@ -304,7 +324,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -328,7 +348,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 110, @@ -421,7 +441,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:52d1089da764e5f52b4edf339dbd74b7f419ca704b10f6661633d8e9e220ecb9", + "digest": "sha256:9b56aeae4c91105ad4b8b2c9f0ae0255292f3d62d4d30c7f067c22ed30823024", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-1", "executionRef": { @@ -433,7 +453,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -457,7 +477,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 1110, @@ -548,7 +568,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:29d6ae794827b4a764cad22533f9e6795ac1379dc448341c16e92b55be56fb43", + "digest": "sha256:ec0a2eea90dbccc5b45a49a8108f785da8e7113ac582576d96ec76e5a47b4b22", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-1", "executionRef": { @@ -560,7 +580,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -584,7 +604,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 1110, @@ -677,7 +697,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:5a7186c9b471c4f23d2f3cbccbe7958f8d4c8233fb2ee09ee66c5499cfaffaa1", + "digest": "sha256:81179ac7f44a5fb2d7235090b42d0f4a3323a559e8e223a3455a8351a85376d5", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-2", "executionRef": { @@ -689,7 +709,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -713,7 +733,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 2110, @@ -804,7 +824,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:bef149104e0f613fc1b734ca951c57b2251bbec315ac3937094d98bd1be0621e", + "digest": "sha256:e9435761e04ad9b55f189c4193da92db86d95dbd50c4070b6c39d08d6199f1a6", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-2", "executionRef": { @@ -816,7 +836,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -840,7 +860,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 2110, @@ -933,7 +953,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:930e0e0df73a8be4e8d260c76a6926a425a5226b1363e0bf4beab1ccc44149de", + "digest": "sha256:9814d56f62fe89eb2117c0d601277942dc3b39d1180ca2c4d99ce5eeb993855c", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-3", "executionRef": { @@ -945,7 +965,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -969,7 +989,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 3110, @@ -1060,7 +1080,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:252b13293837d3f1375d3f4ba8ab062a98fff6474eb5a88a10c493c728a35d1b", + "digest": "sha256:779310f3803ac4b11af3953f5ef0e4ed78a28b8623f296890d7bedfc2291241f", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-3", "executionRef": { @@ -1072,7 +1092,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -1096,7 +1116,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 3110, @@ -1189,7 +1209,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:d7fec1bb98d479d00bc1e887ee81b0e221a3e307404388a3f124966a5a0ee604", + "digest": "sha256:cacfbed32553c79cf82eaecb785ca862c08fe3574fb94c8959991deb441e6fef", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-4", "executionRef": { @@ -1201,7 +1221,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -1225,7 +1245,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 4110, @@ -1316,7 +1336,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:1f26865bfd63cb64e333b5c78fa1a9113191ae5ed76d5d918c6b3c0c6b3088bb", + "digest": "sha256:51730613cefb40c07f29ce0fe8c540ed05e7ace01425d0bd2ed5454329265f63", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-4", "executionRef": { @@ -1328,7 +1348,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -1352,7 +1372,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 4110, @@ -1445,7 +1465,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:447f2531ea14a8d81cd083f9136eb874c49f90f968ad760c69d9ea4a482d8d27", + "digest": "sha256:8ccb242c4ea5f495acae43066d834c1e8727e8dd26e87e55a347fc7cfbbce92b", "digestAlgorithm": "rfc8785-sha256", "executionId": "baseline-5", "executionRef": { @@ -1457,7 +1477,7 @@ "dimensions": [ { "name": "quality", - "score": 0.2 + "score": 0 } ], "evidence": { @@ -1481,7 +1501,7 @@ "version": "1" }, "passed": true, - "score": 0.2, + "score": 0, "timing": { "durationMs": 10, "endedAtMs": 5110, @@ -1572,7 +1592,7 @@ "kind": "platform-billing" } ], - "digest": "sha256:25f5a04b59c28ff22cccda9e24e6bfca7b8a20b3d298e85f1f3a514917f762c5", + "digest": "sha256:a310c0e890411558f35d66d9b7b6c8adfce8c9c8f9faa6b8b043de53acbb5f6e", "digestAlgorithm": "rfc8785-sha256", "executionId": "candidate-5", "executionRef": { @@ -1584,7 +1604,7 @@ "dimensions": [ { "name": "quality", - "score": 0.6 + "score": 1 } ], "evidence": { @@ -1608,7 +1628,7 @@ "version": "1" }, "passed": true, - "score": 0.6, + "score": 1, "timing": { "durationMs": 10, "endedAtMs": 5110, @@ -1700,41 +1720,41 @@ "objectives": [ { "availability": "measured", - "baseline": 0.2, - "candidate": 0.6, + "baseline": 0, + "candidate": 1, "confidenceInterval": { "level": 0.95, - "lower": 0.29999999999999993, + "lower": 1, "method": "paired-bootstrap", "resamples": 100, "statistic": "mean", - "upper": 0.5 + "upper": 1 }, - "delta": 0.39999999999999997, + "delta": 1, "direction": "higher-is-better", "kind": "objective", "n": 6, - "name": "quality", + "name": "benchmark-score", "unit": "score" }, { "availability": "measured", - "baseline": 0.2, - "candidate": 0.6, + "baseline": 0, + "candidate": 1, "confidenceInterval": { "level": 0.95, - "lower": 0.29999999999999993, + "lower": 1, "method": "paired-bootstrap", "resamples": 100, "statistic": "mean", - "upper": 0.5 + "upper": 1 }, - "delta": 0.39999999999999997, + "delta": 1, "direction": "higher-is-better", "kind": "dimension", "n": 6, "name": "quality", - "objective": "quality", + "objective": "benchmark-score", "unit": "score" }, { @@ -1743,11 +1763,11 @@ "candidate": 1.1e-7, "confidenceInterval": { "level": 0.95, - "lower": -0.1, + "lower": 0, "method": "paired-bootstrap", "resamples": 100, "statistic": "mean", - "upper": 0.1 + "upper": 0 }, "delta": 0, "direction": "lower-is-better", @@ -1762,11 +1782,11 @@ "candidate": 110, "confidenceInterval": { "level": 0.95, - "lower": -0.1, + "lower": 0, "method": "paired-bootstrap", "resamples": 100, "statistic": "mean", - "upper": 0.1 + "upper": 0 }, "delta": 0, "direction": "lower-is-better", @@ -1777,17 +1797,17 @@ } ], "overall": { - "baseline": 0.2, - "candidate": 0.6, + "baseline": 0, + "candidate": 1, "confidenceInterval": { "level": 0.95, - "lower": 0.29999999999999993, + "lower": 1, "method": "paired-bootstrap", "resamples": 100, "statistic": "mean", - "upper": 0.5 + "upper": 1 }, - "delta": 0.39999999999999997, + "delta": 1, "direction": "higher-is-better", "n": 6, "name": "composite", @@ -1795,9 +1815,9 @@ }, "power": { "confidenceLevel": 0.95, - "minimumDetectableDelta": 0.1, + "minimumDetectableDelta": 0, "n": 6, - "reason": "6 paired held-out runs", + "reason": "observed paired uncertainty gives a minimum detectable delta of 0.000 at 0.95 confidence from 6 runs. The same scorer evaluated both arms, so scorer bias is not measured.", "scaleAssumed": true, "sharedScorerChannel": true, "sufficient": true @@ -1806,9 +1826,9 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:44465d9d1b5b2f7697ad7a4617d7da80ccad97538b55e8a41b016cddc9c66bcf", + "recordDigest": "sha256:18649e2d7221df7654aed639416108172bf5797cc34fa7bdf4b933ecd3ec445d", "runId": "profile-improvement-1", - "schema": "agent-eval/profile-matrix/v1" + "schema": "agent-profile-improvement-experiment" } }, "findings": [], diff --git a/tests/helpers/profile-improvement-fixture.ts b/tests/helpers/profile-improvement-fixture.ts index 01f1cc8c..a2991aeb 100644 --- a/tests/helpers/profile-improvement-fixture.ts +++ b/tests/helpers/profile-improvement-fixture.ts @@ -1,11 +1,15 @@ import { minimumPairsForPairedDeltaTest } from '@tangle-network/agent-eval' -import type { AgentProfileImprovementExperimentExecutionInput } from '@tangle-network/agent-eval/contract' +import { + type AgentProfileImprovementExperimentExecutionInput, + measuredComparisonFromAgentProfileImprovementExperiment, + verifyAgentProfileImprovementExperimentComparison, +} from '@tangle-network/agent-eval/contract' import { type AgentImprovementEvaluation, type AgentProfile, type AgentProfileDiff, + type AgentProfileImprovementMeasuredComparison, type AgentProfileImprovementRunReceipt, - agentProfileImprovementMeasuredComparisonSchema, applyAgentProfileDiff, canonicalCandidateDigest, defineInlineResource, @@ -143,7 +147,9 @@ export interface ProfileImprovementFixture { } /** A valid opaque profile comparison: it contains changes and state hashes, never profiles. */ -export function createProfileImprovementFixture(): ProfileImprovementFixture { +export function createProfileImprovementFixture( + options: { metadata?: AgentProfileImprovementMeasuredComparison['metadata'] } = {}, +): ProfileImprovementFixture { const task = signed({ kind: 'agent-profile-improvement-task' as const, digestAlgorithm: 'rfc8785-sha256' as const, @@ -304,92 +310,16 @@ export function createProfileImprovementFixture(): ProfileImprovementFixture { }) } - const estimate = (baselineValue: number, candidateValue: number) => ({ - baseline: baselineValue, - candidate: candidateValue, - delta: candidateValue - baselineValue, - confidenceInterval: { - level: 0.95, - lower: candidateValue - baselineValue - 0.1, - upper: candidateValue - baselineValue + 0.1, - method: 'paired-bootstrap' as const, - statistic: 'mean' as const, - resamples: 100, - }, - n: pairedRunCount, - }) - const comparison = agentProfileImprovementMeasuredComparisonSchema.parse({ - kind: 'agent-profile-improvement-measured-comparison', - experiment, - measurements: Array.from({ length: pairedRunCount }, (_, repetition) => ({ - baseline: receipt('baseline', repetition, 0.2), - candidate: receipt('candidate', repetition, 0.6), - })), - overall: { - name: 'composite', - ...estimate(0.2, 0.6), - direction: 'higher-is-better', - unit: 'score', - }, - objectives: [ - { - kind: 'objective', - name: 'quality', - direction: 'higher-is-better', - unit: 'score', - availability: 'measured', - ...estimate(0.2, 0.6), - }, - { - kind: 'dimension', - objective: 'quality', - name: 'quality', - direction: 'higher-is-better', - unit: 'score', - availability: 'measured', - ...estimate(0.2, 0.6), - }, - { - kind: 'cost', - name: 'cost', - direction: 'lower-is-better', - unit: 'usd', - availability: 'measured', - ...estimate(0.00000011, 0.00000011), - }, - { - kind: 'latency', - name: 'latency', - direction: 'lower-is-better', - unit: 'milliseconds', - availability: 'measured', - ...estimate(110, 110), - }, - ], - decision: { - outcome: 'ship', - reasons: ['paired comparison passed'], - contributingChecks: [{ name: 'paired-significance', passed: true }], - }, - power: { - sufficient: true, - n: pairedRunCount, - minimumDetectableDelta: 0.1, - confidenceLevel: 0.95, - scaleAssumed: true, - sharedScorerChannel: true, - reason: `${pairedRunCount} paired held-out runs`, - }, - provenance: { - kind: 'agent-eval-loop', - schema: 'agent-eval/profile-matrix/v1', + const measurements = Array.from({ length: pairedRunCount }, (_, repetition) => ({ + baseline: receipt('baseline', repetition, 0), + candidate: receipt('candidate', repetition, 1), + })) + const measurementCostUsd = (pairedRunCount * 2 * 110) / 1_000_000_000 + const comparison = verifyAgentProfileImprovementExperimentComparison( + measuredComparisonFromAgentProfileImprovementExperiment({ + experiment, + measurements, runId: 'profile-improvement-1', - recordDigest: canonicalCandidateDigest({ record: 'profile-improvement-1' }), - baselineContentHash: baseline.stateDigest, - candidateContentHash: candidate.stateDigest, - }, - diff: 'prompt: add source and uncertainty instructions', - evaluation: { generationsExplored: 1, preparation: { wallDurationMs: 0, @@ -397,15 +327,11 @@ export function createProfileImprovementFixture(): ProfileImprovementFixture { }, measurement: { wallDurationMs: 0, - workDurationMs: pairedRunCount * 2 * 110, - cost: { usd: (pairedRunCount * 2 * 110) / 1_000_000_000, provenance: 'observed' as const }, + cost: { usd: measurementCostUsd, provenance: 'observed' as const }, }, - total: { - wallDurationMs: 0, - cost: { usd: (pairedRunCount * 2 * 110) / 1_000_000_000, provenance: 'observed' as const }, - }, - }, - }) + ...(options.metadata ? { metadata: options.metadata } : {}), + }), + ) return { evaluation: comparison, baselineProfile, diff --git a/tests/improvement-cycle.test.ts b/tests/improvement-cycle.test.ts index dd5e293d..01bd33a3 100644 --- a/tests/improvement-cycle.test.ts +++ b/tests/improvement-cycle.test.ts @@ -5,9 +5,14 @@ import { } from '@tangle-network/agent-eval' import { campaignScenarioIdentity } from '@tangle-network/agent-eval/campaign' import { + measuredComparisonFromAgentProfileImprovementExperiment, + runAgentProfileImprovementExperiment, + sealAgentProfileImprovementExperiment, + sealAgentProfileImprovementSuite, sealAgentProfileImprovementTask, sealCandidateBenchmarkSuite, sealCandidateBenchmarkTask, + verifyAgentProfileImprovementExperimentComparison, } from '@tangle-network/agent-eval/contract' import type { AgentProfile } from '@tangle-network/agent-interface' import { afterEach, describe, expect, it } from 'vitest' @@ -39,6 +44,7 @@ import { verifyCandidateExecutionEvidence, } from '../src/intelligence/improvement-cycle' import { + agentImprovementProfileDiffs, agentImprovementTargetDigest, agentImprovementTargetInput, deriveChangedSurfaces, @@ -287,6 +293,193 @@ describe('agent improvement lifecycle', { timeout: 30_000 }, () => { ).toEqual(restoreResult) }) + it('measures and activates a directly authored complete profile without improve()', async () => { + const template = createProfileImprovementFixture() + const task = template.evaluation.experiment.benchmark.tasks[0] + if (!task) throw new Error('expected a profile improvement task') + const identity = 'tenant/research/profile' + const baselineProfile: AgentProfile = { + name: 'researcher', + prompt: { systemPrompt: 'Investigate the question.' }, + model: { default: 'provider/old-model' }, + resources: { + failOnError: true, + skills: [ + { + kind: 'github', + repository: 'owner/research-agents', + path: 'skills/research/SKILL.md', + ref: '0123456789abcdef', + }, + ], + }, + metadata: { lineage: 'baseline' }, + } + const candidateProfile: AgentProfile = { + ...baselineProfile, + model: { default: 'provider/new-model', reasoningEffort: 'high' }, + harness: 'codex', + tools: { Read: true, Bash: true }, + mcp: { literature: { command: 'literature-server' } }, + hooks: { Stop: [{ command: 'node verify-result.mjs', blocking: true }] }, + metadata: { lineage: 'direct-reflection', reflectionRun: 'reflection-1' }, + } + const stateDigest = ({ + identity: profileIdentity, + profile, + }: { + identity: string + profile: AgentProfile + }) => canonicalCandidateDigest({ identity: profileIdentity, profile }) + const baselineStateDigest = stateDigest({ identity, profile: baselineProfile }) + const candidateStateDigest = stateDigest({ identity, profile: candidateProfile }) + const change = agentImprovementProfileDiffs(baselineProfile, candidateProfile, { + id: 'direct-reflection-1', + source: { + kind: 'frontier-author', + artifacts: ['traces://reflection-1'], + notes: ['Candidate authored directly from a trace autopsy.'], + }, + }) + const profileDiffIds = change.map((step) => { + if (!step.id) throw new Error('expected a profile diff id') + return step.id + }) + const reps = minimumPairedRuns + const seeds = Array.from({ length: reps }, (_, index) => 101 + index) as [number, ...number[]] + const benchmark = sealAgentProfileImprovementSuite({ + splitDigest: canonicalCandidateDigest({ split: 'direct-profile-release' }), + tasks: [task], + reps, + seeds, + }) + const executionRef = { + kind: 'agent-profile-improvement-execution-ref' as const, + identity: 'direct-profile-runner', + digest: canonicalCandidateDigest({ runner: 'direct-profile-runner', revision: 1 }), + } + const experiment = sealAgentProfileImprovementExperiment({ + kind: 'agent-profile-improvement-experiment', + digestAlgorithm: 'rfc8785-sha256', + source: { + kind: 'platform-agent-profile', + sourceIdentity: identity, + sourceDigest: baselineStateDigest, + sourceRevision: 1, + }, + executionRef, + baseline: { stateDigest: baselineStateDigest }, + candidate: { stateDigest: candidateStateDigest }, + change, + candidateLineage: { + source: 'optimizer', + parentDigests: [baselineStateDigest], + runIds: ['reflection-1'], + profileDiffIds, + developmentSplitDigest: canonicalCandidateDigest({ split: 'direct-profile-development' }), + }, + benchmark, + policy: { + ...template.evaluation.experiment.policy, + minProductiveRuns: reps, + }, + }) + const profilesByDigest = new Map([ + [baselineStateDigest, baselineProfile], + [candidateStateDigest, candidateProfile], + ]) + const observedProfiles: AgentProfile[] = [] + const run = await runAgentProfileImprovementExperiment({ + experiment, + async execute(input) { + const profile = profilesByDigest.get(input.stateDigest) + if (!profile) throw new Error('executor received an unknown profile state') + observedProfiles.push(profile) + const candidateVariation = ((input.runCell.repetition % 3) - 1) * 0.02 + const score = input.arm === 'baseline' ? 0.2 : 0.8 + candidateVariation + return createProfileImprovementRunReceipt(input, score) + }, + }) + const evaluation = verifyAgentProfileImprovementExperimentComparison( + measuredComparisonFromAgentProfileImprovementExperiment({ + experiment, + measurements: run.measurements, + preparation: { + wallDurationMs: 1, + cost: { usd: 0, provenance: 'observed' }, + }, + measurement: run.measurement, + runId: 'direct-profile-improvement-1', + candidate: { + label: 'direct reflection', + rationale: 'A worker reflected on its trace and authored the complete candidate.', + }, + generationsExplored: 1, + }), + ) + const proposal = createAgentImprovementProposal({ + runId: 'direct-profile-improvement-1', + findings: [productionFinding], + evaluation, + now: () => new Date('2026-07-28T00:00:00.000Z'), + }) + const review = reviewAgentImprovementProposal(proposal, { + decision: 'approve', + reviewedBy: 'reviewer@example.com', + reason: 'The direct complete-profile candidate passed paired release work.', + now: () => new Date('2026-07-28T00:01:00.000Z'), + }) + const activation = createAgentImprovementActivation(proposal, review, { + intent: 'activate-candidate', + targets: [ + { surface: 'tools', identity }, + { surface: 'mcp', identity }, + { surface: 'hooks', identity }, + { surface: 'agent-profile', identity }, + ], + executionRef, + fundingOwner: 'tenant/research', + authorizedBy: 'operator@example.com', + expiresAt: '2026-07-28T00:10:00.000Z', + now: () => new Date('2026-07-28T00:02:00.000Z'), + }) + let activeProfile = baselineProfile + const result = await executeAgentImprovementActivation( + { proposal, review, activation }, + { + transition: async (input) => { + if (input.kind !== 'profile-improvement') { + throw new Error('expected a profile improvement transition') + } + const prepared = prepareAgentImprovementProfileActivation({ + currentByIdentity: new Map([[identity, activeProfile]]), + profileTransition: input, + stateDigest, + }) + if (prepared.status !== 'apply') throw new Error('expected a profile replacement') + activeProfile = prepared.replacements[0].profile + return createAgentImprovementActivationResult(input, { + completedAt: '2026-07-28T00:03:00.000Z', + outcome: { + status: 'applied', + transactionId: 'profile-version:2', + targets: prepared.targets, + }, + }) + }, + now: () => new Date('2026-07-28T00:03:00.000Z'), + }, + ) + + expect(observedProfiles).toHaveLength(reps * 2) + expect(evaluation.decision.outcome).toBe('ship') + expect(proposal.changedSurfaces).toEqual(['tools', 'mcp', 'hooks', 'agent-profile']) + expect(activation.executionRef).toEqual(executionRef) + expect(result.outcome.status).toBe('applied') + expect(activeProfile).toEqual(candidateProfile) + expect(change.every((step) => step.set?.resources === undefined)).toBe(true) + }) + it('builds a source-bound profile proposal without product experiment wiring', async () => { const template = createProfileImprovementFixture() const task = template.evaluation.experiment.benchmark.tasks[0] @@ -313,9 +506,9 @@ describe('agent improvement lifecycle', { timeout: 30_000 }, () => { await expect( proposeAgentProfileImprovement({ source, - improvement: { surface: 'tools' }, + improvement: { surface: 'memory' }, } as never), - ).rejects.toThrow(/supports prompt or skills/) + ).rejects.toThrow(/supports profile surfaces or a complete agent profile/) const { agent: rejectedOptimize, executionRef: rejectedExecutionDigest, @@ -608,19 +801,70 @@ describe('agent improvement lifecycle', { timeout: 30_000 }, () => { it('rejects malformed optimizer evidence on profile comparisons', () => { const fixture = createProfileImprovementFixture() + if (fixture.evaluation.kind !== 'agent-profile-improvement-measured-comparison') { + throw new Error('expected a profile improvement comparison') + } + const changed = { + ...fixture.evaluation, + metadata: { optimizationReceipt: { kind: 'caller-authored' } }, + } + const { recordDigest: _recordDigest, ...provenance } = changed.provenance + const evaluation = { + ...changed, + provenance: { + ...provenance, + recordDigest: canonicalCandidateDigest({ ...changed, provenance }), + }, + } expect(() => createAgentImprovementProposal({ runId: 'profile-improvement-1', findings: [productionFinding], - evaluation: { - ...fixture.evaluation, - metadata: { optimizationReceipt: { kind: 'caller-authored' } }, - }, + evaluation, }), ).toThrow(/optimization receipt/) }) + it('rejects a profile comparison that was not recomputed from its receipts', () => { + const fixture = createProfileImprovementFixture() + if (fixture.evaluation.kind !== 'agent-profile-improvement-measured-comparison') { + throw new Error('expected a profile improvement comparison') + } + const changed = { ...fixture.evaluation, diff: 'caller-authored result' } + const { recordDigest: _recordDigest, ...provenance } = changed.provenance + const evaluation = { + ...changed, + provenance: { + ...provenance, + recordDigest: canonicalCandidateDigest({ ...changed, provenance }), + }, + } + + expect(() => + createAgentImprovementProposal({ + runId: 'profile-improvement-1', + findings: [productionFinding], + evaluation, + }), + ).toThrow(/does not match.*receipts/) + + const validProposal = createAgentImprovementProposal({ + runId: 'profile-improvement-1', + findings: [productionFinding], + evaluation: fixture.evaluation, + }) + const { digest: _digest, ...proposalMaterial } = validProposal + const invalidMaterial = { ...proposalMaterial, evaluation } + const invalidProposal = { + ...invalidMaterial, + digest: canonicalCandidateDigest(invalidMaterial), + } + expect(() => verifyAgentImprovementProposal(invalidProposal)).toThrow( + /does not match.*receipts/, + ) + }) + it('rejects unmetered proposal sources before analysis runs', async () => { let registryCalls = 0 let proposalCalls = 0 diff --git a/tests/improvement-surfaces.test.ts b/tests/improvement-surfaces.test.ts index 78919db1..4f04f6ae 100644 --- a/tests/improvement-surfaces.test.ts +++ b/tests/improvement-surfaces.test.ts @@ -330,7 +330,7 @@ describe('agent improvement profile delivery', () => { ]) }) - it('derives measured prompt and skill changes and rejects other profile changes', () => { + it('derives granular profile surfaces plus exact complete-profile authority', () => { const baseline: AgentProfile = { name: 'support-agent', prompt: { systemPrompt: 'Old prompt' }, @@ -361,21 +361,220 @@ describe('agent improvement profile delivery', () => { expect(agentProfileSchema.parse(applied)).toEqual(agentProfileSchema.parse(candidate)) expect(profileImprovementChangedSurfaces(changes)).toEqual(['prompt', 'skills']) - expect(() => - agentImprovementProfileDiffs( - baseline, - { - ...candidate, + const completeCandidate: AgentProfile = { + name: 'complete-agent', + description: 'Exercises the complete profile transition.', + version: '2.0.0', + tags: ['research', 'review'], + prompt: { systemPrompt: 'Measured prompt', instructions: ['Report evidence.'] }, + model: { + default: 'provider/model', + small: 'provider/small-model', + provider: 'provider', + reasoningEffort: 'high', + metadata: { route: 'research' }, + }, + harness: 'codex', + permissions: { shell: 'deny', files: { read: 'allow', write: 'ask' } }, + tools: { Read: true, Bash: false }, + mcp: { + docs: { + transport: 'stdio', + command: 'node', + cwd: 'tools', + enabled: true, + }, + }, + connections: [{ connectionId: 'docs', capabilities: ['read'] }], + subagents: { + reviewer: { + description: 'Checks the result.', + prompt: 'Review before answering', + model: 'provider/model', tools: { Read: true }, - mcp: { docs: { command: 'node', args: ['docs-server.js'] } }, - hooks: { Stop: [{ command: 'node check.mjs' }] }, - subagents: { reviewer: { prompt: 'Review before answering' } }, + permissions: { shell: 'deny' }, + maxSteps: 4, + metadata: { role: 'critic' }, }, - { - id: 'unsupported-profile-change', + }, + resources: { + failOnError: true, + files: [ + { + path: 'context.md', + resource: defineInlineResource('context.md', 'Research context'), + }, + ], + tools: [defineInlineResource('read.tool.md', 'Use Read')], + skills: [defineInlineResource('measured.SKILL.md', 'Use the evidence first')], + agents: [defineInlineResource('reviewer.md', 'Review instructions')], + commands: [defineInlineResource('research.md', 'Run the research workflow')], + instructions: defineInlineResource('instructions.md', 'Keep an audit trail'), + }, + hooks: { + Stop: [ + { + command: 'node check.mjs', + timeoutMs: 1_000, + blocking: true, + matcher: 'complete', + }, + ], + }, + modes: { + review: { + description: 'Review mode.', + model: 'provider/model', + prompt: 'Review only.', + tools: { Read: true }, + permissions: { shell: 'deny' }, + metadata: { depth: 'deep' }, }, - ), - ).toThrow(/measured profile contract cannot apply/) + }, + confidential: { tee: 'tdx', sealed: true }, + metadata: { tenant: 'research' }, + extensions: { codex: { feature: true } }, + } + const completeChanges = agentImprovementProfileDiffs(baseline, completeCandidate, { + id: 'complete-profile-change', + }) + const completeApplied = completeChanges.reduce( + (profile, change) => applyAgentProfileDiff(profile, change), + baseline, + ) + + expect(agentProfileSchema.parse(completeApplied)).toEqual( + agentProfileSchema.parse(completeCandidate), + ) + expect(profileImprovementChangedSurfaces(completeChanges)).toEqual([ + 'prompt', + 'skills', + 'tools', + 'mcp', + 'hooks', + 'subagents', + 'agent-profile', + ]) + }) + + it('omits unchanged external resources from a complete-profile change', () => { + const externalSkill = { + kind: 'github' as const, + repository: 'owner/research-agents', + path: 'skills/research/SKILL.md', + ref: '0123456789abcdef', + } + const baseline: AgentProfile = { + name: 'researcher', + model: { default: 'provider/old-model' }, + resources: { failOnError: true, skills: [externalSkill] }, + } + const candidate: AgentProfile = { + ...baseline, + model: { default: 'provider/new-model', reasoningEffort: 'high' }, + } + const changes = agentImprovementProfileDiffs(baseline, candidate, { + id: 'model-only-change', + }) + + expect(changes).toMatchObject([ + { remove: { model: true } }, + { set: { model: candidate.model } }, + ]) + for (const change of changes) { + expect(change.set?.resources).toBeUndefined() + expect(change.remove?.resources).toBeUndefined() + } + expect( + changes.reduce((profile, change) => applyAgentProfileDiff(profile, change), baseline), + ).toEqual(candidate) + expect(profileImprovementChangedSurfaces(changes)).toEqual(['agent-profile']) + }) + + it('does not copy unchanged external resources into direct-field changes', () => { + const external = { + kind: 'github' as const, + repository: 'owner/research-agents', + path: 'profiles/resource.md', + ref: '0123456789abcdef', + } + const cases: Array<{ baseline: AgentProfile; candidate: AgentProfile }> = [ + { + baseline: { + name: 'tool-user', + tools: { Read: true }, + resources: { tools: [external] }, + }, + candidate: { + name: 'tool-user', + tools: { Read: true, Bash: false }, + resources: { tools: [external] }, + }, + }, + { + baseline: { + name: 'manager', + subagents: { reviewer: { prompt: 'Review.' } }, + resources: { agents: [external] }, + }, + candidate: { + name: 'manager', + subagents: { + reviewer: { prompt: 'Review.' }, + researcher: { prompt: 'Research.' }, + }, + resources: { agents: [external] }, + }, + }, + ] + + for (const [index, { baseline, candidate }] of cases.entries()) { + const changes = agentImprovementProfileDiffs(baseline, candidate, { + id: `direct-field-${index}`, + }) + + for (const change of changes) { + expect(change.set?.resources).toBeUndefined() + expect(change.remove?.resources).toBeUndefined() + } + expect( + changes.reduce((profile, change) => applyAgentProfileDiff(profile, change), baseline), + ).toEqual(candidate) + } + }) + + it('reproduces a complete profile using removal-only changes', () => { + const baseline: AgentProfile = { + name: 'retired-agent', + description: 'Every populated axis should be removable.', + version: '1.0.0', + tags: ['retired'], + prompt: { systemPrompt: 'Old prompt' }, + model: { default: 'provider/old-model' }, + harness: 'codex', + tools: { Bash: true }, + resources: { + failOnError: true, + skills: [defineInlineResource('old.SKILL.md', 'Old skill')], + }, + metadata: { owner: 'old-team' }, + } + const candidate: AgentProfile = {} + const changes = agentImprovementProfileDiffs(baseline, candidate, { + id: 'remove-complete-profile', + }) + + expect(changes).toHaveLength(1) + expect(changes[0]?.set).toBeUndefined() + expect( + changes.reduce((profile, change) => applyAgentProfileDiff(profile, change), baseline), + ).toEqual(candidate) + expect(profileImprovementChangedSurfaces(changes)).toEqual([ + 'prompt', + 'skills', + 'tools', + 'agent-profile', + ]) }) it('accepts every profile surface shape produced by Runtime', () => { @@ -470,7 +669,12 @@ describe('agent improvement profile delivery', () => { expect(isAgentImprovementProfileSurface('knowledge')).toBe(false) expect(isAgentProfileMeasuredSurface('prompt')).toBe(true) expect(isAgentProfileMeasuredSurface('skills')).toBe(true) - expect(isAgentProfileMeasuredSurface('tools')).toBe(false) + expect(isAgentProfileMeasuredSurface('tools')).toBe(true) + expect(isAgentProfileMeasuredSurface('mcp')).toBe(true) + expect(isAgentProfileMeasuredSurface('hooks')).toBe(true) + expect(isAgentProfileMeasuredSurface('subagents')).toBe(true) + expect(isAgentProfileMeasuredSurface('agent-profile')).toBe(true) + expect(isAgentProfileMeasuredSurface('memory')).toBe(false) }) it('rejects input that does not match Runtime surface shape', () => { From ff1a728b61ee3eaab1fa079abc352f40bedbc809 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 31 Jul 2026 18:27:25 -0600 Subject: [PATCH 2/2] test(intelligence): pass optional comparison metadata directly --- tests/helpers/profile-improvement-fixture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/profile-improvement-fixture.ts b/tests/helpers/profile-improvement-fixture.ts index a2991aeb..a9bee91b 100644 --- a/tests/helpers/profile-improvement-fixture.ts +++ b/tests/helpers/profile-improvement-fixture.ts @@ -329,7 +329,7 @@ export function createProfileImprovementFixture( wallDurationMs: 0, cost: { usd: measurementCostUsd, provenance: 'observed' as const }, }, - ...(options.metadata ? { metadata: options.metadata } : {}), + metadata: options.metadata, }), ) return {