Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions docs/api/intelligence.md
Original file line number Diff line number Diff line change
Expand Up @@ -3881,7 +3881,7 @@ Usage class for billing. Base-stream tokens bill `'inference'`; every

##### surface

> **surface**: [`AgentImprovementProfileSurface`](#agentimprovementprofilesurface)
> **surface**: [`AgentProfileMeasuredSurface`](#agentprofilemeasuredsurface)

***

Expand All @@ -3893,7 +3893,7 @@ Usage class for billing. Base-stream tokens bill `'inference'`; every

##### surface

> **surface**: [`AgentImprovementProfileSurface`](#agentimprovementprofilesurface)
> **surface**: [`AgentProfileMeasuredSurface`](#agentprofilemeasuredsurface)

***

Expand Down Expand Up @@ -4072,10 +4072,11 @@ Agent improvement surfaces delivered as exact `AgentProfileDiff` replacements.

### AGENT\_PROFILE\_MEASURED\_SURFACES

> `const` **AGENT\_PROFILE\_MEASURED\_SURFACES**: readonly \[`"prompt"`, `"skills"`\]
> `const` **AGENT\_PROFILE\_MEASURED\_SURFACES**: readonly \[`"prompt"`, `"skills"`, `"tools"`, `"mcp"`, `"hooks"`, `"subagents"`, `"agent-profile"`\]

Portable profile surfaces eligible for shared measured comparisons.
Other profile settings can contain credentials or executable configuration.
Profile changes eligible for the product-owned measured comparison path.
The six directly deliverable profile surfaces retain their granular labels;
any residual profile axis also adds the complete `agent-profile` surface.

## Functions

Expand Down Expand Up @@ -4760,7 +4761,7 @@ surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills"

### isAgentProfileMeasuredSurface()

> **isAgentProfileMeasuredSurface**(`surface`): surface is "prompt" \| "skills"
> **isAgentProfileMeasuredSurface**(`surface`): surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile"

Return whether a surface is eligible for shared profile measurement.

Expand All @@ -4772,7 +4773,7 @@ Return whether a surface is eligible for shared profile measurement.

#### Returns

surface is "prompt" \| "skills"
surface is "tools" \| "mcp" \| "subagents" \| "hooks" \| "prompt" \| "skills" \| "agent-profile"

***

Expand Down
2 changes: 1 addition & 1 deletion docs/api/primitive-catalog.md
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports.
| `verifyCandidateExecutionEvidence` | function | Recheck one Runtime receipt against its exact signed experiment cell. |
| `withIntelligence` | function | Wrap an agent so it (a) RECEIVES the tenant's certified profile — the prompt |
| `AGENT_IMPROVEMENT_PROFILE_SURFACES` | const | Agent improvement surfaces delivered as exact `AgentProfileDiff` replacements. |
| `AGENT_PROFILE_MEASURED_SURFACES` | const | Portable profile surfaces eligible for shared measured comparisons. |
| `AGENT_PROFILE_MEASURED_SURFACES` | const | Profile changes eligible for the product-owned measured comparison path. |
| `defaultEffortTier` | const | The default tier when a client declares no effort. `'standard'` turns |
| `exactProcessCandidateExperimentExecutionSupport` | const | Candidate surfaces implemented by the neutral exact-process executor. |
| `AgentCandidateExperimentCellExecutionError` | class | A failed baseline or candidate cell with its complete Runtime failure result. |
Expand Down
16 changes: 7 additions & 9 deletions scripts/generate-agent-improvement-proposal-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,18 +150,16 @@ try {
fixturePath,
`${JSON.stringify(runtime.verifyAgentImprovementProposal(proposal), null, 2)}\n`,
)
const profileFixture = profileFixtures.createProfileImprovementFixture()
const profileFixture = profileFixtures.createProfileImprovementFixture({
metadata: {
fixture: 'agent-profile-improvement-proposal',
runtimeVersion: packageJson.version,
},
})
const profileProposal = runtime.createAgentImprovementProposal({
runId: profileFixture.evaluation.provenance.runId,
findings: [],
evaluation: {
...profileFixture.evaluation,
metadata: {
...(profileFixture.evaluation.metadata ?? {}),
fixture: 'agent-profile-improvement-proposal',
runtimeVersion: packageJson.version,
},
},
evaluation: profileFixture.evaluation,
now: () => new Date('2026-07-10T01:30:00.000Z'),
})
const serializedProfileProposalFixture = formatFixture(
Expand Down
5 changes: 2 additions & 3 deletions src/intelligence/improvement-cycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ import {
agentImprovementSourceSchema,
agentProfileImprovementArmSchema,
agentProfileImprovementExecutionRefSchema,
agentProfileImprovementMeasuredComparisonSchema,
candidateExecutionEvidenceSchema,
numbersApproximatelyEqual,
} from '@tangle-network/agent-interface'
Expand Down Expand Up @@ -765,7 +764,7 @@ export async function proposeAgentProfileImprovement<TScenario extends Scenario,
const surface = options.improvement.surface ?? 'prompt'
if (!isAgentProfileMeasuredSurface(surface)) {
throw new Error(
'measured profile improvement supports prompt or skills; use the sealed-candidate path for this surface',
'measured profile improvement supports profile surfaces or a complete agent profile; use the sealed-candidate path for this surface',
)
}
assertMeasuredAnalysisOptions(options.analysis)
Expand Down Expand Up @@ -1207,7 +1206,7 @@ function verifyAgentImprovementEvaluation(input: unknown): AgentImprovementEvalu
'kind' in input &&
input.kind === 'agent-profile-improvement-measured-comparison'
) {
const evaluation = agentProfileImprovementMeasuredComparisonSchema.parse(input)
const evaluation = verifyAgentProfileImprovementExperimentComparison(input)
optimizationActivationReceiptFromMetadata(evaluation.metadata)
return evaluation
}
Expand Down
249 changes: 224 additions & 25 deletions src/intelligence/improvement-surfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,57 @@
export type AgentImprovementProfileSurface = (typeof AGENT_IMPROVEMENT_PROFILE_SURFACES)[number]

/**
* Portable profile surfaces eligible for shared measured comparisons.
* Other profile settings can contain credentials or executable configuration.
* Profile changes eligible for the product-owned measured comparison path.
* The six directly deliverable profile surfaces retain their granular labels;
* any residual profile axis also adds the complete `agent-profile` surface.
*/
export const AGENT_PROFILE_MEASURED_SURFACES = [
'prompt',
'skills',
] as const satisfies readonly AgentImprovementProfileSurface[]
...AGENT_IMPROVEMENT_PROFILE_SURFACES,
'agent-profile',
] as const satisfies readonly AgentImprovementSurface[]

export type AgentProfileMeasuredSurface = (typeof AGENT_PROFILE_MEASURED_SURFACES)[number]

type ProfileDifferHandledField =
| 'name'
| 'description'
| 'version'
| 'tags'
| 'prompt'
| 'model'
| 'harness'
| 'permissions'
| 'tools'
| 'mcp'
| 'connections'
| 'subagents'
| 'resources'
| 'hooks'
| 'modes'
| 'confidential'
| 'metadata'
| 'extensions'
type MissingProfileDifferField = Exclude<keyof AgentProfile, ProfileDifferHandledField>
const profileDifferIsExhaustive: MissingProfileDifferField extends never ? true : never = true
void profileDifferIsExhaustive

type ProfileResourceDifferHandledField =
| 'files'
| 'tools'
| 'skills'
| 'agents'
| 'commands'
| 'instructions'
| 'failOnError'
type MissingProfileResourceDifferField = Exclude<
keyof NonNullable<AgentProfile['resources']>,
ProfileResourceDifferHandledField
>
const profileResourceDifferIsExhaustive: MissingProfileResourceDifferField extends never
? true
: never = true
void profileResourceDifferIsExhaustive

export interface AgentImprovementTargetProfileDiffOptions {
id: string
source?: AgentProfileDiff['source']
Expand Down Expand Up @@ -163,7 +204,7 @@
isAgentProfileMeasuredSurface(surface),
)
) {
throw new Error('profile improvement experiment does not change a supported surface')

Check failure on line 207 in src/intelligence/improvement-surfaces.ts

View workflow job for this annotation

GitHub Actions / ci

tests/improvement-surfaces.test.ts > agent improvement profile delivery > omits unchanged external resources from a complete-profile change

Error: profile improvement experiment does not change a supported surface ❯ profileImprovementChangedSurfaces src/intelligence/improvement-surfaces.ts:207:11 ❯ tests/improvement-surfaces.test.ts:491:12
}
return surfaces as [AgentProfileMeasuredSurface, ...AgentProfileMeasuredSurface[]]
}
Expand Down Expand Up @@ -355,31 +396,189 @@
omitUndefinedObjectFields(candidateInput, 'profile improvement candidate'),
'profile improvement candidate',
)
const surfaces = AGENT_PROFILE_MEASURED_SURFACES.filter(
(surface) =>
agentImprovementProfileSurfaceDigest(baseline, surface) !==
agentImprovementProfileSurfaceDigest(candidate, surface),
)
if (surfaces.length === 0) {
throw new Error('profile improvement candidate does not change a deliverable profile surface')
}
const changes = surfaces.flatMap((surface) =>
agentImprovementTargetProfileDiffs(
{ surface, desiredInput: agentImprovementProfileSurfaceInput(candidate, surface) },
options,
),
) as [AgentProfileDiff, ...AgentProfileDiff[]]
const applied = changes.reduce(
if (canonicalCandidateDigest(baseline) === canonicalCandidateDigest(candidate)) {
throw new Error('profile improvement candidate does not change the profile')
}
const completeChanges = completeAgentProfileReplacementDiffs(baseline, candidate, options)
const applied = completeChanges.reduce(
(profile, change) =>
applyExactAgentProfileDiff(profile, change, 'profile improvement candidate change'),
applyExactAgentProfileDiff(profile, change, 'complete profile improvement candidate change'),
baseline,
)
if (canonicalCandidateDigest(applied) !== canonicalCandidateDigest(candidate)) {
throw new Error(
'profile improvement candidate changes fields that the measured profile contract cannot apply',
)
throw new Error('complete profile improvement change did not reproduce the candidate')
}
return completeChanges
}

function completeAgentProfileReplacementDiffs(
baseline: AgentProfile,
candidate: AgentProfile,
options: AgentImprovementTargetProfileDiffOptions,
): [AgentProfileDiff, ...AgentProfileDiff[]] {
const remove: AgentProfileDiffRemoval = {}
const set: AgentProfile = {}
if (
profileValuesDiffer(baseline.name, candidate.name) ||
profileValuesDiffer(baseline.description, candidate.description) ||
profileValuesDiffer(baseline.version, candidate.version)
) {
remove.identity = true
if (candidate.name !== undefined) set.name = candidate.name
if (candidate.description !== undefined) set.description = candidate.description
if (candidate.version !== undefined) set.version = candidate.version
}
return changes
if (profileValuesDiffer(baseline.tags, candidate.tags)) {
remove.tags = true
if (candidate.tags !== undefined) set.tags = candidate.tags
}
if (profileValuesDiffer(baseline.prompt, candidate.prompt)) {
remove.prompt = true
if (candidate.prompt !== undefined) set.prompt = candidate.prompt
}
if (profileValuesDiffer(baseline.model, candidate.model)) {
remove.model = true
if (candidate.model !== undefined) set.model = candidate.model
}
if (profileValuesDiffer(baseline.harness, candidate.harness)) {
remove.harness = true
if (candidate.harness !== undefined) set.harness = candidate.harness
}
if (profileValuesDiffer(baseline.permissions, candidate.permissions)) {
remove.permissions = true
if (candidate.permissions !== undefined) set.permissions = candidate.permissions
}
if (profileValuesDiffer(baseline.tools, candidate.tools)) {
remove.tools = true
if (candidate.tools !== undefined) set.tools = candidate.tools
}
if (profileValuesDiffer(baseline.mcp, candidate.mcp)) {
remove.mcp = true
if (candidate.mcp !== undefined) set.mcp = candidate.mcp
}
if (profileValuesDiffer(baseline.connections, candidate.connections)) {
remove.connections = true
if (candidate.connections !== undefined) set.connections = candidate.connections
}
if (profileValuesDiffer(baseline.subagents, candidate.subagents)) {
remove.subagents = true
if (candidate.subagents !== undefined) set.subagents = candidate.subagents
}
replaceChangedProfileResources(baseline.resources, candidate.resources, remove, set)
if (profileValuesDiffer(baseline.hooks, candidate.hooks)) {
remove.hooks = true
if (candidate.hooks !== undefined) set.hooks = candidate.hooks
}
if (profileValuesDiffer(baseline.modes, candidate.modes)) {
remove.modes = true
if (candidate.modes !== undefined) set.modes = candidate.modes
}
if (profileValuesDiffer(baseline.confidential, candidate.confidential)) {
remove.confidential = true
if (candidate.confidential !== undefined) set.confidential = candidate.confidential
}
if (profileValuesDiffer(baseline.metadata, candidate.metadata)) {
remove.metadata = true
if (candidate.metadata !== undefined) set.metadata = candidate.metadata
}
if (profileValuesDiffer(baseline.extensions, candidate.extensions)) {
remove.extensions = true
if (candidate.extensions !== undefined) set.extensions = candidate.extensions
}
if (Object.keys(remove).length === 0) {
throw new Error('complete profile differ found no changed fields')
}
const common = {
kind: 'agent-profile-diff' as const,
...(options.source ? { source: options.source } : {}),
metadata: {
...(options.metadata ?? {}),
surface: 'agent-profile',
},
}
const reset = agentProfileDiffSchema.parse(
defineAgentProfileDiff({
...common,
id: `${options.id}:agent-profile:reset`,
title: 'Reset changed agent profile fields',
remove,
}),
) as AgentProfileDiff
if (Object.keys(set).length === 0) return [reset]
const replacement = agentProfileDiffSchema.parse(
defineAgentProfileDiff({
...common,
id: `${options.id}:agent-profile:set`,
title: 'Set changed agent profile fields',
set,
}),
) as AgentProfileDiff
return [reset, replacement]
}

function replaceChangedProfileResources(
baseline: AgentProfile['resources'],
candidate: AgentProfile['resources'],
remove: AgentProfileDiffRemoval,
set: AgentProfile,
): void {
if (!profileValuesDiffer(baseline, candidate)) return
const resourceRemoval: Exclude<AgentProfileDiffRemoval['resources'], true | undefined> = {}
const resourceSet: NonNullable<AgentProfile['resources']> = {}
let changedSubfields = 0

if (profileValuesDiffer(baseline?.files, candidate?.files)) {
changedSubfields += 1
resourceRemoval.files = true
if (candidate?.files !== undefined) resourceSet.files = candidate.files
}
if (profileValuesDiffer(baseline?.tools, candidate?.tools)) {
changedSubfields += 1
resourceRemoval.tools = true
if (candidate?.tools !== undefined) resourceSet.tools = candidate.tools
}
if (profileValuesDiffer(baseline?.skills, candidate?.skills)) {
changedSubfields += 1
resourceRemoval.skills = true
if (candidate?.skills !== undefined) resourceSet.skills = candidate.skills
}
if (profileValuesDiffer(baseline?.agents, candidate?.agents)) {
changedSubfields += 1
resourceRemoval.agents = true
if (candidate?.agents !== undefined) resourceSet.agents = candidate.agents
}
if (profileValuesDiffer(baseline?.commands, candidate?.commands)) {
changedSubfields += 1
resourceRemoval.commands = true
if (candidate?.commands !== undefined) resourceSet.commands = candidate.commands
}
if (profileValuesDiffer(baseline?.instructions, candidate?.instructions)) {
changedSubfields += 1
resourceRemoval.instructions = true
if (candidate?.instructions !== undefined) resourceSet.instructions = candidate.instructions
}
if (profileValuesDiffer(baseline?.failOnError, candidate?.failOnError)) {
changedSubfields += 1
resourceRemoval.failOnError = true
if (candidate?.failOnError !== undefined) resourceSet.failOnError = candidate.failOnError
}

if (changedSubfields === 0) {
remove.resources = true
if (candidate !== undefined) set.resources = candidate
return
}
remove.resources = resourceRemoval
if (Object.keys(resourceSet).length > 0) set.resources = resourceSet
}

function profileValuesDiffer(baseline: unknown, candidate: unknown): boolean {
const comparable = (value: unknown) =>
value === undefined ? { present: false } : { present: true, value }
return (
canonicalCandidateDigest(comparable(baseline)) !==
canonicalCandidateDigest(comparable(candidate))
)
}

function improvementSurfaceReplacement(target: {
Expand Down
Loading
Loading