Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .evolve/skill-runs.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,5 @@
{"skill":"/critical-audit","ts":"2026-07-29T22:47:03Z","project":"agent-eval-control-integrity","target":"agent-eval PR 487 control-integrity repair, 17 changed paths","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/verify","ts":"2026-07-29T22:47:04Z","project":"agent-eval-control-integrity","target":"agent-eval PR 487: 4244 tests, build, packed exports, runtime import","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/simplify","ts":"2026-07-29T22:54:49Z","project":"agent-eval-control-integrity","target":"PR 487 supervisor-run integrity modularization","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/agent-eval","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"expandProfileAxes canonical harness propagation","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null}
{"skill":"/verify","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"agent-eval expandProfileAxes harness source fix","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null}
15 changes: 15 additions & 0 deletions src/agent-profile.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,21 @@ describe('expandProfileAxes', () => {
expect(harnessAxisOf(p as AgentProfile)).toEqual({ harness: 'opencode', model: 'm1' })
})

it('writes the selected harness onto each canonical profile and overrides the base harness', () => {
const baseWithHarness: AgentProfile = { ...axisBase, harness: 'claude-code' }
const profiles = expandProfileAxes({
base: baseWithHarness,
harnesses: ['opencode', 'codex'],
models: ['m1'],
})

expect(profiles.map((profile) => profile.harness)).toEqual(['opencode', 'codex'])
for (const profile of profiles) {
expect(profile.harness).toBe(harnessAxisOf(profile)?.harness)
}
expect(baseWithHarness.harness).toBe('claude-code')
})

it('fails loud on no harnesses / no models — but snaps (never throws) on all-incompatible', () => {
expect(() => expandProfileAxes({ base: axisBase, harnesses: [] })).toThrow(/no harnesses/)
expect(() => expandProfileAxes({ base: { name: 'x' } })).toThrow(/no models/)
Expand Down
14 changes: 8 additions & 6 deletions src/agent-profile.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,13 @@ export const HARNESS_NATIVE_MODEL = 'default'
* or column→profile mapping (the pattern that let those copies drift and silently
* break the harness pivot).
*
* Each cell clones `base`, sets `model.default`, and stamps `metadata.harness` +
* `metadata.harnessModel` (both hash-bearing, so every cell gets a distinct
* `agentProfileId` row and results join back by harness/model via {@link harnessAxisOf}
* with no hand-recomputed key). A vendor-locked harness snaps to its family's swept
* models — or its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none —
* so every requested harness runs; `keepIncompatible` forces every pair verbatim.
* Each cell clones `base`, sets the canonical top-level `harness` and `model.default`,
* and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both
* metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row
* and results join back by harness/model via {@link harnessAxisOf} with no
* hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or
* its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every
* requested harness runs; `keepIncompatible` forces every pair verbatim.
*
* Omit `harnesses`/`models` to sweep the full default set — the "turn it on for
* everything we care about" switch, identical in shape whether one harness or all.
Expand Down Expand Up @@ -89,6 +90,7 @@ export function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[] {
for (const model of effective) {
const profile: AgentProfile = {
...spec.base,
harness,
name: `${spec.base.name ?? 'agent'}/${harness}/${model}`,
model: { ...spec.base.model, default: model },
metadata: { ...(spec.base.metadata ?? {}), harness, harnessModel: model },
Expand Down
13 changes: 12 additions & 1 deletion tests/campaign/run-profile-matrix.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -378,11 +378,22 @@ describe('runProfileMatrix', () => {
models: ['test-model@2025-01-01'],
})
expect(profiles).toHaveLength(2)
expect(profiles.map((profile) => profile.harness)).toEqual(['opencode', 'codex'])

const result = await runProfileMatrix({ ...baseOpts(), profiles, dispatch: realDispatch })
const dispatchedHarnesses = new Set<AgentProfile['harness']>()
const dispatch: ProfileDispatchFn<FakeScenario, FakeArtifact> = async (
profile,
scenario,
ctx,
) => {
dispatchedHarnesses.add(profile.harness)
return realDispatch(profile, scenario, ctx)
}
const result = await runProfileMatrix({ ...baseOpts(), profiles, dispatch })

// Every record carries the canonical cell, and its harness is the one the
// generator stamped — no metadata smuggling, no hand-recomputed key.
expect(dispatchedHarnesses).toEqual(new Set(['opencode', 'codex']))
for (const rec of result.records) {
expect(rec.agentProfile?.harness?.id).toMatch(/^(opencode|codex)$/)
}
Expand Down