From 0e478684870a0faf8ec5d4e053fb3cfeba10b041 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Thu, 30 Jul 2026 06:01:01 -0600 Subject: [PATCH] fix(profile): preserve harness across matrix expansion --- .evolve/skill-runs.jsonl | 2 ++ src/agent-profile.test.ts | 15 +++++++++++++++ src/agent-profile.ts | 14 ++++++++------ tests/campaign/run-profile-matrix.test.ts | 13 ++++++++++++- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 21288b50..63db1a40 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -37,3 +37,5 @@ {"skill":"/critical-audit","ts":"2026-07-29T22:47:03Z","project":"agent-eval-control-integrity","target":"agent-eval PR 487 control-integrity repair, 17 changed paths","operatorPrompt":"","durationMin":null,"verdict":"APPROVE","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/verify","ts":"2026-07-29T22:47:04Z","project":"agent-eval-control-integrity","target":"agent-eval PR 487: 4244 tests, build, packed exports, runtime import","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/simplify","ts":"2026-07-29T22:54:49Z","project":"agent-eval-control-integrity","target":"PR 487 supervisor-run integrity modularization","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/agent-eval","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"expandProfileAxes canonical harness propagation","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"agent-eval expandProfileAxes harness source fix","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/src/agent-profile.test.ts b/src/agent-profile.test.ts index 79a64999..4c88eb29 100644 --- a/src/agent-profile.test.ts +++ b/src/agent-profile.test.ts @@ -263,6 +263,21 @@ describe('expandProfileAxes', () => { expect(harnessAxisOf(p as AgentProfile)).toEqual({ harness: 'opencode', model: 'm1' }) }) + it('writes the selected harness onto each canonical profile and overrides the base harness', () => { + const baseWithHarness: AgentProfile = { ...axisBase, harness: 'claude-code' } + const profiles = expandProfileAxes({ + base: baseWithHarness, + harnesses: ['opencode', 'codex'], + models: ['m1'], + }) + + expect(profiles.map((profile) => profile.harness)).toEqual(['opencode', 'codex']) + for (const profile of profiles) { + expect(profile.harness).toBe(harnessAxisOf(profile)?.harness) + } + expect(baseWithHarness.harness).toBe('claude-code') + }) + it('fails loud on no harnesses / no models — but snaps (never throws) on all-incompatible', () => { expect(() => expandProfileAxes({ base: axisBase, harnesses: [] })).toThrow(/no harnesses/) expect(() => expandProfileAxes({ base: { name: 'x' } })).toThrow(/no models/) diff --git a/src/agent-profile.ts b/src/agent-profile.ts index ffcd06d8..7699f534 100644 --- a/src/agent-profile.ts +++ b/src/agent-profile.ts @@ -51,12 +51,13 @@ export const HARNESS_NATIVE_MODEL = 'default' * or column→profile mapping (the pattern that let those copies drift and silently * break the harness pivot). * - * Each cell clones `base`, sets `model.default`, and stamps `metadata.harness` + - * `metadata.harnessModel` (both hash-bearing, so every cell gets a distinct - * `agentProfileId` row and results join back by harness/model via {@link harnessAxisOf} - * with no hand-recomputed key). A vendor-locked harness snaps to its family's swept - * models — or its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — - * so every requested harness runs; `keepIncompatible` forces every pair verbatim. + * Each cell clones `base`, sets the canonical top-level `harness` and `model.default`, + * and stamps `metadata.harness` + `metadata.harnessModel` for matrix grouping. Both + * metadata fields are hash-bearing, so every cell gets a distinct `agentProfileId` row + * and results join back by harness/model via {@link harnessAxisOf} with no + * hand-recomputed key. A vendor-locked harness snaps to its family's swept models — or + * its native default ({@link HARNESS_NATIVE_MODEL}) when it supports none — so every + * requested harness runs; `keepIncompatible` forces every pair verbatim. * * Omit `harnesses`/`models` to sweep the full default set — the "turn it on for * everything we care about" switch, identical in shape whether one harness or all. @@ -89,6 +90,7 @@ export function expandProfileAxes(spec: ProfileAxisSpec): AgentProfile[] { for (const model of effective) { const profile: AgentProfile = { ...spec.base, + harness, name: `${spec.base.name ?? 'agent'}/${harness}/${model}`, model: { ...spec.base.model, default: model }, metadata: { ...(spec.base.metadata ?? {}), harness, harnessModel: model }, diff --git a/tests/campaign/run-profile-matrix.test.ts b/tests/campaign/run-profile-matrix.test.ts index 782841ea..1b9d4114 100644 --- a/tests/campaign/run-profile-matrix.test.ts +++ b/tests/campaign/run-profile-matrix.test.ts @@ -378,11 +378,22 @@ describe('runProfileMatrix', () => { models: ['test-model@2025-01-01'], }) expect(profiles).toHaveLength(2) + expect(profiles.map((profile) => profile.harness)).toEqual(['opencode', 'codex']) - const result = await runProfileMatrix({ ...baseOpts(), profiles, dispatch: realDispatch }) + const dispatchedHarnesses = new Set() + const dispatch: ProfileDispatchFn = async ( + profile, + scenario, + ctx, + ) => { + dispatchedHarnesses.add(profile.harness) + return realDispatch(profile, scenario, ctx) + } + const result = await runProfileMatrix({ ...baseOpts(), profiles, dispatch }) // Every record carries the canonical cell, and its harness is the one the // generator stamped — no metadata smuggling, no hand-recomputed key. + expect(dispatchedHarnesses).toEqual(new Set(['opencode', 'codex'])) for (const rec of result.records) { expect(rec.agentProfile?.harness?.id).toMatch(/^(opencode|codex)$/) }