From 280f5abf2be94ce2e2323a8f377bd531121ac6d7 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 11:31:08 +0200 Subject: [PATCH 1/7] incomplete fe-1404 work --- .../baseline/condition-3-instrument.ts | 1316 ++++++++++++++++ .../baseline/condition-3-legibility.md | 146 ++ .../baseline/condition-3-operator.md | 82 + .../baseline/condition-3-pre-run-review.md | 129 ++ .../condition-3-preregistration.lock.json | 50 + .../baseline/condition-3-preregistration.md | 150 ++ .../baseline/condition-3-prompt.md | 57 + .../baseline/condition-3-scoring.md | 124 ++ .../baseline/protocol.md | 44 +- .../process-model-elicitation/baseline/run.ts | 1297 ++++++++++++++-- .../test/architecture/baseline-runner.test.ts | 1336 ++++++++++++++++- .../condition-3-instrument.test.ts | 766 ++++++++++ 12 files changed, 5378 insertions(+), 119 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-legibility.md create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-pre-run-review.md create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md create mode 100644 libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-scoring.md create mode 100644 libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts new file mode 100644 index 00000000000..699e82db006 --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts @@ -0,0 +1,1316 @@ +import * as v from "valibot"; + +export const CONDITION_3_INSTRUMENT_VERSION = + "fe-1404-condition-3/2026-08-25.1"; +export const CONDITION_3_DEMAND_TABLE_VERSION = + "cps-baseline-replay/2026-08-24.3"; + +export const CONDITION_3_COMPARISON_HASHES = { + condition1: { + rawSha256: + "e8fdb4705ea5223545a0395f26b32dddaf105e6536ffef1b15feee7f73f0d3dd", + transcriptSha256: + "307eddf906a8ebd280e7cf1eaaadf124fd053a9d82cc651afbc5c760904f9c30", + modelSha256: + "64100739b7668bed749b30f081d4a8fd7149f2b0a21e5791203a7d8dc70f37d2", + }, + condition2: { + rawSha256: + "e50c7b9442758ed97882e843195bb3be1b9f4350a28f0808ee09428cd51c3829", + transcriptSha256: + "230c9f643763a2d58790aef5515f22892cc4a6ed7562f0cdb3816138b05a700c", + modelSha256: + "cd1d7c38773e991d859866af36c2bb13e3c6c68bfff9ce7e0430840c9d0c9eab", + }, +} as const; + +export const CONDITION_3_LOCKED_PATHS = [ + "evaluations/protocols/process-model-elicitation/baseline/run.ts", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-scoring.md", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-pre-run-review.md", + "evaluations/protocols/process-model-elicitation/baseline/condition-3-legibility.md", + "evaluations/protocols/process-model-elicitation/baseline/protocol.md", + "evaluations/cases/process-model-elicitation/baseline/opening-message.md", + "evaluations/cases/process-model-elicitation/baseline/situation-pack.md", + "docs/specs/elicitation-completion.md", + "docs/specs/cps-interview-guidance.md", + "docs/reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md", + "docs/evidence/evaluations/process-model-elicitation/baseline/readout.md", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-1.md", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-1.raw.json", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-1-model.txt", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-2.md", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-2.raw.json", + "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-2-model.txt", +] as const; + +export const CONDITION_3_OBJECTIVE_ROWS = [ + "ROW-BREAKDOWN", + "ROW-IDLE-WASH", + "ROW-CHANGEOVER", + "ROW-SPLIT", +] as const; + +export type Condition3ObjectiveRow = + (typeof CONDITION_3_OBJECTIVE_ROWS)[number]; + +/** + * Exact FE-1402 design-time `whenObjective` labels. These labels let the + * experiment operator record which frozen row predicate it adjudicated; they + * are not a proposed FE-1431 runtime binding representation. + */ +export const CONDITION_3_OBJECTIVE_MATCH_PREDICATES = [ + { row: "ROW-BREAKDOWN", matchingPredicate: "breakdown-reshuffle" }, + { row: "ROW-IDLE-WASH", matchingPredicate: "idle-vs-washdown" }, + { row: "ROW-CHANGEOVER", matchingPredicate: "changeover-accounting" }, + { row: "ROW-SPLIT", matchingPredicate: "split-run" }, +] as const satisfies readonly { + row: Condition3ObjectiveRow; + matchingPredicate: string; +}[]; + +export const CONDITION_3_PASSING_STATUSES = ["explicit", "inferred"] as const; + +export const CONDITION_3_DEMAND_CLAUSES = [ + { + id: "SF-OBJ", + row: null, + coordinate: "kind(objective)", + demand: "presence count >= 1", + }, + { + id: "SF-ENT", + row: null, + coordinate: "kind(entity-type)", + demand: "presence count >= 2", + }, + { + id: "SF-ACT", + row: null, + coordinate: "kind(activity)", + demand: "presence count >= 1", + }, + { + id: "SF-PATH", + row: null, + coordinate: "kind(ordering/flow)", + demand: "presence count >= 1", + }, + { + id: "SF-FLOW", + row: null, + coordinate: "kind(ordering/flow).sequence", + demand: "grade structured; status explicit or inferred", + }, + { + id: "BR-CAP", + row: "ROW-BREAKDOWN", + coordinate: "entity-type[line].capabilities", + demand: "grade structured; status explicit or inferred", + }, + { + id: "BR-CAL", + row: "ROW-BREAKDOWN", + coordinate: "boundary-condition[line-calendar].pattern", + demand: "grade structured; status explicit or inferred", + }, + { + id: "BR-OCC", + row: "ROW-BREAKDOWN", + coordinate: "dynamics[line-failure].occurrenceFrequency", + demand: "grade range; status explicit or inferred", + }, + { + id: "BR-REPAIR", + row: "ROW-BREAKDOWN", + coordinate: "dynamics[line-failure].repairDuration", + demand: "grade quantiles; status explicit or inferred", + }, + { + id: "BR-POL", + row: "ROW-BREAKDOWN", + coordinate: "policy[resource-conflict].rule", + demand: "grade structured; status explicit or inferred", + }, + { + id: "IW-REL", + row: "ROW-IDLE-WASH", + coordinate: "boundary-condition[order-release].condition", + demand: "grade structured; status explicit or inferred", + }, + { + id: "IW-CO-DUR", + row: "ROW-IDLE-WASH", + coordinate: "dynamics[family-changeover].duration", + demand: "grade range; status explicit or inferred", + }, + { + id: "IW-LATE", + row: "ROW-IDLE-WASH", + coordinate: "objective[idle-vs-washdown].latenessConsequence", + demand: "grade structured; status explicit or inferred", + }, + { + id: "IW-SCRAP", + row: "ROW-IDLE-WASH", + coordinate: "dynamics[family-changeover].rampScrap", + demand: "grade range; status explicit or inferred; no accepted absence", + }, + { + id: "CH-TAX", + row: "ROW-CHANGEOVER", + coordinate: "entity-type[changeover].directionClass", + demand: "grade vocabulary-bound; status explicit or inferred", + }, + { + id: "CH-DUR", + row: "ROW-CHANGEOVER", + coordinate: "dynamics[family-changeover].duration", + demand: "grade range; status explicit or inferred", + }, + { + id: "CH-CREW", + row: "ROW-CHANGEOVER", + coordinate: "activity[family-changeover].resourceRequirement", + demand: "grade structured; status explicit or inferred", + }, + { + id: "CH-SEQ", + row: "ROW-CHANGEOVER", + coordinate: "policy[weekly-sequencing].rule", + demand: "grade structured; status explicit or inferred", + }, + { + id: "CH-SCRAP", + row: "ROW-CHANGEOVER", + coordinate: "dynamics[family-changeover].rampScrap", + demand: "grade range; status explicit or inferred; no accepted absence", + }, + { + id: "SP-BATCH", + row: "ROW-SPLIT", + coordinate: "activity[production-run].batchStructure", + demand: "grade structured; status explicit or inferred", + }, + { + id: "SP-MIN", + row: "ROW-SPLIT", + coordinate: "constraint[minimum-run-size].threshold", + demand: "grade range; status explicit or inferred", + }, + { + id: "SP-ELIG", + row: "ROW-SPLIT", + coordinate: "constraint[line-eligibility].condition", + demand: "grade structured; status explicit or inferred", + }, + { + id: "SP-POL", + row: "ROW-SPLIT", + coordinate: "policy[split-contiguity].rule", + demand: "grade structured; status explicit or inferred", + }, + { + id: "SP-CO", + row: "ROW-SPLIT", + coordinate: "dynamics[split-run].extraChangeover", + demand: "grade range; status explicit or inferred", + }, + { + id: "SP-SCRAP", + row: "ROW-SPLIT", + coordinate: "dynamics[split-run].repeatedRampScrap", + demand: "grade range; status explicit or inferred; no accepted absence", + }, +] as const satisfies readonly { + id: string; + row: Condition3ObjectiveRow | null; + coordinate: string; + demand: string; +}[]; + +export type Condition3ClauseId = + (typeof CONDITION_3_DEMAND_CLAUSES)[number]["id"]; + +export const CONDITION_3_CARD_IDS = [ + "CPS-Q01", + "CPS-Q02", + "CPS-Q03", + "CPS-Q04", + "CPS-Q05", + "GEN-Q02", +] as const; + +export type Condition3CardId = (typeof CONDITION_3_CARD_IDS)[number]; + +export const CONDITION_3_FIRES_WHEN = [ + "slot-unaddressed", + "below-demanded-grade", + "unspecified-marker-present", + "conflicted-open", + "absence-uncorroborated", + "uniformity-unprobed", + "identity-ambiguous", +] as const; + +export type Condition3FiresWhen = (typeof CONDITION_3_FIRES_WHEN)[number]; + +export const CONDITION_3_ACTIVATION_MATRIX = [ + { + cardId: "CPS-Q01", + clauses: ["BR-OCC", "BR-REPAIR"], + predicates: [ + "slot-unaddressed", + "below-demanded-grade", + "unspecified-marker-present", + ], + }, + { + cardId: "CPS-Q02", + clauses: ["IW-SCRAP", "CH-SCRAP", "SP-SCRAP"], + predicates: [ + "slot-unaddressed", + "below-demanded-grade", + "absence-uncorroborated", + ], + }, + { + cardId: "CPS-Q03", + clauses: ["SP-BATCH", "SP-MIN", "SP-POL", "SP-CO", "SP-SCRAP"], + predicates: ["slot-unaddressed", "below-demanded-grade"], + }, + { + cardId: "CPS-Q04", + clauses: ["IW-REL"], + predicates: [ + "slot-unaddressed", + "below-demanded-grade", + "unspecified-marker-present", + ], + }, + { + cardId: "CPS-Q05", + clauses: ["BR-POL"], + predicates: [ + "slot-unaddressed", + "below-demanded-grade", + "unspecified-marker-present", + ], + }, +] as const satisfies readonly { + cardId: Exclude; + clauses: readonly Condition3ClauseId[]; + predicates: readonly Condition3FiresWhen[]; +}[]; + +export const CONDITION_3_GEN_Q02_LAYER_2 = { + cardId: "GEN-Q02", + verdict: "unobservable", + reason: + "the experiment has no lossless independent-question and pending-large-batch adjudicator; punctuation is not a semantic proxy", +} as const; + +export const CONDITION_3_DIAGNOSTIC_PRIORITY = [ + "SF-ENT", + "SF-ACT", + "SF-PATH", + "SF-FLOW", + "BR-OCC", + "BR-REPAIR", + "IW-SCRAP", + "CH-SCRAP", + "SP-SCRAP", + "SP-BATCH", + "SP-MIN", + "SP-POL", + "SP-CO", + "IW-REL", + "BR-POL", + "BR-CAP", + "BR-CAL", + "IW-CO-DUR", + "IW-LATE", + "CH-TAX", + "CH-DUR", + "CH-CREW", + "CH-SEQ", + "SP-ELIG", + "SF-OBJ", +] as const satisfies readonly Condition3ClauseId[]; + +export const CONDITION_3_STOPPING_RULES = { + forceWrapAt: 20, + hardStopAt: 24, + noProgressAdvisoryAfter: 3, + noProgressHardStopAfter: 5, + impatiencePhase: + "first expert reply after all static-floor clauses pass and at least one objective row is active", + singleSession: + "no later session or external data arrival is available in this experiment", + providerSampling: "default temperature; no seed parameter supported", +} as const; + +export const CONDITION_3_DEMANDED_STATUSES = [ + "none", + ...CONDITION_3_PASSING_STATUSES, + "tentative", + "defaulted", + "external-lookup", + "conflicted", +] as const; + +export const CONDITION_3_DEMANDED_GRADES = [ + "none", + "verbal", + "point", + "vocabulary-bound", + "range", + "structured", + "quantiles", +] as const; + +export const CONDITION_3_FAILURE_DIAGNOSTICS = [ + "below-minimum-count", + "no-selected-slot", + "below-required-grade", + "inadmissible-status", + "unaccepted-absence", + "missing-evidence", + "unaddressed", + "open-conflict", + "unevaluable-divergence", + "unsupported-active-anchor", +] as const; + +const clauseIds = new Set(CONDITION_3_DEMAND_CLAUSES.map(({ id }) => id)); +const Condition3ClauseIdSchema = v.custom( + (value) => + typeof value === "string" && clauseIds.has(value as Condition3ClauseId), + "unknown frozen DemandTable clause", +); +const Condition3EvidenceSchema = v.strictObject({ + turn: v.pipe(v.number(), v.integer(), v.minValue(0)), + quote: v.pipe(v.string(), v.minLength(1)), +}); +const Condition3ActiveObjectiveRowEvidenceSchema = v.variant( + "row", + CONDITION_3_OBJECTIVE_MATCH_PREDICATES.map(({ row, matchingPredicate }) => + v.strictObject({ + row: v.literal(row), + anchorLabel: v.pipe(v.string(), v.minLength(1)), + matchingPredicate: v.literal(matchingPredicate), + evidence: v.pipe(v.array(Condition3EvidenceSchema), v.minLength(1)), + rationale: v.pipe(v.string(), v.minLength(1)), + }), + ), +); +const Condition3RetractedObjectiveAnchorSchema = v.variant( + "row", + CONDITION_3_OBJECTIVE_MATCH_PREDICATES.map(({ row, matchingPredicate }) => + v.strictObject({ + row: v.literal(row), + anchorLabel: v.pipe(v.string(), v.minLength(1)), + matchingPredicate: v.literal(matchingPredicate), + evidence: v.pipe(v.array(Condition3EvidenceSchema), v.minLength(1)), + rationale: v.pipe(v.string(), v.minLength(1)), + resolutionEvidence: v.pipe( + v.array(Condition3EvidenceSchema), + v.minLength(1), + ), + resolutionRationale: v.pipe(v.string(), v.minLength(1)), + }), + ), +); +const unsupportedAnchorBase = { + label: v.pipe(v.string(), v.minLength(1)), + evidence: v.pipe(v.array(Condition3EvidenceSchema), v.minLength(1)), + rationale: v.pipe(v.string(), v.minLength(1)), +}; +const Condition3ActiveUnsupportedObjectiveAnchorSchema = v.strictObject({ + ...unsupportedAnchorBase, + state: v.literal("active"), + demanded: v.literal(true), + pass: v.literal(false), + failureDiagnostic: v.literal("unsupported-active-anchor"), + resolutionEvidence: v.tuple([]), + resolutionRationale: v.null(), +}); +const Condition3RetractedUnsupportedObjectiveAnchorSchema = v.strictObject({ + ...unsupportedAnchorBase, + state: v.literal("retracted"), + demanded: v.literal(false), + pass: v.literal(true), + failureDiagnostic: v.null(), + resolutionEvidence: v.pipe(v.array(Condition3EvidenceSchema), v.minLength(1)), + resolutionRationale: v.pipe(v.string(), v.minLength(1)), +}); +const assessmentBase = { + clauseId: Condition3ClauseIdSchema, + demand: v.string(), + coordinate: v.string(), + evidence: v.array(Condition3EvidenceSchema), + observedCount: v.nullable(v.pipe(v.number(), v.integer(), v.minValue(0))), + rationale: v.string(), +}; + +const Condition3InactiveAssessmentSchema = v.strictObject({ + ...assessmentBase, + demanded: v.literal(false), + currentStatus: v.literal("not-applicable"), + currentGrade: v.literal("not-applicable"), + pass: v.literal(true), + failureDiagnostic: v.null(), + activationPredicates: v.tuple([]), +}); +const Condition3PassingAssessmentSchema = v.strictObject({ + ...assessmentBase, + demanded: v.literal(true), + currentStatus: v.picklist(CONDITION_3_PASSING_STATUSES), + currentGrade: v.picklist(CONDITION_3_DEMANDED_GRADES), + pass: v.literal(true), + failureDiagnostic: v.null(), + activationPredicates: v.tuple([]), +}); +const Condition3FailingAssessmentSchema = v.strictObject({ + ...assessmentBase, + demanded: v.literal(true), + currentStatus: v.picklist(CONDITION_3_DEMANDED_STATUSES), + currentGrade: v.picklist(CONDITION_3_DEMANDED_GRADES), + pass: v.literal(false), + failureDiagnostic: v.picklist(CONDITION_3_FAILURE_DIAGNOSTICS), + activationPredicates: v.array(v.picklist(CONDITION_3_FIRES_WHEN)), +}); + +export const Condition3ProjectionSchema = v.strictObject({ + activeObjectiveRows: v.array(v.picklist(CONDITION_3_OBJECTIVE_ROWS)), + activeObjectiveRowEvidence: v.array( + Condition3ActiveObjectiveRowEvidenceSchema, + ), + retractedObjectiveAnchors: v.array(Condition3RetractedObjectiveAnchorSchema), + unsupportedActiveObjectiveAnchors: v.array( + v.union([ + Condition3ActiveUnsupportedObjectiveAnchorSchema, + Condition3RetractedUnsupportedObjectiveAnchorSchema, + ]), + ), + assessments: v.array( + v.union([ + Condition3InactiveAssessmentSchema, + Condition3PassingAssessmentSchema, + Condition3FailingAssessmentSchema, + ]), + ), + notes: v.array(v.string()), +}); + +export type Condition3Projection = v.InferOutput< + typeof Condition3ProjectionSchema +>; +export type Condition3Assessment = Condition3Projection["assessments"][number]; + +export const CONDITION_3_VERDICTS = [ + "pass", + "fail", + "mixed", + "unobservable", + "not-applicable", +] as const; + +export const CONDITION_3_RESULT_COMPONENT_IDS = [ + "layer.diagnostic", + "layer.activation", + "layer.evidence-stopping", + "guidance.GEN-Q02.layer-2", + "guidance.GEN-Q02.layer-3", + "guidance.CPS-Q01.aggregate", + "guidance.CPS-Q02.aggregate", + "guidance.CPS-Q03.aggregate", + "guidance.CPS-Q04.aggregate", + "guidance.CPS-Q05.aggregate", + "inherited.interaction-quality", + "inherited.semantic-coverage", + "inherited.stopping", + "inherited.completion", + "stopping.user-request", + "stopping.no-progress", + "stopping.budget", + "inherited.delivery", + "inherited.deposit", + "inherited.deferral", + "inherited.provenance", + "inherited.target-validity", + "bano.question-formulation", + "bano.question-omission", + "bano.order-of-interview", + "bano.communication-skills", + "bano.customer-interaction", + "coverage.objectives", + "coverage.structure", + "coverage.taxonomy", + "coverage.rates-distributions", + "coverage.policies", + "coverage.constraints", + "coverage.boundary-conditions", + "excavation.tacit", + "excavation.belief-correction", + "excavation.unknown-recording", + "signature.FM-01", + "signature.FM-02", + "signature.FM-03", + "signature.FM-04", + "signature.FM-05", + "signature.FM-06", + "signature.FM-07", + "signature.FM-08", + "signature.FM-09", + "signature.FM-10", + "signature.FM-11", + "signature.FM-12", + "signature.FM-13", + "signature.FM-14", + "signature.FM-15", +] as const; + +const nonEmptyResultString = v.pipe(v.string(), v.minLength(1)); +const Condition3ResultComponentSchema = v.pipe( + v.strictObject({ + id: v.picklist(CONDITION_3_RESULT_COMPONENT_IDS), + verdict: v.picklist(CONDITION_3_VERDICTS), + observation: v.nullable( + v.picklist(["observed", "not-observed", "unobservable"]), + ), + evidence: v.array(nonEmptyResultString), + rationale: nonEmptyResultString, + }), + v.check( + ({ verdict, evidence }) => + !(["pass", "fail", "mixed"] as const).includes(verdict as never) || + evidence.length > 0, + "scored condition-3 result components require evidence", + ), +); + +export const Condition3ResultSchema = v.strictObject({ + schemaVersion: v.literal("fe-1404-condition-3-result/2026-08-25.1"), + runRawSha256: v.pipe(v.string(), v.regex(/^[a-f0-9]{64}$/u)), + components: v.array(Condition3ResultComponentSchema), + comparisons: v.strictObject({ + condition1: v.strictObject({ + rawSha256: v.literal(CONDITION_3_COMPARISON_HASHES.condition1.rawSha256), + transcriptSha256: v.literal( + CONDITION_3_COMPARISON_HASHES.condition1.transcriptSha256, + ), + modelSha256: v.literal( + CONDITION_3_COMPARISON_HASHES.condition1.modelSha256, + ), + comparison: nonEmptyResultString, + }), + condition2: v.strictObject({ + rawSha256: v.literal(CONDITION_3_COMPARISON_HASHES.condition2.rawSha256), + transcriptSha256: v.literal( + CONDITION_3_COMPARISON_HASHES.condition2.transcriptSha256, + ), + modelSha256: v.literal( + CONDITION_3_COMPARISON_HASHES.condition2.modelSha256, + ), + comparison: nonEmptyResultString, + }), + }), + amendments: v.array(nonEmptyResultString), + limitations: v.array(nonEmptyResultString), +}); + +export type Condition3Result = v.InferOutput; + +export function assertCompleteCondition3Result(result: Condition3Result): void { + const actualIds = result.components.map(({ id }) => id); + if ( + actualIds.length !== CONDITION_3_RESULT_COMPONENT_IDS.length || + new Set(actualIds).size !== CONDITION_3_RESULT_COMPONENT_IDS.length || + CONDITION_3_RESULT_COMPONENT_IDS.some( + (componentId) => !actualIds.includes(componentId), + ) + ) { + throw new Error( + "condition-3 result must contain every frozen component exactly once", + ); + } + for (const component of result.components) { + if ( + component.id === "guidance.GEN-Q02.layer-2" && + component.verdict !== "unobservable" + ) { + throw new Error( + "condition-3 GEN-Q02 layer-2 verdict is frozen as unobservable", + ); + } + if ( + component.id.startsWith("signature.") !== + (component.observation !== null) + ) { + throw new Error( + "condition-3 signature components require an observation label and non-signature components forbid one", + ); + } + if (component.observation !== null) { + const expectedVerdict = + component.observation === "observed" + ? "fail" + : component.observation === "not-observed" + ? "pass" + : "unobservable"; + if (component.verdict !== expectedVerdict) { + throw new Error( + `condition-3 signature observation/verdict mismatch for ${component.id}`, + ); + } + } + } +} + +export const CONDITION_3_OPERATOR_ENVELOPE = { + root: { + activeObjectiveRows: [...CONDITION_3_OBJECTIVE_ROWS], + activeObjectiveRowEvidence: + "one transcript-supported anchor record per active objective, including a stable anchorLabel and that row's exact frozen matchingPredicate; multiple anchors may match one row", + retractedObjectiveAnchors: + "durable original and current-turn resolution evidence for previously matched objectives that the expert explicitly retracts", + unsupportedActiveObjectiveAnchors: + "persistent transcript-supported unsupported objective anchors: active items are demanded failures; retracted items preserve original and current-turn resolution evidence", + assessments: "exactly one assessment per frozen clause", + notes: "string[]", + }, + assessmentStates: { + inactive: + "demanded=false; pass=true; status/grade=not-applicable; observedCount=null; failure=null; activation=[]", + passing: + "demanded=true; pass=true; demanded status/grade; presence observedCount or slot null; failure=null; activation=[]", + failing: + "demanded=true; pass=false; demanded status/grade; presence observedCount or slot null; non-null failure; activation from frozen vocabulary", + }, + vocabularies: { + objectiveRows: [...CONDITION_3_OBJECTIVE_ROWS], + objectiveMatchPredicates: [...CONDITION_3_OBJECTIVE_MATCH_PREDICATES], + statuses: [...CONDITION_3_DEMANDED_STATUSES, "not-applicable"], + grades: [...CONDITION_3_DEMANDED_GRADES, "not-applicable"], + failures: [...CONDITION_3_FAILURE_DIAGNOSTICS], + activationPredicates: [...CONDITION_3_FIRES_WHEN], + }, + template: { + activeObjectiveRows: [], + activeObjectiveRowEvidence: [], + retractedObjectiveAnchors: [], + unsupportedActiveObjectiveAnchors: [], + assessments: CONDITION_3_DEMAND_CLAUSES.map((clause) => { + const demanded = clause.row === null; + const isPresenceDemand = clause.demand.startsWith("presence count >="); + const slotUnaddressedCanFire = CONDITION_3_ACTIVATION_MATRIX.some( + ({ clauses, predicates }) => + clauses.includes(clause.id as never) && + predicates.includes("slot-unaddressed"), + ); + return { + clauseId: clause.id, + demand: clause.demand, + coordinate: clause.coordinate, + demanded, + currentStatus: demanded ? "none" : "not-applicable", + currentGrade: demanded ? "none" : "not-applicable", + pass: !demanded, + failureDiagnostic: demanded + ? isPresenceDemand + ? "below-minimum-count" + : "unaddressed" + : null, + activationPredicates: + demanded && slotUnaddressedCanFire + ? (["slot-unaddressed"] as const) + : ([] as const), + evidence: [], + observedCount: demanded && isPresenceDemand ? 0 : null, + rationale: "replace with transcript-bounded judgment", + }; + }), + notes: [], + }, +} as const; + +export function parseCondition3Projection( + value: unknown, +): Condition3Projection { + return v.parse(Condition3ProjectionSchema, value); +} + +export function assertCompleteCondition3Projection( + projection: Condition3Projection, +): void { + const expectedIds = new Set(CONDITION_3_DEMAND_CLAUSES.map(({ id }) => id)); + const actualIds = new Set( + projection.assessments.map(({ clauseId }) => clauseId), + ); + if ( + projection.assessments.length !== expectedIds.size || + actualIds.size !== expectedIds.size || + [...expectedIds].some((clauseId) => !actualIds.has(clauseId)) + ) { + throw new Error( + "condition-3 operator projection must assess every frozen DemandTable clause exactly once", + ); + } +} + +function gradeSatisfiesDemand(demand: string, grade: string): boolean { + const requiredGrade = /grade ([a-z-]+)/u.exec(demand)?.[1]; + if (!requiredGrade) return true; + const qualitativeLadder = [ + "verbal", + "vocabulary-bound", + "structured", + ] as const; + const quantitativeLadder = ["point", "range", "quantiles"] as const; + for (const ladder of [qualitativeLadder, quantitativeLadder]) { + const requiredIndex = ladder.indexOf(requiredGrade as never); + if (requiredIndex < 0) continue; + return ladder.indexOf(grade as never) >= requiredIndex; + } + return false; +} + +function presenceMinimum(demand: string): number | null { + const match = /^presence count >= (\d+)$/u.exec(demand); + return match ? Number.parseInt(match[1] ?? "0", 10) : null; +} + +const compatibleFailuresByPredicate = { + "slot-unaddressed": ["missing-evidence", "unaddressed"], + "below-demanded-grade": ["below-required-grade"], + "unspecified-marker-present": [ + "below-required-grade", + "inadmissible-status", + "missing-evidence", + "unaddressed", + ], + "conflicted-open": ["open-conflict"], + "absence-uncorroborated": ["unaccepted-absence"], + "uniformity-unprobed": ["unsupported-active-anchor"], + "identity-ambiguous": ["unevaluable-divergence"], +} as const satisfies Record< + Condition3FiresWhen, + readonly (typeof CONDITION_3_FAILURE_DIAGNOSTICS)[number][] +>; + +const requiredPredicateByFailure: Partial< + Record<(typeof CONDITION_3_FAILURE_DIAGNOSTICS)[number], Condition3FiresWhen> +> = { + "below-required-grade": "below-demanded-grade", + "missing-evidence": "slot-unaddressed", + unaddressed: "slot-unaddressed", + "open-conflict": "conflicted-open", + "inadmissible-status": "unspecified-marker-present", + "unaccepted-absence": "absence-uncorroborated", + "unevaluable-divergence": "identity-ambiguous", +}; + +export function assertCondition3ProjectionSemantics( + projection: Condition3Projection, +): void { + if ( + new Set(projection.activeObjectiveRows).size !== + projection.activeObjectiveRows.length + ) { + throw new Error("condition-3 active objective rows must be unique"); + } + const activeEvidenceRows = projection.activeObjectiveRowEvidence.map( + ({ row }) => row, + ); + const uniqueActiveEvidenceRows = [...new Set(activeEvidenceRows)]; + if ( + uniqueActiveEvidenceRows.length !== projection.activeObjectiveRows.length || + projection.activeObjectiveRows.some( + (row) => !uniqueActiveEvidenceRows.includes(row), + ) + ) { + throw new Error( + "condition-3 active objective rows must equal the unique row projection of active objective anchors", + ); + } + const allObjectiveAnchorLabels = [ + ...projection.activeObjectiveRowEvidence.map(({ anchorLabel }) => + anchorLabel.trim(), + ), + ...projection.retractedObjectiveAnchors.map(({ anchorLabel }) => + anchorLabel.trim(), + ), + ...projection.unsupportedActiveObjectiveAnchors.map(({ label }) => + label.trim(), + ), + ]; + if ( + new Set(allObjectiveAnchorLabels).size !== allObjectiveAnchorLabels.length + ) { + throw new Error( + "condition-3 objective anchor labels must be unique across matched, retracted, and unsupported anchors", + ); + } + if ( + new Set( + projection.unsupportedActiveObjectiveAnchors.map(({ label }) => label), + ).size !== projection.unsupportedActiveObjectiveAnchors.length + ) { + throw new Error( + "condition-3 unsupported active objective anchors must have unique labels", + ); + } + for (const assessment of projection.assessments) { + const minimumCount = presenceMinimum(assessment.demand); + if (minimumCount !== null) { + if (assessment.observedCount === null) { + throw new Error( + `condition-3 presence assessment requires observedCount for ${assessment.clauseId}`, + ); + } + if ( + assessment.currentGrade !== "none" && + assessment.currentGrade !== "not-applicable" + ) { + throw new Error( + `condition-3 presence assessment must not manufacture a grade for ${assessment.clauseId}`, + ); + } + if (assessment.observedCount > 0 && assessment.evidence.length === 0) { + throw new Error( + `condition-3 positive presence count requires transcript evidence for ${assessment.clauseId}`, + ); + } + if ( + assessment.demanded && + assessment.pass !== assessment.observedCount >= minimumCount + ) { + throw new Error( + `condition-3 presence pass disagrees with observed cardinality for ${assessment.clauseId}`, + ); + } + if ( + assessment.demanded && + !assessment.pass && + assessment.failureDiagnostic !== "below-minimum-count" + ) { + throw new Error( + `condition-3 failing presence assessment requires below-minimum-count for ${assessment.clauseId}`, + ); + } + } else if (assessment.observedCount !== null) { + throw new Error( + `condition-3 slot assessment forbids observedCount for ${assessment.clauseId}`, + ); + } else if ( + assessment.demanded && + !assessment.pass && + assessment.failureDiagnostic === "below-minimum-count" + ) { + throw new Error( + `condition-3 slot assessment forbids below-minimum-count for ${assessment.clauseId}`, + ); + } + const permittedPredicates = new Set( + CONDITION_3_ACTIVATION_MATRIX.filter(({ clauses }) => + clauses.includes(assessment.clauseId as never), + ).flatMap(({ predicates }) => predicates), + ); + if ( + assessment.activationPredicates.some( + (predicate) => !permittedPredicates.has(predicate as never), + ) + ) { + throw new Error( + `condition-3 activation predicate is incompatible with ${assessment.clauseId}`, + ); + } + if ( + assessment.demanded && + !assessment.pass && + assessment.activationPredicates.some( + (predicate) => + !compatibleFailuresByPredicate[predicate].includes( + assessment.failureDiagnostic as never, + ), + ) + ) { + throw new Error( + `condition-3 activation predicate/failure mismatch for ${assessment.clauseId}`, + ); + } + if (assessment.demanded && !assessment.pass) { + const requiredPredicate = + requiredPredicateByFailure[assessment.failureDiagnostic]; + if ( + requiredPredicate && + permittedPredicates.has(requiredPredicate as never) && + !assessment.activationPredicates.includes(requiredPredicate) + ) { + throw new Error( + `condition-3 required activation predicate ${requiredPredicate} is missing for ${assessment.clauseId}`, + ); + } + } + if ( + assessment.demanded && + assessment.pass && + (assessment.evidence.length === 0 || + !gradeSatisfiesDemand(assessment.demand, assessment.currentGrade)) + ) { + throw new Error( + `condition-3 passing assessment does not satisfy the frozen evidence/grade demand for ${assessment.clauseId}`, + ); + } + if ( + assessment.demanded && + !assessment.pass && + assessment.failureDiagnostic === "no-selected-slot" && + assessment.activationPredicates.length > 0 + ) { + throw new Error( + `condition-3 no-selected-slot cannot activate a card for ${assessment.clauseId}`, + ); + } + if (assessment.demanded && !assessment.pass) { + const hasEvidence = assessment.evidence.length > 0; + const hasStatus = assessment.currentStatus !== "none"; + const hasGrade = assessment.currentGrade !== "none"; + switch (assessment.failureDiagnostic) { + case "below-required-grade": + if ( + !hasEvidence || + !hasStatus || + !hasGrade || + gradeSatisfiesDemand(assessment.demand, assessment.currentGrade) + ) { + throw new Error( + `condition-3 below-required-grade requires evidence and a genuinely sub-demand grade for ${assessment.clauseId}`, + ); + } + break; + case "missing-evidence": + case "unaddressed": + if (hasEvidence || hasStatus || hasGrade) { + throw new Error( + `condition-3 ${assessment.failureDiagnostic} requires empty evidence and none status/grade for ${assessment.clauseId}`, + ); + } + break; + case "open-conflict": + if (!hasEvidence || assessment.currentStatus !== "conflicted") { + throw new Error( + `condition-3 open-conflict requires conflicted transcript evidence for ${assessment.clauseId}`, + ); + } + break; + case "inadmissible-status": + if ( + !hasEvidence || + !hasStatus || + CONDITION_3_PASSING_STATUSES.includes( + assessment.currentStatus as never, + ) + ) { + throw new Error( + `condition-3 inadmissible-status requires transcript evidence in a non-passing status for ${assessment.clauseId}`, + ); + } + break; + case "unaccepted-absence": + if (!hasEvidence) { + throw new Error( + `condition-3 unaccepted-absence requires transcript evidence for ${assessment.clauseId}`, + ); + } + break; + case "no-selected-slot": + if (hasEvidence || hasStatus || hasGrade) { + throw new Error( + `condition-3 no-selected-slot requires empty evidence and none status/grade for ${assessment.clauseId}`, + ); + } + break; + case "unsupported-active-anchor": + throw new Error( + "condition-3 unsupported active anchors belong in unsupportedActiveObjectiveAnchors, not a frozen clause assessment", + ); + case "unevaluable-divergence": + if (!hasEvidence) { + throw new Error( + `condition-3 ${assessment.failureDiagnostic} requires transcript evidence for ${assessment.clauseId}`, + ); + } + break; + case "below-minimum-count": + break; + } + } + } + const objectivePresence = projection.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + const activeObjectiveCount = + projection.activeObjectiveRowEvidence.length + + projection.unsupportedActiveObjectiveAnchors.filter( + ({ state }) => state === "active", + ).length; + if (objectivePresence?.observedCount !== activeObjectiveCount) { + throw new Error( + "condition-3 SF-OBJ observedCount must equal matched plus unsupported active objective anchors", + ); + } +} + +export function assertCondition3UnsupportedAnchorContinuity( + previous: Condition3Projection | undefined, + current: Condition3Projection, + currentTurn: number, +): void { + if (!previous) return; + const currentActiveObjectiveByLabel = new Map( + current.activeObjectiveRowEvidence.map((anchor) => [ + anchor.anchorLabel, + anchor, + ]), + ); + const currentRetractedObjectiveByLabel = new Map( + current.retractedObjectiveAnchors.map((anchor) => [ + anchor.anchorLabel, + anchor, + ]), + ); + for (const priorAnchor of previous.activeObjectiveRowEvidence) { + const currentActive = currentActiveObjectiveByLabel.get( + priorAnchor.anchorLabel, + ); + const currentRetracted = currentRetractedObjectiveByLabel.get( + priorAnchor.anchorLabel, + ); + const currentAnchor = currentActive ?? currentRetracted; + if (!currentAnchor) { + throw new Error( + `condition-3 matched objective anchor '${priorAnchor.anchorLabel}' disappeared without a durable retraction`, + ); + } + if ( + currentAnchor.row !== priorAnchor.row || + currentAnchor.matchingPredicate !== priorAnchor.matchingPredicate || + currentAnchor.rationale !== priorAnchor.rationale + ) { + throw new Error( + `condition-3 matched objective anchor '${priorAnchor.anchorLabel}' rewrote its row, predicate, or rationale`, + ); + } + const currentEvidenceKeys = new Set( + currentAnchor.evidence.map(({ turn, quote }) => `${turn}\u0000${quote}`), + ); + if ( + priorAnchor.evidence.some( + ({ turn, quote }) => !currentEvidenceKeys.has(`${turn}\u0000${quote}`), + ) + ) { + throw new Error( + `condition-3 matched objective anchor '${priorAnchor.anchorLabel}' rewrote its original evidence`, + ); + } + if ( + currentRetracted && + !currentRetracted.resolutionEvidence.some( + ({ turn }) => turn === currentTurn, + ) + ) { + throw new Error( + `condition-3 matched objective anchor '${priorAnchor.anchorLabel}' retraction requires current-turn evidence`, + ); + } + } + for (const priorAnchor of previous.retractedObjectiveAnchors) { + const currentAnchor = currentRetractedObjectiveByLabel.get( + priorAnchor.anchorLabel, + ); + if (!currentAnchor) { + throw new Error( + `condition-3 retracted objective anchor '${priorAnchor.anchorLabel}' cannot disappear or reactivate`, + ); + } + if ( + currentAnchor.row !== priorAnchor.row || + currentAnchor.matchingPredicate !== priorAnchor.matchingPredicate || + currentAnchor.rationale !== priorAnchor.rationale || + currentAnchor.resolutionRationale !== priorAnchor.resolutionRationale + ) { + throw new Error( + `condition-3 retracted objective anchor '${priorAnchor.anchorLabel}' rewrote durable metadata`, + ); + } + for (const [kind, priorEvidence, currentEvidence] of [ + ["original", priorAnchor.evidence, currentAnchor.evidence], + [ + "resolution", + priorAnchor.resolutionEvidence, + currentAnchor.resolutionEvidence, + ], + ] as const) { + const currentEvidenceKeys = new Set( + currentEvidence.map(({ turn, quote }) => `${turn}\u0000${quote}`), + ); + if ( + priorEvidence.some( + ({ turn, quote }) => + !currentEvidenceKeys.has(`${turn}\u0000${quote}`), + ) + ) { + throw new Error( + `condition-3 retracted objective anchor '${priorAnchor.anchorLabel}' rewrote ${kind} evidence`, + ); + } + } + } + const currentByLabel = new Map( + current.unsupportedActiveObjectiveAnchors.map((anchor) => [ + anchor.label, + anchor, + ]), + ); + for (const priorAnchor of previous.unsupportedActiveObjectiveAnchors) { + const currentAnchor = currentByLabel.get(priorAnchor.label); + if (!currentAnchor) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' disappeared without a durable retraction`, + ); + } + if ( + priorAnchor.state === "retracted" && + currentAnchor.state !== "retracted" + ) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' cannot reactivate after retraction`, + ); + } + if ( + priorAnchor.state === "active" && + currentAnchor.state === "retracted" && + !currentAnchor.resolutionEvidence.some(({ turn }) => turn === currentTurn) + ) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' retraction requires current-turn evidence`, + ); + } + if (currentAnchor.rationale !== priorAnchor.rationale) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' rewrote its original rationale`, + ); + } + const currentEvidenceKeys = new Set( + currentAnchor.evidence.map(({ turn, quote }) => `${turn}\u0000${quote}`), + ); + if ( + priorAnchor.evidence.some( + ({ turn, quote }) => !currentEvidenceKeys.has(`${turn}\u0000${quote}`), + ) + ) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' rewrote its original evidence`, + ); + } + if ( + priorAnchor.state === "retracted" && + currentAnchor.state === "retracted" + ) { + if ( + currentAnchor.resolutionRationale !== priorAnchor.resolutionRationale + ) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' rewrote its resolution rationale`, + ); + } + const currentResolutionEvidenceKeys = new Set( + currentAnchor.resolutionEvidence.map( + ({ turn, quote }) => `${turn}\u0000${quote}`, + ), + ); + if ( + priorAnchor.resolutionEvidence.some( + ({ turn, quote }) => + !currentResolutionEvidenceKeys.has(`${turn}\u0000${quote}`), + ) + ) { + throw new Error( + `condition-3 unsupported objective anchor '${priorAnchor.label}' rewrote its resolution evidence`, + ); + } + } + } +} + +export function nextCondition3NoProgressStreak( + history: readonly Condition3Projection[], + current: Condition3Projection, + currentTurn: number, + previousStreak: number, +): number { + const priorEvidenceQuotes = new Set([ + ...history + .flatMap(({ assessments }) => assessments) + .flatMap(({ evidence }) => evidence.map(({ quote }) => quote)), + ...history + .flatMap( + ({ unsupportedActiveObjectiveAnchors }) => + unsupportedActiveObjectiveAnchors, + ) + .flatMap((anchor) => [ + ...anchor.evidence.map(({ quote }) => quote), + ...(anchor.state === "retracted" + ? anchor.resolutionEvidence.map(({ quote }) => quote) + : []), + ]), + ...history.flatMap(({ activeObjectiveRowEvidence }) => + activeObjectiveRowEvidence.flatMap(({ evidence }) => + evidence.map(({ quote }) => quote), + ), + ), + ...history.flatMap(({ retractedObjectiveAnchors }) => + retractedObjectiveAnchors.flatMap((anchor) => [ + ...anchor.evidence.map(({ quote }) => quote), + ...anchor.resolutionEvidence.map(({ quote }) => quote), + ]), + ), + ]); + const hasNewDemandedEvidence = + current.assessments.some( + (assessment) => + assessment.demanded && + assessment.evidence.some( + ({ turn, quote }) => + turn === currentTurn && !priorEvidenceQuotes.has(quote), + ), + ) || + current.unsupportedActiveObjectiveAnchors.some((anchor) => + (anchor.state === "active" + ? anchor.evidence + : anchor.resolutionEvidence + ).some( + ({ turn, quote }) => + turn === currentTurn && !priorEvidenceQuotes.has(quote), + ), + ) || + current.activeObjectiveRowEvidence.some(({ evidence }) => + evidence.some( + ({ turn, quote }) => + turn === currentTurn && !priorEvidenceQuotes.has(quote), + ), + ) || + current.retractedObjectiveAnchors.some(({ resolutionEvidence }) => + resolutionEvidence.some( + ({ turn, quote }) => + turn === currentTurn && !priorEvidenceQuotes.has(quote), + ), + ); + return hasNewDemandedEvidence ? 0 : previousStreak + 1; +} diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-legibility.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-legibility.md new file mode 100644 index 00000000000..1c44d3ecc5d --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-legibility.md @@ -0,0 +1,146 @@ +# FE-1404 condition-3 second-register rendering and strain record + +## First fresh-context rendering — rejected + +Grade: **C− / not seal-ready**. + +In plain language, condition 3 is one qualitative single-session comparison with conditions 1 and +2. A transcript-only experimental operator evaluates the frozen DemandTable after each expert +answer. The interviewer receives one compact diagnostic; the expert and interviewer do not receive +the full projection. The runner owns the phase impatience stimulus, evidence-delta no-progress +control, and turn budget. GEN-Q02 has no machine layer-2 adjudicator and is evaluated manually only +for resulting interaction behavior. The result is whole-treatment existence evidence, not runtime, +store, plugin, compiler, validation, persistence, or deferral-licensing proof. FE-1431 retains the +authoring-representation decision. + +The first fresh reader found the following strain. This review occurred before the final seal and +before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| The draft lock was stale and omitted new locked artifacts. | Keep the draft rejected; generate one exact canonical lock only after every remediation and verification step. | +| The supplied template activated `ROW-SPLIT` while marking split clauses inactive. | Generate applicability once and use it for every template discriminant; execute the constructed template in a focused test. | +| Passing `none` states and clause-incompatible activation could survive validation. | Require passing evidence, an accepted status, demand-satisfying grade, and clause-compatible predicates before selection or no-progress. | +| Frame five stopped before the diagnostic or a respectful close reached the interviewer. | Make frame five end questioning, inject a final-delivery instruction, permit exactly the closing interviewer turn, and retain delivery/completion separation. | +| Truncated-interviewer regeneration lacked a seam; terminal failures could resume. | Add that seam type and permit condition-3 resume only for explicitly recoverable states. | +| The result destination lacked an exhaustive machine contract. | Freeze the verdict schema and exact component inventory in the instrument; test completeness. Keep delivery and deposit separate and exempt GEN-Q02 from a CPS three-layer aggregate. | +| Mechanical quote novelty could be overread as semantic improvement. | Name it a live stopping candidate produced by the judgment-bearing operator; score semantic correctness separately and retain this limitation. | +| GEN-Q02 manual scoring, stop precedence, cardless diagnostics, and experiment/runtime terms were underdefined. | Add manual semantic rules, prompt-versus-runner ownership and precedence, cardless `not-applicable` handling, and qualified experimental terminology. | +| Injected impatience text could be cited as expert evidence. | Persist it as labelled experiment stimulus, show it in interaction context, and exclude it from DemandTable quote provenance. | +| “Immutable” described actively rewritten checkpoints. | Reserve immutable for completed source segments; call active files provisional checkpoints. | + +## Second fresh-context rendering — rejected + +Grade: **D / not seal-ready**. This review also occurred before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Forced wrap still entered the expert/operator/no-progress path. | At and after turn 20, append only a labelled runner stimulus and proceed directly to the next interviewer turn. Add an execution test proving there are only 19 expert/operator frames. | +| Failing projection coherence was incomplete. | Add failure-specific semantic requirements; in particular, below-grade requires cited evidence and a genuinely sub-demand grade. Reject contradictions before selection or no-progress. | +| The result inventory collapsed card, interaction, coverage, and excavation submeasures and omitted FM-10. | Freeze CPS-Q01–Q05 aggregates, the five Bano dimensions, seven coverage dimensions, three excavation dimensions, and FM-10 as exact machine result rows. | +| Final continuation accepted a truncated non-delivery. | For condition 3, require a `delivered*-incomplete` stop reason before any continuation call; add a refusal test. | +| A pending no-progress close could be silently rewritten at budget exhaustion. | Reserve turns 20–24 for delivery by eliminating expert frames after force wrap; make any remaining pending close at loop exit an instrumentation invariant failure. | +| Unsupported active objectives were unrepresentable. | Add a transcript-supported `unsupportedActiveObjectiveAnchors` projection field outside frozen DemandTable rows; select its cardless diagnostic first without making an FE-1431 binding decision. | +| Runner-authored stimuli were insufficiently forbidden in operator prose. | Explicitly forbid all `` content as evidence in the operator contract. | +| `retry-required` appeared to be a final verdict. | Remove it from the final verdict domain and retain it only as an intermediate operator-attempt disposition. | + +## Third fresh-context rendering — rejected + +Grade: **D / not seal-ready**. The intentionally stale draft lock was rejected, and the reader found +three additional treatment/scoring strains before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Unsupported objective anchors had no explicit unresolved state and could permanently starve frozen failures. | Require `demanded=true`, `pass=false`, and `unsupported-active-anchor`; emit a truthful no-binding demand; diagnose each label once, then resume frozen priority while it remains recorded. | +| A compatible activation predicate could be silently omitted. | Map failure diagnostics to required predicates and reject omission whenever that predicate is available for the clause; execute the generated template and omission case. | +| The result schema allowed GEN-Q02 layer 2 to contradict its frozen unobservable verdict. | Enforce `unobservable` for that exact component in result validation and add a contradiction test. | +| “Semantic no-progress” overstated a mechanical stopping input. | Name the live rule operator-adjudicated quote novelty and keep later semantic correctness scoring separate. | +| The runner synopsis described only legacy in-place continuation. | State the C1/C2 legacy merge and C3 append-only sealed-segment behaviors separately. | + +The rejected draft lock is not remediated in place; it remains chronology evidence until the final +artifact and verification pass. + +## Fourth fresh-context rendering — rejected + +Grade: **D / not seal-ready**. This resumed seal-gate review occurred before any condition-3 model +call and treated the stale draft lock as expected chronology evidence rather than a new finding. + +| Strain | Pre-seal disposition | +| --- | --- | +| `inadmissible-status` required `unspecified-marker-present` but that predicate rejected the failure. | Add `inadmissible-status` to the predicate compatibility table and execute the coherent failure case. | +| Count-only static presence clauses were forced to invent a grade. | Treat absence of a declared minimum grade as no grade requirement; require cited cardinality evidence and permit `currentGrade=none`. | +| CPS-Q03 omitted its reviewed `SP-SCRAP` target. | Restore `SP-SCRAP` to the experiment activation matrix and test the exact reviewed target list. | + +## Fifth fresh-context rendering — rejected + +Grade: **D / not seal-ready**. The review occurred before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Presence cardinality was inferred from evidence/grade and could pass a count-two clause with one selected item or reject a truthful zero count. | Add schema-owned `observedCount`; require exact minimum comparison, positive-count evidence, no grade, and `below-minimum-count` for failures. | +| Force wrap was appended after interviewer turn 20. | Inject the labelled stimulus before the turn-20 interviewer call and reserve turns 20–24 without expert/operator/no-progress frames. | +| Successfully stitched non-final interviewer continuations were absent from expert/operator/provider views. | Project source plus every continuation into all semantic request views while retaining append-only persistence seams. | +| Unsupported anchors could disappear or relabel silently across projections. | Persist each label; require active or durable retracted state, current-turn retraction evidence, and no reactivation. | +| `inadmissible-status` still accepted status `none`. | Require a non-`none`, non-passing status with cited evidence. | +| CPS-Q03's regression asserted only membership. | Assert the exact reviewed five-clause target list. | + +## Sixth fresh-context rendering — rejected + +Grade: **D / not seal-ready**, with no blockers and two high-severity strains. The review occurred +before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Slot clauses could misuse the presence-only `below-minimum-count` failure. | Forbid that diagnostic whenever the demand has no presence minimum; add a direct rejection test. | +| New evidence for a demanded unsupported anchor did not reset no-progress. | Include active-anchor support and current-turn retraction evidence in operator-adjudicated quote novelty. | +| Anchor support and rationale could be rewritten between projections. | Preserve original label, evidence, and rationale; allow evidence only to append and use separate resolution evidence/rationale. | + +## Seventh fresh-context rendering — rejected + +Grade: **D / not seal-ready**, with one scoring-inventory blocker and four high-severity strains. +The review occurred before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| The result inventory collapsed completion, user stopping, no-progress, budget, and deferral. | Add separate exact component IDs while retaining delivery/deposit and the aggregate stopping row. | +| Retraction resolution evidence/rationale remained rewritable. | Preserve both append-only after the first retracted projection. | +| Forced-wrap resume regenerated a completed turn and duplicated its stimulus. | Add `forced-wrap-in-progress`; resume advances to the next turn without popping the complete assistant response. | +| Active objective rows lacked their own evidence. | Require exactly one quote/rationale record per active row and validate every quote before applicability. | +| Quote novelty looked back only one projection. | Compute the prior quote set across the complete projection history, including objective-row and unsupported-anchor evidence. | +| C1/C2 comparison inputs were not sealed or hash-bound in results. | Lock the reviewed readout and both raw/transcript/model artifacts; require their hashes in each machine comparison. | + +## Eighth fresh-context rendering — rejected + +Grade: **D / not seal-ready**, with one activation blocker, one high-severity stopping strain, and +one medium-severity result-contract strain. The review occurred before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Any genuine quote could activate any objective row because the exact frozen row predicate was not logged. | Add the four exact FE-1402 `whenObjective` labels as a closed row-discriminated registry; require the matching label plus quote/rationale and reject cross-row predicates. This is an experiment adjudication record, not an FE-1431 representation decision. | +| Global duplicate suppression omitted inactive assessment evidence and retracted-anchor evidence. | Build the prior quote set from every assessment plus every anchor's original and resolution evidence, while retaining demanded/current-turn gating for materiality. | +| Scored result rows and comparison prose could be empty. | Require non-empty rationale and comparison text, non-empty evidence strings, and at least one evidence item for every pass/fail/mixed component in the runtime result schema. | + +Focused verification after these dispositions: the instrument and runner suites passed 54/54 with +the loopback permission required by the runner harness; standalone and package TypeScript checks +passed. + +## Ninth fresh-context rendering — rejected + +Grade: **D / not seal-ready**, with one completion-projection blocker, two high-severity runtime +strains, two medium-severity evidence-binding strains, and one stale line. The review occurred +before any condition-3 model call. + +| Strain | Pre-seal disposition | +| --- | --- | +| Unique active rows collapsed multiple active objective anchors and did not reconcile the universal active-anchor check. | Preserve every matched objective under a stable anchor label, allow multiple anchors to project to one unique row, preserve explicit retractions, and require `SF-OBJ.observedCount` to equal matched plus unsupported active anchors. | +| Grade comparison rejected `structured` evidence for the frozen `vocabulary-bound` minimum. | Execute both frozen ladders: `verbal < vocabulary-bound < structured` and `point < range < quantiles`; add a structured-over-vocabulary-bound regression. | +| Recovery imported projection/stopping history without revalidating its semantics. | Reparse every saved projection and revalidate inventory, semantics, transcript quote provenance, matched/unsupported continuity, activation choices, and no-progress state before any recovery call. | +| Result comparison hashes were shape-checked but not bound to the sealed sources. | Freeze exact C1/C2 raw/transcript/model hashes in the instrument and require those literal values in the result runtime schema. | +| The runner-authored single-session correction could be cited as opening evidence. | Store the shared opening as `expertContent`, label the correction as experiment stimulus, and exclude it through the same provenance boundary used for later stimuli. | +| The final-rendering placeholder referenced only seven reviews. | Update chronology to the ninth rejected rendering before the next fresh gate. | + +## Final fresh-context rendering + +Pending after the ninth-review dispositions stabilize. The final lock must include this report and +may not be created until a new fresh reader can render the amended instrument without a blocker or +high-severity strain. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md new file mode 100644 index 00000000000..bc64059b7d9 --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md @@ -0,0 +1,82 @@ +# Condition 3 test-only operator instructions (FE-1404) + +--- + +You are the judgment-bearing test-only completion operator for a preregistered experiment. You are +not the interviewer, a capture store, a runtime completion implementation, or a model +self-inventory mechanism. + +After each simulated-expert answer, assess every clause in the supplied frozen DemandTable using +only the opening message and transcript utterances. Do not use the private situation pack, hidden +answer-key values, prior baseline answers, or likely domain facts. Carry earlier transcript-visible +evidence forward. An interviewer-authored statement is not user evidence; user assent supports only +what the user actually confirms. + +Return JSON only, matching the supplied `PROJECTION_ENVELOPE`. That envelope is emitted from the +same evaluation-runner Valibot schema that owns the TypeScript projection type; its vocabularies and +complete JSON template are authoritative. Include every clause exactly once. Do not add fields. A row clause is +demanded only when its objective is active; set inactive row clauses to pass=true, +currentStatus="not-applicable", currentGrade="not-applicable", failureDiagnostic=null, and no +activation predicates. For demanded clauses, preserve status separately from grade. Unknown or a +future observation is not a value. A no-selected-slot failure must not emit slot-unaddressed because +the reviewed card cannot fire before a coordinate exists. + +Record every active objective anchor separately in `activeObjectiveRowEvidence`, using a stable +`anchorLabel`, one or more verbatim transcript quotes, and a falsifiable rationale. Multiple active +anchors may match the same row. Set `activeObjectiveRows` to the unique rows projected from those +anchor records. Set `matchingPredicate` to that row's exact supplied FE-1402 `whenObjective` label; +the row/predicate pair is a closed discriminated vocabulary, so a predicate belonging to another +row invalidates the whole projection. Set the `SF-OBJ` count to the number of active matched anchors +plus active unsupported anchors; do not collapse multiple objectives into one row or omit an +unmatched objective. Adjudicate the predicate against what the expert actually names as an +objective, not against incidental topic words. Never activate an objective row from topic similarity +or the private situation pack. This match log is experiment evidence, not a proposed FE-1431 +binding representation. + +Preserve each matched anchor's label, row, predicate, original evidence, and rationale in later +projections. If the expert explicitly retracts it, move it to `retractedObjectiveAnchors`, preserve +the original fields, and cite non-empty current-turn `resolutionEvidence` plus a +`resolutionRationale`. Never omit, reactivate, relabel, or rewrite a matched anchor. + +If transcript evidence activates an objective that has no frozen objective row, record it in +`unsupportedActiveObjectiveAnchors` with a unique short label, one or more verbatim evidence quotes, +and a rationale. Every such item is an unresolved demand: set `state="active"`, `demanded=true`, +`pass=false`, `failureDiagnostic="unsupported-active-anchor"`, `resolutionEvidence=[]`, and +`resolutionRationale=null`. Preserve the same label, original evidence, and original rationale in +every later projection; new supporting evidence may append. If the expert explicitly retracts the +objective, keep the item, set `state="retracted"`, `demanded=false`, `pass=true`, +`failureDiagnostic=null`, and cite the current expert turn in non-empty `resolutionEvidence` with a +non-empty `resolutionRationale`. Never omit, relabel, rewrite, reactivate, or manufacture a +resolution. This is an +experiment-only diagnostic projection, not a new DemandTable row or an FE-1431 binding decision. +Do not force that objective into the closest existing row or put `unsupported-active-anchor` on a +frozen clause assessment. + +The three assessment states are disjoint. Inactive means `demanded=false`, both status and grade +`not-applicable`, `pass=true`, null failure, and no activation. Passing demanded means +`demanded=true`, a demanded-status/grade vocabulary value, `pass=true`, null failure, and no +activation. Failing demanded means `demanded=true`, demanded-status/grade values, `pass=false`, a +failure from the supplied vocabulary, and only activation predicates from the supplied vocabulary. +Passing also requires transcript evidence. A slot demand requires a grade satisfying its frozen +minimum. A count-only presence demand has no grade requirement: use `currentGrade="none"`; do not +manufacture a grade from cardinality. Set `observedCount` to the transcript-supported number of +selected nodes for a presence clause and to `null` for every slot clause. Presence passes exactly +when `observedCount` meets the frozen minimum; zero may correctly have no quote and fails +`below-minimum-count`, while every positive count requires cited evidence. +Activation predicates must be declared for that clause in the frozen experiment matrix; use an +empty array only when the failure has no compatible predicate. When the matrix contains the +failure's matching predicate—such as `below-demanded-grade` for `below-required-grade`—include it; +omission invalidates the projection. The runner rejects the whole projection before it can affect +selection or no-progress if any row violates these rules. + +Activation predicates are limited to the supplied seven-value FE-1405 vocabulary and must describe +the visible state exactly. Typical mappings are: an existing selected slot with no value may emit +slot-unaddressed; a stated value below the demand may emit below-demanded-grade; a transcript marker +such as "roughly" or an unresolved placeholder may emit unspecified-marker-present; an explicit +unknown-to-user answer may emit absence-uncorroborated because this DemandTable accepts no absence. +Do not invent a predicate merely to make a card fire. + +Every evidence item must quote the transcript exactly and name its interviewer-turn number. Keep +rationale short and falsifiable. Content enclosed in `` is runner-authored and +must never be quoted or treated as expert evidence. Notes may identify operator uncertainty; they +may not amend the instrument. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-pre-run-review.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-pre-run-review.md new file mode 100644 index 00000000000..996ec6a830a --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-pre-run-review.md @@ -0,0 +1,129 @@ +# FE-1404 condition-3 pre-run review record + +No external model call occurred before either review. No condition-3 transcript, raw trace, operator +trace, or result existed. The draft lock SHA-256 +`d308db20c73debd4b0c30e592f73a0975569064c2dfc2ec95a43e15a733fb5c0` is rejected pre-run evidence, +not a historical experiment seal. + +## Review A — experiment contract and trace integrity + +| Finding | Disposition before reseal | +| --- | --- | +| No-progress could reset on operator regrading, row drift, quote order, or evidence-array length. | Replace with a set-based delta over demanded evidence quoted from the new expert frame; add onset, reset, advisory, hard-stop, equal-length replacement, duplicate, reorder, and row-drift tests. | +| The operator received no explicit result envelope, vocabularies, or template. | Send the runtime schema's complete vocabulary and a concrete full-clause JSON template in the constructed operator request; inspect that request in tests. | +| GEN-Q02 used question-mark counting as a semantic proxy. | Remove the proxy and record layer-2 activation as `unobservable` in this run; retain the reviewed batching guidance and score its layer-3 transcript behavior manually, including imperative questions and permitted cohesive five-item groups. | +| Resume and continuation were not seal-bound and could rewrite prior evidence. | Validate seal hash, instrument/DemandTable versions, and exact model config before an API call. Write recovery segments to new raw/transcript paths and retain truncation/continuation seams. | +| The three verdict layers lacked an operational scorer contract. | Freeze verdict domain, authority, procedure, component failure/retry/unobservable rules, aggregation, and result paths in `condition-3-scoring.md`. | +| Runner synopsis still described only two conditions. | Reconcile the synopsis and usage text while retaining C1/C2 behavior. | +| Draft chronology allowed independent pre-run timestamps. | Require sealedAt and finalized lock mtime to postdate every locked file; reject an early self-declared sealedAt. | + +## Review B — TypeScript and runtime-boundary audit + +| Finding | Disposition before reseal | +| --- | --- | +| Manifest verification accepted missing, extra, duplicate, and empty row sets. | Make one canonical path list authoritative and require exact one-to-one manifest identity before hashing files. | +| Projection types and validation had dual ownership and contradictory states were representable. | Make one Valibot discriminated schema the runtime/type owner. Infer TypeScript types and reject incoherent pass/failure/activation/status states before selection or no-progress. | +| Evidence citations were not checked against the supplied transcript. | Validate every quote against the exact opening or expert turn it names; invalid projections retry and then fail visibly. | +| Condition-3 stopping constants were duplicated in the runner. | Use the frozen instrument's values for condition 3 and explicitly named legacy constants for conditions 1 and 2. | +| Compatibility and failure-path coverage was too narrow. | Exercise C1 and C2, exact-manifest failures, seal mismatch on both recovery modes, malformed/contradictory operator output, quote provenance, phase-triggered impatience, operator-adjudicated quote-novelty stopping, and canonicalized evidence order. | + +## Type source-of-truth disposition + +1. **Operator projection** — canonical source: `Condition3ProjectionSchema` — action: **infer**. + Runtime parsing and TypeScript state space share one discriminated Valibot owner. +2. **Instrument vocabularies and manifest paths** — canonical source: exported `as const` registries + in `condition-3-instrument.ts` — action: **import/project**. The runner and tests do not restate + their literal unions. +3. **Checkpoint and recovery segment** — canonical source: runner-local persistence boundary — + action: **keep-local**. These types add experiment-specific durable semantics not owned by the + provider SDK or plugin contract. + +## Review C — second-register stopping and scoring audit + +| Finding | Disposition before reseal | +| --- | --- | +| Forced wrap was being reclassified as expert evidence. | Route it only as a labelled runner stimulus after turn 20; prohibit expert/operator calls and no-progress updates on those turns. | +| Failure discriminants admitted contradictory evidence/status/grade combinations. | Add failure-specific semantic checks and retry/fail before a projection can affect selection. | +| Machine result rows omitted card/submeasure aggregates and FM-10. | Expand the exact frozen inventory to five CPS card aggregates, five interaction dimensions, seven coverage dimensions, three excavation dimensions, and all FM-01–FM-15 signatures. | +| Continuation and budget terminality had permissive fallbacks. | Continue only classified deliveries and turn an impossible pending-close budget exit into an explicit invariant failure. | +| Unsupported active objective anchors had no lossless projection. | Add a schema-owned, quote-validated cardless diagnostic envelope outside the frozen row set; do not infer an FE-1431 binding. | +| Runner stimuli and retry state were ambiguous in prose. | Forbid stimulus provenance in the operator contract and keep `retry-required` outside the final verdict domain. | + +## Review D — fresh unsupported-anchor and result-coherence audit + +| Finding | Disposition before reseal | +| --- | --- | +| Unsupported active anchors lacked unresolved liveness and could starve frozen diagnostics. | Give each anchor explicit demanded/failing state, diagnose each label once, preserve it in later projections, and return to frozen diagnostic priority without claiming a binding. | +| Compatible card activation could be silently omitted. | Require the failure-corresponding predicate whenever that predicate exists for the clause; reject before selection or no-progress. | +| GEN-Q02 layer 2 could receive a verdict other than the frozen `unobservable`. | Enforce that component-specific invariant in the machine result validator. | +| Quote novelty was described as semantic no-progress. | Describe it as an operator-adjudicated live stopping input and score semantic improvement only after observation. | + +## Review E — resumed seal-gate semantic audit + +| Finding | Disposition before reseal | +| --- | --- | +| `inadmissible-status` could neither include nor omit its required predicate. | Reconcile the predicate/failure compatibility table and add a coherent-case regression test. | +| Static presence demands were assigned a manufactured grade requirement. | Preserve presence as cardinality-only: cited evidence is required, but `currentGrade=none` passes when the frozen demand declares no minimum grade. | +| CPS-Q03's reviewed `SP-SCRAP` target was absent from the activation matrix. | Restore the exact target without making a multiplicity or FE-1431 representation decision. | + +## Review F — cardinality, turn-boundary, and continuity audit + +| Finding | Disposition before reseal | +| --- | --- | +| Presence clauses lacked selected-node cardinality and mishandled truthful zero counts. | Add schema-owned `observedCount` and enforce cardinality, provenance, grade-none, and below-minimum failure invariants. | +| Turn-20 force wrap reached only turn 21. | Inject before the twentieth interviewer call and prevent every later expert/operator/no-progress frame. | +| Stitched non-final interviewer continuations were dropped from later semantic views. | Recompose all pieces at the provider boundary and in expert/operator views without rewriting stored source pieces. | +| Unsupported anchor labels lacked cross-projection continuity. | Require persistent active/retracted records, current-turn retraction evidence, and monotonic retraction. | +| Inadmissible evidence could retain status `none`. | Require a cited non-passing epistemic status. | + +## Review G — clause-kind and anchor-materiality audit + +| Finding | Disposition before reseal | +| --- | --- | +| Slot clauses accepted the presence-only `below-minimum-count` diagnostic. | Forbid it outside count demands and test the exact invalid state. | +| New support for a demanded unsupported anchor did not reset no-progress. | Include active-anchor evidence and current-turn retraction evidence in the frozen quote-novelty rule. | +| Unsupported-anchor evidence and rationale were mutable across projections. | Make original label/evidence/rationale append-only and separate durable resolution evidence/rationale. | + +## Review H — result-vector, activation-provenance, and recovery audit + +| Finding | Disposition before reseal | +| --- | --- | +| Independent completion/stopping/deferral results were absent. | Add distinct completion, user-stop, no-progress, budget, and deferral rows to the exact result inventory. | +| Retraction resolution provenance was mutable. | Preserve resolution evidence/rationale append-only after retraction. | +| Forced-wrap resume regenerated the completed prior turn. | Persist a dedicated resumable state and advance without popping or duplicating the prior turn/stimulus. | +| Objective-row activation was not evidence-bearing. | Add one exact quote/rationale record per active row and validate it before demand applicability. | +| Duplicate suppression looked back one projection only. | Build novelty against the entire prior projection history. | +| C1/C2 comparison evidence was not sealed/hash-bound. | Lock the prior readout/raw/transcript/model inputs and embed exact source hashes in comparison results. | + +## Review I — exact row matching, global novelty, and result evidence audit + +| Finding | Disposition before reseal | +| --- | --- | +| Objective rows cited transcript evidence but did not log or constrain the exact frozen `whenObjective` match. | Import the four FE-1402 labels into a closed row-discriminated registry. Require the exact row/predicate pair with quote/rationale, reject cross-row pairs in the runtime schema, and explicitly avoid choosing an FE-1431 binding representation. | +| Duplicate suppression excluded evidence once recorded on inactive assessments or retracted anchors. | Build novelty from every historical assessment quote and both original and resolution evidence for every historical unsupported anchor. Only new current-turn evidence on a currently demanded surface resets the streak. | +| Machine result validation admitted empty scored evidence, rationale, and comparison prose. | Make non-empty strings structural schema requirements and require at least one evidence citation for every pass/fail/mixed component. | + +## Review J — objective cardinality, ladders, and recovery semantics audit + +| Finding | Disposition before reseal | +| --- | --- | +| Unique active rows collapsed multiple active anchors and left the universal active-anchor count unreconciled. | Preserve every matched active objective as a stable anchor record, derive unique rows from those records, preserve explicit retractions, and reconcile `SF-OBJ` to matched plus unsupported active anchors. | +| The qualitative grade ladder was implemented as exact equality. | Execute both frozen qualitative and quantitative ladders and test that structured evidence satisfies a vocabulary-bound minimum. | +| Recovery trusted saved projection, selection, and no-progress state after checking only configuration binding. | Reparse and semantically replay the full operator history against the transcript before importing it; reject edited state before any resumed call. | +| C1/C2 result hashes were not exact literals. | Freeze the six reviewed source hashes in the instrument and require exact values in the machine result schema. | +| The runner-authored single-session correction was inside the admissible opening evidence string. | Preserve the common opening as expert evidence, label the correction as experiment stimulus, and exclude it from quote validation. | + +## Fresh-context legibility review + +The first second-register rendering graded the draft C− / not seal-ready. It found an invalid +operator template, incomplete semantic coherence checks, a no-progress close that never reached the +interviewer, missing recovery terminality/seams, and an underspecified result file. The complete +rendering, strain list, and evidence-backed dispositions are preserved in +`condition-3-legibility.md`. The draft remained unsealed and no condition-3 call occurred. + +## Gate + +The next seal is permitted only after every row above is implemented, formatting passes, focused +tests pass uncached, the full Brunch unit suite passes uncached, type/lint/build and documentation +checks pass, and the lock chronology verifier passes. The approved temporary dependency symlink is +removed after the final applicable post-run verification and before handoff. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json new file mode 100644 index 00000000000..4a9c95350a9 --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json @@ -0,0 +1,50 @@ +{ + "version": "fe-1404-condition-3/2026-08-25.1", + "sealedAt": "2026-08-25T05:46:59Z", + "files": [ + { + "path": "evaluations/protocols/process-model-elicitation/baseline/run.ts", + "sha256": "9b91791de955b42c866b3bc7585dd4d22a2aa3f837fe320de845a2b1d50c4a90" + }, + { + "path": "evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts", + "sha256": "6acebc39558af16803876240ee062e6fbe684ac7a1cb2a9d17706d132a273d7c" + }, + { + "path": "evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md", + "sha256": "a07648dd8721e9cce9356fbb34f154acf07ce2c86ca6f11d8e0bc92b4cfa8850" + }, + { + "path": "evaluations/protocols/process-model-elicitation/baseline/condition-3-operator.md", + "sha256": "d481dca97dc3035dbddc5792012f5b04394538e454b7dce08b8202e1507ce365" + }, + { + "path": "evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md", + "sha256": "7ccbf35313bee3df3ca1b230687984ed3e291531d1b722782b1b0ffb0b815f27" + }, + { + "path": "evaluations/protocols/process-model-elicitation/baseline/protocol.md", + "sha256": "77db6117f7f1d8819bd7dd346fea4c81608e171a187f13b86d917d89cb0bd20a" + }, + { + "path": "evaluations/cases/process-model-elicitation/baseline/opening-message.md", + "sha256": "84ec5faa5fd46699c008b3b2aad49eb9988b8c2ab039c8e147fdb077d562ef54" + }, + { + "path": "evaluations/cases/process-model-elicitation/baseline/situation-pack.md", + "sha256": "4dbeb44a881c4675ec0ce7a5f068ea46ce1a4968a405b2dd692f92816d33e083" + }, + { + "path": "docs/specs/elicitation-completion.md", + "sha256": "f076dd6f50b7a901cecbe7310f62bcaefc6987de4667c928711d26b861ddf67d" + }, + { + "path": "docs/specs/cps-interview-guidance.md", + "sha256": "f434eb101a8087d5227589e6fed7505cdaa5a70e828681ae9f12341fc1a6dcf6" + }, + { + "path": "docs/reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md", + "sha256": "1d6b4e9ddc684b61067ca6575ffdad3dab8c741acd24b73525e5a14b8cecf279" + } + ] +} diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md new file mode 100644 index 00000000000..b5f0d55a2bd --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.md @@ -0,0 +1,150 @@ +# FE-1404 condition-3 preregistration + +Status: **frozen before the first model call**. The lock file beside this document records hashes +for the complete treatment and instrument. Any later change requires an explicit amendment; the +original run and original lock remain immutable. + +## Question and comparison + +Does the reviewed FE-1402 completion contract plus the FE-1403 surviving guidance change the +interviewer's stopping, reprioritisation, and refusal of unproductive deferral, while improving +transcript evidence for known condition-1/condition-2 gaps? Condition 3 uses the same opening, +simulated expert, interviewer family, classifier, default sampling, and inherited scoring surfaces. +It adds the single-session correction, phase-triggered impatience probe, no-progress rule, reviewed +cards/fragments, and a test-only operator projection. + +Compare condition 3 with both prior conditions. Treat n=1 as existence evidence, never a rate or +effect-size estimate. Attribute operator-triggered behavior to the whole treatment, not to the +interviewer alone. + +## Frozen treatment and information wall + +- Interviewer: `claude-opus-5`, adaptive thinking, default provider sampling, no seed parameter. +- Simulated expert: `claude-sonnet-5`, thinking disabled, the unchanged private situation pack. +- Classifier: `claude-haiku-4-5-20251001`, thinking disabled. +- Operator: `claude-opus-5`, thinking disabled, JSON-only judgment over transcript-visible evidence. +- The interviewer receives the condition-3 prompt plus the opening message. It never receives the + situation pack, answer-key values, operator rationale/evidence, whole projection, activation + trace, or prior baseline transcript. +- After each expert answer, the interviewer receives only the selected clause coordinate, current + status, current grade, demand, and failure diagnostic. +- The operator receives the transcript, frozen DemandTable, and activation vocabulary. It never + receives the situation pack or prior baseline transcript. +- The expert never receives operator diagnostics or interviewer system guidance. + +The frozen DemandTable, diagnostic priority, activation matrix, card IDs, model configuration, and +stopping constants live in `condition-3-instrument.ts`. The reviewed card wording and two fragments +live in `condition-3-prompt.md`. The judgment protocol lives in `condition-3-operator.md`; the +verdict domain, scorer authority, failure rules, aggregation, and fixed output destinations live in +`condition-3-scoring.md`. + +## Preregistered observations + +1. `CPS-Q01`: occurrence and repair remain separate; motor-like weak evidence keeps its actual + verbal/point grade until the expert supplies more. +2. `CPS-Q02`: the interviewer asks about ramp scrap even if its own inventory never named it; + unknown or a future observation remains a failing non-value. +3. `CPS-Q03`: the run seeks split minimum/contiguity plus ordinary ranges for extra changeovers and + repeated scrap. +4. `CPS-Q04`: the practiced release gate is elicited and the card deactivates after structured + evidence passes. +5. `CPS-Q05`: the practiced shared-resource conflict rule is elicited rather than inferred from a + schedule. +6. `GEN-Q02`: layer-2 activation/deactivation is `unobservable` because this experiment has no + lossless independent-question and pending-large-batch adjudicator. Layer-3 behavior is scored + manually and must preserve cohesive five-item frames and recognize imperative independent + questions; punctuation is never a semantic proxy and four is not a universal optimum. +7. Respectful close keeps completion, user stopping, no progress, delivery, budget, and deferral + distinct and makes no recoverable-re-entry or durable-delivery promise. + +E19 quick-rinse provenance is residual only. It is not a DemandTable clause, activation predicate, +or card. GEN-Q01 is absent. The activation matrix is an experiment-only evaluator of frozen +design-time disjunctions, not a FE-1405 amendment or compilable manifest. + +## Measures + +Score diagnostic correctness for every selected diagnostic. Score activation/deactivation and +card-result behavior only when that selected clause has a frozen card/predicate match; otherwise +those card-specific components are `not-applicable` and generic prompt behavior is described +without attributing it to a card: + +1. **Diagnostic correctness:** clause/coordinate selection, transcript-visible status, actual grade, + demand, failure, and evidence quote are correct under the frozen FE-1402 oracle. +2. **Activation/deactivation correctness:** every matrix match is lossless; the selected card and + predicate match; cards do not fire before a coordinate exists or after its demanded state passes. +3. **Evidence/stopping behavior:** the next interviewer move seeks the card's smallest evidence + delta, later evidence improves or honestly remains absent, and stop/delivery behavior follows the + frozen distinctions. + +Also score the inherited Bano/Ferrari dimensions, seven-category asked/probed/output coverage, +silent assumptions, excavation of tacit/belief/unknown facts, output target sanity, and stopping +discipline. Keep interaction quality, semantic coverage, stopping, delivery/deposit, provenance, +and target validity separate. + +Applicable FE-1407 signatures: FM-01 through FM-15. For each, report observed, +not observed, or unobservable. Machinery-owned prevention is unobservable when this +evaluation-runner protocol lacks production store, sweep, support-link, persistence, projection +validation, compilation, simulation, affordance, or controller authority. + +## Stopping rules + +- The single-session constraint is stated before the first interviewer call. +- The impatience line is appended to the first expert reply after all static-floor clauses pass and + at least one objective row is active. It is phase-triggered, not exchange-number-triggered. +- The live operator-adjudicated quote-novelty rule treats an expert frame as material when it + supplies at least one new or replacement exact quote, attributed to that new expert turn, for a + clause demanded in the new projection. This is a stopping input, not proof of semantic + improvement; semantic correctness is scored after observation. Regrading, active-row drift, + reordered or duplicate quotes, and evidence-array length alone never reset the streak. A new + demanded quote—including new support or current-turn retraction evidence for an unsupported + objective anchor—resets it even when it replaces an old quote at equal array length. Delivery is a + separate terminal event, not an expert-frame reset. Plans, promises, burden cues, + acknowledgements, and operator-only changes are non-material. +- Static-floor presence is cardinality-only: the projection records `observedCount`, requires cited + transcript evidence for positive counts, assigns no grade, and passes exactly at the frozen + minimum. Unsupported objective anchors persist under one label until a current-turn explicit + retraction is recorded; they never disappear or become an FE-1431 binding implicitly. +- Every active objective anchor has its own stable label, transcript quote/rationale record, and + exact FE-1402 `whenObjective` label as `matchingPredicate`; multiple anchors may project to one + unique active row. The operator reconciles `SF-OBJ.observedCount` to matched plus unsupported + active anchors and preserves explicit retractions durably. The row/predicate pair is + discriminated and closed; a mismatch is invalid before it can demand clauses or activate + guidance. The operator adjudicates the predicate from a transcript-visible objective rather than + incidental topic similarity. This experiment log does not choose an FE-1431 binding + representation. +- Raise `NP` at three consecutive non-material expert frames. Keep it raised until material expert + evidence resets it. At five consecutive non-material expert frames, end questioning and require + exactly one closing interviewer response containing the best useful result and explicit gaps. + Whether that response is a delivery is recorded separately; this changes no completion assessment + and does not retroactively reset the streak. +- Force a delivery request at interviewer turn 20 and hard-stop at turn 24, preserving inherited + budget comparability. At and after turn 20 the runner supplies only a labeled experiment stimulus; + it must not call the expert or operator and must not update no-progress. Delivery never asserts + completion. +- The interviewer prompt instructs it to honor a user stop regardless of completion or licensing; + the runner has no independent semantic user-stop detector, so compliance is scored behavior, not + a machinery guarantee. Terminal precedence is: classified delivery; the one response after a + no-progress hard stop; forced-wrap request at turn 20; hard budget at turn 24. This protocol cannot + license deferral or prove durable delivery. + +## Recovery binding and raw-evidence preservation + +Before a resume or final continuation can call any model, the runner must match the checkpoint's +exact preregistration seal hash, instrument version, DemandTable version, and complete model +configuration to the current frozen values. It then reparses every saved projection through the +runtime schema and revalidates exact clause inventory, semantics, quote provenance, anchor +continuity, activation/card choice, and the complete no-progress history before importing any +checkpoint state. Condition-3 recovery writes a new numbered raw, +transcript, operator, and model segment. It hashes and names the source checkpoint, records the +truncation seam (expert regeneration, interviewer regeneration, or final continuation), leaves the +source file unchanged, and retains the original truncation marker. Only `in-progress`, +`forced-wrap-in-progress`, either truncation state, and `no-progress-hard-stop-pending-delivery` are +resumable; the forced-wrap state advances to the next interviewer turn without regenerating the +completed prior turn. Instrumentation exhaustion and terminal stops are not. + +## Amendment rule + +Do not change treatment, scoring, predicates, measures, or stopping after observing the run. If a +defect makes the run uninterpretable, preserve the failed run, add a dated amendment naming the +defect and expected consequence, seal a new lock, and label all later analysis exploratory or a new +preregistered run. Do not retroactively edit a transcript or raw trace. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md new file mode 100644 index 00000000000..8663995a744 --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-prompt.md @@ -0,0 +1,57 @@ +# Condition 3 interviewer treatment (FE-1404) + +--- + +You are an expert process-model elicitor. Interview the domain expert and then produce the best +process-model specification supportable in this one session. + +This is a single-session experiment. No later session or external data arrival is available. Never +promise recoverable re-entry, durable capture, or later delivery. A test-only operator will append +one completion diagnostic after each expert answer. The diagnostic names a frozen DemandTable +coordinate, its transcript-visible status and grade, the demanded grade, and the failure. Treat it +as experiment instrumentation: use it to choose the next question, but do not claim you detected or +adjudicated the gap yourself. Do not ask the expert what you have failed to ask as a substitute for +the diagnostic. + +Keep these facts distinct: evidence coverage, interaction quality, completion, session stopping, +user-requested quiet, delivery, no progress, budget exhaustion, and deferral. Neither a smooth +conversation nor a delivered specification makes the target complete. If the expert stops, honor +the stop. State the best useful result and consequential gaps. Do not claim the experiment can +settle, sweep, persist, validate, or license deferral. + +Use only these reviewed cards and fragments: + +- **CPS-Q01 — separate failure occurrence from repair.** When a line-failure occurrence or repair + coordinate is unaddressed, below grade, or unspecified, treat the coordinates independently. + Ask for an ordinary occurrence range for each named failure. Then ask for a plausible low, high, + best guess, and interval confidence for repair. If quantiles are still demanded, ask for median + and conditional quartiles. Preserve verbal, point, range, and quantile grades as actually stated. +- **CPS-Q02 — elicit changeover loss, including ramp scrap.** When family-changeover or split-run + ramp scrap is unaddressed, below grade, or an absence is uncorroborated, ask for ordinary + low-to-high scrap after a named transition and for the repeated loss from a split. If the expert + does not know, ask for the least-burdensome source they recognize as authoritative. Never turn an + unknown, promised observation, or invented threshold into a value. +- **CPS-Q03 — bound the split-run policy.** When the split objective is active and batch structure, + minimum run, contiguity, or extra-changeover evidence is weak, ask for ordinary minimum-run + ranges and family exceptions; the contiguity/interleaving rule; the ordinary low-to-high count of + extra changeovers or cleans; and the ordinary low-to-high ramp scrap repeated by each extra start. +- **CPS-Q04 — state the order-release gate.** Replace a time-shaped approximation with the practiced + state or event that makes an order runnable, who or what changes it, and where it is observable. + Preserve prescribed and practiced variants separately if they diverge. +- **CPS-Q05 — elicit the resource-conflict rule.** When simultaneous demands need one shared + resource, ask which wins, what overrides the priority, how ties break, and which recent borderline + case shows the practiced rule. Do not infer the rule from a schedule. +- **GEN-Q02 — bound a conversational question batch.** Default to two to four related questions. + A cohesive five-item response frame is only a soft warning while the expert remains engaged. + Never repeat condition 1's 29-question opening battery. +- **HINT-STATUS-GRADE.** Name the coordinate, current status, current grade, demanded grade, and + missing evidence. Ask for the smallest evidence delta. Explicitness and numerical precision do + not by themselves satisfy grade. +- **HINT-RESPECTFUL-CLOSE.** Honor a stop, open no new topic, state the best useful result and gaps, + and report that durability, delivery validation, re-entry, and deferral licensing are + unobservable in this protocol. + +Produce the final specification in the most faithful representation supportable by the transcript. +Include a visible evidence/assumption ledger and a loss section. Do not claim it is loadable or +validated because this protocol has no Petrinaut compile or simulation authority. + diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-scoring.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-scoring.md new file mode 100644 index 00000000000..46f8131623d --- /dev/null +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-scoring.md @@ -0,0 +1,124 @@ +# FE-1404 condition-3 frozen scoring contract + +Status: preregistered before observation. This contract scores one qualitative run; it does not +estimate an effect size or reliability rate. + +## Fixed destinations + +- Narrative comparison and signature verdicts: + `docs/evidence/evaluations/process-model-elicitation/baseline/condition-3-readout.md` +- Machine-readable component and aggregate results: + `docs/evidence/evaluations/process-model-elicitation/baseline/condition-3-result.json` +- Provisional active checkpoints and immutable completed source segments, transcript, operator + projections/attempts, and delivered model: + `docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-3*` + +## Verdict domain + +Every scored component receives exactly one verdict: + +- `pass`: all frozen checks for the component are supported by the named evidence. +- `fail`: at least one frozen check is contradicted and no retry rule applies. +- `mixed`: separately named subchecks contain both pass and fail results; never average them away. +- `unobservable`: the protocol lacks the authority or signal needed to judge the claim. +- `not-applicable`: the triggering state never occurs, so no behavior was demanded. + +`retry-required` is an intermediate operator-attempt disposition, not a final result verdict. +Preserve the failed attempt and apply the preregistered retry rule. A later valid projection makes +the diagnostic-instrumentation component `mixed`; three failed attempts make it `fail`. + +## Scorer and authority + +The FE-1404 experiment producer performs the first fixed scoring pass from the immutable transcript, +raw trace, operator trace, frozen DemandTable, prior condition transcripts/readout, and FE-1407 +catalogue. The producer may not use the situation pack to repair operator diagnostics. The situation +pack is used only for the inherited excavation checks after the transcript is fixed. + +The coordinator's independent experiment/replay review is the acceptance authority. A disagreement +is recorded per component and adjudicated against quoted evidence; it is not resolved by changing +the scoring contract or transcript. The producer's first-pass result remains visible. + +## Layer procedures + +### Layer 1 — diagnostic correctness + +For every operator projection, check the selected clause and all changed demanded assessments: + +1. the objective row is active from transcript-visible objectives; +2. the coordinate and demand exactly match the frozen table; +3. status and grade describe the cited expert evidence independently; +4. failure diagnostic follows the FE-1402 evaluation rule; +5. every quote is verbatim at its named opening/expert turn; and +6. no interviewer-authored, operator-authored, situation-pack, or prior-run value supplies evidence. + +Any invalid projection has the intermediate disposition `retry-required` until the third failed +attempt. Exhaustion is `fail` for +instrumentation and stops the run before the projection influences selection or no-progress. Later +valid retry output does not erase the failed attempt: the diagnostic-instrumentation component is +`mixed`, while the validated projection proceeds to ordinary layer scoring. + +### Layer 2 — activation/deactivation correctness + +For each frozen CPS binding, evaluate every declared predicate against the validated projection. +`pass` requires lossless clause and predicate match, correct deactivation after the demand passes, +and no pre-coordinate fire on `no-selected-slot`. A mismatch is `fail`. No card activation verdict +is derived from topical relevance. + +GEN-Q02 has no lossless independent-question/pending-large-batch adjudicator in this experiment. +Its layer-2 semantic activation is therefore always `unobservable`; question-mark punctuation is +not a substitute. Its reviewed batching guidance remains part of the treatment and is scored only +at layer 3. E19 quick-rinse provenance and GEN-Q01 remain outside the matrix. + +### Layer 3 — resulting evidence and stopping behavior + +For each selected CPS diagnostic, compare the next interviewer move and later expert evidence with +the card's smallest expected delta. Score whether the question applies the reviewed card without +inventing evidence, whether the demanded status/grade changes, and whether the card deactivates. +An honest `unknown-to-user` may be a behavioral pass while the DemandTable clause remains failing. + +Score GEN-Q02 manually from semantic independent questions, response-frame cohesion, and expert +burden. Requests are independent when each can be answered without the others; imperative or +colon-led requests count without `?`. A frame is cohesive when one shared scope and answer shape +lets the expert answer it as one artifact. A cohesive five-item frame may pass. More than five is +not automatically a failure, and two-to-four is not a universal optimum. Mark burden observed only +from an explicit expert cue or a clear abandoned/partial response; otherwise say not observed. + +Score no-progress from the live operator-adjudicated quote-novelty rule over demanded evidence in +each new expert frame. Quote novelty has stopping consequences but does not itself prove semantic +improvement. Regrading, row drift, quote reorder/duplication, plans, promises, acknowledgements, and +burden cues do not reset it. New support for a demanded unsupported objective anchor, or +current-turn evidence explicitly retracting it, is material; anchor labels, original evidence, and +original rationale remain append-only across projections. +Delivery terminates the session and never retroactively resets the expert-frame streak or changes +completion. Score user stopping, no-progress, budget, delivery, deposit, and deferral separately. + +From interviewer turn 20 onward, force-wrap content is a labeled runner-authored experiment +stimulus, not an expert frame. It cannot be submitted to the operator or update no-progress. This +reserves turns 20–24 for delivery; a fifth non-material expert frame can arise no later than turn 19, +so its one required closing interviewer response remains inside the frozen budget. + +## Failure, unobservable, and aggregation rules + +- Operator parse/schema/provenance failure: retry up to three attempts; then stop with preserved raw + attempts. Do not select a diagnostic or update no-progress from invalid output. +- Classifier or provider truncation: preserve the original marker. Resume or continuation must use a + new seal-bound segment; otherwise `fail` instrumentation and make no call. +- Store, sweep, support-link, durable delivery, re-entry, deferral licensing, production projection + validation, compilation, simulation, and production affordance claims are `unobservable` here. +- A frozen CPS card's aggregate is `pass` only when layers 1, 2, and 3 pass. It is `fail` if layer 1 or 2 fails, + or layer 3 contradicts the card. It is `mixed` when valid activations have different layer-3 + results. `unobservable` does not convert to pass. GEN-Q02 has no three-layer card aggregate: report + its frozen layer-2 `unobservable` and manual layer-3 verdict separately. +- The run aggregate is a vector, never one numeric score: diagnostic layer, activation layer, + evidence/stopping layer, completion, user-requested stopping, no-progress stopping, budget + stopping, inherited interaction, semantic coverage, delivery, deposit, deferral, provenance, and + target validity. The exact verdict vocabulary, result schema, and exhaustive component IDs + (including each applicable FE-1407 signature) are owned by `Condition3ResultSchema` and + `CONDITION_3_RESULT_COMPONENT_IDS` in `condition-3-instrument.ts`; + `assertCompleteCondition3Result` rejects missing or duplicate rows. Report every component plus + comparison with C1 and C2. Each comparison embeds the exact prior raw, transcript, and model + SHA-256 values frozen as literal schema inputs; a merely well-shaped placeholder is invalid. + Those prior artifacts and the reviewed readout are sealed inputs. Signature rows + also carry their required observational label: + `observed` maps to verdict `fail`, `not-observed` maps to `pass`, and `unobservable` maps to + `unobservable`; non-signature rows forbid that label. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md index 592461460ef..f2d58f45018 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/protocol.md @@ -1,8 +1,7 @@ # Baseline control — experiment protocol (FE-1361) -What does one-shot / lightly-prompted AI elicitation already achieve? Two conditions of the -three-condition design (condition 3, kernel harness + real plugin, is out of this ticket's -scope). The read-out lives in the immutable +What does one-shot / guided AI elicitation already achieve, and what changes when reviewed +completion diagnostics drive the guidance? The read-out lives in the immutable [evaluation evidence](../../../../docs/evidence/evaluations/process-model-elicitation/baseline/readout.md). ## Conditions @@ -11,11 +10,16 @@ scope). The read-out lives in the immutable | --- | --------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | | 1 | `claude-opus-5` | none | the incumbent: a strong model told to interview-then-build (the Petrinaut assistant's prompt already mandates interview-first, per the FE-1358 survey) | | 2 | `claude-opus-5` | [v0-prompt.md](v0-prompt.md) | the degenerate plugin: the seven-category elicitation surface as pure guidance, no machinery | +| 3 | `claude-opus-5` | [condition-3-prompt.md](condition-3-prompt.md) | the reviewed completion-and-guidance treatment plus a labelled test-only operator projection; still no production harness or plugin runtime | -Both conditions receive the identical opening user message +Conditions 1 and 2 receive the identical opening user message ([opening-message.md](../../../cases/process-model-elicitation/baseline/opening-message.md)); the -v0 system prompt is the only difference, so -the 1→2 delta measures what pack content alone buys. +v0 system prompt is the only difference between conditions 1 and 2, so the 1→2 delta measures what +pack content alone buys. Condition 3 uses the same base opening plus its preregistered +single-session treatment sentence, and adds the other preregistered corrections +and instrument recorded in +[condition-3-preregistration.md](condition-3-preregistration.md); its delta measures the complete +experimental treatment, including operator intervention, rather than a model-only effect. ## Subject and interviewee @@ -35,28 +39,44 @@ rather than fill). - Alternating API calls; each side sees only its own history. The interviewer never sees the situation pack; the expert never sees the v0 prompt. +- In condition 3, a separate operator sees only transcript-visible evidence and the frozen FE-1402 + DemandTable. After every expert answer it emits a complete judgment trace, while the interviewer + receives only the selected clause/coordinate/status/grade/demand/failure diagnostic. Operator + diagnostics are removed from the expert's history. - A `claude-haiku-4-5` classifier checks each interviewer turn for the final model deliverable; delivery ends the run. - **Impatience probe**: on exchange 8 the runner appends a scripted time-pressure line to the expert's reply, identically in both conditions (LLMREI found LLM interviewers end too readily on impatience cues; ReqElicitGym found the opposite failure of exhausting the budget — the - probe plus the budget makes both observable). + probe plus the budget makes both observable). Conditions 1 and 2 retain that inherited placement; + condition 3 triggers it on the first expert reply after the static floor passes and one objective + row is active. +- Condition 3 raises a test-only `NP` advisory after three consecutive non-material expert frames + and ends questioning after five, then permits exactly one interviewer response to deliver the best + supportable result and explicit gaps. Only a new/replacement demanded evidence quote from the new + expert turn resets the streak; regrading, row drift, quote order/duplication, and array length do + not. It keeps completion unchanged and logs the intervention. - **Turn budget**: forced wrap-up at 20 interviewer turns ("produce the model now"), hard stop at 24. Delivering only at the forced wrap is itself a stopping-discipline finding. - The interviewer keeps the model's default adaptive thinking (part of "vanilla Claude"); the expert and classifier run with thinking disabled. When a final delivery is cut off at the - response budget, the runner stitches continuation responses into one message - (`--continue-final` repairs an already-finished run the same way); the checkpoint file - (`.raw.json`) is written after every exchange and `--resume` continues an interrupted run. + response budget, the legacy runner stitches continuation responses into one message + (`--continue-final` repairs an already-finished run the same way). Condition 3 instead preserves + each truncation seam and writes seal-bound resume/continuation output to a new numbered segment, + leaving its source raw checkpoint and marker unchanged. A checkpoint is written after every + exchange. - Sampling is default-temperature; runs are single-shot (n=1 per condition), so treat every read-out claim as existence evidence, not a rate estimate. Rerun from the HASH root with `turbo run baseline:run --filter '@hashintel/brunch-agent' -- 1` / -`turbo run baseline:run --filter '@hashintel/brunch-agent' -- 2` (needs `ANTHROPIC_API_KEY`). +`turbo run baseline:run --filter '@hashintel/brunch-agent' -- 2` / +`turbo run baseline:run --filter '@hashintel/brunch-agent' -- 3` (needs `ANTHROPIC_API_KEY`). Production transcripts land in `docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/`. Tests set -`BRUNCH_BASELINE_TEST_OUTPUT_DIR` to an isolated directory and never write committed evidence. + `BRUNCH_BASELINE_TEST_OUTPUT_DIR` to an isolated directory and never write committed evidence. + Condition 3 additionally writes an operator trace and refuses a production run whose frozen + preregistration lock does not match the treatment files. ## Instruments (scored in the read-out) diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/run.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/run.ts index b75d45614e2..01a7d5dd7b4 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/run.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/run.ts @@ -1,37 +1,62 @@ -// Baseline-control interview runner (FE-1361). +// Baseline interview runner (FE-1361 and FE-1404). // -// Runs one condition of the two-condition experiment: an interviewer model is asked to -// "interview me, then produce the model" against a simulated domain expert defined by -// the baseline case's situation-pack.md. Condition 1 sends no interviewer system prompt (bare Claude); -// condition 2 sends v0-prompt.md (the seven-category elicitation surface as guidance). +// Conditions 1 and 2 preserve the reviewed FE-1361 controls: bare Claude and +// the v0 elicitation prompt. Condition 3 adds the frozen FE-1404 completion and +// guidance instrument plus a test-only, transcript-bounded operator projection. // -// Usage: ANTHROPIC_API_KEY=... node --experimental-strip-types run.ts <1|2> [--resume|--continue-final] +// Usage: ANTHROPIC_API_KEY=... node --experimental-strip-types run.ts <1|2|3> [--resume|--continue-final|--verify-seal] // --resume continue an interrupted run from its checkpoint // --continue-final ask the interviewer to finish a final delivery that was cut off at -// max_tokens, and rewrite the artifacts with the merged message +// max_tokens; C1/C2 merge legacy output, while C3 appends a sealed segment +// --verify-seal validate condition 3's exact manifest and chronology without a model call // // Production outputs, under docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/: // condition-.md readable transcript with run metadata // condition-.raw.json full message arrays + per-call token usage (also the checkpoint) // condition--model.txt the final delivery message, verbatim (delivered runs only) +// Condition-3 recovery uses numbered `.segment-NNN-{resume,continuation}` stems and never +// overwrites its source raw trace. +import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; +import { + assertCompleteCondition3Projection, + assertCondition3ProjectionSemantics, + assertCondition3UnsupportedAnchorContinuity, + CONDITION_3_ACTIVATION_MATRIX, + CONDITION_3_DEMAND_CLAUSES, + CONDITION_3_DEMAND_TABLE_VERSION, + CONDITION_3_DIAGNOSTIC_PRIORITY, + CONDITION_3_GEN_Q02_LAYER_2, + CONDITION_3_INSTRUMENT_VERSION, + CONDITION_3_LOCKED_PATHS, + CONDITION_3_OPERATOR_ENVELOPE, + CONDITION_3_STOPPING_RULES, + nextCondition3NoProgressStreak, + parseCondition3Projection, + type Condition3CardId, + type Condition3ClauseId, + type Condition3FiresWhen, + type Condition3Projection, +} from "./condition-3-instrument.ts"; + import type Anthropic from "@anthropic-ai/sdk"; const INTERVIEWER_MODEL = "claude-opus-5"; const EXPERT_MODEL = "claude-sonnet-5"; const CLASSIFIER_MODEL = "claude-haiku-4-5-20251001"; +const OPERATOR_MODEL = "claude-opus-5"; // Interviewer turns, not exchanges. ReqElicitGym budgets 20; we force a wrap-up at 20 and // hard-stop at 24 in case the model keeps talking instead of delivering. -const FORCE_WRAP_AT = 20; -const HARD_STOP_AT = 24; +const LEGACY_FORCE_WRAP_AT = 20; +const LEGACY_HARD_STOP_AT = 24; // The scripted impatience probe (LLMREI: interviewers end too readily on impatience cues). // Appended to the expert's reply on this exchange, identically in both conditions. -const IMPATIENCE_AT = 8; +const LEGACY_IMPATIENCE_AT = 8; const IMPATIENCE_LINE = "(Sorry — I've just seen the time, I have the floor huddle in ten minutes. How much more do you need?)"; const FORCED_WRAP_MESSAGE = @@ -44,6 +69,18 @@ type ChatMessage = Omit & { // Present only when the API ended this model-generated message at its token limit. // Older checkpoints and human-authored messages legitimately omit it. truncated?: true; + // Condition 3 appends an operator diagnostic to the interviewer-facing user + // message. The expert's next call must see only its own original answer. + expertContent?: string; + operatorDiagnostic?: string; + experimentStimulus?: string; + // Condition 3 recovery is append-only. The source message and its truncation + // marker remain intact; later pieces are separate durable seam records. + continuations?: Array<{ + content: string; + truncated: boolean; + recordedAt: string; + }>; }; type Usage = Pick & @@ -55,7 +92,7 @@ type Usage = Pick & >; interface CallRecord { - agent: "interviewer" | "expert" | "classifier"; + agent: "interviewer" | "expert" | "classifier" | "operator"; model: Anthropic.Model; usage: Usage; } @@ -63,58 +100,148 @@ interface CallRecord { interface CallResult { text: string; truncated: boolean; + sourceText?: string; + continuations?: ChatMessage["continuations"]; +} + +interface Condition3ActivationMatch { + cardId: Exclude; + clauseId: Condition3ClauseId; + predicate: Condition3FiresWhen; +} + +interface Condition3ProjectionRecord extends Condition3Projection { + turn: number; + recordedAt: string; + selectedClauseId: Condition3ClauseId | null; + selectedUnsupportedAnchorLabel: string | null; + selectedCardId: Condition3CardId | null; + selectedPredicate: Condition3FiresWhen | null; + activationMatches: Condition3ActivationMatch[]; + noProgressStreak: number; + noProgressAdvisory: boolean; +} + +interface Condition3Preregistration { + path: string; + sha256: string; + sealedAt: string; + modifiedAt: string; + verifiedBeforeRun: boolean; +} + +interface Condition3OperatorAttempt { + turn: number; + attempt: number; + recordedAt: string; + rawText: string; + parseError: string | null; } interface RawCheckpoint { startedAt: string; - condition: "1" | "2"; + condition: "1" | "2" | "3"; stopReason: string; calls: CallRecord[]; interviewerMessages: ChatMessage[]; + instrumentVersion?: string; + demandTableVersion?: string; + modelConfiguration?: { + interviewer: string; + expert: string; + classifier: string; + operator: string; + sampling: string; + seed: null; + seedSupport: false; + }; + preregistration?: Condition3Preregistration; + operatorProjections?: Condition3ProjectionRecord[]; + operatorAttempts?: Condition3OperatorAttempt[]; + impatienceProbeTurn?: number; + genQ02Layer2?: typeof CONDITION_3_GEN_Q02_LAYER_2; + recovery?: { + mode: "resume" | "continue-final"; + sourceRawPath: string; + sourceSha256: string; + seams: Array<{ + kind: + | "truncated-expert-regeneration" + | "truncated-interviewer-regeneration" + | "final-continuation"; + sourceHadTruncationMarker: true; + sourceContent: string; + recordedAt: string; + }>; + }; } +const CONDITION_3_MODEL_CONFIGURATION = { + interviewer: INTERVIEWER_MODEL, + expert: EXPERT_MODEL, + classifier: CLASSIFIER_MODEL, + operator: OPERATOR_MODEL, + sampling: CONDITION_3_STOPPING_RULES.providerSampling, + seed: null, + seedSupport: false, +} as const; + +function usage(): never { + console.error( + "usage: node run.ts <1|2|3> [--resume|--continue-final|--verify-seal]", + ); + process.exit(1); +} + +const conditionArg = process.argv[2]; +const mode = process.argv[3] ?? "fresh"; +if (conditionArg !== "1" && conditionArg !== "2" && conditionArg !== "3") + usage(); +if ( + mode !== "fresh" && + mode !== "--resume" && + mode !== "--continue-final" && + mode !== "--verify-seal" +) + usage(); +if (mode === "--verify-seal" && conditionArg !== "3") usage(); +const condition = conditionArg; const clientModule = process.env["BRUNCH_BASELINE_ANTHROPIC_MODULE"]; const testOutputDirectory = process.env["BRUNCH_BASELINE_TEST_OUTPUT_DIR"]; -if (testOutputDirectory && !clientModule) { +const apiKey = process.env["ANTHROPIC_API_KEY"]; +if (mode !== "--verify-seal" && testOutputDirectory && !clientModule) { console.error( "BRUNCH_BASELINE_TEST_OUTPUT_DIR requires BRUNCH_BASELINE_ANTHROPIC_MODULE", ); process.exit(1); } -const apiKey = process.env["ANTHROPIC_API_KEY"]; -if (!apiKey && !clientModule) { +if (mode !== "--verify-seal" && !apiKey && !clientModule) { console.error("ANTHROPIC_API_KEY is not set"); process.exit(1); } +interface BaselineAnthropicClient { + messages: { + create( + request: Anthropic.MessageCreateParamsNonStreaming, + ): Promise; + }; +} -// The SDK owns transport robustness the hand-rolled fetch client got wrong: -// network-level failures (TCP reset, DNS blip) are retried rather than -// crashing the run hours in, backoff honours retry-after, and the typed -// usage carries the cache-token fields the hand-typed response omitted. -// The timeout is explicit because without one the SDK refuses non-streaming -// requests whose max_tokens imply more than 10 minutes — which the -// empty-text retry's doubled budget does, so the retry path would crash -// instead of retrying. -const anthropic: Pick = clientModule - ? ((await import(clientModule)).default as Pick) - : new (await import("@anthropic-ai/sdk")).default({ - apiKey, - maxRetries: 5, - timeout: 30 * 60 * 1000, - }); +let anthropic: BaselineAnthropicClient | undefined; -function usage(): never { - console.error("usage: node run.ts <1|2> [--resume|--continue-final]"); - process.exit(1); +async function getAnthropic(): Promise { + if (anthropic) return anthropic; + const resolvedClient = clientModule + ? ((await import(clientModule)).default as BaselineAnthropicClient) + : (new (await import("@anthropic-ai/sdk")).default({ + apiKey, + maxRetries: 5, + timeout: 30 * 60 * 1000, + }) as BaselineAnthropicClient); + anthropic = resolvedClient; + return resolvedClient; } -const conditionArg = process.argv[2]; -const mode = process.argv[3] ?? "fresh"; -if (conditionArg !== "1" && conditionArg !== "2") usage(); -if (mode !== "fresh" && mode !== "--resume" && mode !== "--continue-final") - usage(); -const condition = conditionArg; - const baseDir = fileURLToPath(new URL(".", import.meta.url)); const caseDir = fileURLToPath( new URL( @@ -130,8 +257,22 @@ const transcriptDir = import.meta.url, ), ); -const rawPath = `${transcriptDir}/condition-${condition}.raw.json`; const calls: CallRecord[] = []; +const operatorProjections: Condition3ProjectionRecord[] = []; +const operatorAttempts: Condition3OperatorAttempt[] = []; + +function completeMessageContent(message: ChatMessage): string { + return ( + message.content + + (message.continuations ?? []) + .map((continuation) => continuation.content) + .join("") + ); +} + +function sha256(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} async function callClaude( agent: CallRecord["agent"], @@ -148,17 +289,21 @@ async function callClaude( // handles the empty-text case, which is a budget problem rather than a transport one. let tokenBudget = maxTokens; for (let attempt = 1; attempt <= 5; attempt++) { - const response = await anthropic.messages.create({ + const response = await ( + await getAnthropic() + ).messages.create({ model, max_tokens: tokenBudget, ...(options.allowThinking ? {} : { thinking: { type: "disabled" as const } }), ...(system ? { system } : {}), - // `truncated` is checkpoint-only recovery metadata. The provider accepts - // just role/content message objects, so keep persistence concerns outside - // the request boundary. - messages: messages.map(({ role, content }) => ({ role, content })), + // Project persistence metadata out of the provider request while retaining + // every append-only continuation piece in the message's semantic content. + messages: messages.map((message) => ({ + role: message.role, + content: completeMessageContent(message), + })), }); calls.push({ agent, @@ -207,6 +352,8 @@ async function callInterviewer( allowThinking: true, }, ); + const sourceText = result.text; + const continuations: NonNullable = []; let text = result.text; for (let piece = 1; result.truncated && piece <= 4; piece++) { console.error(` interviewer: truncated, requesting continuation ${piece}`); @@ -222,12 +369,24 @@ async function callInterviewer( 16_000, { allowThinking: true }, ); + continuations.push({ + content: result.text, + truncated: result.truncated, + recordedAt: new Date().toISOString(), + }); // No separator at the seam: the cut usually lands mid-line or mid-token // and the model is instructed to continue exactly from where it stopped, // so an injected newline would corrupt the merged document. text += result.text; } - return { text, truncated: result.truncated }; + return condition === "3" && continuations.length > 0 + ? { + text, + truncated: result.truncated, + sourceText, + continuations, + } + : { text, truncated: result.truncated }; } async function loadSection(file: string): Promise { @@ -241,6 +400,546 @@ async function loadSection(file: string): Promise { : raw.slice(separatorIndex + 5).trim(); } +async function loadCondition3Preregistration( + runStartedAt: string, +): Promise { + const path = `${baseDir}condition-3-preregistration.lock.json`; + const contextRoot = fileURLToPath(new URL("../../../../", import.meta.url)); + const [content, metadata] = await Promise.all([ + readFile(path, "utf8"), + stat(path), + ]); + const lock = JSON.parse(content) as { + version?: unknown; + sealedAt?: unknown; + files?: unknown; + }; + if ( + lock.version !== CONDITION_3_INSTRUMENT_VERSION || + typeof lock.sealedAt !== "string" || + !Number.isFinite(Date.parse(lock.sealedAt)) || + !Array.isArray(lock.files) || + lock.files.length === 0 + ) { + throw new Error("condition-3 preregistration lock has an invalid envelope"); + } + const rows: Array<{ path: string; sha256: string }> = []; + for (const item of lock.files) { + if ( + typeof item !== "object" || + item === null || + !("path" in item) || + typeof item.path !== "string" || + !("sha256" in item) || + typeof item.sha256 !== "string" + ) { + throw new Error( + "condition-3 preregistration lock has an invalid file row", + ); + } + rows.push({ path: item.path, sha256: item.sha256 }); + } + const actualPaths = rows.map(({ path: lockedPath }) => lockedPath); + const duplicatePaths = actualPaths.filter( + (lockedPath, index) => actualPaths.indexOf(lockedPath) !== index, + ); + const missingPaths = CONDITION_3_LOCKED_PATHS.filter( + (lockedPath) => !actualPaths.includes(lockedPath), + ); + const extraPaths = actualPaths.filter( + (lockedPath) => + !CONDITION_3_LOCKED_PATHS.includes( + lockedPath as (typeof CONDITION_3_LOCKED_PATHS)[number], + ), + ); + const pathsAreInCanonicalOrder = actualPaths.every( + (lockedPath, index) => lockedPath === CONDITION_3_LOCKED_PATHS[index], + ); + if ( + actualPaths.length !== CONDITION_3_LOCKED_PATHS.length || + duplicatePaths.length > 0 || + missingPaths.length > 0 || + extraPaths.length > 0 || + !pathsAreInCanonicalOrder + ) { + throw new Error( + `condition-3 preregistration manifest is not canonical: missing=${missingPaths.join(",") || "none"}; extra=${extraPaths.join(",") || "none"}; duplicate=${duplicatePaths.join(",") || "none"}; order=${pathsAreInCanonicalOrder ? "canonical" : "noncanonical"}`, + ); + } + let newestLockedMtimeMs = Number.NEGATIVE_INFINITY; + for (const item of rows) { + const lockedFilePath = contextRoot + item.path; + const [lockedContent, lockedMetadata] = await Promise.all([ + readFile(lockedFilePath, "utf8"), + stat(lockedFilePath), + ]); + const actualHash = sha256(lockedContent); + if (actualHash !== item.sha256) { + throw new Error( + `condition-3 preregistration mismatch for ${item.path}: expected ${item.sha256}, got ${actualHash}`, + ); + } + newestLockedMtimeMs = Math.max(newestLockedMtimeMs, lockedMetadata.mtimeMs); + } + const sealedAtMs = Date.parse(lock.sealedAt); + if ( + sealedAtMs <= newestLockedMtimeMs || + metadata.mtimeMs <= newestLockedMtimeMs || + metadata.mtimeMs < sealedAtMs + ) { + throw new Error( + "condition-3 preregistration chronology is invalid: sealedAt and finalized lock mtime must postdate every locked file, and lock mtime must not predate sealedAt", + ); + } + const modifiedAt = metadata.mtime.toISOString(); + const verifiedBeforeRun = + metadata.mtimeMs <= Date.parse(runStartedAt) && + sealedAtMs <= Date.parse(runStartedAt); + if (!verifiedBeforeRun) { + throw new Error( + "condition-3 preregistration lock does not predate the run", + ); + } + return { + path, + sha256: sha256(content), + sealedAt: lock.sealedAt, + modifiedAt, + verifiedBeforeRun, + }; +} + +function assertCondition3CheckpointBinding( + checkpoint: RawCheckpoint, + currentPreregistration: Condition3Preregistration, +): void { + const expectedConfigurationEntries = Object.entries( + CONDITION_3_MODEL_CONFIGURATION, + ); + const modelConfigurationMatches = + checkpoint.modelConfiguration !== undefined && + Object.keys(checkpoint.modelConfiguration).length === + expectedConfigurationEntries.length && + expectedConfigurationEntries.every( + ([key, value]) => + checkpoint.modelConfiguration?.[ + key as keyof typeof checkpoint.modelConfiguration + ] === value, + ); + if ( + checkpoint.condition !== "3" || + checkpoint.instrumentVersion !== CONDITION_3_INSTRUMENT_VERSION || + checkpoint.demandTableVersion !== CONDITION_3_DEMAND_TABLE_VERSION || + checkpoint.preregistration?.sha256 !== currentPreregistration.sha256 || + !modelConfigurationMatches + ) { + throw new Error( + "condition-3 checkpoint binding mismatch: seal, instrument, DemandTable, and model configuration must match exactly before recovery", + ); + } +} + +async function resolveArtifactPaths(): Promise<{ + artifactStem: string; + rawPath: string; + sourceRawPath?: string; +}> { + const baseStem = `condition-${condition}`; + const baseRawPath = `${transcriptDir}/${baseStem}.raw.json`; + if (condition !== "3" || mode === "fresh") { + return { artifactStem: baseStem, rawPath: baseRawPath }; + } + const entries = await readdir(transcriptDir); + const candidates = entries.flatMap((name) => { + if (name === "condition-3.raw.json") { + return [{ sequence: 0, path: `${transcriptDir}/${name}` }]; + } + const match = + /^condition-3\.segment-(\d{3})-(?:resume|continuation)\.raw\.json$/u.exec( + name, + ); + return match + ? [ + { + sequence: Number.parseInt(match[1] ?? "0", 10), + path: `${transcriptDir}/${name}`, + }, + ] + : []; + }); + const latest = candidates.sort( + (left, right) => right.sequence - left.sequence, + )[0]; + if (!latest) { + throw new Error("condition-3 recovery has no source raw checkpoint"); + } + const nextSequence = String(latest.sequence + 1).padStart(3, "0"); + const recoveryLabel = mode === "--continue-final" ? "continuation" : "resume"; + const artifactStem = `condition-3.segment-${nextSequence}-${recoveryLabel}`; + return { + artifactStem, + rawPath: `${transcriptDir}/${artifactStem}.raw.json`, + sourceRawPath: latest.path, + }; +} + +function parseOperatorJson(text: string): Condition3Projection { + const withoutFence = text + .trim() + .replace(/^```(?:json)?\s*/u, "") + .replace(/\s*```$/u, ""); + const projection = parseCondition3Projection( + JSON.parse(withoutFence) as unknown, + ); + assertCompleteCondition3Projection(projection); + assertCondition3ProjectionSemantics(projection); + + const clausesById = new Map( + CONDITION_3_DEMAND_CLAUSES.map((clause) => [clause.id, clause]), + ); + for (const assessment of projection.assessments) { + const clause = clausesById.get(assessment.clauseId as Condition3ClauseId); + if (!clause) { + throw new Error( + `condition-3 operator returned unknown clause ${assessment.clauseId}`, + ); + } + if ( + assessment.coordinate !== clause.coordinate || + assessment.demand !== clause.demand + ) { + throw new Error( + `condition-3 operator changed frozen metadata for ${assessment.clauseId}`, + ); + } + const demanded = + clause.row === null || + projection.activeObjectiveRows.includes(clause.row); + if (assessment.demanded !== demanded) { + throw new Error( + `condition-3 operator demand applicability disagrees with the active rows for ${assessment.clauseId}`, + ); + } + if ( + !demanded && + (!assessment.pass || + assessment.currentStatus !== "not-applicable" || + assessment.currentGrade !== "not-applicable" || + assessment.failureDiagnostic !== null || + assessment.activationPredicates.length !== 0) + ) { + throw new Error( + `condition-3 operator did not mark inactive clause ${assessment.clauseId} inapplicable`, + ); + } + } + return projection; +} + +function validateProjectionEvidence( + projection: Condition3Projection, + messages: ChatMessage[], +): void { + const visibleTextByTurn = new Map(); + messages.forEach((message, index) => { + if (message.role !== "user") return; + const turn = Math.ceil(index / 2); + const visibleText = message.expertContent ?? message.content; + const texts = visibleTextByTurn.get(turn) ?? []; + texts.push(visibleText); + visibleTextByTurn.set(turn, texts); + }); + for (const rowEvidence of projection.activeObjectiveRowEvidence) { + for (const evidence of rowEvidence.evidence) { + const suppliedAtTurn = visibleTextByTurn.get(evidence.turn) ?? []; + if (!suppliedAtTurn.some((text) => text.includes(evidence.quote))) { + throw new Error( + `condition-3 activation evidence quote for ${rowEvidence.row} does not occur in supplied transcript turn ${evidence.turn}`, + ); + } + } + } + for (const anchor of projection.retractedObjectiveAnchors) { + for (const evidence of [...anchor.evidence, ...anchor.resolutionEvidence]) { + const suppliedAtTurn = visibleTextByTurn.get(evidence.turn) ?? []; + if (!suppliedAtTurn.some((text) => text.includes(evidence.quote))) { + throw new Error( + `condition-3 evidence quote for retracted objective anchor '${anchor.anchorLabel}' does not occur in supplied transcript turn ${evidence.turn}`, + ); + } + } + } + for (const assessment of projection.assessments) { + for (const evidence of assessment.evidence) { + const suppliedAtTurn = visibleTextByTurn.get(evidence.turn) ?? []; + if (!suppliedAtTurn.some((text) => text.includes(evidence.quote))) { + throw new Error( + `condition-3 evidence quote for ${assessment.clauseId} does not occur in supplied transcript turn ${evidence.turn}`, + ); + } + } + } + for (const anchor of projection.unsupportedActiveObjectiveAnchors) { + const allAnchorEvidence = [ + ...anchor.evidence, + ...(anchor.state === "retracted" ? anchor.resolutionEvidence : []), + ]; + for (const evidence of allAnchorEvidence) { + const suppliedAtTurn = visibleTextByTurn.get(evidence.turn) ?? []; + if (!suppliedAtTurn.some((text) => text.includes(evidence.quote))) { + throw new Error( + `condition-3 evidence quote for unsupported active objective anchor '${anchor.label}' does not occur in supplied transcript turn ${evidence.turn}`, + ); + } + } + } +} + +function operatorTranscript(messages: ChatMessage[]): string { + return messages + .map((message, index) => { + const speaker = + message.role === "assistant" + ? "INTERVIEWER" + : message.experimentStimulus && !message.expertContent + ? "EXPERIMENT_STIMULUS" + : index === 0 + ? "OPENING" + : "EXPERT"; + const visibleContent = message.expertContent + ? `${message.expertContent}${message.experimentStimulus ? `\n\n${message.experimentStimulus}` : ""}` + : completeMessageContent(message); + return `[${speaker} turn=${Math.ceil(index / 2)}]\n${visibleContent}`; + }) + .join("\n\n"); +} + +async function callCondition3Operator( + operatorSystem: string, + messages: ChatMessage[], + turn: number, +): Promise { + let priorError = ""; + let lastValidationError = "not recorded"; + for (let attempt = 1; attempt <= 3; attempt++) { + const result = await callClaude( + "operator", + OPERATOR_MODEL, + operatorSystem, + [ + { + role: "user", + content: + "Return the complete projection JSON for this transcript.\n\n" + + operatorTranscript(messages) + + priorError, + }, + ], + 12_000, + ); + try { + const projection = parseOperatorJson(result.text); + validateProjectionEvidence(projection, messages); + assertCondition3UnsupportedAnchorContinuity( + operatorProjections.at(-1), + projection, + turn, + ); + operatorAttempts.push({ + turn, + attempt, + recordedAt: new Date().toISOString(), + rawText: result.text, + parseError: null, + }); + return projection; + } catch (error) { + const parseError = error instanceof Error ? error.message : String(error); + lastValidationError = parseError; + operatorAttempts.push({ + turn, + attempt, + recordedAt: new Date().toISOString(), + rawText: result.text, + parseError, + }); + console.error( + `condition-3 operator projection rejected at turn ${turn}, attempt ${attempt}: ${parseError}`, + ); + priorError = `\n\nYour previous response failed validation: ${parseError}. Return a corrected complete JSON projection.`; + } + } + throw new Error( + `condition-3 operator exhausted projection-validation attempts; last error: ${lastValidationError}`, + ); +} + +function activationMatches( + projection: Condition3Projection, +): Condition3ActivationMatch[] { + const matches: Condition3ActivationMatch[] = []; + for (const assessment of projection.assessments) { + for (const binding of CONDITION_3_ACTIVATION_MATRIX) { + if (!binding.clauses.includes(assessment.clauseId as never)) continue; + for (const predicate of assessment.activationPredicates) { + if (binding.predicates.includes(predicate as never)) { + matches.push({ + cardId: binding.cardId, + clauseId: assessment.clauseId as Condition3ClauseId, + predicate, + }); + } + } + } + } + + return matches; +} + +function selectedAssessment(projection: Condition3Projection) { + for (const clauseId of CONDITION_3_DIAGNOSTIC_PRIORITY) { + const assessment = projection.assessments.find( + (candidate) => candidate.clauseId === clauseId && !candidate.pass, + ); + if (assessment) return assessment; + } + return null; +} + +function assertCondition3CheckpointSemantics(checkpoint: RawCheckpoint): void { + if ( + !Array.isArray(checkpoint.interviewerMessages) || + checkpoint.interviewerMessages.some( + (message) => + (message.role !== "user" && message.role !== "assistant") || + typeof message.content !== "string", + ) + ) { + throw new Error("condition-3 checkpoint has malformed interview messages"); + } + if ( + !Array.isArray(checkpoint.operatorProjections) || + !Array.isArray(checkpoint.operatorAttempts) + ) { + throw new Error("condition-3 checkpoint lacks an operator trace"); + } + const validatedHistory: Condition3Projection[] = []; + const selectedUnsupportedLabels = new Set(); + for (const [index, record] of checkpoint.operatorProjections.entries()) { + if ( + record.turn !== index + 1 || + typeof record.recordedAt !== "string" || + !Number.isInteger(record.noProgressStreak) || + typeof record.noProgressAdvisory !== "boolean" + ) { + throw new Error( + "condition-3 checkpoint projection metadata is malformed or non-sequential", + ); + } + const projection = parseCondition3Projection({ + activeObjectiveRows: record.activeObjectiveRows, + activeObjectiveRowEvidence: record.activeObjectiveRowEvidence, + retractedObjectiveAnchors: record.retractedObjectiveAnchors, + unsupportedActiveObjectiveAnchors: + record.unsupportedActiveObjectiveAnchors, + assessments: record.assessments, + notes: record.notes, + }); + assertCompleteCondition3Projection(projection); + assertCondition3ProjectionSemantics(projection); + assertCondition3UnsupportedAnchorContinuity( + validatedHistory.at(-1), + projection, + record.turn, + ); + validateProjectionEvidence( + projection, + checkpoint.interviewerMessages.slice(0, record.turn * 2 + 1), + ); + const expectedStreak = nextCondition3NoProgressStreak( + validatedHistory, + projection, + record.turn, + validatedHistory.length === 0 + ? 0 + : (checkpoint.operatorProjections[index - 1]?.noProgressStreak ?? 0), + ); + const expectedMatches = activationMatches(projection); + const selectedUnsupportedAnchor = + projection.unsupportedActiveObjectiveAnchors.find( + ({ label, state }) => + state === "active" && !selectedUnsupportedLabels.has(label), + ); + const selected = selectedUnsupportedAnchor + ? null + : selectedAssessment(projection); + const selectedMatch = selected + ? expectedMatches.find((match) => match.clauseId === selected.clauseId) + : undefined; + if ( + record.noProgressStreak !== expectedStreak || + record.noProgressAdvisory !== + expectedStreak >= CONDITION_3_STOPPING_RULES.noProgressAdvisoryAfter || + JSON.stringify(record.activationMatches) !== + JSON.stringify(expectedMatches) || + record.selectedUnsupportedAnchorLabel !== + (selectedUnsupportedAnchor?.label ?? null) || + record.selectedClauseId !== (selected?.clauseId ?? null) || + record.selectedCardId !== (selectedMatch?.cardId ?? null) || + record.selectedPredicate !== (selectedMatch?.predicate ?? null) + ) { + throw new Error( + `condition-3 checkpoint projection semantics disagree at turn ${record.turn}`, + ); + } + if (selectedUnsupportedAnchor) { + selectedUnsupportedLabels.add(selectedUnsupportedAnchor.label); + } + validatedHistory.push(projection); + } + if ( + checkpoint.operatorAttempts.some( + (attempt) => + !Number.isInteger(attempt.turn) || + !Number.isInteger(attempt.attempt) || + typeof attempt.recordedAt !== "string" || + typeof attempt.rawText !== "string" || + (attempt.parseError !== null && typeof attempt.parseError !== "string"), + ) + ) { + throw new Error("condition-3 checkpoint operator attempts are malformed"); + } +} + +function renderOperatorDiagnostic( + projection: Condition3ProjectionRecord, +): string { + if (projection.selectedUnsupportedAnchorLabel !== null) { + return [ + "clause=unsupported-active-anchor", + `coordinate=${projection.selectedUnsupportedAnchorLabel}`, + "status=explicit", + "grade=verbal", + "demand=record the active objective as unsupported by the frozen row set; do not invent a binding", + "failure=unsupported-active-anchor", + ].join("; "); + } + const selected = projection.assessments.find( + (assessment) => assessment.clauseId === projection.selectedClauseId, + ); + if (!selected) { + return "clause=none; coordinate=none; status=not-applicable; grade=not-applicable; demand=none; failure=none"; + } + return [ + `clause=${selected.clauseId}`, + `coordinate=${selected.coordinate}`, + `status=${selected.currentStatus}`, + `grade=${selected.currentGrade}`, + `demand=${selected.demand}`, + `failure=${selected.failureDiagnostic ?? "none"}`, + ].join("; "); +} + async function isFinalModel(message: string): Promise { const verdict = await callClaude( "classifier", @@ -263,14 +962,42 @@ async function isFinalModel(message: string): Promise { return verdict.text.trim().toUpperCase().startsWith("YES"); } -const openingMessage = await loadSection(`${caseDir}opening-message.md`); -const v0Prompt = - condition === "2" ? await loadSection("v0-prompt.md") : undefined; +const sharedOpeningMessage = await loadSection(`${caseDir}opening-message.md`); +const singleSessionStimulus = + "This is the only session available for this experiment. No external data or later follow-up will arrive. Please produce the best useful result supportable in this session."; +const openingMessage = + condition === "3" + ? `${sharedOpeningMessage}\n\n${singleSessionStimulus}` + : sharedOpeningMessage; +const interviewerSystem = + condition === "2" + ? await loadSection("v0-prompt.md") + : condition === "3" + ? await loadSection("condition-3-prompt.md") + : undefined; +const operatorSystem = + condition === "3" + ? `${await loadSection("condition-3-operator.md")}\n\n\n${JSON.stringify(CONDITION_3_OPERATOR_ENVELOPE, null, 2)}\n\n\n\n${JSON.stringify(CONDITION_3_DEMAND_CLAUSES)}\n\n\n\n${JSON.stringify(CONDITION_3_ACTIVATION_MATRIX)}\n` + : undefined; const situationPack = await readFile(`${caseDir}situation-pack.md`, "utf8"); -await mkdir(transcriptDir, { recursive: true }); +const forceWrapAt = + condition === "3" + ? CONDITION_3_STOPPING_RULES.forceWrapAt + : LEGACY_FORCE_WRAP_AT; +const hardStopAt = + condition === "3" + ? CONDITION_3_STOPPING_RULES.hardStopAt + : LEGACY_HARD_STOP_AT; let interviewerMessages: ChatMessage[] = [ - { role: "user", content: openingMessage }, + condition === "3" + ? { + role: "user", + content: openingMessage, + expertContent: sharedOpeningMessage, + experimentStimulus: singleSessionStimulus, + } + : { role: "user", content: openingMessage }, ]; let stopReason = "hard-stop"; @@ -281,11 +1008,35 @@ function expertView(): ChatMessage[] { return interviewerMessages.slice(1).map((message) => ({ role: message.role === "assistant" ? ("user" as const) : ("assistant" as const), - content: message.content, + content: message.expertContent + ? `${message.expertContent}${message.experimentStimulus ? `\n\n${message.experimentStimulus}` : ""}` + : completeMessageContent(message), })); } let interviewerTurns = 0; let startedAt = new Date().toISOString(); +let preregistration = + condition === "3" + ? await loadCondition3Preregistration(startedAt) + : undefined; +if (mode === "--verify-seal") { + console.error( + `condition-3 seal verified: ${preregistration?.sha256} (sealed ${preregistration?.sealedAt}; lock mtime ${preregistration?.modifiedAt})`, + ); + process.exit(0); +} +await mkdir(transcriptDir, { recursive: true }); +const { artifactStem, rawPath, sourceRawPath } = await resolveArtifactPaths(); +let recovery: RawCheckpoint["recovery"] = sourceRawPath + ? { + mode: mode === "--continue-final" ? "continue-final" : "resume", + sourceRawPath, + sourceSha256: sha256(await readFile(sourceRawPath, "utf8")), + seams: [], + } + : undefined; +let impatienceProbeTurn: number | undefined; +let noProgressClosePending = false; if (mode === "fresh" && existsSync(rawPath)) { // The checkpoint is also the run's only record; an unguarded fresh run @@ -299,8 +1050,15 @@ if (mode === "fresh" && existsSync(rawPath)) { if (mode !== "fresh") { const checkpoint = JSON.parse( - await readFile(rawPath, "utf8"), + await readFile(sourceRawPath ?? rawPath, "utf8"), ) as RawCheckpoint; + if (condition === "3") { + if (!preregistration) { + throw new Error("condition-3 current preregistration is unavailable"); + } + assertCondition3CheckpointBinding(checkpoint, preregistration); + assertCondition3CheckpointSemantics(checkpoint); + } interviewerMessages = checkpoint.interviewerMessages; calls.push(...checkpoint.calls); interviewerTurns = interviewerMessages.filter( @@ -308,6 +1066,22 @@ if (mode !== "fresh") { ).length; startedAt = checkpoint.startedAt; stopReason = checkpoint.stopReason; + if (condition === "3") { + if ( + !checkpoint.preregistration || + !checkpoint.operatorProjections || + !checkpoint.operatorAttempts + ) { + throw new Error( + "condition-3 checkpoint lacks preregistration or operator trace", + ); + } + operatorProjections.push(...checkpoint.operatorProjections); + operatorAttempts.push(...checkpoint.operatorAttempts); + impatienceProbeTurn = checkpoint.impatienceProbeTurn; + noProgressClosePending = + checkpoint.stopReason === "no-progress-hard-stop-pending-delivery"; + } } function writeCheckpoint(reason: string): Promise { @@ -317,6 +1091,21 @@ function writeCheckpoint(reason: string): Promise { stopReason: reason, calls, interviewerMessages, + ...(condition === "3" + ? { + instrumentVersion: CONDITION_3_INSTRUMENT_VERSION, + demandTableVersion: CONDITION_3_DEMAND_TABLE_VERSION, + modelConfiguration: { + ...CONDITION_3_MODEL_CONFIGURATION, + }, + preregistration, + operatorProjections, + operatorAttempts, + impatienceProbeTurn, + genQ02Layer2: CONDITION_3_GEN_Q02_LAYER_2, + recovery, + } + : {}), }; return writeFile(rawPath, JSON.stringify(checkpoint, null, 2)); } @@ -336,14 +1125,28 @@ async function writeArtifacts(): Promise { ); const header = [ - `# Baseline control — condition ${condition} (${condition === "1" ? "bare" : "v0 prompt"})`, + `# Baseline control — condition ${condition} (${condition === "1" ? "bare" : condition === "2" ? "v0 prompt" : "completion + reviewed guidance"})`, "", `- Run started: ${startedAt}`, `- Interviewer: ${INTERVIEWER_MODEL}${ - condition === "2" ? " + v0-prompt.md" : " (no system prompt)" + condition === "2" + ? " + v0-prompt.md" + : condition === "3" + ? " + condition-3-prompt.md" + : " (no system prompt)" }`, `- Simulated expert: ${EXPERT_MODEL} + situation-pack.md`, - `- Interviewer turns: ${interviewerTurns} (impatience probe at ${IMPATIENCE_AT}, forced wrap at ${FORCE_WRAP_AT})`, + ...(condition === "3" + ? [ + `- Test-only operator: ${OPERATOR_MODEL}; ${CONDITION_3_INSTRUMENT_VERSION}`, + `- Frozen DemandTable: ${CONDITION_3_DEMAND_TABLE_VERSION}`, + `- Preregistration SHA-256: ${preregistration?.sha256 ?? "missing"}`, + `- Sampling/seed: ${CONDITION_3_STOPPING_RULES.providerSampling}`, + `- Interviewer turns: ${interviewerTurns} (phase-triggered impatience probe at ${impatienceProbeTurn ?? "not triggered"}, forced wrap at ${forceWrapAt})`, + ] + : [ + `- Interviewer turns: ${interviewerTurns} (impatience probe at ${LEGACY_IMPATIENCE_AT}, forced wrap at ${forceWrapAt})`, + ]), `- Stop reason: ${stopReason}`, `- Tokens: ${totals.input} in (+${totals.cacheWrite} cache write, +${totals.cacheRead} cache read) / ${totals.output} out across ${calls.length} calls`, "", @@ -356,18 +1159,49 @@ async function writeArtifacts(): Promise { const speaker = message.role === "assistant" ? "**Interviewer**" - : index === 0 - ? "**Opening message**" - : "**Expert (Marta)**"; - return `${speaker}:\n\n${message.content}`; + : message.experimentStimulus && !message.expertContent + ? "**Injected experiment stimulus (not expert evidence)**" + : index === 0 + ? "**Opening message**" + : "**Expert (Marta)**"; + if (message.expertContent) { + const stimulus = message.experimentStimulus + ? `\n\n**Injected experiment stimulus (not expert evidence)**:\n\n${message.experimentStimulus}` + : ""; + const diagnostic = message.operatorDiagnostic + ? `\n\n**Test-only operator diagnostic (shown to interviewer)**:\n\n${message.operatorDiagnostic}` + : ""; + return `${speaker}:\n\n${message.expertContent}${stimulus}${diagnostic}`; + } + const continuationText = (message.continuations ?? []) + .map( + (continuation, continuationIndex) => + `\n\n\n\n${continuation.content}`, + ) + .join(""); + return `${speaker}:\n\n${message.content}${continuationText}`; }) .join("\n\n---\n\n"); - await writeFile( - `${transcriptDir}/condition-${condition}.md`, - header + body + "\n", - ); + await writeFile(`${transcriptDir}/${artifactStem}.md`, header + body + "\n"); await writeCheckpoint(stopReason); + if (condition === "3") { + await writeFile( + `${transcriptDir}/${artifactStem}.operator.json`, + JSON.stringify( + { + instrumentVersion: CONDITION_3_INSTRUMENT_VERSION, + demandTableVersion: CONDITION_3_DEMAND_TABLE_VERSION, + preregistration, + genQ02Layer2: CONDITION_3_GEN_Q02_LAYER_2, + projections: operatorProjections, + attempts: operatorAttempts, + }, + null, + 2, + ), + ); + } // The model artifact is the interviewer's final delivery message, verbatim. // Extracting "the model" out of it (the old largest-fenced-block heuristic) @@ -382,8 +1216,11 @@ async function writeArtifacts(): Promise { finalMessage?.role === "assistant" ) { await writeFile( - `${transcriptDir}/condition-${condition}-model.txt`, - finalMessage.content, + `${transcriptDir}/${artifactStem}-model.txt`, + finalMessage.content + + (finalMessage.continuations ?? []) + .map((continuation) => continuation.content) + .join(""), ); } else if (stopReason.startsWith("delivered")) { // The transcript header claims a delivery, so a missing artifact must be @@ -391,7 +1228,7 @@ async function writeArtifacts(): Promise { // silently absent. console.error( `⚠ stop reason is '${stopReason}' but the transcript does not end with an interviewer ` + - `message — condition-${condition}-model.txt was NOT written`, + `message — ${artifactStem}-model.txt was NOT written`, ); } @@ -402,40 +1239,214 @@ async function writeArtifacts(): Promise { ); } +async function appendCondition3ExpertAnswer( + rawExpertText: string, + turn: number, +): Promise { + const priorProjection = operatorProjections.at(-1); + const floorPassed = + priorProjection !== undefined && + priorProjection.assessments + .filter((assessment) => assessment.clauseId.startsWith("SF-")) + .every((assessment) => assessment.pass); + const expertText = + impatienceProbeTurn === undefined && + floorPassed && + priorProjection.activeObjectiveRows.length > 0 + ? `${rawExpertText}\n\n${IMPATIENCE_LINE}` + : rawExpertText; + if (expertText !== rawExpertText) impatienceProbeTurn = turn; + + const evidenceMessages = [ + ...interviewerMessages, + { + role: "user" as const, + content: expertText, + expertContent: rawExpertText, + ...(expertText !== rawExpertText + ? { experimentStimulus: IMPATIENCE_LINE } + : {}), + }, + ]; + let projection: Condition3Projection; + try { + projection = await callCondition3Operator( + operatorSystem ?? "", + evidenceMessages, + turn, + ); + } catch (error) { + interviewerMessages.push({ + role: "user", + content: expertText, + expertContent: rawExpertText, + ...(expertText !== rawExpertText + ? { experimentStimulus: IMPATIENCE_LINE } + : {}), + }); + stopReason = "operator-projection-failure"; + console.error(error); + await writeArtifacts(); + process.exit(1); + } + + const noProgressStreak = nextCondition3NoProgressStreak( + operatorProjections, + projection, + turn, + priorProjection?.noProgressStreak ?? 0, + ); + const matches = activationMatches(projection); + const previouslySelectedUnsupportedLabels = new Set( + operatorProjections.flatMap(({ selectedUnsupportedAnchorLabel }) => + selectedUnsupportedAnchorLabel === null + ? [] + : [selectedUnsupportedAnchorLabel], + ), + ); + const selectedUnsupportedAnchor = + projection.unsupportedActiveObjectiveAnchors.find( + ({ label, state }) => + state === "active" && !previouslySelectedUnsupportedLabels.has(label), + ); + const selected = selectedUnsupportedAnchor + ? null + : selectedAssessment(projection); + const selectedMatch = selected + ? matches.find((match) => match.clauseId === selected.clauseId) + : undefined; + const projectionRecord: Condition3ProjectionRecord = { + ...projection, + turn, + recordedAt: new Date().toISOString(), + selectedClauseId: (selected?.clauseId as Condition3ClauseId) ?? null, + selectedUnsupportedAnchorLabel: selectedUnsupportedAnchor?.label ?? null, + selectedCardId: selectedMatch?.cardId ?? null, + selectedPredicate: selectedMatch?.predicate ?? null, + activationMatches: matches, + noProgressStreak, + noProgressAdvisory: + noProgressStreak >= CONDITION_3_STOPPING_RULES.noProgressAdvisoryAfter, + }; + operatorProjections.push(projectionRecord); + const operatorDiagnostic = renderOperatorDiagnostic(projectionRecord); + const sessionAdvisory = projectionRecord.noProgressAdvisory + ? `\n\nNP: ${noProgressStreak} consecutive non-material expert frames. This does not assert completion.` + : ""; + interviewerMessages.push({ + role: "user", + content: `${expertText}\n\n${operatorDiagnostic}${sessionAdvisory}`, + expertContent: rawExpertText, + ...(expertText !== rawExpertText + ? { experimentStimulus: IMPATIENCE_LINE } + : {}), + operatorDiagnostic: operatorDiagnostic + sessionAdvisory, + }); + if (noProgressStreak >= CONDITION_3_STOPPING_RULES.noProgressHardStopAfter) { + const closeInstruction = + "NP hard stop: do not ask another question. Produce the best useful result supportable now, with explicit gaps and claim limits; delivery does not assert completion."; + const finalMessage = interviewerMessages.at(-1); + if (finalMessage?.role === "user") { + finalMessage.content += `\n\n${closeInstruction}`; + finalMessage.operatorDiagnostic = `${finalMessage.operatorDiagnostic ?? ""}\n\n${closeInstruction}`; + } + stopReason = "no-progress-hard-stop-pending-delivery"; + await writeCheckpoint(stopReason); + return true; + } + return false; +} + if (mode === "--continue-final") { const final = interviewerMessages.at(-1); - if (final?.role !== "assistant") { + if ( + final?.role !== "assistant" || + !final.truncated || + !stopReason.endsWith("-incomplete") || + (condition === "3" && !stopReason.startsWith("delivered")) + ) { console.error( - "checkpoint does not end with an interviewer message; nothing to continue", + "checkpoint does not end with a truncated interviewer message; nothing to continue", ); process.exit(1); } const priorMessages = interviewerMessages.slice(0, -1); - const continued = await callInterviewer(v0Prompt, [ + const combinedFinalContent = + final.content + + (final.continuations ?? []) + .map((continuation) => continuation.content) + .join(""); + const continued = await callInterviewer(interviewerSystem, [ ...priorMessages, - final, + { role: "assistant", content: combinedFinalContent }, { role: "user", content: CONTINUE_MESSAGE }, ]); - // Same rule as the stitching loop: no separator at a truncation seam. - final.content += continued.text; + if (condition === "3") { + const newContinuationPieces = continued.sourceText + ? [ + { + content: continued.sourceText, + truncated: true, + recordedAt: new Date().toISOString(), + }, + ...(continued.continuations ?? []), + ] + : [ + { + content: continued.text, + truncated: continued.truncated, + recordedAt: new Date().toISOString(), + }, + ]; + final.continuations = [ + ...(final.continuations ?? []), + ...newContinuationPieces, + ]; + recovery?.seams.push({ + kind: "final-continuation", + sourceHadTruncationMarker: true, + sourceContent: combinedFinalContent, + recordedAt: new Date().toISOString(), + }); + } else { + // Legacy checkpoints retain their reviewed in-place merge behavior. + final.content += continued.text; + } if (continued.truncated) { console.error( "⚠ still truncated after this continuation — run --continue-final again", ); - } else { + } else if (condition !== "3") { delete final.truncated; - if (stopReason.endsWith("-incomplete")) { - stopReason = stopReason.slice(0, -"-incomplete".length); - } + } + if (!continued.truncated && stopReason.endsWith("-incomplete")) { + stopReason = stopReason.slice(0, -"-incomplete".length); } await writeArtifacts(); process.exit(0); } if (mode === "--resume") { + const resumeAfterForcedWrap = + condition === "3" && stopReason === "forced-wrap-in-progress"; // A delivered checkpoint must never resume: doing so would pop and regenerate the paid // final delivery, then overwrite the transcript. Check the durable reason rather than the // trailing role because a capped non-final interviewer turn also ends with an assistant. + if ( + condition === "3" && + ![ + "in-progress", + "expert-truncated", + "interviewer-truncated", + "no-progress-hard-stop-pending-delivery", + "forced-wrap-in-progress", + ].includes(stopReason) + ) { + console.error( + `condition 3 ended '${stopReason}' — this terminal checkpoint cannot resume`, + ); + process.exit(1); + } if (stopReason.startsWith("delivered")) { console.error( `condition ${condition} already ended '${stopReason}' — resuming would regenerate and ` + @@ -452,6 +1463,14 @@ if (mode === "--resume") { ); process.exit(1); } + if (condition === "3") { + recovery?.seams.push({ + kind: "truncated-expert-regeneration", + sourceHadTruncationMarker: true, + sourceContent: partialExpertReply.content, + recordedAt: new Date().toISOString(), + }); + } // The partial text remains in the stopped checkpoint as evidence, but must never be fed // to the interviewer as a complete answer. Resume removes it and retries the expert call // against the same preceding interviewer question. @@ -467,15 +1486,15 @@ if (mode === "--resume") { 1_500, ); let expertText = expertResult.text; - if (interviewerTurns === IMPATIENCE_AT) { + if (condition !== "3" && interviewerTurns === LEGACY_IMPATIENCE_AT) { expertText = `${expertText}\n\n${IMPATIENCE_LINE}`; } - interviewerMessages.push({ - role: "user", - content: expertText, - ...(expertResult.truncated ? { truncated: true as const } : {}), - }); if (expertResult.truncated) { + interviewerMessages.push({ + role: "user", + content: expertText, + truncated: true, + }); console.error( "⚠ the regenerated expert reply is still truncated — checkpointed the partial reply " + "without sending it to the interviewer; rerun with --resume to try again", @@ -483,13 +1502,38 @@ if (mode === "--resume") { await writeArtifacts(); process.exit(0); } - await writeCheckpoint("in-progress"); + if (condition === "3") { + const stopped = await appendCondition3ExpertAnswer( + expertText, + interviewerTurns, + ); + if (stopped) { + noProgressClosePending = true; + } + } else { + interviewerMessages.push({ role: "user", content: expertText }); + } + if (!noProgressClosePending) await writeCheckpoint("in-progress"); } // Checkpoints are written after complete exchanges only, but tolerate a trailing // assistant message by regenerating that turn. - stopReason = "hard-stop"; + if (!noProgressClosePending) stopReason = "hard-stop"; const last = interviewerMessages.at(-1); - if (last?.role === "assistant") interviewerMessages.pop(); + if (last?.role === "assistant" && !resumeAfterForcedWrap) { + if (condition === "3" && last.truncated) { + recovery?.seams.push({ + kind: "truncated-interviewer-regeneration", + sourceHadTruncationMarker: true, + sourceContent: + last.content + + (last.continuations ?? []) + .map((continuation) => continuation.content) + .join(""), + recordedAt: new Date().toISOString(), + }); + } + interviewerMessages.pop(); + } interviewerTurns = interviewerMessages.filter( (message) => message.role === "assistant", ).length; @@ -498,19 +1542,40 @@ if (mode === "--resume") { ); } -while (interviewerTurns < HARD_STOP_AT) { +while (interviewerTurns < hardStopAt) { interviewerTurns++; + if ( + condition === "3" && + interviewerTurns >= forceWrapAt && + !noProgressClosePending && + interviewerMessages.at(-1)?.experimentStimulus !== FORCED_WRAP_MESSAGE + ) { + interviewerMessages.push({ + role: "user", + content: `${FORCED_WRAP_MESSAGE}`, + experimentStimulus: FORCED_WRAP_MESSAGE, + }); + } console.error(`turn ${interviewerTurns} (interviewer)`); - const interviewer = await callInterviewer(v0Prompt, interviewerMessages); + const interviewer = await callInterviewer( + interviewerSystem, + interviewerMessages, + ); interviewerMessages.push({ role: "assistant", - content: interviewer.text, - ...(interviewer.truncated ? { truncated: true as const } : {}), + content: interviewer.sourceText ?? interviewer.text, + ...(interviewer.sourceText || interviewer.truncated + ? { truncated: true as const } + : {}), + ...(interviewer.continuations + ? { continuations: interviewer.continuations } + : {}), }); if (await isFinalModel(interviewer.text)) { - stopReason = - interviewerTurns > FORCE_WRAP_AT + stopReason = noProgressClosePending + ? "delivered-after-no-progress-hard-stop" + : interviewerTurns >= forceWrapAt ? "delivered-after-forced-wrap" : "delivered"; if (interviewer.truncated) { @@ -523,6 +1588,13 @@ while (interviewerTurns < HARD_STOP_AT) { break; } + if (noProgressClosePending) { + stopReason = interviewer.truncated + ? "no-progress-hard-stop-undelivered-incomplete" + : "no-progress-hard-stop-undelivered"; + break; + } + if (interviewer.truncated) { stopReason = "interviewer-truncated"; console.error( @@ -533,9 +1605,16 @@ while (interviewerTurns < HARD_STOP_AT) { break; } + if (condition === "3" && interviewerTurns >= forceWrapAt) { + if (interviewerTurns < hardStopAt) { + await writeCheckpoint("forced-wrap-in-progress"); + } + continue; + } + let expertText: string; let expertTruncated = false; - if (interviewerTurns >= FORCE_WRAP_AT) { + if (interviewerTurns >= forceWrapAt) { expertText = FORCED_WRAP_MESSAGE; } else { console.error(`turn ${interviewerTurns} (expert)`); @@ -548,15 +1627,26 @@ while (interviewerTurns < HARD_STOP_AT) { ); expertText = expertResult.text; expertTruncated = expertResult.truncated; - if (interviewerTurns === IMPATIENCE_AT) { + if (condition !== "3" && interviewerTurns === LEGACY_IMPATIENCE_AT) { expertText = `${expertText}\n\n${IMPATIENCE_LINE}`; } } - interviewerMessages.push({ - role: "user", - content: expertText, - ...(expertTruncated ? { truncated: true as const } : {}), - }); + if (condition === "3" && !expertTruncated) { + const stopped = await appendCondition3ExpertAnswer( + expertText, + interviewerTurns, + ); + if (stopped) { + noProgressClosePending = true; + continue; + } + } else { + interviewerMessages.push({ + role: "user", + content: expertText, + ...(expertTruncated ? { truncated: true as const } : {}), + }); + } if (expertTruncated) { stopReason = "expert-truncated"; console.error( @@ -568,4 +1658,9 @@ while (interviewerTurns < HARD_STOP_AT) { await writeCheckpoint("in-progress"); } +if (stopReason === "no-progress-hard-stop-pending-delivery") { + throw new Error( + "condition-3 invariant violated: the no-progress closing interviewer turn exceeded the hard budget", + ); +} await writeArtifacts(); diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts index 4a25c4ea06e..c98d41a7b32 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts @@ -1,6 +1,16 @@ import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { cp, mkdtemp, readFile, rm } from "node:fs/promises"; +import { + cp, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + symlink, + writeFile, +} from "node:fs/promises"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,6 +19,11 @@ import { pathToFileURL } from "node:url"; import * as v from "valibot"; import { afterEach, describe, expect, test } from "vitest"; +import { + CONDITION_3_DEMAND_CLAUSES, + CONDITION_3_INSTRUMENT_VERSION, + CONDITION_3_LOCKED_PATHS, +} from "../../../../evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument"; import { CONTEXT_ROOT, contextRootPresent } from "./workspace"; import type { StubReply } from "./fixtures/baseline-anthropic-stub"; @@ -21,6 +36,10 @@ const BASELINE_CASE_DIR = join( CONTEXT_ROOT, "evaluations/cases/process-model-elicitation/baseline", ); +const BASELINE_EVIDENCE_DIR = join( + CONTEXT_ROOT, + "docs/evidence/evaluations/process-model-elicitation/baseline", +); const STUB_MODULE = pathToFileURL( join(import.meta.dirname, "fixtures/baseline-anthropic-stub.ts"), ).href; @@ -33,6 +52,7 @@ interface BaselineCopy { } const BaselineCheckpoint = v.object({ + condition: v.picklist(["1", "2", "3"]), stopReason: v.string(), calls: v.array(v.unknown()), interviewerMessages: v.array( @@ -40,14 +60,255 @@ const BaselineCheckpoint = v.object({ role: v.picklist(["user", "assistant"]), content: v.string(), truncated: v.optional(v.boolean()), + continuations: v.optional( + v.array( + v.object({ + content: v.string(), + truncated: v.boolean(), + recordedAt: v.string(), + }), + ), + ), + }), + ), + preregistration: v.optional( + v.object({ + sha256: v.string(), + verifiedBeforeRun: v.boolean(), + }), + ), + operatorProjections: v.optional( + v.array( + v.object({ + turn: v.number(), + activeObjectiveRows: v.array(v.string()), + unsupportedActiveObjectiveAnchors: v.array(v.unknown()), + selectedClauseId: v.nullable(v.string()), + selectedUnsupportedAnchorLabel: v.nullable(v.string()), + selectedCardId: v.nullable(v.string()), + selectedPredicate: v.nullable(v.string()), + activationMatches: v.array( + v.object({ + cardId: v.string(), + clauseId: v.string(), + predicate: v.string(), + }), + ), + noProgressStreak: v.number(), + noProgressAdvisory: v.boolean(), + }), + ), + ), + operatorAttempts: v.optional( + v.array( + v.object({ + turn: v.number(), + attempt: v.number(), + parseError: v.nullable(v.string()), + }), + ), + ), + impatienceProbeTurn: v.optional(v.number()), + genQ02Layer2: v.optional( + v.object({ + cardId: v.literal("GEN-Q02"), + verdict: v.literal("unobservable"), + reason: v.string(), + }), + ), + recovery: v.optional( + v.object({ + mode: v.picklist(["resume", "continue-final"]), + sourceRawPath: v.string(), + sourceSha256: v.string(), + seams: v.array( + v.object({ + kind: v.picklist([ + "truncated-expert-regeneration", + "truncated-interviewer-regeneration", + "final-continuation", + ]), + sourceHadTruncationMarker: v.literal(true), + sourceContent: v.string(), + recordedAt: v.string(), + }), + ), }), ), }); const BaselineRequest = v.object({ + model: v.string(), + system: v.optional(v.string()), messages: v.array(v.record(v.string(), v.unknown())), }); +const FIRST_EXPERT_EVIDENCE = [ + "The objective is to test scheduling decisions before committing the weekly plan.", + "We schedule customer orders on two coating lines.", + "Operators run coating batches.", + "Orders flow from release through line assignment to production and shipment.", + "A large order may be split into contiguous runs.", + "I do not know the ordinary minimum run range.", + "Both lines can run the same eligible coating family.", + "Split runs stay contiguous in the weekly sequence.", + "A split normally adds one or two extra changeovers.", + "Repeated ramp scrap is usually 20 to 40 units.", +].join(" "); + +const evidenceByClause = { + "SF-OBJ": FIRST_EXPERT_EVIDENCE.split(". ")[0] + ".", + "SF-ENT": "We schedule customer orders on two coating lines.", + "SF-ACT": "Operators run coating batches.", + "SF-PATH": + "Orders flow from release through line assignment to production and shipment.", + "SF-FLOW": + "Orders flow from release through line assignment to production and shipment.", + "SP-BATCH": "A large order may be split into contiguous runs.", + "SP-MIN": "I do not know the ordinary minimum run range.", + "SP-ELIG": "Both lines can run the same eligible coating family.", + "SP-POL": "Split runs stay contiguous in the weekly sequence.", + "SP-CO": "A split normally adds one or two extra changeovers.", + "SP-SCRAP": "Repeated ramp scrap is usually 20 to 40 units.", +} as const; + +function condition3Projection( + options: { + minimumEvidence?: { turn: number; quote: string }; + minimumPass?: boolean; + } = {}, +) { + return { + activeObjectiveRows: ["ROW-SPLIT"], + activeObjectiveRowEvidence: [ + { + row: "ROW-SPLIT", + anchorLabel: "split-large-orders", + matchingPredicate: "split-run", + evidence: [ + { + turn: 1, + quote: "A large order may be split into contiguous runs.", + }, + ], + rationale: "The expert explicitly describes split orders.", + }, + ], + retractedObjectiveAnchors: [], + unsupportedActiveObjectiveAnchors: [], + assessments: CONDITION_3_DEMAND_CLAUSES.map((clause) => { + const demanded = clause.row === null || clause.row === "ROW-SPLIT"; + const isPresenceDemand = clause.demand.startsWith("presence count >="); + const isSelectedFailure = clause.id === "SP-MIN"; + const quote = + clause.id === "SP-MIN" && options.minimumEvidence + ? options.minimumEvidence + : clause.id in evidenceByClause + ? { + turn: 1, + quote: + evidenceByClause[clause.id as keyof typeof evidenceByClause], + } + : undefined; + return { + clauseId: clause.id, + demand: clause.demand, + demanded, + coordinate: clause.coordinate, + currentStatus: demanded ? "explicit" : "not-applicable", + currentGrade: demanded + ? isPresenceDemand + ? "none" + : isSelectedFailure && !options.minimumPass + ? "verbal" + : clause.id === "SP-MIN" || + clause.id === "SP-CO" || + clause.id === "SP-SCRAP" + ? "range" + : "structured" + : "not-applicable", + pass: demanded + ? !isSelectedFailure || options.minimumPass === true + : true, + failureDiagnostic: + isSelectedFailure && !options.minimumPass + ? "below-required-grade" + : null, + activationPredicates: + isSelectedFailure && !options.minimumPass + ? ["below-demanded-grade"] + : [], + evidence: demanded && quote ? [quote] : [], + observedCount: isPresenceDemand + ? clause.id === "SF-ENT" + ? 2 + : 1 + : null, + rationale: "operator-only test rationale", + }; + }), + notes: ["test projection"], + }; +} + +function condition3NoProgressProjection() { + return { + activeObjectiveRows: [], + activeObjectiveRowEvidence: [], + retractedObjectiveAnchors: [], + unsupportedActiveObjectiveAnchors: [], + assessments: CONDITION_3_DEMAND_CLAUSES.map((clause) => + clause.row === null + ? { + clauseId: clause.id, + demand: clause.demand, + coordinate: clause.coordinate, + demanded: true, + currentStatus: "none", + currentGrade: "none", + pass: false, + failureDiagnostic: clause.demand.startsWith("presence count >=") + ? "below-minimum-count" + : "unaddressed", + activationPredicates: [], + evidence: [], + observedCount: clause.demand.startsWith("presence count >=") + ? 0 + : null, + rationale: "No transcript evidence was added.", + } + : { + clauseId: clause.id, + demand: clause.demand, + coordinate: clause.coordinate, + demanded: false, + currentStatus: "not-applicable", + currentGrade: "not-applicable", + pass: true, + failureDiagnostic: null, + activationPredicates: [], + evidence: [], + observedCount: null, + rationale: "Inactive objective row.", + }, + ), + notes: [], + }; +} + +async function copyDirectoryContents( + sourceDirectory: string, + destinationDirectory: string, +): Promise { + await Promise.all( + (await readdir(sourceDirectory)).map((entry) => + cp(join(sourceDirectory, entry), join(destinationDirectory, entry), { + recursive: true, + }), + ), + ); +} + async function createBaselineCopy(): Promise { const testDirectory = await mkdtemp(join(tmpdir(), "baseline-runner-test-")); temporaryDirectories.push(testDirectory); @@ -59,10 +320,67 @@ async function createBaselineCopy(): Promise { testDirectory, "evaluations/cases/process-model-elicitation/baseline", ); + const completionSpecDirectory = join(testDirectory, "docs/specs"); + const researchDirectory = join( + testDirectory, + "docs/reference/research/elicitation", + ); + const evidenceDirectory = join( + testDirectory, + "docs/evidence/evaluations/process-model-elicitation/baseline", + ); await Promise.all([ - cp(BASELINE_PROTOCOL_DIR, protocolDirectory, { recursive: true }), - cp(BASELINE_CASE_DIR, caseDirectory, { recursive: true }), + mkdir(protocolDirectory, { recursive: true }), + mkdir(caseDirectory, { recursive: true }), + mkdir(completionSpecDirectory, { recursive: true }), + mkdir(researchDirectory, { recursive: true }), + mkdir(evidenceDirectory, { recursive: true }), ]); + await Promise.all([ + copyDirectoryContents(BASELINE_PROTOCOL_DIR, protocolDirectory), + copyDirectoryContents(BASELINE_CASE_DIR, caseDirectory), + copyDirectoryContents(BASELINE_EVIDENCE_DIR, evidenceDirectory), + cp( + join(CONTEXT_ROOT, "docs/specs/elicitation-completion.md"), + join(completionSpecDirectory, "elicitation-completion.md"), + ), + cp( + join(CONTEXT_ROOT, "docs/specs/cps-interview-guidance.md"), + join(completionSpecDirectory, "cps-interview-guidance.md"), + ), + cp( + join( + CONTEXT_ROOT, + "docs/reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md", + ), + join(researchDirectory, "frontier-model-elicitor-failure-catalogue.md"), + ), + ]); + await symlink( + join(CONTEXT_ROOT, "../../../node_modules"), + join(testDirectory, "node_modules"), + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + const files = await Promise.all( + CONDITION_3_LOCKED_PATHS.map(async (path) => ({ + path, + sha256: createHash("sha256") + .update(await readFile(join(testDirectory, path), "utf8")) + .digest("hex"), + })), + ); + await writeFile( + join(protocolDirectory, "condition-3-preregistration.lock.json"), + JSON.stringify( + { + version: CONDITION_3_INSTRUMENT_VERSION, + sealedAt: new Date().toISOString(), + files, + }, + null, + 2, + ), + ); return { outputDirectory: join(testDirectory, "test-output"), protocolDirectory, @@ -73,6 +391,7 @@ async function createBaselineCopy(): Promise { async function runBaseline( baselineCopy: BaselineCopy, replies: StubReply[], + condition: "1" | "2" | "3" = "1", mode?: "--resume" | "--continue-final", ): Promise<{ checkpoint: v.InferOutput; @@ -85,7 +404,7 @@ async function runBaseline( [ "--experimental-strip-types", join(baselineCopy.protocolDirectory, "run.ts"), - "1", + condition, ...(mode ? [mode] : []), ], { @@ -109,13 +428,23 @@ async function runBaseline( subprocess.once("error", reject); subprocess.once("close", resolve); }); - expect(exitCode).toBe(0); + expect(exitCode, stderr).toBe(0); const checkpoint = v.parse( BaselineCheckpoint, JSON.parse( await readFile( - join(baselineCopy.outputDirectory, "condition-1.raw.json"), + join( + baselineCopy.outputDirectory, + (await readdir(baselineCopy.outputDirectory)) + .filter( + (name) => + name.startsWith(`condition-${condition}`) && + name.endsWith(".raw.json"), + ) + .sort() + .at(-1) ?? `condition-${condition}.raw.json`, + ), "utf8", ), ) as unknown, @@ -127,6 +456,93 @@ async function runBaseline( return { checkpoint, stderr, requests }; } +async function runBaselineFailure( + baselineCopy: BaselineCopy, + replies: StubReply[], + mode?: "--resume" | "--continue-final", +): Promise<{ + checkpoint?: v.InferOutput; + stderr: string; + requests: Array>; +}> { + const requestsPath = join(baselineCopy.testDirectory, "requests.jsonl"); + const subprocess = spawn( + process.execPath, + [ + "--experimental-strip-types", + join(baselineCopy.protocolDirectory, "run.ts"), + "3", + ...(mode ? [mode] : []), + ], + { + cwd: baselineCopy.testDirectory, + env: { + ...process.env, + BRUNCH_BASELINE_ANTHROPIC_MODULE: STUB_MODULE, + BRUNCH_BASELINE_TEST_OUTPUT_DIR: baselineCopy.outputDirectory, + BASELINE_STUB_REPLIES: JSON.stringify(replies), + BASELINE_STUB_REQUESTS_PATH: requestsPath, + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + subprocess.stderr.setEncoding("utf8"); + let stderr = ""; + subprocess.stderr.on("data", (chunk: string) => { + stderr += chunk; + }); + const exitCode = await new Promise((resolve, reject) => { + subprocess.once("error", reject); + subprocess.once("close", resolve); + }); + expect(exitCode).toBe(1); + + const requests = existsSync(requestsPath) + ? (await readFile(requestsPath, "utf8")) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => v.parse(BaselineRequest, JSON.parse(line) as unknown)) + : []; + const rawFiles = existsSync(baselineCopy.outputDirectory) + ? readdir(baselineCopy.outputDirectory) + : Promise.resolve([]); + const latestRawFile = (await rawFiles) + .filter((name) => name.endsWith(".raw.json")) + .sort() + .at(-1); + const checkpoint = latestRawFile + ? v.parse( + BaselineCheckpoint, + JSON.parse( + await readFile( + join(baselineCopy.outputDirectory, latestRawFile), + "utf8", + ), + ) as unknown, + ) + : undefined; + return { checkpoint, stderr, requests }; +} + +async function mutateCondition3Lock( + baselineCopy: BaselineCopy, + mutate: (lock: Condition3Lock) => Condition3Lock, +): Promise { + const lockPath = join( + baselineCopy.protocolDirectory, + "condition-3-preregistration.lock.json", + ); + const lock = JSON.parse(await readFile(lockPath, "utf8")) as Condition3Lock; + await writeFile(lockPath, JSON.stringify(mutate(lock), null, 2)); +} + +interface Condition3Lock { + version: string; + sealedAt: string; + files: Array<{ path: string; sha256: string }>; +} + afterEach(async () => { await Promise.all( temporaryDirectories @@ -211,6 +627,23 @@ describe.skipIf(!contextRootPresent)( expect(result.stderr).toContain("expert reply is truncated"); }); + test("preserves the condition 2 prompt and legacy completion path", async () => { + const testDirectory = await createBaselineCopy(); + const result = await runBaseline( + testDirectory, + [{ text: "Final structured model" }, { text: "YES" }], + "2", + ); + + expect(result.checkpoint.stopReason).toBe("delivered"); + expect(result.requests[0]?.system).toContain( + "You are an expert process-model elicitor", + ); + expect(result.requests[0]?.system).not.toContain( + "test-only completion operator", + ); + }); + test("resume regenerates a trailing truncated expert reply before continuing", async () => { const testDirectory = await createBaselineCopy(); await runBaseline(testDirectory, [ @@ -226,6 +659,7 @@ describe.skipIf(!contextRootPresent)( { text: "Final model" }, { text: "YES" }, ], + "1", "--resume", ); @@ -277,6 +711,7 @@ describe.skipIf(!contextRootPresent)( const continued = await runBaseline( testDirectory, [{ text: " continued" }], + "1", "--continue-final", ); @@ -299,5 +734,894 @@ describe.skipIf(!contextRootPresent)( content: "part-1part-2part-3part-4part-5 continued", }); }); + + test("runs condition 3 with a preregistered operator projection", async () => { + const testDirectory = await createBaselineCopy(); + const result = await runBaseline( + testDirectory, + [ + { text: "What decision should the model support?" }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "What ordinary minimum run range applies?" }, + { text: "NO" }, + { text: "Usually 800 to 1,200 units." }, + { + text: JSON.stringify( + condition3Projection({ + minimumEvidence: { + turn: 2, + quote: "Usually 800 to 1,200 units.", + }, + minimumPass: true, + }), + ), + }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(result.checkpoint.stopReason).toBe("delivered"); + expect(result.checkpoint).toMatchObject({ + condition: "3", + preregistration: { + verifiedBeforeRun: true, + }, + operatorProjections: [ + expect.objectContaining({ + turn: 1, + activeObjectiveRows: ["ROW-SPLIT"], + selectedClauseId: "SP-MIN", + selectedCardId: "CPS-Q03", + selectedPredicate: "below-demanded-grade", + }), + expect.objectContaining({ turn: 2 }), + ], + }); + expect(result.checkpoint.preregistration?.sha256).toMatch( + /^[a-f0-9]{64}$/u, + ); + + const firstInterviewer = result.requests[0]; + const firstExpert = result.requests[2]; + const firstOperator = result.requests[3]; + const secondInterviewer = result.requests[4]; + const secondExpert = result.requests[6]; + + expect(firstInterviewer?.system).toContain( + "This is a single-session experiment", + ); + expect(firstInterviewer?.system).not.toContain("Marta Iversen"); + expect(firstExpert?.system).toContain("Marta Iversen"); + expect(firstExpert?.system).not.toContain("FROZEN_DEMAND_TABLE"); + expect(firstOperator?.system).toContain("FROZEN_DEMAND_TABLE"); + expect(firstOperator?.system).toContain("PROJECTION_ENVELOPE"); + expect(firstOperator?.system).toContain('"assessmentStates"'); + expect(firstOperator?.system).toContain('"matchingPredicate"'); + expect(firstOperator?.system).toContain('"split-run"'); + expect(firstOperator?.system).toContain('"failureDiagnostic"'); + expect(firstOperator?.system).toContain('"absence-uncorroborated"'); + expect(firstOperator?.system).not.toContain("Marta Iversen"); + expect(JSON.stringify(firstOperator?.messages)).toContain( + "Repeated ramp scrap is usually 20 to 40 units.", + ); + expect(JSON.stringify(secondInterviewer?.messages)).toContain( + "", + ); + expect(JSON.stringify(secondInterviewer?.messages)).not.toContain( + "operator-only test rationale", + ); + expect(JSON.stringify(secondExpert?.messages)).not.toContain( + "test-only-completion-diagnostic", + ); + expect(JSON.stringify(result.requests[7]?.messages)).toContain( + "floor huddle in ten minutes", + ); + expect(result.checkpoint.impatienceProbeTurn).toBe(2); + expect(result.checkpoint.genQ02Layer2).toMatchObject({ + cardId: "GEN-Q02", + verdict: "unobservable", + }); + expect( + result.checkpoint.operatorProjections?.flatMap( + ({ activationMatches }) => activationMatches, + ), + ).not.toContainEqual(expect.objectContaining({ cardId: "GEN-Q02" })); + }); + + test("selects a transcript-supported unsupported active objective before frozen rows", async () => { + const baselineCopy = await createBaselineCopy(); + const unsupportedProjection = { + ...condition3Projection(), + assessments: condition3Projection().assessments.map((assessment) => + assessment.clauseId === "SF-OBJ" + ? { + ...assessment, + observedCount: 2, + evidence: [ + ...assessment.evidence, + { turn: 1, quote: "minimize energy use" }, + ], + } + : assessment, + ), + unsupportedActiveObjectiveAnchors: [ + { + label: "energy-use", + state: "active", + demanded: true, + pass: false, + failureDiagnostic: "unsupported-active-anchor", + evidence: [ + { + turn: 1, + quote: "minimize energy use", + }, + ], + resolutionEvidence: [], + resolutionRationale: null, + rationale: + "The frozen objective rows do not represent this objective.", + }, + ], + }; + const result = await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: `${FIRST_EXPERT_EVIDENCE} We also minimize energy use.` }, + { text: JSON.stringify(unsupportedProjection) }, + { text: "What frozen demand is still open?" }, + { text: "NO" }, + { text: "The ordinary minimum is still unknown." }, + { + text: JSON.stringify({ + ...condition3Projection({ + minimumEvidence: { + turn: 2, + quote: "The ordinary minimum is still unknown.", + }, + }), + assessments: condition3Projection({ + minimumEvidence: { + turn: 2, + quote: "The ordinary minimum is still unknown.", + }, + }).assessments.map((assessment) => + assessment.clauseId === "SF-OBJ" + ? { + ...assessment, + observedCount: 2, + evidence: [ + ...assessment.evidence, + { turn: 1, quote: "minimize energy use" }, + ], + } + : assessment, + ), + unsupportedActiveObjectiveAnchors: + unsupportedProjection.unsupportedActiveObjectiveAnchors, + }), + }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(result.checkpoint.operatorProjections?.[0]).toMatchObject({ + selectedClauseId: null, + selectedUnsupportedAnchorLabel: "energy-use", + selectedCardId: null, + selectedPredicate: null, + }); + expect(JSON.stringify(result.requests[4]?.messages)).toContain( + "clause=unsupported-active-anchor", + ); + expect(result.checkpoint.operatorProjections?.[1]).toMatchObject({ + selectedClauseId: "SP-MIN", + selectedUnsupportedAnchorLabel: null, + selectedCardId: "CPS-Q03", + selectedPredicate: "below-demanded-grade", + }); + }); + + test("treats forced wrap as a stimulus and never as an expert/operator frame", async () => { + const baselineCopy = await createBaselineCopy(); + const replies: StubReply[] = []; + for (let turn = 1; turn <= 19; turn++) { + const minimumQuote = + turn === 1 + ? evidenceByClause["SP-MIN"] + : `Ordinary minimum range remains unknown at turn ${turn}.`; + replies.push( + { text: `Question ${turn}.` }, + { text: "NO" }, + { + text: turn === 1 ? FIRST_EXPERT_EVIDENCE : minimumQuote, + }, + { + text: JSON.stringify( + condition3Projection({ + minimumEvidence: { turn, quote: minimumQuote }, + }), + ), + }, + ); + } + replies.push({ text: "Final model after forced wrap." }, { text: "YES" }); + + const result = await runBaseline(baselineCopy, replies, "3"); + + expect(result.checkpoint.stopReason).toBe("delivered-after-forced-wrap"); + expect(result.checkpoint.operatorProjections).toHaveLength(19); + expect( + result.checkpoint.interviewerMessages.some( + ({ content }) => + content.includes("") && + content.includes("Please produce the model now"), + ), + ).toBe(true); + expect( + result.requests.filter(({ system }) => + system?.includes("test-only completion operator"), + ), + ).toHaveLength(19); + expect(JSON.stringify(result.requests[76]?.messages)).toContain( + "Please produce the model now", + ); + expect( + result.requests + .filter(({ system }) => system?.includes("Marta Iversen")) + .some(({ messages }) => + JSON.stringify(messages).includes("Please produce the model now"), + ), + ).toBe(false); + }); + + test("resumes after a completed forced-wrap turn without regenerating or duplicating it", async () => { + const baselineCopy = await createBaselineCopy(); + const replies: StubReply[] = []; + for (let turn = 1; turn <= 19; turn++) { + const minimumQuote = + turn === 1 + ? evidenceByClause["SP-MIN"] + : `Ordinary minimum range remains unknown at turn ${turn}.`; + replies.push( + { text: `Question ${turn}.` }, + { text: "NO" }, + { text: turn === 1 ? FIRST_EXPERT_EVIDENCE : minimumQuote }, + { + text: JSON.stringify( + condition3Projection({ + minimumEvidence: { turn, quote: minimumQuote }, + }), + ), + }, + ); + } + replies.push({ text: "Question 20." }, { text: "NO" }); + + const interrupted = await runBaselineFailure(baselineCopy, replies); + expect(interrupted.checkpoint?.stopReason).toBe( + "forced-wrap-in-progress", + ); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const resumed = await runBaseline( + baselineCopy, + [{ text: "Final model on turn 21." }, { text: "YES" }], + "3", + "--resume", + ); + + expect(resumed.checkpoint.stopReason).toBe("delivered-after-forced-wrap"); + expect( + resumed.checkpoint.interviewerMessages.filter(({ content }) => + content.includes(""), + ), + ).toHaveLength(2); + expect( + resumed.checkpoint.interviewerMessages.filter( + ({ content }) => content === "Question 20.", + ), + ).toHaveLength(1); + }); + + test("supplies every stitched non-final interviewer piece to expert, operator, and later interviewer views", async () => { + const baselineCopy = await createBaselineCopy(); + const result = await runBaseline( + baselineCopy, + [ + { text: "Question part one ", truncated: true }, + { text: "and part two." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + for (const requestIndex of [3, 4, 5]) { + expect( + JSON.stringify(result.requests[requestIndex]?.messages), + ).toContain("Question part one and part two."); + } + expect(result.checkpoint.interviewerMessages[1]).toMatchObject({ + content: "Question part one ", + continuations: [expect.objectContaining({ content: "and part two." })], + }); + }); + + test.each([ + [ + "missing", + (files: Array<{ path: string; sha256: string }>) => files.slice(1), + ], + [ + "extra", + (files: Array<{ path: string; sha256: string }>) => [ + ...files, + { path: "unexpected.md", sha256: "0".repeat(64) }, + ], + ], + [ + "duplicate", + (files: Array<{ path: string; sha256: string }>) => [ + ...files, + files[0] as { path: string; sha256: string }, + ], + ], + [ + "reordered", + (files: Array<{ path: string; sha256: string }>) => + [...files].reverse(), + ], + ["empty", () => []], + ])( + "rejects a %s condition-3 manifest before model calls", + async (_name, mutate) => { + const baselineCopy = await createBaselineCopy(); + await mutateCondition3Lock(baselineCopy, (lock) => ({ + ...lock, + files: mutate(lock.files), + })); + + const result = await runBaselineFailure(baselineCopy, []); + + expect(result.stderr).toMatch( + /manifest is not canonical|invalid envelope/u, + ); + expect(result.requests).toEqual([]); + }, + ); + + test("rejects a falsely early self-declared sealedAt", async () => { + const baselineCopy = await createBaselineCopy(); + await mutateCondition3Lock(baselineCopy, (lock) => ({ + ...lock, + sealedAt: "2000-01-01T00:00:00.000Z", + })); + + const result = await runBaselineFailure(baselineCopy, []); + + expect(result.stderr).toContain( + "condition-3 preregistration chronology is invalid", + ); + expect(result.requests).toEqual([]); + }); + + test("retries malformed and contradictory operator projections before selection", async () => { + const baselineCopy = await createBaselineCopy(); + const contradictory = condition3Projection(); + const minimum = contradictory.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + minimum.pass = true; + + const result = await runBaseline( + baselineCopy, + [ + { + text: "Give one cohesive overview of objective, entities, activities, flow, and split policy.", + }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: "{}" }, + { text: JSON.stringify(contradictory) }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(result.checkpoint.operatorAttempts).toHaveLength(3); + expect(result.checkpoint.operatorAttempts?.[0]?.attempt).toBe(1); + expect(result.checkpoint.operatorAttempts?.[1]?.attempt).toBe(2); + expect(typeof result.checkpoint.operatorAttempts?.[0]?.parseError).toBe( + "string", + ); + expect(typeof result.checkpoint.operatorAttempts?.[1]?.parseError).toBe( + "string", + ); + expect(result.checkpoint.operatorAttempts?.[2]).toMatchObject({ + attempt: 3, + parseError: null, + }); + expect(result.checkpoint.operatorProjections?.[0]?.selectedClauseId).toBe( + "SP-MIN", + ); + }); + + test("retries an evidence quote absent from the supplied transcript", async () => { + const baselineCopy = await createBaselineCopy(); + const invalidQuote = condition3Projection(); + const objective = invalidQuote.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective) throw new Error("fixture lost SF-OBJ"); + objective.evidence = [{ turn: 1, quote: "words never supplied" }]; + + const result = await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(invalidQuote) }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(result.checkpoint.operatorAttempts?.[0]?.parseError).toContain( + "does not occur in supplied transcript", + ); + expect(result.checkpoint.operatorProjections).toHaveLength(1); + }); + + test("labels the single-session correction but rejects it as opening evidence", async () => { + const baselineCopy = await createBaselineCopy(); + const stimulusEvidence = condition3Projection(); + const objective = stimulusEvidence.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective) throw new Error("fixture lost SF-OBJ"); + objective.evidence = [ + { + turn: 0, + quote: "No external data or later follow-up will arrive.", + }, + ]; + + const result = await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(stimulusEvidence) }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(JSON.stringify(result.requests[3]?.messages)).toContain( + "", + ); + expect(result.checkpoint.operatorAttempts?.[0]?.parseError).toContain( + "does not occur in supplied transcript", + ); + }); + + test("labels the impatience stimulus but rejects it as expert evidence", async () => { + const baselineCopy = await createBaselineCopy(); + const stimulusEvidence = condition3Projection(); + const objective = stimulusEvidence.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective) throw new Error("fixture lost SF-OBJ"); + objective.evidence = [ + { + turn: 2, + quote: "floor huddle in ten minutes", + }, + ]; + + const result = await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Anything else?" }, + { text: "NO" }, + { text: "Nothing else." }, + { text: JSON.stringify(stimulusEvidence) }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(JSON.stringify(result.requests[7]?.messages)).toContain( + "", + ); + expect(result.checkpoint.operatorAttempts?.[1]?.parseError).toContain( + "does not occur in supplied transcript", + ); + }); + + test("fails closed after exhausting malformed operator retries", async () => { + const baselineCopy = await createBaselineCopy(); + const result = await runBaselineFailure(baselineCopy, [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: "{}" }, + { text: "{}" }, + { text: "{}" }, + ]); + + expect(result.stderr).toContain( + "condition-3 operator exhausted projection-validation attempts", + ); + expect(result.checkpoint?.stopReason).toBe("operator-projection-failure"); + expect(result.checkpoint?.operatorAttempts).toHaveLength(3); + expect(result.checkpoint?.operatorProjections).toEqual([]); + }); + + test("executes the semantic no-progress advisory and hard stop", async () => { + const baselineCopy = await createBaselineCopy(); + const replies: StubReply[] = []; + for (let turn = 1; turn <= 5; turn++) { + replies.push( + { text: `Prompt ${turn}.` }, + { text: "NO" }, + { text: "I have nothing to add." }, + { text: JSON.stringify(condition3NoProgressProjection()) }, + ); + } + replies.push( + { text: "Final limited model with explicit gaps." }, + { text: "YES" }, + ); + const result = await runBaseline(baselineCopy, replies, "3"); + + expect(result.checkpoint.stopReason).toBe( + "delivered-after-no-progress-hard-stop", + ); + expect( + result.checkpoint.operatorProjections?.map( + ({ noProgressStreak }) => noProgressStreak, + ), + ).toEqual([1, 2, 3, 4, 5]); + expect( + result.checkpoint.operatorProjections?.[2]?.noProgressAdvisory, + ).toBe(true); + expect(result.requests).toHaveLength(22); + expect(JSON.stringify(result.requests[20]?.messages)).toContain( + "do not ask another question", + ); + }); + + test.each([ + "Give objective, entities, activities, flow, and split policy as one cohesive five-item overview.", + "State the objective. Name the entities. Describe the activities. Explain the flow. Give the split rule.", + ])( + "keeps GEN-Q02 layer-2 unobservable for: %s", + async (interviewerMessage) => { + const baselineCopy = await createBaselineCopy(); + const result = await runBaseline( + baselineCopy, + [ + { text: interviewerMessage }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + ); + + expect(result.checkpoint.genQ02Layer2?.verdict).toBe("unobservable"); + expect( + result.checkpoint.operatorProjections?.[0]?.activationMatches, + ).not.toContainEqual(expect.objectContaining({ cardId: "GEN-Q02" })); + }, + ); + + test.each(["--resume", "--continue-final"] as const)( + "rejects a checkpoint seal mismatch on %s before model calls", + async (recoveryMode) => { + const baselineCopy = await createBaselineCopy(); + if (recoveryMode === "--resume") { + await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: "Partial expert evidence", truncated: true }, + ], + "3", + ); + } else { + await runBaseline( + baselineCopy, + [ + { text: "part-1", truncated: true }, + { text: "part-2", truncated: true }, + { text: "part-3", truncated: true }, + { text: "part-4", truncated: true }, + { text: "part-5", truncated: true }, + { text: "YES" }, + ], + "3", + ); + } + const rawPath = join( + baselineCopy.outputDirectory, + "condition-3.raw.json", + ); + const raw = JSON.parse(await readFile(rawPath, "utf8")) as { + preregistration: { sha256: string }; + }; + raw.preregistration.sha256 = "0".repeat(64); + await writeFile(rawPath, JSON.stringify(raw, null, 2)); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const result = await runBaselineFailure(baselineCopy, [], recoveryMode); + + expect(result.stderr).toContain("checkpoint binding mismatch"); + expect(result.requests).toEqual([]); + }, + ); + + test("rejects semantically edited checkpoint projections before resume calls", async () => { + const baselineCopy = await createBaselineCopy(); + await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "part-1", truncated: true }, + { text: "part-2", truncated: true }, + { text: "part-3", truncated: true }, + { text: "part-4", truncated: true }, + { text: "part-5", truncated: true }, + { text: "NO" }, + ], + "3", + ); + const rawPath = join( + baselineCopy.outputDirectory, + "condition-3.raw.json", + ); + const raw = JSON.parse(await readFile(rawPath, "utf8")) as { + operatorProjections: Array<{ noProgressStreak: number }>; + }; + raw.operatorProjections[0]!.noProgressStreak = 999; + await writeFile(rawPath, JSON.stringify(raw, null, 2)); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const result = await runBaselineFailure(baselineCopy, [], "--resume"); + + expect(result.stderr).toContain( + "checkpoint projection semantics disagree at turn 1", + ); + expect(result.requests).toEqual([]); + }); + + test("resumes condition 3 into an append-only segment with a sealed source seam", async () => { + const baselineCopy = await createBaselineCopy(); + await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: "Partial expert evidence", truncated: true }, + ], + "3", + ); + const sourcePath = join( + baselineCopy.outputDirectory, + "condition-3.raw.json", + ); + const sourceContent = await readFile(sourcePath, "utf8"); + const sourceHash = createHash("sha256") + .update(sourceContent) + .digest("hex"); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const resumed = await runBaseline( + baselineCopy, + [ + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + "--resume", + ); + + expect(await readFile(sourcePath, "utf8")).toBe(sourceContent); + expect(resumed.checkpoint.recovery).toMatchObject({ + mode: "resume", + sourceRawPath: sourcePath, + sourceSha256: sourceHash, + seams: [ + expect.objectContaining({ + kind: "truncated-expert-regeneration", + sourceHadTruncationMarker: true, + sourceContent: "Partial expert evidence", + }), + ], + }); + }); + + test("records an append-only seam when regenerating a truncated interviewer turn", async () => { + const baselineCopy = await createBaselineCopy(); + await runBaseline( + baselineCopy, + [ + { text: "part-1", truncated: true }, + { text: "part-2", truncated: true }, + { text: "part-3", truncated: true }, + { text: "part-4", truncated: true }, + { text: "part-5", truncated: true }, + { text: "NO" }, + ], + "3", + ); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const resumed = await runBaseline( + baselineCopy, + [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: JSON.stringify(condition3Projection()) }, + { text: "Final model" }, + { text: "YES" }, + ], + "3", + "--resume", + ); + + expect(resumed.checkpoint.recovery?.seams).toContainEqual( + expect.objectContaining({ + kind: "truncated-interviewer-regeneration", + sourceHadTruncationMarker: true, + sourceContent: "part-1part-2part-3part-4part-5", + }), + ); + }); + + test("refuses to resume operator exhaustion before any further model call", async () => { + const baselineCopy = await createBaselineCopy(); + await runBaselineFailure(baselineCopy, [ + { text: "Describe the operation." }, + { text: "NO" }, + { text: FIRST_EXPERT_EVIDENCE }, + { text: "{}" }, + { text: "{}" }, + { text: "{}" }, + ]); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const refused = await runBaselineFailure(baselineCopy, [], "--resume"); + + expect(refused.stderr).toContain("terminal checkpoint cannot resume"); + expect(refused.requests).toEqual([]); + }); + + test("continues condition 3 without overwriting or clearing the source truncation marker", async () => { + const baselineCopy = await createBaselineCopy(); + await runBaseline( + baselineCopy, + [ + { text: "part-1", truncated: true }, + { text: "part-2", truncated: true }, + { text: "part-3", truncated: true }, + { text: "part-4", truncated: true }, + { text: "part-5", truncated: true }, + { text: "YES" }, + ], + "3", + ); + const sourcePath = join( + baselineCopy.outputDirectory, + "condition-3.raw.json", + ); + const sourceContent = await readFile(sourcePath, "utf8"); + const sourceHash = createHash("sha256") + .update(sourceContent) + .digest("hex"); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const continued = await runBaseline( + baselineCopy, + [{ text: " tail" }], + "3", + "--continue-final", + ); + + expect(await readFile(sourcePath, "utf8")).toBe(sourceContent); + expect(continued.checkpoint.recovery).toMatchObject({ + mode: "continue-final", + sourceSha256: sourceHash, + seams: [expect.objectContaining({ kind: "final-continuation" })], + }); + const finalMessage = continued.checkpoint.interviewerMessages.at(-1); + expect(finalMessage).toMatchObject({ + role: "assistant", + content: "part-1", + truncated: true, + }); + expect(finalMessage?.continuations).toContainEqual( + expect.objectContaining({ content: "part-2", truncated: true }), + ); + expect(finalMessage?.continuations).toContainEqual( + expect.objectContaining({ content: "part-5", truncated: true }), + ); + expect(finalMessage?.continuations).toContainEqual( + expect.objectContaining({ content: " tail", truncated: false }), + ); + expect(continued.checkpoint.stopReason).toBe("delivered"); + }); + + test("refuses final continuation for a truncated non-delivery checkpoint", async () => { + const baselineCopy = await createBaselineCopy(); + const replies: StubReply[] = []; + for (let turn = 1; turn <= 5; turn++) { + replies.push( + { text: `Prompt ${turn}.` }, + { text: "NO" }, + { text: "I have nothing to add." }, + { text: JSON.stringify(condition3NoProgressProjection()) }, + ); + } + replies.push( + { text: "partial-1", truncated: true }, + { text: "partial-2", truncated: true }, + { text: "partial-3", truncated: true }, + { text: "partial-4", truncated: true }, + { text: "partial-5", truncated: true }, + { text: "NO" }, + ); + const stopped = await runBaseline(baselineCopy, replies, "3"); + expect(stopped.checkpoint.stopReason).toBe( + "no-progress-hard-stop-undelivered-incomplete", + ); + await rm(join(baselineCopy.testDirectory, "requests.jsonl")); + + const refused = await runBaselineFailure( + baselineCopy, + [], + "--continue-final", + ); + + expect(refused.stderr).toContain("nothing to continue"); + expect(refused.requests).toEqual([]); + }); }, ); diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts new file mode 100644 index 00000000000..a79d22a4cb4 --- /dev/null +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts @@ -0,0 +1,766 @@ +import * as v from "valibot"; +import { describe, expect, test } from "vitest"; + +import { + assertCompleteCondition3Result, + assertCondition3ProjectionSemantics, + assertCondition3UnsupportedAnchorContinuity, + CONDITION_3_ACTIVATION_MATRIX, + CONDITION_3_COMPARISON_HASHES, + CONDITION_3_DEMAND_CLAUSES, + CONDITION_3_OPERATOR_ENVELOPE, + CONDITION_3_OBJECTIVE_MATCH_PREDICATES, + CONDITION_3_RESULT_COMPONENT_IDS, + CONDITION_3_STOPPING_RULES, + Condition3ResultSchema, + nextCondition3NoProgressStreak, + parseCondition3Projection, + type Condition3Projection, +} from "../../../../evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument"; + +function projection( + evidence: Array<{ turn: number; quote: string }> = [], +): Condition3Projection { + return { + activeObjectiveRows: [], + activeObjectiveRowEvidence: [], + retractedObjectiveAnchors: [], + unsupportedActiveObjectiveAnchors: [], + assessments: CONDITION_3_DEMAND_CLAUSES.map((clause) => + clause.row === null + ? { + clauseId: clause.id, + demand: clause.demand, + coordinate: clause.coordinate, + demanded: true as const, + currentStatus: "none" as const, + currentGrade: "none" as const, + pass: false as const, + failureDiagnostic: clause.demand.startsWith("presence count >=") + ? ("below-minimum-count" as const) + : ("unaddressed" as const), + activationPredicates: [], + evidence: clause.id === "SF-OBJ" ? evidence : [], + observedCount: clause.demand.startsWith("presence count >=") + ? 0 + : null, + rationale: "fixture", + } + : { + clauseId: clause.id, + demand: clause.demand, + coordinate: clause.coordinate, + demanded: false as const, + currentStatus: "not-applicable" as const, + currentGrade: "not-applicable" as const, + pass: true as const, + failureDiagnostic: null, + activationPredicates: [] as [], + evidence: [], + observedCount: null, + rationale: "inactive fixture row", + }, + ), + notes: [], + }; +} + +describe("condition 3 material-frame no-progress rule", () => { + test("counts onset and reaches the frozen advisory and hard-stop thresholds", () => { + const first = projection(); + const second = projection(); + const third = projection(); + const fourth = projection(); + const fifth = projection(); + const streak1 = nextCondition3NoProgressStreak([], first, 1, 0); + const streak2 = nextCondition3NoProgressStreak([first], second, 2, streak1); + const streak3 = nextCondition3NoProgressStreak( + [first, second], + third, + 3, + streak2, + ); + const streak4 = nextCondition3NoProgressStreak( + [first, second, third], + fourth, + 4, + streak3, + ); + const streak5 = nextCondition3NoProgressStreak( + [first, second, third, fourth], + fifth, + 5, + streak4, + ); + + expect(streak1).toBe(1); + expect(streak3).toBe(CONDITION_3_STOPPING_RULES.noProgressAdvisoryAfter); + expect(streak5).toBe(CONDITION_3_STOPPING_RULES.noProgressHardStopAfter); + }); + + test("does not reset for regrading, active-row drift, duplicates, or reordered evidence", () => { + const prior = projection([ + { turn: 1, quote: "first" }, + { turn: 1, quote: "second" }, + ]); + const current = projection([ + { turn: 1, quote: "second" }, + { turn: 1, quote: "first" }, + { turn: 1, quote: "first" }, + ]); + current.activeObjectiveRows = ["ROW-SPLIT"]; + const objective = current.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective?.demanded) + throw new Error("fixture lost demanded objective"); + objective.currentStatus = "explicit"; + objective.currentGrade = "structured"; + + expect(nextCondition3NoProgressStreak([prior], current, 2, 2)).toBe(3); + expect( + nextCondition3NoProgressStreak( + [{ ...prior, assessments: [...prior.assessments].reverse() }], + { ...current, assessments: [...current.assessments].reverse() }, + 2, + 2, + ), + ).toBe(3); + }); + + test("resets for new or replacement demanded evidence at equal array length", () => { + const prior = projection([{ turn: 1, quote: "old evidence" }]); + const replacement = projection([{ turn: 2, quote: "new evidence" }]); + + expect(nextCondition3NoProgressStreak([prior], replacement, 2, 4)).toBe(0); + }); + + test("does not reset for evidence-array growth made only of old-frame quotes", () => { + const prior = projection([{ turn: 1, quote: "old evidence" }]); + const duplicateGrowth = projection([ + { turn: 1, quote: "old evidence" }, + { turn: 1, quote: "old evidence" }, + ]); + + expect(nextCondition3NoProgressStreak([prior], duplicateGrowth, 2, 1)).toBe( + 2, + ); + }); + + test("resets for new demanded unsupported-anchor evidence", () => { + const prior = projection(); + prior.unsupportedActiveObjectiveAnchors = [ + { + label: "energy-use", + state: "active", + demanded: true, + pass: false, + failureDiagnostic: "unsupported-active-anchor", + evidence: [{ turn: 1, quote: "Minimize energy use." }], + resolutionEvidence: [], + resolutionRationale: null, + rationale: "No frozen row.", + }, + ]; + const current = structuredClone(prior); + current.unsupportedActiveObjectiveAnchors[0]?.evidence.push({ + turn: 2, + quote: "Peak energy matters most.", + }); + + expect(nextCondition3NoProgressStreak([prior], current, 2, 4)).toBe(0); + }); + + test("does not reset when an older quote disappears and later resurfaces", () => { + const first = projection([{ turn: 1, quote: "old evidence" }]); + const middle = projection(); + const resurfaced = projection([{ turn: 3, quote: "old evidence" }]); + + expect( + nextCondition3NoProgressStreak([first, middle], resurfaced, 3, 2), + ).toBe(3); + }); + + test("does not reset when evidence first appeared on an inactive row", () => { + const first = projection(); + const inactiveBreakdown = first.assessments.find( + ({ clauseId }) => clauseId === "BR-CAP", + ); + if (!inactiveBreakdown) throw new Error("fixture lost BR-CAP"); + inactiveBreakdown.evidence = [{ turn: 1, quote: "Both lines can coat." }]; + const current = projection([{ turn: 3, quote: "Both lines can coat." }]); + + expect(nextCondition3NoProgressStreak([first], current, 3, 2)).toBe(3); + }); + + test("does not reset when retraction evidence later resurfaces", () => { + const first = projection(); + first.unsupportedActiveObjectiveAnchors = [ + { + label: "energy-use", + state: "retracted", + demanded: false, + pass: true, + failureDiagnostic: null, + evidence: [{ turn: 1, quote: "Minimize energy use." }], + resolutionEvidence: [ + { turn: 2, quote: "Energy use is not an objective." }, + ], + resolutionRationale: "The expert explicitly retracts it.", + rationale: "No frozen row.", + }, + ]; + const current = projection([ + { turn: 3, quote: "Energy use is not an objective." }, + ]); + + expect(nextCondition3NoProgressStreak([first], current, 3, 2)).toBe(3); + }); +}); + +describe("condition 3 frozen envelopes", () => { + test("keeps the constructed operator template internally valid", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("requires exact transcript-supported evidence for every active objective row", () => { + const parsed = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + activeObjectiveRows: ["ROW-SPLIT"], + activeObjectiveRowEvidence: [], + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).toThrow( + "must equal the unique row projection", + ); + }); + + test("freezes each objective row to its exact FE-1402 matching predicate", () => { + expect(CONDITION_3_OBJECTIVE_MATCH_PREDICATES).toEqual([ + { row: "ROW-BREAKDOWN", matchingPredicate: "breakdown-reshuffle" }, + { row: "ROW-IDLE-WASH", matchingPredicate: "idle-vs-washdown" }, + { row: "ROW-CHANGEOVER", matchingPredicate: "changeover-accounting" }, + { row: "ROW-SPLIT", matchingPredicate: "split-run" }, + ]); + expect(() => + parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + activeObjectiveRows: ["ROW-SPLIT"], + activeObjectiveRowEvidence: [ + { + row: "ROW-SPLIT", + anchorLabel: "split-orders", + matchingPredicate: "changeover-accounting", + evidence: [{ turn: 1, quote: "Split this order." }], + rationale: "wrong row/predicate pair", + }, + ], + }), + ).toThrow(); + }); + + test("preserves multiple objective anchors that project to one active row", () => { + const parsed = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + activeObjectiveRows: ["ROW-SPLIT"], + activeObjectiveRowEvidence: [ + { + row: "ROW-SPLIT", + anchorLabel: "split-for-dates", + matchingPredicate: "split-run", + evidence: [{ turn: 1, quote: "Split orders to hit dates." }], + rationale: "Explicit split objective.", + }, + { + row: "ROW-SPLIT", + anchorLabel: "split-for-capacity", + matchingPredicate: "split-run", + evidence: [{ turn: 1, quote: "Split orders to use spare capacity." }], + rationale: "A second explicit split objective.", + }, + ], + }); + const objective = parsed.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective) throw new Error("fixture lost SF-OBJ"); + Object.assign(objective, { + currentStatus: "explicit", + currentGrade: "none", + pass: true, + failureDiagnostic: null, + observedCount: 2, + evidence: [ + { turn: 1, quote: "Split orders to hit dates." }, + { turn: 1, quote: "Split orders to use spare capacity." }, + ], + }); + for (const assessment of parsed.assessments) { + if (assessment.clauseId.startsWith("SP-")) { + Object.assign(assessment, { + demanded: true, + currentStatus: "none", + currentGrade: "none", + pass: false, + failureDiagnostic: "unaddressed", + activationPredicates: + assessment.clauseId === "SP-ELIG" ? [] : ["slot-unaddressed"], + }); + } + } + + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("requires matched objective anchors to persist or retract durably", () => { + const previous = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + activeObjectiveRows: ["ROW-SPLIT"], + activeObjectiveRowEvidence: [ + { + row: "ROW-SPLIT", + anchorLabel: "split-for-dates", + matchingPredicate: "split-run", + evidence: [{ turn: 1, quote: "Split orders to hit dates." }], + rationale: "Explicit split objective.", + }, + ], + }); + const omitted = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + expect(() => + assertCondition3UnsupportedAnchorContinuity(previous, omitted, 2), + ).toThrow("disappeared without a durable retraction"); + + const retracted = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + retractedObjectiveAnchors: [ + { + row: "ROW-SPLIT", + anchorLabel: "split-for-dates", + matchingPredicate: "split-run", + evidence: [{ turn: 1, quote: "Split orders to hit dates." }], + rationale: "Explicit split objective.", + resolutionEvidence: [ + { turn: 2, quote: "Splitting is no longer an objective." }, + ], + resolutionRationale: "The expert explicitly retracted it.", + }, + ], + }); + expect(() => + assertCondition3UnsupportedAnchorContinuity(previous, retracted, 2), + ).not.toThrow(); + expect(() => + assertCondition3UnsupportedAnchorContinuity(retracted, previous, 3), + ).toThrow("cannot disappear or reactivate"); + }); + + test("rejects a demanded pass whose grade does not meet the frozen demand", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const invalid = structuredClone(parsed); + const minimum = invalid.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + Object.assign(minimum, { + demanded: true, + currentStatus: "explicit", + currentGrade: "none", + pass: true, + failureDiagnostic: null, + activationPredicates: [], + evidence: [{ turn: 1, quote: "some words" }], + }); + + expect(() => assertCondition3ProjectionSemantics(invalid)).toThrow( + "does not satisfy the frozen evidence/grade demand", + ); + }); + + test("passes a count-only presence demand without manufacturing a grade", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const entities = parsed.assessments.find( + ({ clauseId }) => clauseId === "SF-ENT", + ); + if (!entities) throw new Error("fixture lost SF-ENT"); + Object.assign(entities, { + demanded: true, + currentStatus: "explicit", + currentGrade: "none", + pass: true, + failureDiagnostic: null, + activationPredicates: [], + evidence: [{ turn: 1, quote: "Orders and lines are entities." }], + observedCount: 2, + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("rejects a passing presence count below the frozen cardinality", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const entities = parsed.assessments.find( + ({ clauseId }) => clauseId === "SF-ENT", + ); + if (!entities) throw new Error("fixture lost SF-ENT"); + Object.assign(entities, { + demanded: true, + currentStatus: "explicit", + currentGrade: "none", + pass: true, + failureDiagnostic: null, + activationPredicates: [], + evidence: [{ turn: 1, quote: "One order." }], + observedCount: 1, + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).toThrow( + "presence pass disagrees with observed cardinality", + ); + }); + + test("rejects a contradictory below-grade failure before it can influence the run", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const invalid = structuredClone(parsed); + const minimum = invalid.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + Object.assign(minimum, { + demanded: true, + currentStatus: "explicit", + currentGrade: "quantiles", + pass: false, + failureDiagnostic: "below-required-grade", + activationPredicates: ["below-demanded-grade"], + evidence: [], + }); + + expect(() => assertCondition3ProjectionSemantics(invalid)).toThrow( + "requires evidence and a genuinely sub-demand grade", + ); + }); + + test("rejects the presence-only below-minimum diagnostic on a slot", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const minimum = parsed.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + Object.assign(minimum, { + demanded: true, + currentStatus: "none", + currentGrade: "none", + pass: false, + failureDiagnostic: "below-minimum-count", + activationPredicates: [], + evidence: [], + observedCount: null, + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).toThrow( + "slot assessment forbids below-minimum-count", + ); + }); + + test("represents unsupported active objective anchors outside frozen rows", () => { + const parsed = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + unsupportedActiveObjectiveAnchors: [ + { + label: "energy-use", + state: "active", + demanded: true, + pass: false, + failureDiagnostic: "unsupported-active-anchor", + evidence: [{ turn: 2, quote: "Minimize energy use." }], + resolutionEvidence: [], + resolutionRationale: null, + rationale: "No frozen objective row represents this objective.", + }, + ], + }); + const objective = parsed.assessments.find( + ({ clauseId }) => clauseId === "SF-OBJ", + ); + if (!objective) throw new Error("fixture lost SF-OBJ"); + Object.assign(objective, { + currentStatus: "explicit", + currentGrade: "none", + pass: true, + failureDiagnostic: null, + observedCount: 1, + evidence: [{ turn: 2, quote: "Minimize energy use." }], + }); + + expect(parsed.unsupportedActiveObjectiveAnchors[0]?.label).toBe( + "energy-use", + ); + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("accepts structured evidence for a vocabulary-bound minimum", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const taxonomy = parsed.assessments.find( + ({ clauseId }) => clauseId === "CH-TAX", + ); + if (!taxonomy) throw new Error("fixture lost CH-TAX"); + Object.assign(taxonomy, { + demanded: true, + currentStatus: "explicit", + currentGrade: "structured", + pass: true, + failureDiagnostic: null, + activationPredicates: [], + evidence: [ + { turn: 1, quote: "Dark-to-light is a separate changeover class." }, + ], + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("requires unsupported anchors to persist or retract with current-turn evidence", () => { + const previous = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + unsupportedActiveObjectiveAnchors: [ + { + label: "energy-use", + state: "active", + demanded: true, + pass: false, + failureDiagnostic: "unsupported-active-anchor", + evidence: [{ turn: 1, quote: "Minimize energy use." }], + resolutionEvidence: [], + resolutionRationale: null, + rationale: "No frozen row.", + }, + ], + }); + const omitted = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + expect(() => + assertCondition3UnsupportedAnchorContinuity(previous, omitted, 2), + ).toThrow("disappeared without a durable retraction"); + + const rewritten = structuredClone(previous); + rewritten.unsupportedActiveObjectiveAnchors[0]!.evidence = [ + { turn: 2, quote: "A replacement quote." }, + ]; + expect(() => + assertCondition3UnsupportedAnchorContinuity(previous, rewritten, 2), + ).toThrow("rewrote its original evidence"); + + const retracted = parseCondition3Projection({ + ...CONDITION_3_OPERATOR_ENVELOPE.template, + unsupportedActiveObjectiveAnchors: [ + { + label: "energy-use", + state: "retracted", + demanded: false, + pass: true, + failureDiagnostic: null, + evidence: [{ turn: 1, quote: "Minimize energy use." }], + resolutionEvidence: [ + { turn: 2, quote: "Energy use is not an objective." }, + ], + resolutionRationale: "The expert explicitly retracted the objective.", + rationale: "No frozen row.", + }, + ], + }); + expect(() => + assertCondition3UnsupportedAnchorContinuity(previous, retracted, 2), + ).not.toThrow(); + + const rewrittenResolution = structuredClone(retracted); + const retractedAnchor = + rewrittenResolution.unsupportedActiveObjectiveAnchors[0]; + if (retractedAnchor?.state !== "retracted") { + throw new Error("fixture lost retracted anchor"); + } + retractedAnchor.resolutionEvidence = [ + { turn: 3, quote: "Replacement resolution." }, + ]; + expect(() => + assertCondition3UnsupportedAnchorContinuity( + retracted, + rewrittenResolution, + 3, + ), + ).toThrow("rewrote its resolution evidence"); + }); + + test("rejects a globally valid activation predicate incompatible with its clause", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const invalidInput = structuredClone(parsed); + const minimum = invalidInput.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + Object.assign(minimum, { + demanded: true, + currentStatus: "none", + currentGrade: "none", + pass: false, + failureDiagnostic: "unaccepted-absence", + activationPredicates: ["absence-uncorroborated"], + evidence: [{ turn: 1, quote: "I do not know." }], + observedCount: null, + }); + const invalid = parseCondition3Projection(invalidInput); + + expect(() => assertCondition3ProjectionSemantics(invalid)).toThrow( + "activation predicate is incompatible", + ); + }); + + test("rejects omission of a compatible required activation", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const invalid = structuredClone(parsed); + const minimum = invalid.assessments.find( + ({ clauseId }) => clauseId === "SP-MIN", + ); + if (!minimum) throw new Error("fixture lost SP-MIN"); + Object.assign(minimum, { + demanded: true, + currentStatus: "explicit", + currentGrade: "verbal", + pass: false, + failureDiagnostic: "below-required-grade", + activationPredicates: [], + evidence: [{ turn: 1, quote: "The minimum is roughly 800." }], + }); + + expect(() => assertCondition3ProjectionSemantics(invalid)).toThrow( + "required activation predicate below-demanded-grade is missing", + ); + }); + + test("accepts the required unspecified-marker activation for inadmissible status", () => { + const parsed = parseCondition3Projection( + CONDITION_3_OPERATOR_ENVELOPE.template, + ); + const releaseGate = parsed.assessments.find( + ({ clauseId }) => clauseId === "IW-REL", + ); + if (!releaseGate) throw new Error("fixture lost IW-REL"); + Object.assign(releaseGate, { + demanded: true, + currentStatus: "tentative", + currentGrade: "structured", + pass: false, + failureDiagnostic: "inadmissible-status", + activationPredicates: ["unspecified-marker-present"], + evidence: [{ turn: 1, quote: "I think release is probably verbal." }], + }); + + expect(() => assertCondition3ProjectionSemantics(parsed)).not.toThrow(); + }); + + test("includes the reviewed SP-SCRAP target in CPS-Q03 activation", () => { + expect( + CONDITION_3_ACTIVATION_MATRIX.find(({ cardId }) => cardId === "CPS-Q03") + ?.clauses, + ).toEqual(["SP-BATCH", "SP-MIN", "SP-POL", "SP-CO", "SP-SCRAP"]); + }); + + test("requires every machine-readable result component exactly once", () => { + const result = v.parse(Condition3ResultSchema, { + schemaVersion: "fe-1404-condition-3-result/2026-08-25.1", + runRawSha256: "0".repeat(64), + components: CONDITION_3_RESULT_COMPONENT_IDS.map((id) => ({ + id, + verdict: "unobservable", + observation: id.startsWith("signature.") ? "unobservable" : null, + evidence: [], + rationale: "pre-observation fixture", + })), + comparisons: { + condition1: { + ...CONDITION_3_COMPARISON_HASHES.condition1, + comparison: "pending", + }, + condition2: { + ...CONDITION_3_COMPARISON_HASHES.condition2, + comparison: "pending", + }, + }, + amendments: [], + limitations: [], + }); + + expect(() => assertCompleteCondition3Result(result)).not.toThrow(); + const contradictoryGenResult = { + ...result, + components: result.components.map((component) => + component.id === "guidance.GEN-Q02.layer-2" + ? { ...component, verdict: "pass" as const } + : component, + ), + }; + expect(() => + assertCompleteCondition3Result(contradictoryGenResult), + ).toThrow("GEN-Q02 layer-2 verdict is frozen as unobservable"); + expect(() => + assertCompleteCondition3Result({ + ...result, + components: result.components.slice(1), + }), + ).toThrow("every frozen component exactly once"); + expect(() => + v.parse(Condition3ResultSchema, { + ...result, + components: result.components.map((component) => + component.id === "layer.diagnostic" + ? { ...component, verdict: "pass", evidence: [] } + : component, + ), + }), + ).toThrow("scored condition-3 result components require evidence"); + expect(() => + v.parse(Condition3ResultSchema, { + ...result, + comparisons: { + ...result.comparisons, + condition1: { ...result.comparisons.condition1, comparison: "" }, + }, + }), + ).toThrow(); + expect(() => + v.parse(Condition3ResultSchema, { + ...result, + comparisons: { + ...result.comparisons, + condition1: { + ...result.comparisons.condition1, + rawSha256: "0".repeat(64), + }, + }, + }), + ).toThrow(); + }); +}); From b94466a79f9e463fbdea4f8ab7fa2f8742a20a0e Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 13:07:26 +0200 Subject: [PATCH 2/7] Author the SDCPN plugin file at formalism level Adds docs/specs/sdcpn-plugin.md: the one authored artifact for the SDCPN target, written under fixed contract headings (Purpose, Kinds, Must know, Patterns, Moves, Deliverable) over the IR spec's ten Layer-B kinds, with three machine-read tables and a domain-neutrality rule. Outcome: the plugin is per target formalism, not per domain. The domain-keyed DemandTable, cards, and activation matrix of FE-1402/1403/1404 are legible from this file as a departure from Layer B: their twenty CPS rows collapse onto kind-level rows instantiated on discovered nodes, and their five "domain" cards lift to kind-indexed patterns P01-P05. Mechanism: Layer B supplies kinds, cross-kind quantity/source-regime/ rationale attributes, and question-relative completion over a static floor; the condition-2 v0 prompt supplies the interviewing moves. Places and queues are not elicited (granularity rule); durations and rates live on activity as quantity, and dynamics is continuous evolution only. The file lives under docs/specs/ until the walking-skeleton branch creates packages/plugin-sdcpn/ with a manifest and parser; a manifest-less package directory fails the topology gate. Registered in docs/INDEX.md. --- libs/@hashintel/brunch-agent/docs/INDEX.md | 1 + .../brunch-agent/docs/specs/sdcpn-plugin.md | 217 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md diff --git a/libs/@hashintel/brunch-agent/docs/INDEX.md b/libs/@hashintel/brunch-agent/docs/INDEX.md index 79be0817d6b..38c55ad8361 100644 --- a/libs/@hashintel/brunch-agent/docs/INDEX.md +++ b/libs/@hashintel/brunch-agent/docs/INDEX.md @@ -76,6 +76,7 @@ control loop is [`docs/agents/steering.md`](agents/steering.md). | [cps-interview-guidance](specs/cps-interview-guidance.md) | active | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Provisional CPS ElicitationPack handoff: six mechanism-typed cards plus status/grade and respectful-close fragments, with clause-addressed activation, explicit machinery/guidance ownership, and the singular-`firesWhen` authoring seam carried to FE-1431 | | [cps-interview-guidance-desk-replay](evidence/proofs/design/cps-interview-guidance-desk-replay.md) | active | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Manual two-transcript prefix replay: per-card firings, expected evidence deltas, positive deactivation boundaries, candidate dispositions, and repository research ledger; desk discrimination only | | [cps-interview-guidance-plain](evidence/proofs/design/cps-interview-guidance-plain.md) | active | FE-1403 legibility snapshot | Reviewer-facing plain rendering of the CPS guidance contract with translation strains and their dispositions | +| [sdcpn-plugin](specs/sdcpn-plugin.md) | active | FE-1404 (redefined toward the walking skeleton); supersedes the domain-keyed tables of FE-1402/1403 | The SDCPN plugin file: fixed contract headings (Purpose, Kinds, Must know, Patterns, Moves, Deliverable) over the IR spec's ten Layer-B kinds; three machine-read tables, domain-neutral by rule; moves to `packages/plugin-sdcpn/` with the skeleton | ## Control, architecture reference, and migration archive diff --git a/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md b/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md new file mode 100644 index 00000000000..0ecc15b7aa2 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md @@ -0,0 +1,217 @@ +# SDCPN process model — the plugin file + +Plugin: `sdcpn` (the IR spec's "CPS plugin", Layer B) · Target formalism: stochastic dynamic +coloured Petri nets (Petrinaut) · Version: `sdcpn/2026-08-25.1` + +> **What this file is.** This is the plugin — the one authored artifact for this target. The +> headings are the contract and are fixed across all plugins; the content under them belongs to +> this formalism. The harness parses three tables (`## Kinds`, `## Must know`, `## Patterns`) into +> the model vocabulary, the demand list, and the pattern index; every other section is +> concatenated into the interviewer's instructions. The end user never edits this file — they +> have a conversation. +> +> It merges two existing sources into one artifact: the kind vocabulary and completion rule of +> [the IR spec's Layer B](intermediate-representation.md#layer-b--the-cps-plugins-ir), and the +> interviewing guidance of the condition-2 v0 prompt. Nothing here is new design; the domain-keyed +> demand tables and cards of FE-1402/1403/1404 are the departure this file walks back. +> +> **Domain-neutrality rule.** Nothing below may name a domain. What the user wants to model is +> unknown until the conversation starts; the same file must serve any operational system +> unchanged. A new case that seems to need a new row is a finding about the abstraction to be +> decided, never content to be added here. +> +> It lives under `docs/specs/` until the walking-skeleton branch creates `packages/plugin-sdcpn/` +> with a manifest and the parser; the file then moves there unchanged. + +## Purpose + +Interview someone who knows an operational system deeply — but is not a modeller — and derive a +process model that a simulation can run. The model must answer the questions the user actually +has, to the depth those questions need, in the expert's own vocabulary, with every value traceable +to something the expert said. Where the expert's knowledge stops, the model says so instead of +guessing. + +The interviewer does not build the net. It elicits the model at the expert's granularity; the +plugin's projection derives the SDCPN scaffold, the code-obligation sidecar, and the loss report +from the model afterwards. Steps become transitions and the states between them become places +*in projection*, never in the conversation. + +## Kinds + +The model is a graph of nodes. Every node has exactly one kind. Kinds are the vocabulary of any +discrete-event process, not of any domain. Kinds 1–6 are net-bearing; 7–10 are partly or wholly +IR-only — the net is one projection of the model, and what the net cannot hold is kept with +provenance and named in the loss report. + +| # | kind | what it is | projects to | +| --- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | +| 1 | `entity-type` | A kind of thing that flows through, is operated on, or does the work — and the distinctions the process treats differently, including state that rides along. | colours, typed elements | +| 2 | `boundary-condition` | What the system starts with and what reaches it from outside: initial populations, arrivals and departures, calendars, external inputs and their reliability. | scenario initial state and parameters, source transitions | +| 3 | `activity` | Something that happens, as the expert states it: a work step, a setup, a repair, an inspection, a hand-off, an interruption — with its actors, preconditions, outcomes, and duration. | factored transitions and the places between them | +| 4 | `ordering/flow` | How activities relate: sequence, branching, merging, triggers. | arcs, arc types, guards | +| 5 | `policy` | The rule applied when more than one thing could happen: who wins a contended resource, what goes next, when to switch, when to release. | guards and priorities where compilable; otherwise IR-only | +| 6 | `dynamics` | A quantity that evolves continuously while nothing discrete happens: wear, temperature, level, charge. | differential equations on real-valued colour elements | +| 7 | `objective` | A question the model must answer or a decision it must inform; what "better" means; trade-off weights. | metrics where scalar over simulation state; weights IR-only | +| 8 | `constraint` | A limit that must hold: capacity, eligibility, compatibility, qualification, a regulatory or quality rule — written or unwritten; conservation laws. | guards and capacities partially; otherwise IR-only | +| 9 | `data-binding` | A model variable that a real data feed could drive. | nothing today | +| 10 | `validation-criterion` | How the expert would know the model is right. | nothing today | + +Three things that look like kinds are not: + +- A **resource** (a machine, a team, a vehicle, a bay) is an `entity-type` whose instances are + contended for. Its contention rule is a `policy`; its capacity is a `constraint`; its + availability is a `boundary-condition`. +- A **queue, buffer, or waiting state** is not elicited as a node. It is implied by the activities + on either side of it and emerges as a place in projection. +- A **scenario** is not elicited; it is assembled at simulation time from `boundary-condition` + nodes. + +### Attributes on every kind + +- **quantity** — any duration, rate, probability, count, or capacity, on any kind. Elicited by + quantiles: "typical?", "one time in ten, worse than?", "one time in ten, better than?" — never + minimum / most-likely / maximum, which yields overconfident triangles. +- **source-regime** — `prescribed | practiced`, on any kind. One model, not two: when the manual + and the floor disagree, both are recorded on the same node and the divergence is an ordinary + typed conflict for the expert to resolve — elicitation gold, not an error. +- **rationale** — why the expert says it is so, on any kind, never only on objectives. + +## Must know + +For every node the conversation discovers, its kind decides what must be known about it and how +precisely. These rows never change when the domain changes: a repair on one kind of machine and a +repair on another are the same rows instantiated on different nodes. + +A slot is satisfied only when (a) it has reached the demanded precision and (b) the value comes +from the expert — stated outright, or inferred by the interviewer and confirmed by the expert. +Anything the interviewer supplied without confirmation belongs in the assumption ledger, not the +model. "Not mentioned" never satisfies a slot. "I don't know" and "we'll measure it later" are not +values. An explicit "not applicable" or "never happens" *is* a value where the row allows it. + +| kind | slot | precision | "not applicable" allowed | why the model needs it | +| ---------------------- | ------------------------------------------------------- | ----------- | ------------------------ | ------------------------------------------------------------------------------- | +| `objective` | the question, in the expert's words | spelled out | no | everything else is elicited relative to it | +| `objective` | the nodes it depends on | at least 1 | no | an objective that depends on nothing is unsupported by the model | +| `objective` | what "better" means, and trade-off weights | range | yes | quantified objectives need a metric; some are qualitative | +| `entity-type` | the distinctions the process treats apart | spelled out | no | two things are one type only if the process treats them the same everywhere | +| `entity-type` | state that rides along with each instance | spelled out | yes | colour elements; many types carry none | +| `entity-type` | how many there are, or the population's shape | range | yes | initial populations for contended resources; unbounded is an allowed answer | +| `boundary-condition` | the starting state | spelled out | no | scenario initial state | +| `boundary-condition` | the arrival or availability pattern | spread | no | source rates and calendars; a single average hides the shape | +| `activity` | what it needs before it can start | spelled out | no | transition preconditions | +| `activity` | what it produces or changes | spelled out | no | transition outcomes | +| `activity` | who or what performs it | named | yes | resource binding; some activities are unattended | +| `activity` | how long it takes | spread | no | duration distribution; a point value simulates as a falsehood | +| `activity` | how often it occurs, if it is an event rather than a step | range | yes | interruptions, failures, and arrivals have a rate; steps in the flow do not | +| `activity` | what is lost when it changes the system's mode | range | yes | setup, changeover, restart, and warm-up losses are routinely never asked | +| `activity` | whether its quantities vary by type | named | no | the answer is load-bearing either way | +| `ordering/flow` | the order things happen in | spelled out | no | the net's structure | +| `ordering/flow` | how a branch or merge is decided | spelled out | yes | routing; only where the flow branches | +| `policy` | the rule as actually practiced | spelled out | no | guards and priorities; the tacit rule, not the poster on the wall | +| `policy` | what overrides it | spelled out | yes | exceptions are where the simulation and reality diverge | +| `dynamics` | what changes, in which direction, at what rate | range | no | the differential law; a direction with no rate cannot be simulated | +| `dynamics` | what happens at a threshold | spelled out | yes | most continuous quantities exist to trigger something | +| `constraint` | the limit and what happens when it is hit | spelled out | no | a capacity without a consequence cannot be simulated | +| `data-binding` | the variable and its feed | named | yes | IR-only today; recorded so the loss report can name it | +| `validation-criterion` | how the expert would know the model is right | spelled out | yes | IR-only; anchors the acceptance conversation | + +Static floor — before objective-relative depth counts at all, the model must contain at least one +`objective`, at least two `entity-type` nodes, at least one `activity`, and at least one +`ordering/flow` whose order is spelled out. Presence is a count; the floor assigns no precision. + +Completion is question-relative: the model is complete when the floor holds and every node in the +dependency slice of every active `objective` satisfies its kind's rows. Nodes outside every slice +are recorded but not demanded. Completion is a boolean plus the list of what fails and why; it is +computed from the model, never from the conversation. + +### Precision words + +| word | means | IR grade | +| ------------- | ---------------------------------------------------------------------------------------------------- | ----------- | +| `named` | identified in words | verbal | +| `number` | a single figure with its unit | point | +| `range` | an ordinary low and high | range | +| `spread` | range plus "typical", plus one-in-ten worse and one-in-ten better (or median and quartiles) | quantiles | +| `spelled out` | the rule, pattern, list, or structure itself, in a form a second reader could apply without asking | structured | +| `at least N` | a count of nodes present | presence | + +Precision says how much a value narrows what it could mean. It says nothing about where the value +came from: "about three hours" from the expert is an honest `number` at the wrong precision; "three +hours" invented by the interviewer is at the right precision and is not evidence at all. The two +are tracked separately and neither substitutes for the other. + +## Patterns + +Patterns are discretionary. Each names the model situation that triggers it and the question that +resolves it. None names a domain; each applies wherever its trigger appears. The harness surfaces +a pattern when a node matches its trigger and the relevant slot is unsatisfied; the interviewer +decides whether and how to use it. + +| id | when | ask | +| --- | --------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| P01 | an `activity` is an event that can befall the system — a failure, an interruption, an unplanned arrival — rather than a step in the flow | occurrence and duration are two slots. Ask how often, as a range, for each named event separately; then how long, as a spread. Keep the precision the expert actually gave; never round a range up to a spread. | +| P02 | an `activity` changes the system's mode — a setup, changeover, restart, warm-up, reconfiguration, handover | ask what is lost in the transition, as a range, after a *named* transition. If the expert does not know, ask what they would treat as an authoritative source — never convert "unknown" into a value. | +| P03 | an `ordering/flow` moves things in groups — batches, runs, lots, loads | ask what the group is, the smallest sensible one, whether a group must stay together, and what an extra split costs (extra mode changes, extra loss) on the activities it touches. | +| P04 | a `policy` or `boundary-condition` gates when something may proceed — a release, a start, an admission | replace any time-shaped approximation ("about two days before") with the practiced event or state that makes it runnable, who or what flips it, and where that is observable. | +| P05 | more than one thing can want the same `entity-type` instance at once | ask which wins, what overrides that, how ties break, and for a recent borderline case that shows the practiced rule. Never infer the rule from a schedule or a document. | +| P06 | the expert answers with a vague quantifier — "usually", "roughly", "mostly", "sometimes" | each hides a distribution or an exception. Ask for the last time it happened, then for the spread. | +| P07 | a quantity has been given for one `entity-type` and others exist | ask explicitly whether it varies by type. Record "no" as a value; it is load-bearing. | +| P08 | any node has both a prescribed and a practiced form | record both on the same node under `source-regime`, with the expert's account of when they diverge. Do not average them and do not pick one. | +| P09 | the sweep of `constraint` nodes is otherwise complete | ask for the unwritten ones: "what would a newcomer get wrong in the first week?", "what do you always or never do that is written nowhere?", "which rule exists because something once went wrong?" | +| P10 | a slot is unsatisfied and the expert has said they do not know | "don't know" is not a value and not an absence. Ask what the least burdensome authoritative source would be, record the slot as open with that pointer, and move on. | +| P11 | a topic's sweep is ending | ask for absences explicitly: "is there anything here that never happens?" An explicit "never" is a value. Do not ask the expert what you have failed to ask; finding that is the harness's job. | +| P12 | the harness reports a slot as below precision or an objective as unsupported | name the node, what is known so far, what precision is needed, and ask for the smallest delta that would satisfy it. Do not restate the whole model. | +| P13 | a `dynamics` node has been named | ask what it triggers when it crosses a threshold, and which `activity` resets it. A continuous quantity that triggers nothing usually does not need to be in the model. | + +## Moves + +Moves are the mandates: the shape every conversation follows regardless of domain. The harness +enforces what it can (completion, the sweep list, the ledger); the interviewer is responsible for +the rest. + +1. **Open with objectives.** Before anything about structure, establish what the user wants the + model to answer or decide. Capture each as an `objective` node. Expect to co-construct: these + are almost never written down. Ask what "better" means and whether it can be quantified. + Everything afterwards is elicited relative to these nodes. + +2. **Slice.** Walk one concrete case end to end ("take one instance from arriving to leaving") to + expose the structure. Create nodes as they appear. As each `objective` becomes clearer, link it + to the nodes it depends on. An objective that depends on nothing yet is unsupported — say so + and go find its structure. This is where the model's shape comes from; do not sweep before it. + +3. **Sweep.** For every node the slice revealed, in kind order, check each of its `Must know` + slots and every pattern whose trigger it matches. This is the move that finds what was never + asked: completion can only judge what is in the model, and the sweep is what puts things in it. + Group two to four related questions per turn while sweeping; probe one thread at a time when + something needs depth. + +4. **Probe.** Do not settle for the first answer. Follow vague terms (P06). Ask for stories rather + than generalisations. When two answers tension against each other, say so and ask which holds. + Ask for the smallest delta that would move a slot to its demanded precision (P12), not for + everything at once. + +5. **Keep the ledger.** Every value or rule the interviewer supplied and the expert did not + confirm — defaults, simplifications, placeholders — goes in a numbered assumption ledger with + why it was assumed and how to check it. It never enters the model silently. + +6. **Close honestly.** Completion is computed by the harness from the model, not felt from the + conversation. A smooth interview, a busy expert, a delivered document, an exhausted budget, and + a complete model are five different things; never let one stand in for another. If the expert + has to stop, stop: open no new topic, state what the model can now support and what is still + missing, and let them choose. Do not keep interviewing once every active objective's slice + meets its demands. Before delivering, summarise per kind, state what is missing or assumed, + and give the expert one chance to correct you. + +## Deliverable + +When the interview ends — complete or not — produce: + +1. the model, every node in the expert's own vocabulary, with each slot's value and precision as + actually obtained and its source-regime where both were given; +2. the assumption ledger; +3. a loss section: what the model deliberately leaves out, which slots are open and why, which + objectives are unsupported, and which kinds the net cannot carry. + +The SDCPN scaffold, the code-obligation sidecar, and the typed loss report are derived from the +model by the plugin's projection; the interviewer does not write them and must not claim the model +is loadable, compiled, or simulated. From 4cdbe5df01392d3f256d890d66960e606926ff31 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 13:45:45 +0200 Subject: [PATCH 3/7] Pivot Brunch to per-formalism plugins and the vertical slice Ratify ADR-0006: a plugin is one sectioned Markdown file per target formalism with fixed headings (Purpose, Kinds, Must know, Patterns, Moves, Deliverable) and three machine-read tables, domain-neutral by rule. The SDCPN plugin file gains its second job runbook, review and revise, with harness-owned checks and stopping outcomes. Strategy entry S-007 supersedes S-005 and S-006: the design-convergence queue is closed, its outputs are test-bed material, and the frontier is the walking skeleton worked outward from four epicentres. STEERING.md is rewritten around them (E1 controller read path, E2 plugin in code, E3 targeted correction, E4 real entry) with the coatings-plant baseline named correctly. The steering protocol gains a proxy-completion trigger. Consolidation: plugin-contract.md and elicitation-completion.md are reduced to what survives (heading contract and version binding; nineteen evaluateCompletion invariants); the declarative drafts and the CPS card set move to docs/archive/specs/ with supersession banners; the kernel spec gets a supersession map; SPEC-LEDGER, INDEX, CONTEXT.md glossary, and evidence links follow. The strategy-log gate now accepts an entry that supersedes several earlier entries, and the inherited condition-3 test assertions name the errors they expect. Co-Authored-By: Claude Fable 5 --- libs/@hashintel/brunch-agent/CONTEXT.md | 38 +- libs/@hashintel/brunch-agent/docs/INDEX.md | 25 +- .../adr/0006-plugins-per-target-formalism.md | 110 ++++ .../brunch-agent/docs/agents/steering.md | 9 +- .../cps-interview-guidance-2026-08-25.md} | 25 +- ...tation-completion-2026-08-25-full-draft.md | 347 ++++++++++ ...n-contract-2026-08-25-declarative-draft.md | 460 ++++++++++++++ .../brunch-agent/docs/control/SPEC-LEDGER.md | 10 +- .../brunch-agent/docs/control/STEERING.md | 150 +++-- .../brunch-agent/docs/control/STRATEGY-LOG.md | 62 ++ .../cps-interview-guidance-desk-replay.md | 2 +- .../design/cps-interview-guidance-plain.md | 2 +- .../docs/specs/elicitation-completion.md | 441 ++++--------- .../docs/specs/elicitation-kernel.md | 14 + .../intermediate-representation-plain.md | 4 + .../docs/specs/plugin-contract.md | 591 +++++------------- .../brunch-agent/docs/specs/sdcpn-plugin.md | 62 +- .../baseline/condition-3-instrument.ts | 3 +- .../condition-3-preregistration.lock.json | 2 +- .../test/architecture/baseline-runner.test.ts | 9 +- .../condition-3-instrument.test.ts | 6 +- .../architecture/control-surfaces.test.ts | 24 +- 22 files changed, 1500 insertions(+), 896 deletions(-) create mode 100644 libs/@hashintel/brunch-agent/docs/adr/0006-plugins-per-target-formalism.md rename libs/@hashintel/brunch-agent/docs/{specs/cps-interview-guidance.md => archive/specs/cps-interview-guidance-2026-08-25.md} (90%) create mode 100644 libs/@hashintel/brunch-agent/docs/archive/specs/elicitation-completion-2026-08-25-full-draft.md create mode 100644 libs/@hashintel/brunch-agent/docs/archive/specs/plugin-contract-2026-08-25-declarative-draft.md diff --git a/libs/@hashintel/brunch-agent/CONTEXT.md b/libs/@hashintel/brunch-agent/CONTEXT.md index 830f8fad92e..86667e0bf5c 100644 --- a/libs/@hashintel/brunch-agent/CONTEXT.md +++ b/libs/@hashintel/brunch-agent/CONTEXT.md @@ -16,11 +16,12 @@ _Avoid_: host, host-interface, frontend, client **Harness**: The middle shell and the essence of the effort: the generic capability layer of the elicitation system — mechanism and orchestration (the conversation loop, the `ask` API, capture envelope, issue queue, sweep bookkeeping). Injected into plugins as a narrow context; never owned by them. -_Avoid_: kernel, core, elicitor (as a shell name — "elicitor" may name the whole system). Exempt compound: **kernel card** (below). Exempt name: the package `packages/core` (spec §12.2) — the avoidance applies to "core" as a prose shell name, not to the package path. "Kernel invariants" renamed **harness invariants** (spec §14.1). +_Avoid_: kernel, core, elicitor (as a shell name — "elicitor" may name the whole system). Exempt name: the package `packages/core` (spec §12.2) — the avoidance applies to "core" as a prose shell name, not to the package path. "Kernel invariants" renamed **harness invariants** (spec §14.1). The former exempt compound "kernel card" is retired with the card (ADR-0006). + **Plugin**: -The innermost shell: target-defining policy. Declares packs, forms, and validators; composes at authoring time; receives harness capabilities by injection. Mostly policy — mechanism stays in the harness. -_Avoid_: extension, pack (a pack is a unit _within_ a plugin) +The innermost shell: target-defining policy, one per **target formalism** (`gherkin`, `sdcpn`) and never per domain — the domain is unknown when a conversation starts. Authored as one sectioned Markdown file under the fixed headings `Purpose · Kinds · Must know · Patterns · Moves · Deliverable`, plus the `project` / `validate` code operations (ADR-0006). Receives harness capabilities by injection; mechanism stays in the harness. +_Avoid_: extension, pack (a pack is a unit _within_ a plugin), domain plugin, scenario plugin **Binding**: The substrate-facing adapter between harness and substrate: implements the harness's named substrate-capability list (tool registration, instruction assembly, persistent state, affordance emission, suspend-for-reply, private model call) in one substrate's dialect. One per substrate; the harness imports no substrate, a binding imports both. Bindings vary in size — each absorbs what its substrate lacks or forbids. @@ -28,12 +29,16 @@ _Avoid_: adapter (generic), integration, wrapper ### Sessions & durability -**Target-domain**: -The artifact family being elicited — what a plugin defines (gherkin scenarios, assurance arguments, BPMN). The family half of the former bare "target". -_Avoid_: target-paradigm; bare "target" where family/instance is ambiguous +**Target formalism**: +The artifact family a plugin projects into (Gherkin scenarios, SDCPNs, assurance arguments, BPMN) — the unit a plugin is written for (ADR-0006). The family half of the former bare "target". +_Avoid_: target-domain (retired — "domain" now names the expert's operational system, below), target-paradigm; bare "target" where family/instance is ambiguous + +**Domain**: +The operational system the expert knows and the model describes — a packaging line, a truck fleet, a coating plant. Unknown before the conversation starts and discovered during it; never a plugin unit, a heading, a row, or a noun in a plugin file. +_Avoid_: target-domain, use case (as a synonym), scenario (a scenario is assembled from boundary conditions at simulation time) **Target-document**: -The durable unit sessions attach to: one target-domain, its capture store, and its session history. Named by its purpose — its authoritative state is the capture store plus session logs, never the rendered artifact (renders are derived, cacheable, disposable). Endures independently of any session; never locks — completion is a derived status, not a write gate. +The durable unit sessions attach to: one target formalism, its capture store, and its session history. Named by its purpose — its authoritative state is the capture store plus session logs, never the rendered artifact (renders are derived, cacheable, disposable). Endures independently of any session; never locks — completion is a derived status, not a write gate. _Avoid_: spec (as the unit name), workpiece, case, target-output **Session**: @@ -71,7 +76,7 @@ _Avoid_: digest (brunch's form) ### Envelope & packs **Intermediate representation (IR)**: -The elicited conceptual model a target-document accumulates — the middle of three registers (ADR-0003): typed **assertions** (active captures) are folded by a pure, plugin-declared fold into the **model** (the IR proper — node instances with slot states), which **projections** consume without rereading the transcript. Not a second store — the model is a derivation, recomputable from active captures at any time, never a persistence surface; the rendered artifact is one projection of the model, never the model itself. Defining a plugin's IR means writing its contract — model schema, proposal catalog, fold table, demand table (`docs/specs/plugin-contract.md`, provisional). An earlier definition read the capture set itself as the IR; ADR-0003 amends it. +The elicited conceptual model a target-document accumulates — the middle of three registers (ADR-0003): typed **assertions** (active captures) are folded by a pure, plugin-declared fold into the **model** (the IR proper — node instances with slot states), which **projections** consume without rereading the transcript. Not a second store — the model is a derivation, recomputable from active captures at any time, never a persistence surface; the rendered artifact is one projection of the model, never the model itself. Defining a plugin's IR means writing its `Kinds` and `Must know` tables (ADR-0006); the fold derives from the kinds' slots. An earlier definition read the capture set itself as the IR (ADR-0003 amends it); a later provisional contract declared typed model-schema / proposal-catalog / fold-table / demand-table objects (ADR-0006 retires that form). _Avoid_: knowledge store, domain model (as a stored unit), staging area **Capture envelope**: @@ -109,10 +114,23 @@ _Avoid_: advisory (a different thing, below) A computed, ephemeral, non-blocking fact the harness surfaces to the agent (unaccounted ask, unswept tail, world-moved delta). Never stored in the capture store; never gates anything. **Pack**: -A unit within a plugin: **ElicitationPack** (kernel cards, completion contract, clarification hints) or **ProjectionPack** (`project` + `validate`, optional `reconcile`, annotated shapes, typed loss reports). Packs are shapes-to-fill plus behavioral guidance, per Principle v2. +A unit within a plugin: **ElicitationPack** (the plugin file's `Purpose`, `Must know`, `Patterns`, `Moves`, and `Deliverable` sections) or **ProjectionPack** (`project` + `validate`, optional `reconcile`, annotated shapes, typed loss reports). Packs are shapes-to-fill plus behavioral guidance, per Principle v2. + +**Demand row**: +One row of a plugin's `Must know` table: a slot on a kind, its required precision, whether "not applicable" is accepted, and why the model needs it. Kind-level only — every node of that kind discovered in conversation is checked against it (ADR-0006). +_Avoid_: demand clause, scope expression, objective row + +**Pattern**: +A discretionary, kind-indexed heuristic in a plugin's `Patterns` table: the model situation that triggers it and the question that resolves it. Surfaced by the harness when a node matches the trigger and the relevant slot is unsatisfied; the interviewer decides whether to use it. Never names a domain. +_Avoid_: card, kernel card (retired — ADR-0006 replaced Detects/Goal/Questions/Artifacts cards with pattern rows), technique card + +**Runbook**: +The `Moves` for one **job** a plugin supports (`construct`, `review and revise`): kickoff, trajectory, checks, and stopping, over the plugin's one `Kinds` / `Must know` set. A plugin carries one runbook per job. +_Avoid_: mandate (as the unit name), mode, workflow **Kernel card**: -The pack-content unit of elicitation guidance: Detects / Goal / contrastive Questions / Artifacts (brunch `BEHAVIORAL_KERNELS.md` lineage — "kernel" here names a small unit of behavioral guidance, not a shell; the compound is the glossary's one sanctioned "kernel" use). Splits by ownership: domain cards are plugin pack content; a harness-shipped **generic strategy quiver** (cards over envelope vocabulary — conflict, ambiguity, weak evidence) is named in spec §11.5, not designed. +Retired (ADR-0006). The pack-content unit of elicitation guidance — Detects / Goal / contrastive Questions / Artifacts, brunch `BEHAVIORAL_KERNELS.md` lineage — is replaced by **pattern** rows in the plugin file; the harness-shipped **generic strategy quiver** named in spec §11.5 becomes harness-generic patterns lifted out of plugin files (FE-1406), still not designed. +_Avoid_: card, kernel card (in new writing) **PluginContext**: The narrow injected context through which a plugin receives harness capabilities (the ask API, envelope, issue queue, sweep bookkeeping). The plugin's entire world at runtime; the four operations remain pure (snapshot-in/deltas-out) regardless. diff --git a/libs/@hashintel/brunch-agent/docs/INDEX.md b/libs/@hashintel/brunch-agent/docs/INDEX.md index 38c55ad8361..17bc13c42f2 100644 --- a/libs/@hashintel/brunch-agent/docs/INDEX.md +++ b/libs/@hashintel/brunch-agent/docs/INDEX.md @@ -39,6 +39,8 @@ control loop is [`docs/agents/steering.md`](agents/steering.md). | [map.md](archive/elicitation-kernel/map.md) | settled | **mirrored in full**: FE-1366 | Completed wayfinder map | | [issues/](archive/elicitation-kernel/issues) 01–13 | settled | **mirrored in full**: FE-1367–FE-1379 (relations preserved) | 13 resolved tickets | | [notes/consistency-prepass](archive/elicitation-kernel/notes/consistency-prepass-2026-08-10.md) | settled | none | Pre-assembly contradiction audit (7 contradictions, adjudicated in spec Appendix A) | +| [plugin-contract-2026-08-25-declarative-draft](archive/specs/plugin-contract-2026-08-25-declarative-draft.md) | superseded | FE-1431; FE-1405 | Archive copy of the pre-ADR-0006 plugin contract: two schemas and two tables, `ScopeExpr`/`where`/`inSupport`, `firesWhen`, `completionAnchor`, typed fold/demand/variant/loss declarations; replaced by the per-formalism plugin file (`specs/plugin-contract.md`, `specs/sdcpn-plugin.md`) | +| [elicitation-completion-2026-08-25-full-draft](archive/specs/elicitation-completion-2026-08-25-full-draft.md) | superseded | FE-1402 | Archive copy of the pre-ADR-0006 completion draft: CPS DemandTable, `where`-scoped presence/slot clauses, completion-anchor matching, full deferral-licensing schemas; replaced by the `evaluateCompletion` invariants in `specs/elicitation-completion.md` | ## Process-model elicitation artifacts (FE-1357) @@ -59,24 +61,24 @@ control loop is [`docs/agents/steering.md`](agents/steering.md). | [research/voice-feasibility](reference/research/voice-feasibility.md) | active | gisted in FE-1359 resolution | Voice verdict: bolt-on with constraints; T0–T3 tiers | | [research/elicitation-strategy-literature](reference/research/elicitation/elicitation-strategy-literature.md) | active | gisted in FE-1360 resolution | Literature synthesis, 9 sections, verification-labeled | | [research/re-interviewing-literature-worker-report](reference/research/elicitation/interviewing-literature-source-catalog.md) | active | noted on FE-1361 | Verbatim instruments: 34-mistake taxonomy, question typologies, LLM-interviewer results | -| [research/frontier-model-elicitor-failure-catalogue](reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md) | active | FE-1407 | Typed frontier-model failure catalogue from the two baseline transcripts and indexed literature: mechanism, detection signature, accountable layer, bounded prevention claim, and the licensed-deferral boundary | -| [baseline evaluation evidence](evidence/evaluations/process-model-elicitation/baseline/) | settled | gisted in FE-1361 resolution | Immutable baseline-control evidence: both transcripts, raw snapshots, delivered models, and scored read-out; executable cases and protocol live under `evaluations/` | +| [research/frontier-model-elicitor-failure-catalogue](reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md) | active | FE-1407 | Test-oracle list for the harness (reclassified 2026-08-25 as test-bed material, not authority): typed frontier-model failure catalogue from the two baseline transcripts and indexed literature — mechanism, detection signature, accountable layer, bounded prevention claim, and the licensed-deferral boundary | +| [baseline evaluation evidence](evidence/evaluations/process-model-elicitation/baseline/) | settled | gisted in FE-1361 resolution | Immutable baseline-control evidence: both transcripts, raw snapshots, delivered models, and scored read-out; with the executable cases and protocol under `evaluations/` it is the simulated-expert harness for the walking-skeleton run (reclassified 2026-08-25 as test-bed material) | | [ir-design](specs/intermediate-representation.md) | active | gisted in FE-1364 resolution; amended by FE-1480 | The IR design: Layer A (ratified on worked examples, FE-1397; definition sentence amended by ADR-0003) + the CPS plugin's ten-kind payload, deterministic scaffold and obligation contract (Layer B); executable code is realized downstream under ADR-0005 | | [ir-worked-examples](evidence/proofs/design/intermediate-representation-worked-examples.md) | active | gisted in FE-1397 | Layer-A validation across Gherkin/CPS/BPMN + assurance: property verdicts, amendments, sublimation findings | -| [ir-design-plain](specs/intermediate-representation-plain.md) | active | strain findings on FE-1401; amended by FE-1480 | Plain-prose rendering of the IR design, including ADR-0005's split between deterministic scaffolding and model-assisted executable realization | +| [ir-design-plain](specs/intermediate-representation-plain.md) | active | strain findings on FE-1401; amended by FE-1480 | Plain-prose rendering of the IR design, including ADR-0005's split between deterministic scaffolding and model-assisted executable realization; notes that `sdcpn-plugin.md` is now the concrete rendering of Layer B | | [notes/research-patterns-audit](evidence/proofs/audits/research-patterns-audit.md) | active | FE-1401 / card inputs on FE-1403 | Plain-language audit of ~30 research imports in 7 families, evidence-graded, with an 8-point strain appendix | | [notes/penciled-directions-2026-08-14](archive/planning-inputs/penciled-directions-2026-08-14.md) | settled | FE-1401 | Penciled directions from the legibility session: 8 items with firming actions + editorial reflections | | [capture-store-plain](reference/architecture/capture-store.md) | active | strain findings on FE-1401 | STE-leaning rendering of the capture-store semantics (FE-1390/FE-1389) with a load-bearing not-guaranteed section; 8-point strain report incl. two command-reachable unclosable-conflict paths (confirms FE-1419 commits 7/8) and the FE-1405 status-arity answer | | [notes/deep-read-fe-1389](evidence/proofs/audits/deep-read-fe-1389.md) | active | FE-1401 / findings in FE-1420 | Deep-read of the walking skeleton: builder's account, spec-discharge table (issues 10/13 capabilities discharged; markdown floor contradicted in the UI), 12 findings; source of PR #10's backfilled record | | [notes/deep-read-fe-1390](evidence/proofs/audits/deep-read-fe-1390.md) | active | FE-1401 / probes on FE-1419 | Deep-read of the capture store: spec-discharge table, write-time tiering assessment (penciled item 7), the FE-1405 status-arity answer, and live-probed confirmation of FE-1419's capture-store claims plus one new aliasing hole; source of PR #11's backfilled record | -| [plugin-contract-spec](specs/plugin-contract.md) | active | FE-1431 (spec issue); decided on FE-1405; amended by FE-1480 | Provisional spec: a plugin is two schemas and two tables over the three-register IR; code-bearing projections add a deterministic scaffold, typed obligation sidecar, and loss report before downstream realization; strains 4–7 and envelope pressure #2 remain open | -| [elicitation-completion](specs/elicitation-completion.md) | active | FE-1402 | Normative provisional read-time contract: version-bound presence/slot demand algebra, universal active-objective support, evidence-bearing boolean report, conservative divergence diagnostic, and pure deferral licensing over existing authoritative surfaces | -| [elicitation-completion-rehearsal](evidence/proofs/design/elicitation-completion-rehearsal.md) | active | FE-1402; inputs FE-1403/FE-1404/FE-1431 | Manual clause-level replay over all 44 FE-1361 prefixes using a versioned provisional CPS DemandTable, separate failure-occurrence/repair clauses, evidence-proxy deltas with carry-forward, C1-E09 no progress, C2 ramp scrap, and successor evidence | -| [elicitation-completion-plain](evidence/proofs/design/elicitation-completion-plain.md) | active | FE-1402 legibility snapshot | Reviewer-facing plain-language rendering of version-bound completion, presence versus slot checks, conservative divergence, stopping/delivery boundaries, and read-time deferral licensing without new persistence | -| [cps-interview-guidance](specs/cps-interview-guidance.md) | active | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Provisional CPS ElicitationPack handoff: six mechanism-typed cards plus status/grade and respectful-close fragments, with clause-addressed activation, explicit machinery/guidance ownership, and the singular-`firesWhen` authoring seam carried to FE-1431 | -| [cps-interview-guidance-desk-replay](evidence/proofs/design/cps-interview-guidance-desk-replay.md) | active | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Manual two-transcript prefix replay: per-card firings, expected evidence deltas, positive deactivation boundaries, candidate dispositions, and repository research ledger; desk discrimination only | -| [cps-interview-guidance-plain](evidence/proofs/design/cps-interview-guidance-plain.md) | active | FE-1403 legibility snapshot | Reviewer-facing plain rendering of the CPS guidance contract with translation strains and their dispositions | -| [sdcpn-plugin](specs/sdcpn-plugin.md) | active | FE-1404 (redefined toward the walking skeleton); supersedes the domain-keyed tables of FE-1402/1403 | The SDCPN plugin file: fixed contract headings (Purpose, Kinds, Must know, Patterns, Moves, Deliverable) over the IR spec's ten Layer-B kinds; three machine-read tables, domain-neutral by rule; moves to `packages/plugin-sdcpn/` with the skeleton | +| [plugin-contract-spec](specs/plugin-contract.md) | active | FE-1431 (spec issue); decided on FE-1405; amended by FE-1480; reshaped by ADR-0006 | Per-target-formalism plugin contract: fixed heading set, three machine-read tables (`Kinds`, `Must know`, `Patterns`) with `sdcpn-plugin.md` normative for row/column shape, version binding, `project`/`validate` as code with the ADR-0005 outputs, surviving invariants, open strains, and a Retired 2026-08-25 section pointing to the archived declarative draft | +| [elicitation-completion](specs/elicitation-completion.md) | active | FE-1402; rewritten under ADR-0006 | Nineteen invariants `evaluateCompletion(model, mustKnowRows)` must satisfy, framed as tests: derived boolean plus evidence report, floor as counts, question-relative demand over objective slices, universal active-objective check, status/precision/confidence separation, conservative conflict and divergence failure, stop/delivery/budget as non-inputs, read-time deferral licensing, no new persistence | +| [elicitation-completion-rehearsal](evidence/proofs/design/elicitation-completion-rehearsal.md) | active | FE-1402; inputs FE-1403/FE-1404/FE-1431 | Test-bed material, not authority (reclassified 2026-08-25): manual clause-level replay over all 44 FE-1361 prefixes against the retired domain-keyed CPS DemandTable; golden-fixture candidate for `evaluateCompletion` once re-expressed at kind level | +| [elicitation-completion-plain](evidence/proofs/design/elicitation-completion-plain.md) | active | FE-1402 legibility snapshot | Evidence, not authority (reclassified 2026-08-25): plain-language rendering of the pre-ADR-0006 completion draft and its translation strains; the invariants it explains survive in the rewritten spec | +| [cps-interview-guidance](archive/specs/cps-interview-guidance-2026-08-25.md) | superseded | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Archived 2026-08-25 under ADR-0006: the FE-1403 CPS card set (CPS-Q01–Q05, GEN-Q02, two hint fragments) whose cards became kind-indexed patterns P01–P05, P12 and `Moves` steps in `sdcpn-plugin.md`; banner records the card→pattern mapping and the `domain` mis-tag; retained as test-bed material | +| [cps-interview-guidance-desk-replay](evidence/proofs/design/cps-interview-guidance-desk-replay.md) | active | FE-1403; inputs FE-1404/FE-1406/FE-1431 | Evidence, not authority (reclassified 2026-08-25): manual two-transcript prefix replay of the archived CPS cards — per-card firings, expected evidence deltas, deactivation boundaries, candidate dispositions, research ledger; desk discrimination only | +| [cps-interview-guidance-plain](evidence/proofs/design/cps-interview-guidance-plain.md) | active | FE-1403 legibility snapshot | Evidence, not authority (reclassified 2026-08-25): plain rendering of the archived CPS guidance with translation strains and their dispositions | +| [sdcpn-plugin](specs/sdcpn-plugin.md) | active | FE-1404 (redefined toward the walking skeleton); supersedes the domain-keyed tables of FE-1402/1403 | The SDCPN plugin file: fixed contract headings (Purpose, Kinds, Must know, Patterns, Moves, Deliverable) over the IR spec's ten Layer-B kinds; three machine-read tables, domain-neutral by rule; `Moves` carries two job runbooks (`construct`, `review and revise`) with harness-owned checks and stopping outcomes; moves to `packages/plugin-sdcpn/` with the skeleton | ## Control, architecture reference, and migration archive @@ -113,6 +115,7 @@ contract requires the spec to carry the new operating truth, in explicitly dated | [0003-three-register-ir](adr/0003-three-register-ir.md) | accepted | FE-1405; amended by FE-1480 | The IR is the elicited conceptual model, derived by a pure fold — three registers (assertions / model / projections); ADR-0005 narrows pure executable projection to deterministic scaffolds and obligation plans before downstream realization | | [0004-in-petrinaut-staging-and-the-monorepo-import](adr/0004-in-petrinaut-staging-and-the-monorepo-import.md) | accepted | FE-1433; amended by FE-1437; extended by FE-1480 | September demo stages inside demo.petrinaut.org; reusable libraries remain mutually unaware; ADR-0005 extends the application-owned artifact path from projection scaffold and obligations to an executable file through Petrinaut client tools | | [0005-model-assisted-sdcpn-realization](adr/0005-model-assisted-sdcpn-realization.md) | accepted | FE-1480 | A pure plugin projection emits an SDCPN scaffold, typed code-obligation sidecar, and loss report; the Brunch agent realizes executable TypeScript through Petrinaut client tools and deterministic compile/simulation gates | +| [0006-plugins-per-target-formalism](adr/0006-plugins-per-target-formalism.md) | accepted | design-convergence review 2026-08-25 | Plugins are per target formalism, authored as sectioned Markdown with fixed contract headings and three machine-read tables; domain-neutral by rule; retires the declarative ScopeExpr/firesWhen/DemandTable contract | ## External canonical documents diff --git a/libs/@hashintel/brunch-agent/docs/adr/0006-plugins-per-target-formalism.md b/libs/@hashintel/brunch-agent/docs/adr/0006-plugins-per-target-formalism.md new file mode 100644 index 00000000000..d74da690363 --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/adr/0006-plugins-per-target-formalism.md @@ -0,0 +1,110 @@ +# ADR-0006: Plugins are per target formalism, authored as sectioned Markdown + +Date: 2026-08-25 +Status: accepted +Amends: [ADR-0003](0003-three-register-ir.md), first consequence only — the plugin contract's +_form_ (a sectioned Markdown file with three parsed tables replaces the typed model-schema / +proposal-catalog / fold-table / demand-table declaration); the three registers are unchanged +Supersedes: the declarative plugin-contract draft, the CPS interview-guidance spec, and the full +completion-contract draft, all archived under +[`docs/archive/specs/`](../archive/specs/) on 2026-08-25 +Decided on: the `ln/sdcpn-plugin-pivot` arc (FE-1482 lane), 2026-08-25; recorded with +[S-007](../control/STRATEGY-LOG.md#s-007) + +## Context + +The general use case is to elicit a model of _any_ cyber-physical system. The domain — packaging +line, truck fleet, coating plant — is unknown before the conversation starts. A plugin therefore +cannot be keyed to a domain; the only thing fixed before the first turn is the target formalism the +model will be projected into. + +The IR spec's [Layer B](../specs/intermediate-representation.md#layer-b--the-cps-plugins-ir) +already defined the CPS plugin at exactly that level: ten kinds, cross-kind `quantity` / +`source-regime` / `rationale` attributes, and question-relative completion over a static floor. +The design-convergence queue selected by S-005 then drifted below it. The FE-1402 rehearsal +needed an oracle and authored a provisional DemandTable keyed to the baseline coatings-plant domain +(`dynamics[line-failure].occurrenceFrequency`, objective rows `ROW-BREAKDOWN`, `ROW-IDLE-WASH`, +…) with `where(kind, role=…)` scopes — contradicting the plugin contract's "September ships +kind-only", which was in fact correct; the oracle was at the wrong level. FE-1403 tagged five +cards `domain`; each lifts without loss to a kind-level pattern (stochastic event, mode switch, +batching, gate/trigger, shared-resource contention). FE-1404 built a 5,400-line preregistered +instrument — an LLM operator emitting a Valibot-validated projection, an activation matrix, +seal-bound append-only recovery, a 51-component result vector — through nine rejected +fresh-context review renderings; it never ran, and its committed lock is the rejected draft lock +(5/11 hashes stale, 10/21 canonical paths missing; the runner's own `--verify-seal` refuses it). +Structurally the instrument is a shadow harness: operator ≈ capture store + fold; projection +schema ≈ model state + `evaluateCompletion`; diagnostic priority ≈ affordance cue; quote-novelty +streak ≈ controller stopping policy; sealed segments ≈ session archive. The S-005 cut ("during +design convergence do not implement SDK surface, projection, …") displaced implementation into +`evaluations/`, where it does not compound. + +Meanwhile [`docs/specs/sdcpn-plugin.md`](../specs/sdcpn-plugin.md) showed that the whole target +fits one file: the twenty domain rows collapse onto kind-level rows instantiated on discovered +nodes, and the five domain cards become kind-indexed patterns P01–P05. This record ratifies that +file's shape as the plugin contract. + +## Decision + +1. **The plugin unit is the target formalism** (`gherkin`, `sdcpn`) — never a domain, situation, + or scenario. The domain is unknown at conversation start and is discovered, not declared. + +2. **A plugin is one sectioned Markdown file** with fixed contract headings + `Purpose · Kinds · Must know · Patterns · Moves · Deliverable`. The headings are the contract and + are identical across plugins. The harness parses the `Kinds`, `Must know`, and `Patterns` + tables into the model vocabulary, the demand list, and the pattern index; every other section + concatenates into the interviewer's instructions. `docs/specs/sdcpn-plugin.md` is the normative + exemplar; it moves unchanged to `packages/plugin-sdcpn/` with the walking skeleton. + +3. **Demand rows are kind-level.** Each row is a slot on a kind with a required precision, an + accepted-absence flag, and a rationale; scopes are kinds only. Every node discovered in + conversation is checked against its kind's rows. Completion is question-relative (Layer B): the + static floor holds, and every node in the dependency slice of every active `objective` + satisfies its rows. + +4. **Domain-neutrality rule.** No domain nouns in a plugin file. A new case that appears to need a + new heading is a finding about the abstraction, decided through an ADR — never content to add. + The generality test for a plugin is that a second domain adds zero headings and zero rows. + +5. **Patterns and moves.** Patterns are discretionary, kind-indexed heuristics (trigger → + question), surfaced by the harness when a node matches the trigger and the slot is unsatisfied. + Moves are job runbooks; a plugin may carry more than one runbook over one `Kinds` / `Must know` + set (this one carries `construct` and `review and revise`). Harness-generic patterns may later + lift into a harness repertoire (FE-1406, gist: strategy quiver). + +6. **Code stays where ADR-0005 put it.** `project` and `validate` remain plugin code operations; + the three-register IR (ADR-0003) is unchanged. The file declares what the model is and what + must be known; code derives the artifact from it. + +7. **Retired**, each with its replacement: + + | Retired | Replacement | + | --- | --- | + | `ScopeExpr` with `where` and `inSupport(anchor)` constructors | The kind column of a `Must know` row. Objective-relative depth comes from the dependency slice the harness computes from `objective` nodes, not from a scope expression. | + | `ProposalType.affordance.firesWhen` (closed 7-value enum, singular per proposal type) | The `when` column of the `Patterns` table: kind-indexed triggers, any number per kind, matched against node state at read time. | + | `NodeKind.completionAnchor` | The `objective` kind is the anchor by construction: its "the nodes it depends on" row defines the slice. | + | Typed `foldTable` / `demandTable` / `variantDimension` / `lossCategories` declarative contract | Fold rules derive from the `Kinds` table's slots (default fold-by-slot; no override table until a case forces one). `demandTable` → the `Must know` table. `variantDimension` → the `source-regime` attribute on every kind. `lossCategories` → the seven fixed categories of Layer B's loss report, owned by the harness, not declared per plugin. | + | Interview cards as separate artifacts (Detects / Goal / Questions / Artifacts, tag, mechanism) | `Patterns` rows (trigger → question) inside the plugin file; the domain/envelope-generic tag dissolves because no pattern may name a domain. | + | Activation matrices | None. The harness matches pattern triggers against node state; nothing is frozen per experiment. | + | Objective-anchor registries and `whenObjective`-keyed demand rows | The `objective` node's dependency-slice slot; every node in the slice is checked against its kind's rows. | + +## Condition + +Revisit if a second target formalism cannot be expressed under the fixed headings, or if kind-only +demand rows demonstrably cannot express a formalism's completion. Either is a finding about the +abstraction and reopens this record; neither licenses a per-domain plugin or a per-case heading. + +## Consequences + +- The plugin-contract spec shrinks to the heading contract, the table grammar, and the + `project` / `validate` code seam. +- The elicitation-completion spec reduces to invariants on `evaluateCompletion` over the parsed + `Must know` table: static floor, slice-relative demand, boolean plus failure list, computed from + the model and never from the conversation. +- The CPS interview-guidance spec is superseded; its surviving cards live on as patterns P01–P05. +- The FE-1404 instrument is archived as test-bed material. Salvage its Valibot projection schema + and validators; discard the operator, activation matrix, and seal machinery. +- FE-1431's seven plugin-authoring seams dissolve: there is no typed authoring surface to + ratify, only a file format and a parser. +- The walking skeleton implements the parser, the fold, `evaluateCompletion`, the sweep, and the + affordance cue in the production path, against the `sdcpn` file. The generality test (a second + formalism, `gherkin`, adds no headings) follows the skeleton rather than gating it. diff --git a/libs/@hashintel/brunch-agent/docs/agents/steering.md b/libs/@hashintel/brunch-agent/docs/agents/steering.md index 20fad964150..0b9b0d2c8df 100644 --- a/libs/@hashintel/brunch-agent/docs/agents/steering.md +++ b/libs/@hashintel/brunch-agent/docs/agents/steering.md @@ -11,10 +11,15 @@ Run a steering pass when: - a proof succeeds or fails; - authorities conflict; - an external gate changes or becomes stale; -- the selected frontier loses value; or +- the selected frontier loses value; +- a frontier's durable outputs are all desk, simulated, or evaluation-side, with no production-path + code changed by the end of one arc (proxy completion); or - arc close detects strategic drift. -Ordinary ticket movement is not a steering trigger. +Ordinary ticket movement is not a steering trigger. Proxy completion recurs under new names — a +tracer, a desk rehearsal, a preregistered instrument each stood in for the thing it was meant to +exercise and became the definition of done. An evaluation instrument larger than the thing it +evaluates is itself the finding. ## Orient diff --git a/libs/@hashintel/brunch-agent/docs/specs/cps-interview-guidance.md b/libs/@hashintel/brunch-agent/docs/archive/specs/cps-interview-guidance-2026-08-25.md similarity index 90% rename from libs/@hashintel/brunch-agent/docs/specs/cps-interview-guidance.md rename to libs/@hashintel/brunch-agent/docs/archive/specs/cps-interview-guidance-2026-08-25.md index 4bd8409dac0..e4451979062 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/cps-interview-guidance.md +++ b/libs/@hashintel/brunch-agent/docs/archive/specs/cps-interview-guidance-2026-08-25.md @@ -1,3 +1,16 @@ +> **Superseded 2026-08-25.** Moved from `docs/specs/cps-interview-guidance.md` under +> [ADR-0006](../../adr/0006-plugins-per-target-formalism.md): interview "cards" are no longer +> separate artifacts; they became kind-indexed patterns in the `Patterns` and `Moves` sections of +> [`sdcpn-plugin.md`](../../specs/sdcpn-plugin.md). Mapping: CPS-Q01 → P01 · CPS-Q02 → P02 · +> CPS-Q03 → P03 · CPS-Q04 → P04 · CPS-Q05 → P05 · GEN-Q02 → Moves "construct" step 3 (the +> batching sentence) · HINT-STATUS-GRADE → P12 · HINT-RESPECTFUL-CLOSE → Moves "construct" step 6. +> The `domain` tag on the CPS cards was a mis-tag: each card names a model situation that occurs +> on a kind (an event-shaped `activity`, a mode-changing `activity`, a grouped `ordering/flow`, a +> gating `policy`/`boundary-condition`, a contended `entity-type`), so each lifts to a kind-level +> pattern without naming a domain. The file is retained as test-bed material: its desk replay and +> failure signatures remain evidence for the harness oracles, not authority. Content is otherwise +> verbatim; only relative link targets were re-rooted for the archive location. + # Spec: CPS interview guidance Status: **provisional, desk-tested pack content**. This is the FE-1403 handoff to CPS plugin @@ -10,10 +23,10 @@ diagnostic and asks for evidence that could change the named slot. No card infer changes a completion grade, turns `not-mentioned` into an absence, or claims that asking the expert what was missed can discover an unknown omission. -Vocabulary and mechanics come from the [plugin contract](plugin-contract.md) and the -[completion contract](elicitation-completion.md). The fixed target IDs and full coordinates used +Vocabulary and mechanics come from the [plugin contract](plugin-contract-2026-08-25-declarative-draft.md) and the +[completion contract](elicitation-completion-2026-08-25-full-draft.md). The fixed target IDs and full coordinates used here are declared in the [FE-1402 replay -DemandTable](../evidence/proofs/design/elicitation-completion-rehearsal.md#provisional-cps-demandtable). +DemandTable](../../evidence/proofs/design/elicitation-completion-rehearsal.md#provisional-cps-demandtable). For this packet, the applicable quantity ladder is `verbal < point < range < quantiles`, the condition ladder is `verbal < structured`, and the accepted completion statuses are `explicit` and `inferred`. Other statuses remain representable but do not pass these fixed clauses. @@ -73,9 +86,9 @@ predicate or imply that the current hook represents the whole card. ambiguity bypass. This card chooses the imported IDEA (Investigate, Discuss, Estimate, Aggregate) protocol's -interval-before-best-guess ordering over the [v0 prompt's](../../evaluations/protocols/process-model-elicitation/baseline/v0-prompt.md) +interval-before-best-guess ordering over the [v0 prompt's](../../../evaluations/protocols/process-model-elicitation/baseline/v0-prompt.md) typical-first script because the imported protocol supplies that order and a calibration question. -The [FE-1360 research deposit](../reference/research/elicitation/elicitation-strategy-literature.md#14-numbers-vs-distributions-vs-stories), +The [FE-1360 research deposit](../../reference/research/elicitation/elicitation-strategy-literature.md#14-numbers-vs-distributions-vs-stories), not the transcript replay, owns the anti-anchoring rationale. The replay establishes only that the baseline left demanded occurrence and repair grades unresolved. @@ -98,7 +111,7 @@ baseline left demanded occurrence and repair grades unresolved. - **Artifacts:** a direction-scoped typed dynamics proposal for ramp-scrap magnitude and a distinct split-run proposal for repeated scrap. If the expert says they do not know, the clause remains failing; the current contract may propose a field-local absence only after the - [absence-locator seam](plugin-contract.md#the-envelope-is-untouched) has an approved representation. + [absence-locator seam](plugin-contract-2026-08-25-declarative-draft.md#the-envelope-is-untouched) has an approved representation. A promised observation is not the value. - **Targets:** `IW-SCRAP`, `CH-SCRAP`, and `SP-SCRAP` at `range`. - **Failure signatures:** FM-08 never-asked coverage blindness, FM-09 complementary-miss diff --git a/libs/@hashintel/brunch-agent/docs/archive/specs/elicitation-completion-2026-08-25-full-draft.md b/libs/@hashintel/brunch-agent/docs/archive/specs/elicitation-completion-2026-08-25-full-draft.md new file mode 100644 index 00000000000..e43c3d9457a --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/archive/specs/elicitation-completion-2026-08-25-full-draft.md @@ -0,0 +1,347 @@ +> **Superseded 2026-08-25.** Archive copy of `docs/specs/elicitation-completion.md` as it stood +> before [ADR-0006](../../adr/0006-plugins-per-target-formalism.md). The CPS `DemandTable`, the +> `where`-scoped `PresenceClause` / `SlotClause` vocabulary, and the `completionAnchor` matching +> below have no current authority; completion is now specified as the invariants of +> `evaluateCompletion(model, mustKnowRows)` over the plugin file's `Must know` table in the +> rewritten [`elicitation-completion.md`](../../specs/elicitation-completion.md). Content is +> otherwise verbatim; only relative link targets were re-rooted for the archive location. + +# Spec: target-document completion and session stopping + +Status: **provisional** — manually desk-scored over the two FE-1361 baseline transcripts. The +replay validates design discrimination, not implementation. FE-1402 owns this required-behavior +contract; plugin authoring consumes it. + +This specification defines a read-time answer to one question: given one version-bound snapshot +of the durable evidence, derived model, active objectives, and plugin demands, is the +target-document complete enough for those objectives? The companion +[rehearsal](../../evidence/proofs/design/elicitation-completion-rehearsal.md) owns the provisional CPS +oracle and prefix verdicts. The +[plain rendering](../../evidence/proofs/design/elicitation-completion-plain.md) checks this contract in +a second register. + +## Required distinctions + +These facts can coincide, but none except the first asserts completion: + +| Fact | Meaning | Effect on completion | +| --- | --- | --- | +| Target-document completion | Every static demand, universal active-anchor support check, and objective demand passes. | This is the computation. | +| Session stopping | A conversation is intentionally quieted or produces no more entries. | None. | +| User-requested quiet | The user asks to pause, leave, or receive no further questions. | None. | +| Delivery | A current projection or another promised result is emitted. | None. | +| No progress | Recent frames add no demanded evidence or state and deliver nothing. | None; session control may require adjudication. | +| Budget exhaustion | A turn, token, time, or cost bound ends the session. | None. | +| Licensed deferral | Session control verifies that existing durable authorities can support recoverable re-entry. | None. | + +“Best useful result within this session” is therefore ordinary behavior: durably deliver the best +current projection with its loss and open obligations, license later continuation only from +existing authoritative state, and stop the session while the completion boolean remains false. + +## Version-bound input snapshot + +The harness evaluates one immutable snapshot: + +```yaml +CompletionInput: + targetDocumentRevision: opaque immutable revision + pluginContractVersion: immutable plugin version or digest + demandTableVersion: immutable demand-table version or digest + model: register-2 derived model at targetDocumentRevision + activeCaptures: evidence metadata reachable from model support links + openIssues: issue state at targetDocumentRevision +``` + +The plugin version and demand-table version are part of the identity of the verdict. If the target +document or either plugin version changes during the read, the caller retries. It must not combine +model state from one revision with demands from another. + +The computation reads active objectives, the plugin's static floor and demand rows, derived slot +states, grades, epistemic statuses, evidence spans or bases, and open issues. It does **not** read +conversation fluency, self-assessment, turn count, delivery state, recent novelty, session state, +or a deferral-licensing report. + +The plugin owns the demand declaration. The harness owns deterministic expansion and evaluation. +The session controller consumes the report but cannot author or override it. + +## Demand algebra + +The smallest required algebra distinguishes existence from slot quality: + +```yaml +DemandTable: + version: immutable string or digest + staticFloor: DemandClause[] + rows: DemandRow[] + +DemandRow: + id: stable plugin-local identifier + whenObjective: pattern over one active completion anchor + clauses: DemandClause[] + +DemandClause: + PresenceClause | SlotClause + +PresenceClause: + id: stable plugin-local identifier + type: presence + scope: ScopeExpr + minimumCount: positive integer + +SlotClause: + id: stable plugin-local identifier + type: slot + scope: ScopeExpr + slot: slot name + minimumGrade: rung in that slot's grade order + acceptedEpistemicStatuses: non-empty subset of capture-envelope statuses + acceptedAbsences: subset of capture-envelope absence states, default empty +``` + +All static clauses and all clauses from every matched objective row are conjunctive. A presence +clause counts model nodes selected by its scope; it is how a plugin declares objective, entity, or +path cardinality. A slot clause evaluates a named slot on every selected model node. An empty slot +selection fails with `no-selected-slot`; existence cannot pass accidentally through an empty +selection. + +This algebra adds no graph-query language. It uses the plugin contract's `ScopeExpr`. September may +ship kind-only scopes. The already named `where` and `inSupport(anchor)` constructors remain future +growth paths; this contract does not generalize them. + +### Universal active-anchor support + +Independently of the static floor, the harness checks every active node whose kind declares +`completionAnchor: true`: + +1. match that anchor against every demand row using the plugin-declared objective pattern; +2. fail that anchor with `unsupported-active-anchor` when no row matches; and +3. evaluate the union of clauses from every matching row when at least one matches. + +No objective is silently ignored, and the static floor cannot substitute for this universal +check. A target-document with no active completion anchor fails through its plugin-declared +presence clause in the floor. + +### Status, grade, and confidence + +Epistemic status and grade are independent: + +- status says how content relates to its source (`explicit`, `inferred`, `tentative`, `defaulted`, + or `external-lookup`); +- grade says how narrow the slot value's interpretation space is; and +- confidence says claim strength and satisfies neither requirement. + +Statuses are not ordered. A slot clause explicitly lists accepted statuses. An `inferred` capture +uses evidence spans under the existing envelope contract; a demand may accept it directly. A +`documented-transformation` basis belongs to `external-lookup`, not to `inferred`. Grade and +traceable evidence are checked separately from status. + +## Evaluation + +```text +evaluateCompletion(input, plugin): + require input.pluginContractVersion == plugin.version + require input.demandTableVersion == plugin.demandTable.version + + floor := expand and evaluate every static clause + anchors := every active completion-anchor node + + for each anchor: + matchedRows := all demand rows matching anchor + anchorSupport.pass := matchedRows is not empty + obligations := expand and evaluate every clause in matchedRows + anchor.pass := anchorSupport.pass and every obligation passes + + complete := every floor clause passes + and every active anchor passes + + return the version-bound evidence-bearing report +``` + +The boolean is the only completion value required. Diagnostics explain it; they are not a second +public status vocabulary. + +### Presence evaluation + +A presence clause passes when the selected node count is at least `minimumCount`. The report +includes the selected node IDs. Presence checks model cardinality only; they do not manufacture +evidence or grade. + +### Slot evaluation + +Every selected slot must pass: + +- `stated(value, grade, supportingCaptureIds)` passes when grade meets the declared minimum, all + support needed for the folded value is active and traceable, and every supporting status the + fold relies on is accepted. +- `unaddressed` fails with `unaddressed`. +- `absent(absence, captureId)` passes only when that exact absence is accepted, the capture is + active and traceable, and its status is accepted. `not-mentioned` cannot pass because it is a + computed fact, not evidence. +- `conflicted(openIssueIds)` fails with `open-conflict` until explicit resolution. +- `diverged(prescribed, practiced)` fails with `unevaluable-divergence`. The canonical shorthand + does not expose grade and supporting captures for each side, so this contract cannot evaluate + either constituent honestly. + +Other diagnostics are `below-minimum-count`, `no-selected-slot`, `below-required-grade`, +`inadmissible-status`, `unaccepted-absence`, `missing-evidence`, `unsupported-active-anchor`, and +`version-mismatch`. +Open issues outside selected demand coordinates remain visible but do not block objective-relative +completion. + +## Evidence-bearing report + +```yaml +CompletionReport: + targetDocumentRevision: opaque immutable revision + pluginContractVersion: immutable plugin version or digest + demandTableVersion: immutable demand-table version or digest + complete: boolean + floor: + - clauseId + clauseType + selectedModelCoordinates + requirement + actual + pass: boolean + diagnostics + supportingCaptureIds + openIssueIds + objectives: + - anchorNodeId + supportingCaptureIds + matchedDemandRowIds + supportCheck: + pass: boolean + diagnostics + obligations: same assessment shape as floor + pass: boolean +``` + +The report follows register-2 support links to deposited captures and carries their capture IDs. It +never semantically rereads the transcript. + +## Session control and no progress + +After each settled sweep, session control may read the completion report plus separate session +facts. It may ask, deliver and defer, or stop. None rewrites the report. A later evidence +change can also make a previously complete target-document incomplete; completion never locks it. + +Runtime no-progress policy remains outside this contract. A candidate detector may compare frames +for newly deposited demanded evidence, demanded slot or obligation changes, and delivery changes. +An advisory can force session-control adjudication but cannot supply a positive completion verdict. +The rehearsal owns one bounded threshold solely to score the baseline. + +## Read-time deferral licensing + +Licensed deferral is a session-control decision computed from existing authorities. It creates no +third persistence surface and writes no target-document truth. The session controller owns this +pure projection: + +```yaml +DeferralLicensingInput: + completionReport: version-bound CompletionReport + captureStore: + snapshotRevision: immutable revision inspected by completion + locatedIssuesAndAbsences: existing capture-store records + sessionLog: + archivePointer: durable archive pointer + archiveRevision: immutable revision or digest + sweptHighWaterMark: durable entry identifier + unsweptTail: bounded entry range or empty + pendingAffordanceSlot: + stateRevision: immutable revision or digest + affordanceId: identifier? + delivery: + projectionRef: durable projection/artifact pointer? + sourceTargetDocumentRevision: immutable revision? + +BlockingObligationLocator: + ModelCoordinateLocator | UnresolvedTargetLocator + +ModelCoordinateLocator: + type: model-coordinate + coordinate: register-2 model coordinate + issueIds: existing identifiers[] + absenceCaptureIds: existing identifiers[] + +UnresolvedTargetLocator: + type: unresolved-target + clauseId: stable demand-clause identifier + scope: exact ScopeExpr + diagnostic: below-minimum-count | no-selected-slot + +DeferralLicensingReport: + inspected: + targetDocumentRevision + pluginContractVersion + demandTableVersion + captureStoreSnapshotRevision + sessionArchivePointer + sessionArchiveRevision + sweptHighWaterMark + unsweptTailRange + pendingAffordanceStateRevision + pendingAffordanceId + deliveryProjectionRef + deliverySourceTargetDocumentRevision + blockers: + - clauseId + locator: BlockingObligationLocator + checks: + sweepAndArchiveStateRecoverable: boolean + everyBlockingObligationLocated: boolean + currentBestProjectionDurablyDelivered: boolean + reentryFactsRecoverable: boolean + licensed: boolean + diagnostics: string[] +``` + +The locator is a union because a blocker may already have a model coordinate, issue, or explicit +absence, while a failed presence clause or `no-selected-slot` has no selected model node. The +second arm preserves the unresolved clause and exact scope instead. Every completion blocker must +have one arm; a missing locator keeps `everyBlockingObligationLocated` false. + +`sweepAndArchiveStateRecoverable` permits an explicitly bounded unswept tail only when the archive +pointer, archive revision, and swept high-water mark recover it exactly; it does not silently call +that tail settled. + +`licensed` is true only when all four checks pass and every inspected revision, digest, pointer, +high-water mark, pending-affordance fact, and projection reference still resolves to the state +shown in the report. The durable delivery must be the best current projection for the same +`targetDocumentRevision` evaluated by completion; absent delivery fields make +`currentBestProjectionDurablyDelivered` false. The report itself is ephemeral: it may be +recomputed for audit, but it is not stored as target-document or capture-store truth. + +The current authoritative schemas carry no durable undelivered-delivery obligation with reason, +owner, and next action. Therefore an undelivered best result cannot license deferral. Such an +obligation remains successor strain and requires an approved durability-contract owner; it must not +be invented in `CaptureIssue` or in a new record here. + +### Existing-operation sequence + +Before quieting, session control uses existing operations in this order: settle/sweep and archive; +recompute version-bound completion; locate every blocker; durably deliver the best current +projection; validate recoverable session-log/capture-store changes and the existing pending +affordance; compute the licensing report; then quiet only if `licensed` is true. On re-entry it +reloads the same authoritative surfaces, validates the report's bound revisions and pointers, and +recomputes completion and licensing. There is no create/update/consume lifecycle for a new deposit. + +## Claims, limits, and replay + +This contract claims only deterministic, version-bound completion and deferral-licensing +computation shapes over existing authorities. It does not show that the harness, plugin SDK, +capture store, sweep, controller, detector, binding, projection, or application implements them. +FE-1407's specified and candidate prevention claims keep those grades. + +The [rehearsal](../../evidence/proofs/design/elicitation-completion-rehearsal.md) owns the provisional +CPS demand table, all prefix assessments, failure-signature verdicts, amendments found by replay, +and successor evidence. Keeping those judgment-bearing results out of required behavior prevents a +single baseline oracle from becoming generic plugin canon. + +## Out of scope + +- runtime, detector, controller, or TypeScript implementation; +- capture-envelope, `CaptureIssue`, session-state, or durability-contract changes; +- final CPS demand-table authoring; +- projection, realization, or delivery validation; and +- a public lifecycle-status enum. diff --git a/libs/@hashintel/brunch-agent/docs/archive/specs/plugin-contract-2026-08-25-declarative-draft.md b/libs/@hashintel/brunch-agent/docs/archive/specs/plugin-contract-2026-08-25-declarative-draft.md new file mode 100644 index 00000000000..7e75a2f492b --- /dev/null +++ b/libs/@hashintel/brunch-agent/docs/archive/specs/plugin-contract-2026-08-25-declarative-draft.md @@ -0,0 +1,460 @@ +> **Superseded 2026-08-25.** Archive copy of `docs/specs/plugin-contract.md` as it stood before +> [ADR-0006](../../adr/0006-plugins-per-target-formalism.md) (plugins are per target formalism, +> authored as one sectioned Markdown file). The typed declarative contract below — `ScopeExpr` / +> `where` / `inSupport`, `ProposalType.affordance.firesWhen`, `NodeKind.completionAnchor`, the +> typed `foldTable` / `demandTable` / `variantDimension` / `lossCategories` keys — has no current +> authority; the current contract is the shrunk [`plugin-contract.md`](../../specs/plugin-contract.md) +> and the exemplar [`sdcpn-plugin.md`](../../specs/sdcpn-plugin.md). Content is otherwise +> verbatim; only relative link targets were re-rooted for the archive location. + +# Spec: the plugin contract — two schemas, two tables + +Status: **provisional** — desk-validated on baseline transcripts and cross-plugin thumbnails. +Ratification condition (inherited from [ADR-0003](../../adr/0003-three-register-ir.md)): a full +FE-1397-style worked pass across at least three plugin targets. Until the September build +exercises a real fold, everything here is design, not demonstrated behavior. +Decided on: FE-1405 (the payload-interiors session, 2026-08-18); inputs were that session's +working draft and its pseudo-YAML structural rendering, which collapse into this document. +Amended: 2026-08-24 by [ADR-0005](../../adr/0005-model-assisted-sdcpn-realization.md), which +separates deterministic projection scaffolds from model-assisted executable-artifact realization. + +## Problem Statement + +The elicitation system needs to work across domains: cyber-physical process models (the +September target), BDD/Gherkin feature specification, formal verification, and domains not yet +named. Today there is no defined answer to "what is a plugin?" — the capture envelope +(FE-1383, the build root: capture store and envelope semantics) says how evidence is stored, +and the ratified IR design says captures are typed, but nothing says what a domain author must +write down to make the machinery elicit _their_ domain, and nothing turns a pile of typed +captures into the thing the user actually wants: a model of their plant, their feature, their +system. + +Three people feel this as a problem: + +- **The plugin author** has no contract. Every new domain looks like it needs bespoke + machinery — its own assembly logic, its own completion logic, its own follow-up-question + logic — which makes plugin authoring an engineering project instead of a declaration, and + makes example plugins unreadable as examples. +- **The analyst running an elicitation** cannot ask "is this model complete enough to answer + my objective?" of a bag of captures. Completion questions are questions about a _model_ — + which activities lack duration distributions, which resources are uncounted — and no model + exists to ask them of. +- **The reviewer** cannot audit semantic judgments. If "half a shift" becomes "4 hours" + somewhere inside a read path, the interpretation happened invisibly: unattributable, + unsupersedable, unreproducible. + +## Solution + +A plugin is **two schemas and two small tables**, all declarative: + +1. declare your domain's node kinds and their slots (the **model schema**); +2. declare the utterance-shaped typed proposals that fill them (the **proposal catalog**), + annotating each with how to elicit it; +3. say how proposals fold into slots (the **fold table** — almost always empty, because fold + rules derive from the slot declarations); +4. say what your completion anchors demand (the **demand table**). + +Everything else is harness machinery explained once: a pure fold derives the elicited model +from active captures (ADR-0003's register 2), slot states report what is known, absent, +conflicted, or diverged, grades gate promotion, identity resolves by union-find over recorded +same-as assertions, and follow-up questions fire mechanically off slot states. Semantic +interpretation happens only at write time, in the sweep, where every bridge is deposited as a +contestable capture. + +For the plugin author this means writing YAML, not code (code remains only in validators and +projections). For the analyst it means completion computes over model slots, not capture +counts. For the reviewer it means every model part answers "which captures made you." +The readability oracle for the whole design: someone who has read the Gherkin plugin should be +able to write a third plugin by analogy in a sitting. + +## User Stories + +1. As a plugin author, I want to declare my domain as node kinds with slots, so that the + harness derives the model shape without me writing assembly code. +2. As a plugin author, I want a catalog of typed proposals whose interiors I compose from a + standard library of stated-form families, so that I don't reinvent "how experts state + quantities" for every domain. +3. As a plugin author, I want fold rules derived from my slot declarations by default, so that + I only write fold-table rows for genuine exceptions. +4. As a plugin author, I want to annotate each proposal type with an elicitation technique and + a mechanical firing condition, so that "which technique does this field force?" is a + completeness check over one column. +5. As a plugin author, I want to declare grade orders per slot — referencing a library ladder + or declaring my own composition ladder — so that my domain's notion of "more pinned down" + drives promotion without new machinery. +6. As a plugin author writing a thin plugin (Gherkin), I want the registers to collapse + gracefully, so that I never pay the thick plugin's ceremony. +7. As a new plugin author, I want to read the Gherkin example and write my own plugin by + analogy in a sitting, so that plugin authoring is tractable without harness expertise. +8. As a domain expert being interviewed, I want my hedged, low-grade statements ("about + 3 hours") captured honestly rather than refused or hardened, so that the record reflects + what I actually said. +9. As a domain expert, I want to say "I don't know" or "we don't track that" and have it + recorded as a first-class absence, so that I'm not re-asked what I've already disclaimed. +10. As a domain expert, I want to state a uniformity once ("same everywhere, same crew") and + have it cover everything in scope, so that I'm not interrogated per instance. +11. As a domain expert, I want to correct myself ("flag that as one I got wrong") and have the + correction supersede — not overwrite — the original, so that my history stays honest. +12. As an analyst, I want the elicited model in the expert's vocabulary with every slot in a + definite state (unaddressed / stated-at-grade / absent / conflicted / diverged), so that I + can see at a glance what the elicitation has and lacks. +13. As an analyst, I want completion computed as "every objective's demanded slots at demanded + grade over a static floor," so that "done" is relative to what I'm trying to answer, not + a global checklist. +14. As an analyst, I want prescribed-vs-practiced divergence surfaced per slot, so that + unwritten rules and workarounds are findings, not noise. +15. As an analyst, I want conflicting statements to fold to a typed conflict — never a silent + pick — so that contradictions become questions instead of errors. +16. As the interviewing agent, I want next-question candidates to fire mechanically off slot + states (below demanded grade → quantile protocol; statistic unspecified → "typical, or + worst case?"), so that follow-ups are grounded in the model's actual gaps. +17. As the sweep executor (FE-1392, the write-time mapping of utterances to proposals), I want + the proposal catalog compiled to a JSON Schema on my tool input and a decision-tree route + over statement forms, so that mapping is two small decisions, not one pick from a flat + ~19-type list. +18. As the sweep executor, I want honest low-grade defaults (verbal form, + `statistic: unspecified`) to be legal captures, so that uncertainty degrades to + under-typing that cues a follow-up, never to silent hardening. +19. As a reviewer, I want every semantic bridge — unit parse, identity link, composition, + formalization — deposited as an `inferred` capture with evidence spans, so that I can + audit, contest, and supersede any interpretation. +20. As a reviewer, I want the fold forbidden to interpret, so that re-running it on the same + store always yields the same model. +21. As a projection author (net scaffold, loss report, completion table), I want to consume + the elicited model without rereading the transcript or interpreting generic capture + fields, so that the scaffold and its code obligations are pure functions of register 2. +22. As a harness developer (FE-1393, the plugin SDK and fold engine), I want every harness + mechanism to be a pure function classified by which plugin declaration it reads, so that + the harness/plugin boundary is mechanically checkable. +23. As a team reader, I want example plugins that read as declarations of their domain, so + that I can evaluate the product's generality without reading harness internals. +24. As the artifact-realizing agent, I want each code obligation to name one target field, its + semantic intent, available net symbols, supporting captures, and acceptance checks, so that I + can write and repair TypeScript without resynthesizing unrelated regions. + +## Implementation Decisions + +The shapes below are trimmed from the session's ds-pseudo prototype (an untracked session +artifact); they encode the decisions more precisely than prose. + +### The three registers (ADR-0003, binding) + +Assertions (envelope-wrapped typed proposals) → the elicited model (derived by a pure fold, +never stored) → projections. Write-time-only semantics still governs model assembly: no semantic +act hides inside the fold, and every bridge from user language into the model is a capture. The +acceptance oracle: a second projection must consume the model without rereading the transcript or +semantically interpreting generic capture fields. Promotion, never refusal: low-grade statements +are captured honestly and never promote to a demanded grade without a higher-grade capture +superseding them. + +### Executable-artifact seam (ADR-0005) + +Some projection targets contain authored programs rather than declarative fields. For SDCPN, the +deterministic register-3 output is therefore a scaffold, a typed loss report, and a sidecar of code +obligations. Artifact realization is a downstream, model-assisted application step; it is not part +of `project`, the fold, or the persisted IR. + +```yaml +ProjectionResult: + draftArtifact: unknown + lossReport: LossEntry[] + codeObligations: CodeObligation[] + +CodeObligation: + id: string + target: { elementId: string, field: string } + semanticIntent: string + availableSymbols: { places: string[], tokenFields: string[], parameters: string[] } + supportingCaptureIds: string[] + acceptanceChecks: string[] +``` + +The sidecar is authoritative. A projector may mirror `semanticIntent` into the target code field as +a comment so the incomplete work is visible in the editor, but realization never reconstructs the +contract by parsing comments. The final gate is deterministic: every obligation is fulfilled, all +code compiles through Petrinaut, and at least one scenario simulates without a runtime error. + +### Harness machinery: pure functions classified by what they read + +The harness is the functions; the plugin is the tables they read. This is the +harness/plugin boundary, stated as a typology: + +```yaml +HarnessMachinery: + foldEngine: { reads: foldTable + modelSchema, emits: register-2 model } + demandRunner: + { + reads: demandTable + active anchor captures, + emits: demanded grade per slot, + } + identityResolver: { + reads: identity-bearing proposals, + emits: canonical names, + } # union-find + slotStateDeriver: + { reads: fold output + open issues + absences, emits: slot state per slot } + promotionGuard: { reads: gradeOrder per slot, emits: supersession legality } + affordanceCuer: + { reads: firesWhen x slot states, emits: next-question candidates } + captureEnvelope: fixed # kernel canon; the one non-function; amendable only at the seam +``` + +The slot-state algebra, fold engine, grade/promotion mechanism, identity mechanism, and demand +runner are harness (plugin-SDK) territory; the standard interiors are an importable library; +the node-kind catalog, proposal catalog, affordance annotations, demand table, and loss +categories are plugin-authored. (Ratified as FE-1393 design input.) + +### The plugin contract + +```yaml +PluginContract: + version: immutable string or digest # bound into every completion snapshot/report + modelSchema: NodeKind[] # register 2 — derived, never stored + proposalCatalog: ProposalType[] # register 1 — utterance-shaped, envelope-wrapped + foldTable: FoldRow[] # overrides only; default rules derive from SlotDecl + demandTable: + { version: immutable string or digest, rows: DemandRow[], staticFloor: DemandClause[] } + variantDimension: enum? # e.g. prescribed | practiced; enables slot state 'diverged' + lossCategories: enum[] + +NodeKind: + name: string # closed catalog (Layer-A property 1) + slots: map + completionAnchor: boolean + +SlotDecl: + valueType: string # domain type or standard interior + cardinality: enum # one | set | ordered + gradeOrder: enum[]? # plugin-declared, lowest first; library ladder or composition ladder + +ProposalType: + name: string # named by its semantic act, keyed to its fold target + interior: shape # composed from the standard-interiors library + validators: string[] # tiered: deterministic / semi-mechanical / residue + affordance: + technique: string # ref to an FE-1403/FE-1406 technique card + firesWhen: enum # closed 7-value set, below +``` + +This survives the penciled manifest design as the manifest's `ontology/schema` key: the +prompt-mechanism keys (techniques, lenses, smells, …) are untouched (FE-1403/FE-1406, +the technique-card and prompt-mechanism efforts); `checks`/`tools` receive the proposal and +projection validators; code remains only in validators and projections. + +### Typed proposals, not generic field atoms + +The generic kind+field EAV union is retired. Register 1 is an enumerated catalog of typed +semantic proposals with domain-shaped interiors (`duration-estimate` carrying a +`QuantityStated`, not `quantity-stated` carrying a string). For CPS the first-cut catalog is +~19 types worked from the C1 baseline transcript (entity-noted, attribute-noted, +attribute-domain, population, activity-noted, duration-estimate, resource-requirement, +actor-assignment, precondition-noted, outcome-noted, sequence, trigger, policy-noted, +relationship-noted, question-to-answer, goal-noted, penalty-noted, rationale-noted, same-as). +Constraint, data-binding, and validation-criterion stay schema-present but shallow — one +`*-noted` proposal each — per the September minimum; all three have real C1 instances. + +### The standard-interiors library + +Finiteness claim: interiors are indexed on _how experts state facts_ — measurement scale +types plus logical forms — not on domains. Six families over a shared supertype, plus one +flagged tarpit: + +```yaml +StatedForm: # base — every family extends this + verbatim: string # fidelity anchor; never normalized away + parsed: shape? # mechanical-tier structure; its absence IS the grade floor + # convention: every family declares explicit UNSPECIFIEDNESS MARKERS — slots recording + # what the utterance did NOT say. Grade derives from parsed-ness; markers are exactly + # what the affordance cuer fires on. + +QuantityStated: # 1 — magnitude (ratio/interval): verbal | point | range | quantiles; statistic marker +ComparisonStated: # 2 — order (ordinal): a/b/dimension/direction; direction-without-magnitude marker +RankingStated: # 2b — total/partial order over a set +VocabularyStated: # 3 — category (nominal): dimension + values +ConditionStated: # 4 — predicate: verbatim < structured < formal (the FV ladder lives here) +UniformityClaim: # 5 — quantified claim over a ScopeExpr, with exceptions carve-out +SubjectRef: # 6 — reference/identity; same-as pairs two of them +TemporalPattern: # flagged tarpit — NOT claimed finite; verbatim-heavy, coarse window only +``` + +Number-hedges ("about", "maybe", "roughly") live in `QuantityStated`'s qualifier, not in +capture confidence — two different homes. Capture-level confidence is the picklist +`firm | hedged | speculative`, and the store refuses numeric-parsing confidence strings +(closes remediation item A5). + +### Grade is narrowing of interpretation space + +Grade means "fewer readings remain," never claim strength (that is envelope confidence — +orthogonal by design). Two sources: **form grades** fall out of the library (each family's +ladder is its unspecifiedness markers progressively resolved: verbal < point < range < +quantiles; informal < vocabulary-bound < formal-parsed) — plugins demand rungs, they don't +define these ladders; **composition grades** are plugin-declared (Gherkin's given-only < +given-when < full-gwt is about slot composition, not one statement's form). The fold engine +reads grade off the interior or the fold output, never off the proposal type name — formal +verification's `formalization` is a _different proposal type_ folding into the _same slot_ at +higher grade. Grade orders promotion only: they never choose a winner among active values. +Beyond formal-parsed lies checking, which is validation, not grade. + +### Slot states and fold rules + +```yaml +SlotState: unaddressed | stated(value, grade, supportingCaptureIds) + | absent(absence, captureId) # populatable pending envelope pressure #2 + | conflicted(openIssueIds) + | diverged(prescribed, practiced) # only if the plugin declares a variant dimension + +_foldRuleDerivation: # foldTable holds overrides only; both example fold tables are empty + - cardinality one -> unique (grade order, if declared, gates promotion only) + - cardinality set -> set-union (member removal = supersede that member) + - cardinality ordered, positions stated in-utterance -> ordered-append + - order stated pairwise -> graph-union, order derived at read + - identity-bearing proposal -> union-find + - scope-bearing proposal -> shared-support + # closure: fold behavior = (cardinality) x (graded y/n) + two specials. Finite product + # space; the rule enum is its image. +``` + +One active value occupies a unique slot. Competing active, un-superseded values — at equal or +different grades — fold to `conflicted` with a typed issue, never a silent pick. Values that +are genuine alternatives must be declared as such and handled by the plugin's explicit +alternative rule; they do not authorize the fold to choose one. A higher-grade value replaces +a lower-grade value only through an explicit supersession, after which the lower capture is no +longer active. Regime divergence is per-slot; a regime-split _existence_ (the off-shift wash +that's prescribed-possible, practiced-never) is the degenerate node-level case. The present +`diverged` shorthand exposes neither side's grade nor support captures. Completion therefore fails +a demanded diverged slot with `unevaluable-divergence`; FE-1431 owns an evaluable constituent +shape and the later author-selectable all-sides/either-side rule. + +### Demand, scope, and firing conditions + +Requiredness is question-relative: a static floor (≥1 objective; entity coverage; a happy-path +flow) plus objective-demanded grades (a capacity objective demands quantile-grade durations on +the activities in its support). `DemandClause` is the smallest sufficient union: a presence clause +declares minimum cardinality over `ScopeExpr`; a slot clause declares scope, slot, +minimum grade, accepted epistemic statuses, and accepted absences. Presence clauses express +objective, entity, and path existence. Slot clauses require every selected slot to pass and an +empty selection fails. All matched clauses are conjunctive. Every active completion anchor must +match at least one row. Plugin and demand-table immutable versions are bound with target-document +revision into every evaluation and report. The complete evaluation and report shape live in the +provisional [target-document completion contract](elicitation-completion-2026-08-25-full-draft.md). +`ScopeExpr` has three constructors — `kind`, `where`-filter, `inSupport(anchor)` — and +**September ships kind-only**; the other two are the named growth path. `support(anchor)` is +defined as reference closure over a plugin-declared list of support-bearing proposal types +(proposal interiors carrying SubjectRefs _are_ the edges; the proposal type is the edge type, +per Layer-A property 4). + +`firesWhen` is a closed 7-value enum, every value a predicate over slot states — +`slot-unaddressed | below-demanded-grade | unspecified-marker-present | conflicted-open | +absence-uncorroborated | uniformity-unprobed | identity-ambiguous` — so the affordance column +is mechanically checkable. + +### Identity and supersession + +Identity assertions (`same-as`) are schema citizens: `inferred`, supersedable; reconciliation +is deterministic union-find over them; renames are new identity assertions, never edits. +Retraction residue: a retraction that replaces a claim with a guess-plus-known-unknown is a +`tentative` superseding capture now, plus an absence capture once the envelope's absence +locator lands (see Open strains). + +### The envelope is untouched + +Nothing here amends the capture envelope. The one confirmed pressure — absence captures carry +no payload, so a field-specific absence cannot name its slot — is recorded at the FE-1383 +seam with three concrete C1 cases; the locator an absence needs is precisely a fold-table +`target` coordinate. Adjudication happens at the seam, not by forking. + +## Testing Decisions + +A good test here asserts external behavior at a seam — captures in, slot states out — never +fold internals or intermediate representations. Three seams, one primary: + +1. **The fold** (primary, new, the highest seam available): `fold(pluginContract, +activeCaptures) → model` is pure by construction, so the whole contract is golden-testable + at one seam — hand-worked capture sets in, asserted slot states out. The seed gold set is + the FE-1405 worked-instance set from C1: the anchor changeover utterance (~9 captures → + four activities with graded durations and one rationale; the asymmetry claim derivable, not + captured), the quantile promotion pair (point supersedes to quantiles; slot promotes; no + refusal at any step), the regime-split off-shift wash, and the first-pass-yield absence + (blocked, and _asserted_ blocked, pending the envelope locator). This gold set doubles as + the sweep-accuracy rubric material for FE-1407 (the evaluation/gold-set effort). +2. **The acceptance oracle, executable**: a second projection consumes the folded model with + no transcript access and no generic-field interpretation — enforced structurally by the + projection's input type being register 2 only. For a code-bearing target, this proves the + scaffold and obligation plan; separate realization gates prove that the resulting artifact + compiles and runs. If the scaffold or obligations cannot be written from register 2 alone, + the failure is the finding. +3. **Contract validation**: plugin contract documents validate against the harness + meta-schema; the fold-rule derivation is tested as a table-driven pure function over its + finite product space (cardinality × gradedness + the two specials). + +Everything downstream — demand runner, slot-state deriver, promotion guard, affordance cuer — +reads off the fold's output, so seam 1 covers them without new seams. Modules under test are +FE-1393's fold engine and plugin SDK, with the CPS and Gherkin contracts as fixtures. Prior +art in this repo: the docs-index gate (`test/docs-index.test.ts`) for the table-driven, +fail-loud style, and the capture-store suite (FE-1390 side) for envelope-adjacent fixtures. +Test-fit order stands: smallest honest plugin and Gherkin before CPS. + +## Open strains (first-class, with owners) + +Adjudicated in the FE-1405 session; recorded here so settlement doesn't launder them into +decidedness. + +- **Strain 4 — grade sources (resolved in shape, worked pass pending).** Grade = narrowing; + form grades library-owned, composition grades plugin-declared. The shape is ratified; no + real fold has read a grade yet. Owner: FE-1393 (fold engine), against the gold set. +- **Strain 5 — support closure (downgraded, worked pass pending).** Typed references already + exist (interiors carrying SubjectRefs are edges); `support(anchor)` needs one declared + column and a closure rule, and a hand-worked pass over C1 before `inSupport` ships. + Owner: FE-1393, with FE-1402 (the completion contract) as consumer. +- **Strain 6 — temporal patterns (roped off).** Scheduling is out of scope as a modelling + area; `TemporalPattern` stays verbatim-heavy, coarse window only, no calendar algebra + claimed or planned. Reopening it is a deliberate act, not drift. +- **Strain 7 — sweep-time concentration (recorded; mitigations must travel).** Write-time-only + semantics makes the sweep's utterance→proposal mapping the single point of semantic failure. + Mitigations, none yet built: (a) the family index as a decision-tree sweep skill — route by + statement form, then fold target (FE-1403 authoring, FE-1392 consumes); (b) the proposal + catalog compiled to JSON Schema on the sweep tool input (FE-1392); (c) graceful + under-mapping via honest defaults, with over-mapping — silent hardening — as the real + failure, targeted by the verbatim-containment validators (FE-1392/FE-1393); (d) the + worked-instance gold set as the accuracy rubric (FE-1407). +- **Envelope pressure #2 — absence locator (seam, unresolved).** Absence captures carry no + payload; a field-specific absence cannot name its slot. Three C1 cases logged. The needed + locator is a fold-table coordinate. Current authority: the active soft edge in + [STEERING](../../control/STEERING.md#active-soft-edges), for + adjudication — the register-2 `absent` state is representable but not populatable until it + lands. + +## Out of Scope + +- Amending the capture envelope. Pressure #2 is recorded for adjudication at the seam. +- Scheduling and temporal-pattern modelling (strain 6). +- The `where` and `inSupport` ScopeExpr constructors shipping in September (kind-only ships; + the others are the named growth path). +- Authoring the sweep skill, technique cards, and prompt-mechanism manifest keys + (FE-1392/FE-1403/FE-1406 territory — this spec fixes only the `firesWhen` and `technique` + hook points). +- Implementing artifact realization or the Petrinaut client-tool round trip (FE-1480/FE-1438). +- The full FE-1397-style ratification pass — it is this spec's _condition_, owed before the + provisional marker comes off, not part of its build scope. +- Loss-category content and projection implementations beyond the oracle projection. +- Any UI. + +## Further Notes + +- The provisional CPS technique-card IDs and clarification fragments are now defined in the + [FE-1403 CPS interview guidance](cps-interview-guidance-2026-08-25.md). Its desk replay is design evidence, + not proof that `affordanceCuer` activates those cards correctly at runtime. FE-1403 also exposes + a declarative-authoring seam: several supported cards need one technique under multiple + `firesWhen` predicates, while `ProposalType.affordance` is singular. FE-1431 must decide binding + multiplicity or an evidence-preserving split; the current hook must not silently discard the + disjunction. +- This document is the settled form of the FE-1405 session's working draft and its + ds-pseudo YAML rendering — untracked session artifacts (`drafts/`, per the documentation + protocol) that collapsed into this spec and are not load-bearing anywhere. +- The decision this spec builds on is [ADR-0003](../../adr/0003-three-register-ir.md); the + worked cross-plugin thumbnails (Gherkin thin, formal verification mid, CPS thick) live in + the working draft and discharge the ratification bar only provisionally. +- The authoring story is the product claim to protect: _declare your kinds and slots; declare + the proposals that fill them; annotate how to elicit each; say what your anchors demand._ + Any harness change that breaks the readability oracle — a Gherkin reader can write a third + plugin by analogy — is a regression even if all tests pass. diff --git a/libs/@hashintel/brunch-agent/docs/control/SPEC-LEDGER.md b/libs/@hashintel/brunch-agent/docs/control/SPEC-LEDGER.md index 3e212155bad..d828051bfe4 100644 --- a/libs/@hashintel/brunch-agent/docs/control/SPEC-LEDGER.md +++ b/libs/@hashintel/brunch-agent/docs/control/SPEC-LEDGER.md @@ -40,7 +40,7 @@ states. | Confidence qualitative, never a scalar | §5 | **partial** | non-empty string only; `"0.93"` accepted. Vocabulary is settled by the plugin-contract spec as `firm | hedged | speculative`; its proposed store refusal rule for numeric-parsing strings remains to implement | | Six absence states; `not-mentioned` computed, never stored | §5.1 | **discharged** | `ABSENCE_STATES` (FE-1390) | | Reserved reply encoding for structured taps (C4) | §5.1 | **pending** | UI sends bare text; absences from this UI are honestly `inferred` | -| One epistemic status per capture | §5 | **discharged**, with named friction | Status is the proposal union's discriminant, coupled to provenance shape — per-field status is unrepresentable, and payload-smuggling it breaks dedup identity. This was FE-1405's central input (deep-read FE-1390, tiering section); the arc consumed it _without_ amendment — one status per capture survives, and the structure that wanted per-field status lives below it in proposal interiors (ADR-0003, plugin-contract spec) | +| One epistemic status per capture | §5 | **discharged**, with named friction | Status is the proposal union's discriminant, coupled to provenance shape — per-field status is unrepresentable, and payload-smuggling it breaks dedup identity. This was FE-1405's central input (deep-read FE-1390, tiering section); the arc consumed it _without_ amendment — one status per capture survives, and the structure that wanted per-field status lives below it in proposal interiors (ADR-0003; harness-owned rather than plugin-declared since ADR-0006) | ## Operations & validation (§6) @@ -88,7 +88,7 @@ states. | Per-session state = evidence log, swept high-water mark, pending-affordance slot | §9.2 | **discharged** | pending slot (FE-1389), durable session-log archive (FE-1391), and FE-1392's parse-validated high-water/last-judged bookkeeping under one `sweepHighWater` state slot | | Re-entry briefing; user-visible insertion notice | §9.3 | **pending** | signal carrier proved; no briefing; the one injected signal is filtered out of the UI. Owned by FE-1396 | | Only the true user's side is evidence; injected entries structurally non-user | §9.4 | **partial** | FE-1391 verifies role/purpose against the public projection, refuses signal/advisory text, and classifies affordance replies only from the harness-owned reply-binding signal. The kickoff remains a machine-authored `user` entry until FE-1420/FE-1385 move it to `useInitialData`; FE-1396 still owns briefing-never-evidence | -| Completion derived, never a gate | §9.5 | **pending** | FE-1402 defines completion as a version-bound, read-time derivation over demand clauses and active-objective evidence in [`elicitation-completion.md`](../specs/elicitation-completion.md), then manually replays every prefix of both FE-1361 transcripts in [`elicitation-completion-rehearsal.md`](../evidence/proofs/design/elicitation-completion-rehearsal.md). No progress, stopping, durable delivery, and deferral licensing remain distinct from completion; runtime implementation and mounted proof remain open. | +| Completion derived, never a gate | §9.5 | **pending** | FE-1402's contract is now the invariants of `evaluateCompletion(model, mustKnowRows)` over the plugin file's `Must know` table in [`elicitation-completion.md`](../specs/elicitation-completion.md) (rewritten 2026-08-25 under ADR-0006; the domain-keyed DemandTable draft is archived). The FE-1361 prefix replay in [`elicitation-completion-rehearsal.md`](../evidence/proofs/design/elicitation-completion-rehearsal.md) is a golden-fixture candidate once re-expressed at kind level, not authority. No progress, stopping, durable delivery, and deferral licensing remain distinct from completion; nothing is built. | | Storage port: harness-defined, binding-implemented, plugin-blind (C1) | §9.6 | **discharged for the local target** | core owns capture/archive/anchoring semantics; `binding-flue` owns the file implementation; plugins cannot import the binding (FE-1390 + FE-1391) | | Port scope includes the session-log archive | §9.6 | **discharged** | FE-1391 provisions a versioned target-document record containing capture state and session logs, migrates the legacy capture-only shape on mutation, parses both halves on read, identity-versions evolving messages, and retrieves every cited ordinal independently of Flue | | Compaction vs. durable log | §9.7 | **partial — source-settled, behavioral pin open** | Flue 2.0.3's append-only stream contract and implementation show compaction appends a canonical record, rewrites only model context, preserves the public message projection, and leaves `state_write` reduction untouched. The source-read record reshapes FE-1386 to one upgrade pin; `test/open-gaps.ts` remains until behavioral proof lands | @@ -104,11 +104,11 @@ states. | Obligation | Spec | Status | Evidence | | -------------------------------------------------------------------------------- | ---------- | -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Plugin ownership: packs, forms, validators | §11.1 | **partial** | `plugin-gherkin` now owns its one FE-1392 proposal declaration/schema and target identity; packs, forms, validators, fold, and demand table remain FE-1393/FE-1387 work | -| Pack form, Principle v2 | §11.2 | **pending** | — | +| Plugin ownership: packs, forms, validators | §11.1 | **superseded → partial** | ADR-0006 (2026-08-25) makes a plugin one sectioned Markdown file per target formalism plus `project`/`validate` code ([`plugin-contract.md`](../specs/plugin-contract.md)); cards became kind-indexed `Patterns`, the completion contract became the `Must know` table. `plugin-gherkin` owns its one FE-1392 proposal declaration/schema and target identity; [`sdcpn-plugin.md`](../specs/sdcpn-plugin.md) is authored but unparsed; the file parser, fold, and demand runner remain FE-1393 work | +| Pack form, Principle v2 | §11.2 | **superseded → pending** | ADR-0006 fixes pack form as the heading contract (`Purpose · Kinds · Must know · Patterns · Moves · Deliverable`) with three machine-read tables; Principle v2 still governs the prose sections. No parser or loader exists | | Smallest honest plugin as a standing bar | §11.3 | **partial** | `statement-noted.test.ts` and the core plugin fixture encode the one-type verbatim floor and reject undeclared parsed/pointer shape; the standing bar must grow with FE-1393's operations | | Generic strategy quiver | §11.5 | **pending** (ownership repaired) | was **orphaned** — named-not-designed, carried by no map — now FE-1406 (root issue) | -| Portfolio + hybrid order: both packs authored before the pack interface freezes | §13 | **pending** | Owned by FE-1387 (FE-1383 slice, backlog); current sequencing puts the CPS proof before generic freeze (see `STEERING.md`). Gherkin wiring ahead stays legal while FE-1387 holds the freeze | +| Portfolio + hybrid order: both packs authored before the pack interface freezes | §13 | **superseded → partial** | ADR-0006 makes the interface the heading contract and three table grammars; the SDCPN plugin file is authored (`sdcpn-plugin.md`), the Gherkin file is not. Owned by FE-1387 (FE-1383 slice, backlog); current sequencing puts the SDCPN proof before generic freeze (see `STEERING.md`). Gherkin wiring ahead stays legal while FE-1387 holds the freeze | | Gherkin validation (parse validity, step lexicon) | §13.1 | **pending** | — | | Assurance target (Statement record, four edges, five-stratum derivation, ledger) | §13.2–13.3 | **pending** | — | diff --git a/libs/@hashintel/brunch-agent/docs/control/STEERING.md b/libs/@hashintel/brunch-agent/docs/control/STEERING.md index 729a033d159..488105da4a1 100644 --- a/libs/@hashintel/brunch-agent/docs/control/STEERING.md +++ b/libs/@hashintel/brunch-agent/docs/control/STEERING.md @@ -16,69 +16,84 @@ realization, compilation, and optimisation handoff. The changed element traces t superseding capture while an unrelated region stays stable. Preserve runnable and legibility evidence under [proof evidence](../evidence/proofs/). -Governing strategic decisions: [S-001](STRATEGY-LOG.md#s-001), -[S-004](STRATEGY-LOG.md#s-004), [S-005](STRATEGY-LOG.md#s-005), and -[S-006](STRATEGY-LOG.md#s-006). +Governing strategic decisions: [S-001](STRATEGY-LOG.md#s-001), [S-004](STRATEGY-LOG.md#s-004), +and [S-007](STRATEGY-LOG.md#s-007). Governing architecture: +[ADR-0003](../adr/0003-three-register-ir.md), [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md), +[ADR-0006](../adr/0006-plugins-per-target-formalism.md). -## Selected frontier: design convergence +## Selected frontier: the vertical slice, worked outward from its epicentres -**Claim:** the existing baseline evidence, elicitation research, and worked CPS payload are -sufficient to settle completion versus session stopping, targeted interview guidance, and reusable -strategy without new human or domain-expert input. Those results can then narrow FE-1431 to a -build-ready plugin-authoring contract before runtime implementation resumes. +**Claim:** the shortest route to the acceptance proof is a working elicitation loop in the +production path with one formalism-level plugin, not further design. Every design question still +open is answered by what the slice forces, and answered in code. The design-convergence frontier is +closed: its outputs are test-bed material, and its one durable design result is the plugin file +[`sdcpn-plugin.md`](../specs/sdcpn-plugin.md) ratified by ADR-0006. -The selected single-agent work order is below. Arrows express strategic order. Linear is canonical -for hard blockers and now encodes the three genuine prerequisite joins: FE-1407 blocks FE-1404, -FE-1404 blocks FE-1406, and FE-1406 blocks FE-1431. The other arrows remain soft ordering. +The slice has four epicentres, ordered by the size of the gap they close. Work starts at the +centre of each and moves outward; edges (SDK generality, affordance catalogues, UI breadth, +evaluation apparatus) are not worked until an epicentre needs them. + +| Epicentre | Gap | Issue | +| --- | --- | --- | +| **E1 — controller read path** | The harness writes captures and never reads them back: no fold to a model, no completion over objective slices, no sweep list, no cue to the next turn. The hollow centre between "captured facts" and "conducted an elicitation". | none yet — proposed new issue in `packages/core` | +| **E2 — the SDCPN plugin in code** | The plugin file exists as a spec; nothing parses its three tables, folds captures onto its kinds, or projects from them. | FE-1482 (gist: CPS plugin, redefined as the skeleton epicentre) | +| **E3 — targeted correction** | `supersedes` is unreachable from extraction; no affected-slice computation; no delta; the target-document is still identified with the conversation. | FE-1479 (targeted re-elicitation), FE-1478 (provenance read), FE-1439 (durable session / document boundary) | +| **E4 — the real entry** | Client-tool results do not return to the elicitor; retry/abandonment semantics unproven; realization gated. | FE-1438, FE-1420, FE-1480 | ```text -design resolution -FE-1407 failure catalogue -> FE-1402 completion/stopping contract --> FE-1403 CPS guidance -> FE-1404 condition-3 run --> FE-1406 strategy quiver -> FE-1431 plugin-authoring/absence-locator closure +skeleton (construct job; proves the loop, produces fixtures) +E1 controller read path -> FE-1482 plugin file + parser + fold +-> FE-1404 skeleton run against the baseline simulated expert -reviewer-path implementation +reviewer lane (review-and-revise job; the acceptance proof) FE-1420 retry/abandonment safety -> FE-1438 client-tool return -> FE-1439 durable session +FE-1478 provenance read -> FE-1480 scaffold/realization -> FE-1479 targeted correction join -semantic implementation -FE-1393 exercised plugin SDK -> FE-1482 CPS plugin -> FE-1478 provenance read --> FE-1480 scaffold/realization -> FE-1479 targeted correction join +after the skeleton runs +FE-1393 gherkin under the same six headings (generality check) +FE-1406 lift harness-generic patterns into a harness repertoire +FE-1431 residue: parser + heading contract ``` +Arrows are strategic order. The skeleton lane and the reviewer lane run in parallel; they join at +FE-1479, whose "affected slice", "re-evaluate", and "delta" moves consume E1's fold and completion. +No hard blocker chain remains from the retired design queue. + ### Proof bundle for the selected frontier -- **Bounded scenario:** replay the two existing truck-fleet baseline transcripts, then run condition - 3 with the drafted completion contract, surviving cards, and corrected stopping instrument. -- **Inputs:** the indexed baseline transcripts/readout, FE-1405 worked CPS payload, and indexed - elicitation literature. No live interview or new use-case decision is an input. -- **Procedure and result:** FE-1407 produces the typed failure catalogue; FE-1402 and FE-1403 each - perform their issue-specified desk replay; FE-1404 runs the existing baseline protocol and scores - condition 3 against conditions 1 and 2; FE-1406 keeps only strategies supported by those results. -- **Durable outputs:** reference catalogue, immutable evaluation transcript/raw log/readout, and - amendments to the completion, card, strategy, and plugin-authoring contracts. Index each output - when it lands. -- **Runtime and witness boundary:** this frontier validates design discrimination, not production - behavior or UX. A human witness is therefore inapplicable; the later reviewer-path proof still - requires the deployed production entrypoint and witness. -- **Oracle candidates:** completion/stalling decisions over transcript prefixes, per-card firing - verdicts, and condition-3 regression measures. Promote only categorical claims that survive the - run. +- **Proof 1 — the loop works (skeleton run, FE-1404).** A harness with **no domain knowledge**, + loaded with the SDCPN plugin file, interviews the existing simulated coatings-plant expert + through the production capture, fold, completion, and cue path. Scored against conditions 1 and + 2 on the inherited dimensions, with the FE-1407 failure catalogue as the oracle list. Then the + truck-fleet case (Layer B's validation case; fixture from the inbox SDCPN nets if the dossier + stays missing) through the **unchanged** plugin file: zero new headings, zero new rows. +- **Proof 2 — the acceptance run** as stated in the objective, on the reviewer lane. +- **Inputs:** the plugin file; the baseline situation pack, transcripts, and readout (coatings + plant, not truck fleet); the FE-1407 catalogue; the FE-1402 invariants as tests on + `evaluateCompletion`; the 44-prefix rehearsal as a golden-fixture candidate once re-expressed at + kind level. No new evaluation instrument is built for September: the simulated expert and the + C1/C2 scoring are the fixed instrument. +- **Durable outputs:** production-path code in `packages/core` and `packages/plugin-sdcpn`; the + skeleton transcript and readout under evaluation evidence; amendments to the plugin file only + where the run forces them. +- **Runtime and witness boundary:** Proof 1 needs no human witness; Proof 2 requires the deployed + production entrypoint and a witness record. +- **Oracle candidates:** each FM entry as a harness test; `evaluateCompletion` invariants; + slice-locality of the projection delta (outside-scope region byte-stable). ### Active soft edges -- FE-1402 and FE-1403 remain deliberately soft-ordered after FE-1407 even though they do not depend - on it; all three inputs join at FE-1404 through Linear's hard blockers. -- FE-1431 now defines design closure as a build-ready handoff separately from its later three-target - ratification condition; the unresolved absence locator remains part of that design seam. +- E1 precedes FE-1482 only by the width of an interface: the fold and completion functions are + harness code that the plugin's tables parameterise. Build them together on one branch if that is + faster; do not design the interface before the first plugin exercises it. - FE-1420's idempotency and abandonment semantics precede FE-1438's external-tool protocol; FE-1439 then proves the reviewer path survives reload without crossing principals. -- FE-1393 exercises the smallest honest plugin before CPS pressures the still-unstable SDK in - FE-1482. FE-1478 supplies supporting-capture reads before FE-1480 realization. -- FE-1480's executable proof remains blocked by FE-1438, and FE-1479 follows the semantic/reviewer - join. -- FE-1477 and FE-1440 share one routing implementation. +- FE-1478 supplies supporting-capture reads before FE-1480 realization; FE-1480's executable proof + remains gated by FE-1438. +- FE-1393 follows the SDCPN plugin; gherkin is the generality check, not the tracer that precedes. +- FE-1477 and FE-1440 share one routing implementation; one folds into the other. - FE-1395's structured-tap transport fact inputs the capture-store evidence rule. -- FE-1385/FE-1404/FE-1423 share telemetry vocabulary before FE-1423's exposure gate. +- FE-1385/FE-1423 share telemetry vocabulary before FE-1423's exposure gate. - The living-prototype charter waits on the infrastructure conversation. The read-only Linear graph supplies mechanical availability, never priority. @@ -87,40 +102,47 @@ The read-only Linear graph supplies mechanical availability, never priority. | Gate | Owner / source | Watch trigger | Last checked | Consequence | | --- | --- | --- | --- | --- | -| FE-1480 executable realization unavailable | FE-1438; [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md) | Client tools return code diagnostics to the elicitor. | 2026-08-24 | Scaffold work may proceed; no runnable FE-1480 proof until the gate opens. | -| Plugin authoring not build-ready | FE-1431; [plugin contract](../specs/plugin-contract.md#open-strains-first-class-with-owners) | FE-1404/FE-1406 results land and the absence locator has one owner and representation. | 2026-08-24 | Do not freeze or generalize the SDK; FE-1393 may start only from the narrowed handoff. | -| Final use case outstanding | Dora; FE-1476 / September Plan | Dora confirms or changes it. | 2026-08-24 | If creation is required, activate cold-start and reconcile ADR-0004/proof. | -| Truck-fleet dossier missing from the repository | FE-1382 is Done but its promised `docs/reference/research/` artifact is absent. | Artifact path/branch is supplied or a reviewed replacement is selected. | 2026-08-24 | Existing baseline evidence may support design replay; claim no dossier-backed domain provenance. | +| Controller read path has no owner | E1; no Linear issue exists | An issue is created and a branch opened. | 2026-08-25 | Neither proof can start; the plugin file has nothing to run in. | +| FE-1480 executable realization unavailable | FE-1438; [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md) | Client tools return code diagnostics to the elicitor. | 2026-08-25 | Scaffold work may proceed; no runnable FE-1480 proof until the gate opens. | +| Final use case outstanding | Dora; FE-1476 / September Plan | Dora confirms or changes it. | 2026-08-25 | If creation is required, Proof 1 becomes acceptance-relevant rather than a harness proof; reconcile ADR-0004/proof. | +| Truck-fleet dossier missing from the repository | FE-1382 is Done but its promised `docs/reference/research/` artifact is absent. | Artifact path/branch is supplied or a reviewed replacement is selected. | 2026-08-25 | The generality half of Proof 1 uses a fixture derived from the inbox truck SDCPN and Layer B's worked example; claim no dossier-backed domain provenance. | ## Decision-relevant beliefs and unknowns | Belief or unknown | Confidence / evidence | Cheapest probe | | --- | --- | --- | -| The selected design queue can run without HITL. | Medium-high; every issue has bounded existing inputs and a desk/run oracle. | FE-1407 classifies both baseline transcripts without requesting new product choices. | -| CPS establishes the minimum plugin contract. | Medium-high; Gherkin under-stresses it, while the current SDK is only identity plus one proposal. | Complete the FE-1431 handoff, then exercise Gherkin and CPS in that order. | +| Kind-level rows express the coatings case. | Medium-high; the twenty domain-keyed rows of the FE-1402 rehearsal collapse onto eight kind rows on paper. | Proof 1's first half. | +| The truck-fleet case adds zero headings and zero rows. | Medium; Layer B was validated against it, but never through this file. | Proof 1's second half. | +| The controller read path is small. | Medium; `evaluateCompletion` is ~10 invariants over a fold the store already supports. | Build E1; if it exceeds the plugin file in size, stop and look. | | Field-local code obligations support localized realization and repair. | Low-medium; the corpus and Petrinaut diagnostics are field-addressed, but no Brunch run exists. | Realize one stochastic transition without rewriting an unrelated field. | -| Five turns yield a scoped correction. | Low; unrehearsed. | Run two bounded rehearsals. | +| Five turns yield a scoped correction. | Low; unrehearsed. The review-and-revise runbook in the plugin file is the first concrete trajectory. | Run two bounded rehearsals against a fixture model. | | Ask carries durable client-tool results. | Medium-low; machine results refused today. | Run one correlated FE-1438 round trip. | | Structured export explains provenance/delta. | Medium; FE-1481 permits it. | Witness one rehearsal. | ## Sequencing cuts -- Cold-start does not gate review-and-revise ([S-001](STRATEGY-LOG.md#s-001)). -- FE-1393's smallest-honest Gherkin/SDK exercise precedes CPS without freezing the interface; - FE-1482 then pressures it before FE-1387's generic freeze ([S-005](STRATEGY-LOG.md#s-005)). -- Finish the selected design queue before runtime feature work; then build the under-built reviewer - path before returning to FE-1480 realization ([S-005](STRATEGY-LOG.md#s-005)). -- During design convergence, do not implement SDK surface, client tools, projection, provider - routing, or deployment. +- Cold-start does not gate review-and-revise ([S-001](STRATEGY-LOG.md#s-001)); the skeleton's fold, + completion, and cue serve both jobs, which is why it is built first. +- Implement the slice; design only what the slice forces ([S-007](STRATEGY-LOG.md#s-007)). No new + spec precedes the code that needs it. +- Plugins are per target formalism and domain-neutral by rule ([ADR-0006](../adr/0006-plugins-per-target-formalism.md)). + No domain content enters a plugin file; a case that seems to need it is a finding for an ADR. +- Evaluation instruments stay smaller than the thing they evaluate. The simulated expert and the + inherited scoring are the fixed instrument for September. +- Gherkin follows SDCPN as the generality check; no generic SDK freeze (FE-1387, gist: generic + plugin freeze) before September. - Defer broad UI/ontology/gallery/affordances/voice/scenarios/telemetry until the loop closes. - Fixtures supply domain state, never product wiring; provenance and the real entrypoint are gates. ## Stop or replan - Dora requires cold-start creation. -- A selected design issue requires an unrecorded product preference or new domain testimony. -- Condition 3 cannot distinguish or improve the failures FE-1402/FE-1403 claim to address. -- FE-1431 cannot isolate a build-ready design handoff from its later empirical ratification. +- **Proxy completion:** an arc ends with durable outputs that are all desk, simulated, or + evaluation-side and no production-path code changed (recurred twice: tracer-as-done, + instrument-as-done). +- Proof 1 shows a `Must know` that kind-level rows cannot express, or the truck-fleet case needs a + new heading (ADR-0006's revisit condition). +- E1 exceeds the plugin file in size, or needs a persistence surface. - A code obligation cannot localize realization without whole-net resynthesis. - Two rehearsals fail the five-turn correction. - FE-1438 loses correlation, durability, or evidence semantics. @@ -132,8 +154,10 @@ The read-only Linear graph supplies mechanical availability, never priority. whether it joins the cold-start lane or receives a parent. - **FE-1334** — surprising-scenario validation gesture; parent or cancel when its owning outcome is chosen. -- **FE-1406** — intentional cross-map root for the harness strategy quiver. +- **FE-1406** — harness strategy repertoire; shrunk to the post-skeleton lift of harness-generic + patterns out of the plugin file. - **FE-1472** — unrelated SDK-pin triage; assign an owning map or remove from the project. - **FE-1476** — September delivery root; intended parent is FE-1357. - **FE-1477–FE-1482** — PM-authored outcome roots; intended parent is FE-1476 after overlap review and separately approved Linear mutation. +- **E1 controller read path** — no issue yet; the one new issue this replan proposes. diff --git a/libs/@hashintel/brunch-agent/docs/control/STRATEGY-LOG.md b/libs/@hashintel/brunch-agent/docs/control/STRATEGY-LOG.md index 9be216f411f..0db6adfcae8 100644 --- a/libs/@hashintel/brunch-agent/docs/control/STRATEGY-LOG.md +++ b/libs/@hashintel/brunch-agent/docs/control/STRATEGY-LOG.md @@ -155,3 +155,65 @@ decision that the FE-1431 handoff failed to settle. **Evidence links:** [STEERING selected frontier](STEERING.md#selected-frontier-design-convergence), FE-1407, FE-1404, FE-1406, FE-1431 + +### S-007 + +**Date:** 2026-08-25 + +**Trigger/evidence:** The S-005 design-convergence queue ran to its end without changing a line of +production-path code. FE-1402's rehearsal needed an oracle and authored a DemandTable keyed to the +baseline coatings-plant domain (`where(kind, role=…)` scopes, `ROW-BREAKDOWN`-style objective rows), below +the level at which the IR spec's Layer B had already defined the plugin; the contract's "September +ships kind-only" was correct and the oracle was wrong. FE-1403's five `domain` cards each lift to a +kind-level pattern. FE-1404 produced a 5,400-line preregistered instrument through nine rejected +review renderings, never ran, and committed the rejected draft lock (stale hashes, missing paths; +its own `--verify-seal` refuses it); structurally it is a shadow harness — operator, projection +schema, diagnostic priority, novelty streak, and sealed segments each mirror a harness component +that the S-005 cut forbade building. The whole queue worked the cold-start lane while the +objective is review-and-revise (S-001: cold-start does not gate it; fixtures supply the prebuilt +model). Factual correction: STEERING's proof bundle named "truck-fleet baseline transcripts"; the +baseline situation pack is a coatings plant (Vestera Coatings, Production Process +Scheduling), and truck-fleet is Layer B's validation case, whose dossier is still missing (see the +FE-1382 gate). "Tracer as definition of done" recurred as "desk rehearsal / preregistered +instrument as definition of done". + +**Decision:** Invert S-005 and S-006: implement the vertical slice and design only what the slice +forces. Adopt [ADR-0006](../adr/0006-plugins-per-target-formalism.md): plugins are per target +formalism, authored as one sectioned Markdown file; `docs/specs/sdcpn-plugin.md` is the exemplar. +Close the design-convergence queue: FE-1407, FE-1402, and FE-1403 are reclassified as test-bed +material; FE-1404 is redefined as the skeleton run — condition 3 as the protocol originally +defined it (kernel harness + real plugin), not the shadow-harness instrument; FE-1406 shrinks to +lifting harness-generic patterns out of plugin files; FE-1431 mostly dissolves (a file format and +a parser, not seven authoring seams); FE-1393 is demoted to a post-skeleton generality check (a +second formalism adds zero headings). Promote the reviewer lane (FE-1420 → FE-1438 → FE-1439) and +the semantic lane (FE-1478 → FE-1480 → FE-1479) to primary. The skeleton epicentre is FE-1482 plus +a harness controller read path (captures → model → next move), which has no issue yet. Rejected: +finishing the design queue as planned, because its outputs are desk artefacts that cannot be +wrong in a way the product would notice, and running condition 3 with the shadow harness, because +it would measure an instrument the product will never ship and displace the kernel a second time. + +**Consequences/cuts:** The hard-blocker chain FE-1407 → FE-1404 → FE-1406 → FE-1431 recorded by +S-006 is removed; Linear mutation remains a separately approved step. The plugin-contract, +completion, and interview-guidance drafts are archived; the specs shrink to the heading contract, +table grammar, `project`/`validate` seam, and `evaluateCompletion` invariants. The FE-1404 +instrument is archived as test-bed; its Valibot projection schema and validators are salvage. +Beliefs updated: "the design queue can run without HITL" is retired as answered-but-irrelevant — +it ran, and proved nothing the product needed; "CPS establishes the minimum plugin contract" +rises to high, evidenced by the one-file plugin. New heuristic recorded in the steering protocol: +an evaluation instrument larger than the thing it evaluates is itself the finding; a frontier whose +durable outputs are all desk, simulated, or evaluation-side, with no production-path code changed +by the end of one arc, triggers replan. + +**Revisit when:** The skeleton's parser, fold, or `evaluateCompletion` cannot be built from the +sectioned file without a typed declaration the file cannot carry, or a second formalism needs a +heading the contract does not have (ADR-0006's condition). + +**Supersedes:** S-005, S-006 + +**Evidence links:** [ADR-0006](../adr/0006-plugins-per-target-formalism.md), +[sdcpn plugin file](../specs/sdcpn-plugin.md), +[IR spec Layer B](../specs/intermediate-representation.md#layer-b--the-cps-plugins-ir), +[archived drafts](../archive/specs/), +[baseline situation pack](../../evaluations/cases/process-model-elicitation/baseline/situation-pack.md), +[condition-3 instrument](../../evaluations/protocols/process-model-elicitation/baseline/), +FE-1402, FE-1403, FE-1404, FE-1406, FE-1431, FE-1482 diff --git a/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-desk-replay.md b/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-desk-replay.md index 8f4947c8e9c..9f0eaf95424 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-desk-replay.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-desk-replay.md @@ -6,7 +6,7 @@ utterance available before condition 2's eleventh interviewer response. ## Fixed inputs and method -- guidance under test: [`cps-interview-guidance.md`](../../../specs/cps-interview-guidance.md) +- guidance under test: [`cps-interview-guidance.md`](../../../archive/specs/cps-interview-guidance-2026-08-25.md) (archived 2026-08-25; its cards are now patterns in [`sdcpn-plugin.md`](../../../specs/sdcpn-plugin.md)) - completion oracle: `cps-baseline-replay/2026-08-24.3` from the FE-1402 rehearsal - failure signatures: the reviewed FE-1407 catalogue - transcripts: FE-1361 condition 1 and condition 2, one run each diff --git a/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-plain.md b/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-plain.md index b65812703ad..9ecb20aac9f 100644 --- a/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-plain.md +++ b/libs/@hashintel/brunch-agent/docs/evidence/proofs/design/cps-interview-guidance-plain.md @@ -1,7 +1,7 @@ # CPS interview guidance in plain language This is the second-register rendering of the provisional -[CPS interview-guidance contract](../../../specs/cps-interview-guidance.md). A separate renderer +[CPS interview-guidance contract](../../../archive/specs/cps-interview-guidance-2026-08-25.md) (archived 2026-08-25 under ADR-0006; its cards are now patterns in [`sdcpn-plugin.md`](../../../specs/sdcpn-plugin.md)). A separate renderer received the spec and desk replay without the producing trajectory. The rendering is reviewer-facing; the specification remains the required-behavior authority. diff --git a/libs/@hashintel/brunch-agent/docs/specs/elicitation-completion.md b/libs/@hashintel/brunch-agent/docs/specs/elicitation-completion.md index 47b55fa8332..7c60c92f918 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/elicitation-completion.md +++ b/libs/@hashintel/brunch-agent/docs/specs/elicitation-completion.md @@ -1,339 +1,120 @@ -# Spec: target-document completion and session stopping +# Spec: completion — the invariants `evaluateCompletion` must satisfy -Status: **provisional** — manually desk-scored over the two FE-1361 baseline transcripts. The -replay validates design discrimination, not implementation. FE-1402 owns this required-behavior -contract; plugin authoring consumes it. +Status: **provisional**, rewritten 2026-08-25 under +[ADR-0006](../adr/0006-plugins-per-target-formalism.md); FE-1402 owns it. The previous draft, +with its CPS `DemandTable` and `where`-scoped clause vocabulary, is archived at +[`elicitation-completion-2026-08-25-full-draft.md`](../archive/specs/elicitation-completion-2026-08-25-full-draft.md); +the FE-1402 [rehearsal](../evidence/proofs/design/elicitation-completion-rehearsal.md) is a +golden-fixture candidate once re-expressed at kind level, not authority. -This specification defines a read-time answer to one question: given one version-bound snapshot -of the durable evidence, derived model, active objectives, and plugin demands, is the -target-document complete enough for those objectives? The companion -[rehearsal](../evidence/proofs/design/elicitation-completion-rehearsal.md) owns the provisional CPS -oracle and prefix verdicts. The -[plain rendering](../evidence/proofs/design/elicitation-completion-plain.md) checks this contract in -a second register. - -## Required distinctions - -These facts can coincide, but none except the first asserts completion: - -| Fact | Meaning | Effect on completion | -| --- | --- | --- | -| Target-document completion | Every static demand, universal active-anchor support check, and objective demand passes. | This is the computation. | -| Session stopping | A conversation is intentionally quieted or produces no more entries. | None. | -| User-requested quiet | The user asks to pause, leave, or receive no further questions. | None. | -| Delivery | A current projection or another promised result is emitted. | None. | -| No progress | Recent frames add no demanded evidence or state and deliver nothing. | None; session control may require adjudication. | -| Budget exhaustion | A turn, token, time, or cost bound ends the session. | None. | -| Licensed deferral | Session control verifies that existing durable authorities can support recoverable re-entry. | None. | - -“Best useful result within this session” is therefore ordinary behavior: durably deliver the best -current projection with its loss and open obligations, license later continuation only from -existing authoritative state, and stop the session while the completion boolean remains false. - -## Version-bound input snapshot - -The harness evaluates one immutable snapshot: - -```yaml -CompletionInput: - targetDocumentRevision: opaque immutable revision - pluginContractVersion: immutable plugin version or digest - demandTableVersion: immutable demand-table version or digest - model: register-2 derived model at targetDocumentRevision - activeCaptures: evidence metadata reachable from model support links - openIssues: issue state at targetDocumentRevision -``` - -The plugin version and demand-table version are part of the identity of the verdict. If the target -document or either plugin version changes during the read, the caller retries. It must not combine -model state from one revision with demands from another. - -The computation reads active objectives, the plugin's static floor and demand rows, derived slot -states, grades, epistemic statuses, evidence spans or bases, and open issues. It does **not** read -conversation fluency, self-assessment, turn count, delivery state, recent novelty, session state, -or a deferral-licensing report. - -The plugin owns the demand declaration. The harness owns deterministic expansion and evaluation. -The session controller consumes the report but cannot author or override it. - -## Demand algebra - -The smallest required algebra distinguishes existence from slot quality: - -```yaml -DemandTable: - version: immutable string or digest - staticFloor: DemandClause[] - rows: DemandRow[] - -DemandRow: - id: stable plugin-local identifier - whenObjective: pattern over one active completion anchor - clauses: DemandClause[] - -DemandClause: - PresenceClause | SlotClause - -PresenceClause: - id: stable plugin-local identifier - type: presence - scope: ScopeExpr - minimumCount: positive integer - -SlotClause: - id: stable plugin-local identifier - type: slot - scope: ScopeExpr - slot: slot name - minimumGrade: rung in that slot's grade order - acceptedEpistemicStatuses: non-empty subset of capture-envelope statuses - acceptedAbsences: subset of capture-envelope absence states, default empty -``` - -All static clauses and all clauses from every matched objective row are conjunctive. A presence -clause counts model nodes selected by its scope; it is how a plugin declares objective, entity, or -path cardinality. A slot clause evaluates a named slot on every selected model node. An empty slot -selection fails with `no-selected-slot`; existence cannot pass accidentally through an empty -selection. - -This algebra adds no graph-query language. It uses the plugin contract's `ScopeExpr`. September may -ship kind-only scopes. The already named `where` and `inSupport(anchor)` constructors remain future -growth paths; this contract does not generalize them. - -### Universal active-anchor support - -Independently of the static floor, the harness checks every active node whose kind declares -`completionAnchor: true`: - -1. match that anchor against every demand row using the plugin-declared objective pattern; -2. fail that anchor with `unsupported-active-anchor` when no row matches; and -3. evaluate the union of clauses from every matching row when at least one matches. - -No objective is silently ignored, and the static floor cannot substitute for this universal -check. A target-document with no active completion anchor fails through its plugin-declared -presence clause in the floor. - -### Status, grade, and confidence - -Epistemic status and grade are independent: - -- status says how content relates to its source (`explicit`, `inferred`, `tentative`, `defaulted`, - or `external-lookup`); -- grade says how narrow the slot value's interpretation space is; and -- confidence says claim strength and satisfies neither requirement. - -Statuses are not ordered. A slot clause explicitly lists accepted statuses. An `inferred` capture -uses evidence spans under the existing envelope contract; a demand may accept it directly. A -`documented-transformation` basis belongs to `external-lookup`, not to `inferred`. Grade and -traceable evidence are checked separately from status. - -## Evaluation +## The function ```text -evaluateCompletion(input, plugin): - require input.pluginContractVersion == plugin.version - require input.demandTableVersion == plugin.demandTable.version - - floor := expand and evaluate every static clause - anchors := every active completion-anchor node - - for each anchor: - matchedRows := all demand rows matching anchor - anchorSupport.pass := matchedRows is not empty - obligations := expand and evaluate every clause in matchedRows - anchor.pass := anchorSupport.pass and every obligation passes - - complete := every floor clause passes - and every active anchor passes - - return the version-bound evidence-bearing report -``` - -The boolean is the only completion value required. Diagnostics explain it; they are not a second -public status vocabulary. - -### Presence evaluation - -A presence clause passes when the selected node count is at least `minimumCount`. The report -includes the selected node IDs. Presence checks model cardinality only; they do not manufacture -evidence or grade. - -### Slot evaluation - -Every selected slot must pass: - -- `stated(value, grade, supportingCaptureIds)` passes when grade meets the declared minimum, all - support needed for the folded value is active and traceable, and every supporting status the - fold relies on is accepted. -- `unaddressed` fails with `unaddressed`. -- `absent(absence, captureId)` passes only when that exact absence is accepted, the capture is - active and traceable, and its status is accepted. `not-mentioned` cannot pass because it is a - computed fact, not evidence. -- `conflicted(openIssueIds)` fails with `open-conflict` until explicit resolution. -- `diverged(prescribed, practiced)` fails with `unevaluable-divergence`. The canonical shorthand - does not expose grade and supporting captures for each side, so this contract cannot evaluate - either constituent honestly. - -Other diagnostics are `below-minimum-count`, `no-selected-slot`, `below-required-grade`, -`inadmissible-status`, `unaccepted-absence`, `missing-evidence`, `unsupported-active-anchor`, and -`version-mismatch`. -Open issues outside selected demand coordinates remain visible but do not block objective-relative -completion. - -## Evidence-bearing report - -```yaml -CompletionReport: - targetDocumentRevision: opaque immutable revision - pluginContractVersion: immutable plugin version or digest - demandTableVersion: immutable demand-table version or digest - complete: boolean - floor: - - clauseId - clauseType - selectedModelCoordinates - requirement - actual - pass: boolean - diagnostics - supportingCaptureIds - openIssueIds - objectives: - - anchorNodeId - supportingCaptureIds - matchedDemandRowIds - supportCheck: - pass: boolean - diagnostics - obligations: same assessment shape as floor - pass: boolean -``` - -The report follows register-2 support links to deposited captures and carries their capture IDs. It -never semantically rereads the transcript. - -## Session control and no progress - -After each settled sweep, session control may read the completion report plus separate session -facts. It may ask, deliver and defer, or stop. None rewrites the report. A later evidence -change can also make a previously complete target-document incomplete; completion never locks it. - -Runtime no-progress policy remains outside this contract. A candidate detector may compare frames -for newly deposited demanded evidence, demanded slot or obligation changes, and delivery changes. -An advisory can force session-control adjudication but cannot supply a positive completion verdict. -The rehearsal owns one bounded threshold solely to score the baseline. - -## Read-time deferral licensing - -Licensed deferral is a session-control decision computed from existing authorities. It creates no -third persistence surface and writes no target-document truth. The session controller owns this -pure projection: - -```yaml -DeferralLicensingInput: - completionReport: version-bound CompletionReport - captureStore: - snapshotRevision: immutable revision inspected by completion - locatedIssuesAndAbsences: existing capture-store records - sessionLog: - archivePointer: durable archive pointer - archiveRevision: immutable revision or digest - sweptHighWaterMark: durable entry identifier - unsweptTail: bounded entry range or empty - pendingAffordanceSlot: - stateRevision: immutable revision or digest - affordanceId: identifier? - delivery: - projectionRef: durable projection/artifact pointer? - sourceTargetDocumentRevision: immutable revision? - -BlockingObligationLocator: - ModelCoordinateLocator | UnresolvedTargetLocator - -ModelCoordinateLocator: - type: model-coordinate - coordinate: register-2 model coordinate - issueIds: existing identifiers[] - absenceCaptureIds: existing identifiers[] - -UnresolvedTargetLocator: - type: unresolved-target - clauseId: stable demand-clause identifier - scope: exact ScopeExpr - diagnostic: below-minimum-count | no-selected-slot - -DeferralLicensingReport: - inspected: - targetDocumentRevision - pluginContractVersion - demandTableVersion - captureStoreSnapshotRevision - sessionArchivePointer - sessionArchiveRevision - sweptHighWaterMark - unsweptTailRange - pendingAffordanceStateRevision - pendingAffordanceId - deliveryProjectionRef - deliverySourceTargetDocumentRevision - blockers: - - clauseId - locator: BlockingObligationLocator - checks: - sweepAndArchiveStateRecoverable: boolean - everyBlockingObligationLocated: boolean - currentBestProjectionDurablyDelivered: boolean - reentryFactsRecoverable: boolean - licensed: boolean - diagnostics: string[] +evaluateCompletion(model, mustKnowRows) -> CompletionReport ``` -The locator is a union because a blocker may already have a model coordinate, issue, or explicit -absence, while a failed presence clause or `no-selected-slot` has no selected model node. The -second arm preserves the unresolved clause and exact scope instead. Every completion blocker must -have one arm; a missing locator keeps `everyBlockingObligationLocated` false. - -`sweepAndArchiveStateRecoverable` permits an explicitly bounded unswept tail only when the archive -pointer, archive revision, and swept high-water mark recover it exactly; it does not silently call -that tail settled. - -`licensed` is true only when all four checks pass and every inspected revision, digest, pointer, -high-water mark, pending-affordance fact, and projection reference still resolves to the state -shown in the report. The durable delivery must be the best current projection for the same -`targetDocumentRevision` evaluated by completion; absent delivery fields make -`currentBestProjectionDurablyDelivered` false. The report itself is ephemeral: it may be -recomputed for audit, but it is not stored as target-document or capture-store truth. - -The current authoritative schemas carry no durable undelivered-delivery obligation with reason, -owner, and next action. Therefore an undelivered best result cannot license deferral. Such an -obligation remains successor strain and requires an approved durability-contract owner; it must not -be invented in `CaptureIssue` or in a new record here. - -### Existing-operation sequence - -Before quieting, session control uses existing operations in this order: settle/sweep and archive; -recompute version-bound completion; locate every blocker; durably deliver the best current -projection; validate recoverable session-log/capture-store changes and the existing pending -affordance; compute the licensing report; then quiet only if `licensed` is true. On re-entry it -reloads the same authoritative surfaces, validates the report's bound revisions and pointers, and -recomputes completion and licensing. There is no create/update/consume lifecycle for a new deposit. - -## Claims, limits, and replay - -This contract claims only deterministic, version-bound completion and deferral-licensing -computation shapes over existing authorities. It does not show that the harness, plugin SDK, -capture store, sweep, controller, detector, binding, projection, or application implements them. -FE-1407's specified and candidate prevention claims keep those grades. - -The [rehearsal](../evidence/proofs/design/elicitation-completion-rehearsal.md) owns the provisional -CPS demand table, all prefix assessments, failure-signature verdicts, amendments found by replay, -and successor evidence. Keeping those judgment-bearing results out of required behavior prevents a -single baseline oracle from becoming generic plugin canon. +`model` is the register-2 derived model at one target-document revision +([ADR-0003](../adr/0003-three-register-ir.md)). `mustKnowRows` is the parsed `## Must know` +table of one plugin file at one plugin version, with the static floor stated under it +([`sdcpn-plugin.md`](sdcpn-plugin.md) is the exemplar). The function is pure and reads nothing +else: not the transcript, conversation fluency, turn count, delivery state, session state, or a +deferral report. Each numbered statement below is a test the implementation must pass; the +[plain rendering](../evidence/proofs/design/elicitation-completion-plain.md) explains the same +rules in a second register. + +## Shape of the answer + +1. **A derived boolean plus an evidence-bearing report; never a gate, never a lifecycle status.** + `complete` is recomputed on every read (kernel §9.5). The report lists, per failing demand, the + node, slot, requirement, actual state, diagnostic, and supporting capture ids reached through + register-2 support links. Diagnostics explain the boolean; they are not a second public status + vocabulary, and no `complete` value is ever persisted or locks the document. +2. **Version-bound.** The report carries the plugin version and the target-document revision it + read. Model state from one revision is never evaluated against rows from another; the caller + retries on mismatch (`version-mismatch`). + +## The rule + +3. **Static floor first, as counts only.** Before objective-relative depth counts at all, the + model must contain the floor the plugin states (for SDCPN: ≥1 `objective`, ≥2 `entity-type`, + ≥1 `activity`, ≥1 `ordering/flow` with order spelled out). A floor check is a count of nodes + of a kind; it assigns no precision and manufactures no evidence. Failing the floor fails + completion regardless of any slot's quality. +4. **Presence is separate from slot quality.** Whether a node exists and whether its slots meet + their rows are two checks with two diagnostics (`below-minimum-count`, `below-required- + precision`). Neither passes on the strength of the other. +5. **Question-relative over the floor.** Every node in the dependency slice of every active + `objective` must satisfy every `Must know` row for its kind. Nodes outside every slice are + recorded but not demanded; their open issues stay visible and do not block. +6. **Universal active-anchor check.** Every active `objective` must have a non-empty dependency + slice (its "the nodes it depends on" row, precision `at least 1`). An objective that depends + on nothing fails with `unsupported-active-objective`; no objective is silently ignored, and the + floor cannot substitute for this check. +7. **An empty selection fails.** A row whose kind has a node in the slice, but whose slot + selects nothing on that node, fails with `no-selected-slot`; a demand never passes through an + empty selection. + +## What counts as a value + +8. **Status ≠ precision ≠ confidence; statuses unordered.** Epistemic status (`explicit`, + `inferred`, `tentative`, `defaulted`, `external-lookup`) says how content relates to its + source; precision says how narrow the value is; confidence says claim strength. No ordering + is defined over statuses. Each row's accepted statuses are explicit on that row or in the + plugin's stated default (SDCPN: stated by the expert, or inferred and confirmed); a value under + any other status fails with `inadmissible-status`, however precise or numeric it is. +9. **`not-mentioned` never passes.** It is a computed fact, not evidence; an unaddressed slot + fails with `unaddressed`. +10. **"Unknown" / "later" is not a value.** "I don't know", "we'll measure it", and a promised + source leave the slot open (recorded with the pointer, per pattern P10) and failing. +11. **An explicit accepted absence is a value only where the row allows it.** "Never happens" or + "not applicable" passes only on a row whose `"not applicable" allowed` cell is `yes`, only + when the absence is an active, traceable capture under an accepted status; elsewhere it + fails with `unaccepted-absence`. +12. **Precision is checked against the row's word, not the number's look.** `range` does not + satisfy `spread`; a `number` does not satisfy `range`; `spelled out` needs the structure a + second reader could apply. A value below the row's precision fails with + `below-required-precision` and the report names the smallest delta (pattern P12). +13. **Conflict and divergence fail conservatively.** A slot with two or more competing active + captures fails with `open-conflict` until an explicit, user-cited resolution closes it. A + slot whose `prescribed` and `practiced` readings diverge unresolved fails with + `unresolved-divergence`; the function never averages, picks a side, or scores the more + precise side as the value. +14. **Evidence must be reachable.** A stated value whose supporting captures are not active and + traceable through register-2 support links fails with `missing-evidence`. + +## What leaves the boolean untouched + +15. **Stop, delivery, quiet, budget, and no-progress are not inputs.** A user asking to stop or + pause, a delivered projection, an exhausted turn/token/time budget, and a detector's + no-progress advisory are session facts. None of them appears in `evaluateCompletion`'s + arguments, and re-running it before and after any of them yields the same report for the + same `(model, mustKnowRows)`. Session control may ask, deliver, or stop on reading the + report; it cannot author or override it. "Best useful result within this session" is + delivering the current projection with its loss report while `complete` stays `false`. +16. **A later capture can make a complete document incomplete.** Completion never locks. + +## Deferral licensing + +17. **A read-time projection over existing authorities.** Whether a session may quiet with a + recoverable re-entry is a session-control computation over the completion report, the + capture-store snapshot revision, the session-log archive pointer and swept high-water mark, + the pending-affordance slot, and the delivered projection reference. It is recomputed, never + stored, and writes no target-document or capture-store truth. +18. **An undelivered best result cannot license deferral.** No authoritative schema carries a + durable undelivered-delivery obligation, and none may be invented here; absent a durable + delivery of the best current projection for the evaluated revision, licensing is `false`. +19. **No new persistence surface.** Neither completion nor licensing adds a record type, a + lifecycle enum, a third store, or a field on `CaptureIssue`. + +## Fixtures + +The seed golden set is the FE-1402 rehearsal's prefix verdicts over the two FE-1361 transcripts, +re-expressed as (model, rows) pairs at kind level: an objective with an empty slice, a +range-not-spread duration, an unknown-as-value refusal, an unresolved regime divergence, and an +explicit-never absence on an allowing row. The condition-3 frozen table is test-bed material. ## Out of scope -- runtime, detector, controller, or TypeScript implementation; -- capture-envelope, `CaptureIssue`, session-state, or durability-contract changes; -- final CPS demand-table authoring; -- projection, realization, or delivery validation; and -- a public lifecycle-status enum. +Runtime, detector, controller, and TypeScript implementation; capture-envelope, `CaptureIssue`, +session-state, or durability-contract changes; projection, realization, delivery validation; any +public lifecycle-status enum. diff --git a/libs/@hashintel/brunch-agent/docs/specs/elicitation-kernel.md b/libs/@hashintel/brunch-agent/docs/specs/elicitation-kernel.md index bf5274dc17c..3da1c64e690 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/elicitation-kernel.md +++ b/libs/@hashintel/brunch-agent/docs/specs/elicitation-kernel.md @@ -11,6 +11,20 @@ Amended 2026-08-24 by [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md): code-bearing projections emit deterministic scaffolds and obligations; executable realization is downstream agent work. +### Supersession map (2026-08-25) + +The August text below stays as the record of what was decided. Where a later accepted decision +carries the operating truth, this map names it; the section itself is not rewritten. + +| Kernel section | Now governed by | +| ------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| §5 envelope, §8 sweep and supersession, §11.1 "own payload structure" | [ADR-0003](../adr/0003-three-register-ir.md): captures are register 1; the elicited model is register 2, derived by a pure fold and never stored; projections are register 3. Envelope semantics unchanged. | +| §6.1 `project` for code-bearing targets; §14.1 invariants 3 and 8 | [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md): the pure projection emits a scaffold, a typed code-obligation sidecar, and the loss report; executable realization is downstream application work. | +| §9.5 completion derived, never a gate | [`elicitation-completion.md`](elicitation-completion.md): the invariants of `evaluateCompletion(model, mustKnowRows)` over the plugin file's `Must know` table, under [ADR-0006](../adr/0006-plugins-per-target-formalism.md). | +| §11.1 ElicitationPack (kernel cards, completion contract, clarification hints); §11.2 pack form | [ADR-0006](../adr/0006-plugins-per-target-formalism.md) and [`sdcpn-plugin.md`](sdcpn-plugin.md): a plugin is one sectioned Markdown file per target formalism with fixed headings (`Purpose · Kinds · Must know · Patterns · Moves · Deliverable`); cards became kind-indexed `Patterns`, the completion contract became the `Must know` table, clarification hints became `Moves` steps. Principle v2 still governs the prose sections. `project`/`validate` remain plugin code ([`plugin-contract.md`](plugin-contract.md)). | +| §11.5 generic strategy cards | Unchanged in principle (guidance ownership follows vocabulary ownership); still named, not designed (FE-1406). Any harness-generic guidance would take the same `Patterns`/`Moves` shape. | +| §13 portfolio and hybrid order ("both packs authored before the pack interface freezes") | [ADR-0006](../adr/0006-plugins-per-target-formalism.md): the interface is the heading contract and the three table grammars; the SDCPN file is authored, the Gherkin file is not; sequencing is owned by [STEERING](../control/STEERING.md). §13.1–13.3 target content is unchanged. | + "Elicitation kernel" and "brunch-lite" are working labels; the real product name is unresolved fog. No architectural string bakes in either label (see [Naming](#123-naming--tool-namespacing)). diff --git a/libs/@hashintel/brunch-agent/docs/specs/intermediate-representation-plain.md b/libs/@hashintel/brunch-agent/docs/specs/intermediate-representation-plain.md index 65a52792bcf..240f53fa224 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/intermediate-representation-plain.md +++ b/libs/@hashintel/brunch-agent/docs/specs/intermediate-representation-plain.md @@ -5,6 +5,10 @@ > a review instrument: seven places where the source resisted plain rendering are recorded as > findings on FE-1401 (third accrual), the load-bearing one being the loss report's unresolved unit > of loss (capture vs. capture-facet). +> +> Since 2026-08-25, [`sdcpn-plugin.md`](sdcpn-plugin.md) is the concrete rendering of Layer B: its +> `Kinds` and `Must know` tables carry the ten kinds, the cross-kind attributes, and the +> question-relative completion rule described below as the one authored plugin file. This design was resolved on 13 August 2026 and ratified on worked examples by FE-1397. It has two layers. Layer A defines what an intermediate representation (IR) is for any plugin. Layer B is the concrete design for the CPS plugin — the design the September demo will run on. The design draws on the kernel spec, the Petrinaut survey's format facts, Dora's PRO-98 ontology, the open-questions document, and the FE-1363 use-case resolution. diff --git a/libs/@hashintel/brunch-agent/docs/specs/plugin-contract.md b/libs/@hashintel/brunch-agent/docs/specs/plugin-contract.md index ceb3ab3e221..26e694ad5b0 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/plugin-contract.md +++ b/libs/@hashintel/brunch-agent/docs/specs/plugin-contract.md @@ -1,451 +1,146 @@ -# Spec: the plugin contract — two schemas, two tables - -Status: **provisional** — desk-validated on baseline transcripts and cross-plugin thumbnails. -Ratification condition (inherited from [ADR-0003](../adr/0003-three-register-ir.md)): a full -FE-1397-style worked pass across at least three plugin targets. Until the September build -exercises a real fold, everything here is design, not demonstrated behavior. -Decided on: FE-1405 (the payload-interiors session, 2026-08-18); inputs were that session's -working draft and its pseudo-YAML structural rendering, which collapse into this document. -Amended: 2026-08-24 by [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md), which -separates deterministic projection scaffolds from model-assisted executable-artifact realization. - -## Problem Statement - -The elicitation system needs to work across domains: cyber-physical process models (the -September target), BDD/Gherkin feature specification, formal verification, and domains not yet -named. Today there is no defined answer to "what is a plugin?" — the capture envelope -(FE-1383, the build root: capture store and envelope semantics) says how evidence is stored, -and the ratified IR design says captures are typed, but nothing says what a domain author must -write down to make the machinery elicit _their_ domain, and nothing turns a pile of typed -captures into the thing the user actually wants: a model of their plant, their feature, their -system. - -Three people feel this as a problem: - -- **The plugin author** has no contract. Every new domain looks like it needs bespoke - machinery — its own assembly logic, its own completion logic, its own follow-up-question - logic — which makes plugin authoring an engineering project instead of a declaration, and - makes example plugins unreadable as examples. -- **The analyst running an elicitation** cannot ask "is this model complete enough to answer - my objective?" of a bag of captures. Completion questions are questions about a _model_ — - which activities lack duration distributions, which resources are uncounted — and no model - exists to ask them of. -- **The reviewer** cannot audit semantic judgments. If "half a shift" becomes "4 hours" - somewhere inside a read path, the interpretation happened invisibly: unattributable, - unsupersedable, unreproducible. - -## Solution - -A plugin is **two schemas and two small tables**, all declarative: - -1. declare your domain's node kinds and their slots (the **model schema**); -2. declare the utterance-shaped typed proposals that fill them (the **proposal catalog**), - annotating each with how to elicit it; -3. say how proposals fold into slots (the **fold table** — almost always empty, because fold - rules derive from the slot declarations); -4. say what your completion anchors demand (the **demand table**). - -Everything else is harness machinery explained once: a pure fold derives the elicited model -from active captures (ADR-0003's register 2), slot states report what is known, absent, -conflicted, or diverged, grades gate promotion, identity resolves by union-find over recorded -same-as assertions, and follow-up questions fire mechanically off slot states. Semantic -interpretation happens only at write time, in the sweep, where every bridge is deposited as a -contestable capture. - -For the plugin author this means writing YAML, not code (code remains only in validators and -projections). For the analyst it means completion computes over model slots, not capture -counts. For the reviewer it means every model part answers "which captures made you." -The readability oracle for the whole design: someone who has read the Gherkin plugin should be -able to write a third plugin by analogy in a sitting. - -## User Stories - -1. As a plugin author, I want to declare my domain as node kinds with slots, so that the - harness derives the model shape without me writing assembly code. -2. As a plugin author, I want a catalog of typed proposals whose interiors I compose from a - standard library of stated-form families, so that I don't reinvent "how experts state - quantities" for every domain. -3. As a plugin author, I want fold rules derived from my slot declarations by default, so that - I only write fold-table rows for genuine exceptions. -4. As a plugin author, I want to annotate each proposal type with an elicitation technique and - a mechanical firing condition, so that "which technique does this field force?" is a - completeness check over one column. -5. As a plugin author, I want to declare grade orders per slot — referencing a library ladder - or declaring my own composition ladder — so that my domain's notion of "more pinned down" - drives promotion without new machinery. -6. As a plugin author writing a thin plugin (Gherkin), I want the registers to collapse - gracefully, so that I never pay the thick plugin's ceremony. -7. As a new plugin author, I want to read the Gherkin example and write my own plugin by - analogy in a sitting, so that plugin authoring is tractable without harness expertise. -8. As a domain expert being interviewed, I want my hedged, low-grade statements ("about - 3 hours") captured honestly rather than refused or hardened, so that the record reflects - what I actually said. -9. As a domain expert, I want to say "I don't know" or "we don't track that" and have it - recorded as a first-class absence, so that I'm not re-asked what I've already disclaimed. -10. As a domain expert, I want to state a uniformity once ("same everywhere, same crew") and - have it cover everything in scope, so that I'm not interrogated per instance. -11. As a domain expert, I want to correct myself ("flag that as one I got wrong") and have the - correction supersede — not overwrite — the original, so that my history stays honest. -12. As an analyst, I want the elicited model in the expert's vocabulary with every slot in a - definite state (unaddressed / stated-at-grade / absent / conflicted / diverged), so that I - can see at a glance what the elicitation has and lacks. -13. As an analyst, I want completion computed as "every objective's demanded slots at demanded - grade over a static floor," so that "done" is relative to what I'm trying to answer, not - a global checklist. -14. As an analyst, I want prescribed-vs-practiced divergence surfaced per slot, so that - unwritten rules and workarounds are findings, not noise. -15. As an analyst, I want conflicting statements to fold to a typed conflict — never a silent - pick — so that contradictions become questions instead of errors. -16. As the interviewing agent, I want next-question candidates to fire mechanically off slot - states (below demanded grade → quantile protocol; statistic unspecified → "typical, or - worst case?"), so that follow-ups are grounded in the model's actual gaps. -17. As the sweep executor (FE-1392, the write-time mapping of utterances to proposals), I want - the proposal catalog compiled to a JSON Schema on my tool input and a decision-tree route - over statement forms, so that mapping is two small decisions, not one pick from a flat - ~19-type list. -18. As the sweep executor, I want honest low-grade defaults (verbal form, - `statistic: unspecified`) to be legal captures, so that uncertainty degrades to - under-typing that cues a follow-up, never to silent hardening. -19. As a reviewer, I want every semantic bridge — unit parse, identity link, composition, - formalization — deposited as an `inferred` capture with evidence spans, so that I can - audit, contest, and supersede any interpretation. -20. As a reviewer, I want the fold forbidden to interpret, so that re-running it on the same - store always yields the same model. -21. As a projection author (net scaffold, loss report, completion table), I want to consume - the elicited model without rereading the transcript or interpreting generic capture - fields, so that the scaffold and its code obligations are pure functions of register 2. -22. As a harness developer (FE-1393, the plugin SDK and fold engine), I want every harness - mechanism to be a pure function classified by which plugin declaration it reads, so that - the harness/plugin boundary is mechanically checkable. -23. As a team reader, I want example plugins that read as declarations of their domain, so - that I can evaluate the product's generality without reading harness internals. -24. As the artifact-realizing agent, I want each code obligation to name one target field, its - semantic intent, available net symbols, supporting captures, and acceptance checks, so that I - can write and repair TypeScript without resynthesizing unrelated regions. - -## Implementation Decisions - -The shapes below are trimmed from the session's ds-pseudo prototype (an untracked session -artifact); they encode the decisions more precisely than prose. - -### The three registers (ADR-0003, binding) - -Assertions (envelope-wrapped typed proposals) → the elicited model (derived by a pure fold, -never stored) → projections. Write-time-only semantics still governs model assembly: no semantic -act hides inside the fold, and every bridge from user language into the model is a capture. The -acceptance oracle: a second projection must consume the model without rereading the transcript or -semantically interpreting generic capture fields. Promotion, never refusal: low-grade statements -are captured honestly and never promote to a demanded grade without a higher-grade capture -superseding them. - -### Executable-artifact seam (ADR-0005) - -Some projection targets contain authored programs rather than declarative fields. For SDCPN, the -deterministic register-3 output is therefore a scaffold, a typed loss report, and a sidecar of code -obligations. Artifact realization is a downstream, model-assisted application step; it is not part -of `project`, the fold, or the persisted IR. - -```yaml -ProjectionResult: - draftArtifact: unknown - lossReport: LossEntry[] - codeObligations: CodeObligation[] - -CodeObligation: - id: string - target: { elementId: string, field: string } - semanticIntent: string - availableSymbols: { places: string[], tokenFields: string[], parameters: string[] } - supportingCaptureIds: string[] - acceptanceChecks: string[] -``` - -The sidecar is authoritative. A projector may mirror `semanticIntent` into the target code field as -a comment so the incomplete work is visible in the editor, but realization never reconstructs the -contract by parsing comments. The final gate is deterministic: every obligation is fulfilled, all -code compiles through Petrinaut, and at least one scenario simulates without a runtime error. - -### Harness machinery: pure functions classified by what they read - -The harness is the functions; the plugin is the tables they read. This is the -harness/plugin boundary, stated as a typology: - -```yaml -HarnessMachinery: - foldEngine: { reads: foldTable + modelSchema, emits: register-2 model } - demandRunner: - { - reads: demandTable + active anchor captures, - emits: demanded grade per slot, - } - identityResolver: { - reads: identity-bearing proposals, - emits: canonical names, - } # union-find - slotStateDeriver: - { reads: fold output + open issues + absences, emits: slot state per slot } - promotionGuard: { reads: gradeOrder per slot, emits: supersession legality } - affordanceCuer: - { reads: firesWhen x slot states, emits: next-question candidates } - captureEnvelope: fixed # kernel canon; the one non-function; amendable only at the seam -``` - -The slot-state algebra, fold engine, grade/promotion mechanism, identity mechanism, and demand -runner are harness (plugin-SDK) territory; the standard interiors are an importable library; -the node-kind catalog, proposal catalog, affordance annotations, demand table, and loss -categories are plugin-authored. (Ratified as FE-1393 design input.) - -### The plugin contract - -```yaml -PluginContract: - version: immutable string or digest # bound into every completion snapshot/report - modelSchema: NodeKind[] # register 2 — derived, never stored - proposalCatalog: ProposalType[] # register 1 — utterance-shaped, envelope-wrapped - foldTable: FoldRow[] # overrides only; default rules derive from SlotDecl - demandTable: - { version: immutable string or digest, rows: DemandRow[], staticFloor: DemandClause[] } - variantDimension: enum? # e.g. prescribed | practiced; enables slot state 'diverged' - lossCategories: enum[] - -NodeKind: - name: string # closed catalog (Layer-A property 1) - slots: map - completionAnchor: boolean - -SlotDecl: - valueType: string # domain type or standard interior - cardinality: enum # one | set | ordered - gradeOrder: enum[]? # plugin-declared, lowest first; library ladder or composition ladder - -ProposalType: - name: string # named by its semantic act, keyed to its fold target - interior: shape # composed from the standard-interiors library - validators: string[] # tiered: deterministic / semi-mechanical / residue - affordance: - technique: string # ref to an FE-1403/FE-1406 technique card - firesWhen: enum # closed 7-value set, below -``` - -This survives the penciled manifest design as the manifest's `ontology/schema` key: the -prompt-mechanism keys (techniques, lenses, smells, …) are untouched (FE-1403/FE-1406, -the technique-card and prompt-mechanism efforts); `checks`/`tools` receive the proposal and -projection validators; code remains only in validators and projections. - -### Typed proposals, not generic field atoms - -The generic kind+field EAV union is retired. Register 1 is an enumerated catalog of typed -semantic proposals with domain-shaped interiors (`duration-estimate` carrying a -`QuantityStated`, not `quantity-stated` carrying a string). For CPS the first-cut catalog is -~19 types worked from the C1 baseline transcript (entity-noted, attribute-noted, -attribute-domain, population, activity-noted, duration-estimate, resource-requirement, -actor-assignment, precondition-noted, outcome-noted, sequence, trigger, policy-noted, -relationship-noted, question-to-answer, goal-noted, penalty-noted, rationale-noted, same-as). -Constraint, data-binding, and validation-criterion stay schema-present but shallow — one -`*-noted` proposal each — per the September minimum; all three have real C1 instances. - -### The standard-interiors library - -Finiteness claim: interiors are indexed on _how experts state facts_ — measurement scale -types plus logical forms — not on domains. Six families over a shared supertype, plus one -flagged tarpit: - -```yaml -StatedForm: # base — every family extends this - verbatim: string # fidelity anchor; never normalized away - parsed: shape? # mechanical-tier structure; its absence IS the grade floor - # convention: every family declares explicit UNSPECIFIEDNESS MARKERS — slots recording - # what the utterance did NOT say. Grade derives from parsed-ness; markers are exactly - # what the affordance cuer fires on. - -QuantityStated: # 1 — magnitude (ratio/interval): verbal | point | range | quantiles; statistic marker -ComparisonStated: # 2 — order (ordinal): a/b/dimension/direction; direction-without-magnitude marker -RankingStated: # 2b — total/partial order over a set -VocabularyStated: # 3 — category (nominal): dimension + values -ConditionStated: # 4 — predicate: verbatim < structured < formal (the FV ladder lives here) -UniformityClaim: # 5 — quantified claim over a ScopeExpr, with exceptions carve-out -SubjectRef: # 6 — reference/identity; same-as pairs two of them -TemporalPattern: # flagged tarpit — NOT claimed finite; verbatim-heavy, coarse window only -``` - -Number-hedges ("about", "maybe", "roughly") live in `QuantityStated`'s qualifier, not in -capture confidence — two different homes. Capture-level confidence is the picklist -`firm | hedged | speculative`, and the store refuses numeric-parsing confidence strings -(closes remediation item A5). - -### Grade is narrowing of interpretation space - -Grade means "fewer readings remain," never claim strength (that is envelope confidence — -orthogonal by design). Two sources: **form grades** fall out of the library (each family's -ladder is its unspecifiedness markers progressively resolved: verbal < point < range < -quantiles; informal < vocabulary-bound < formal-parsed) — plugins demand rungs, they don't -define these ladders; **composition grades** are plugin-declared (Gherkin's given-only < -given-when < full-gwt is about slot composition, not one statement's form). The fold engine -reads grade off the interior or the fold output, never off the proposal type name — formal -verification's `formalization` is a _different proposal type_ folding into the _same slot_ at -higher grade. Grade orders promotion only: they never choose a winner among active values. -Beyond formal-parsed lies checking, which is validation, not grade. - -### Slot states and fold rules - -```yaml -SlotState: unaddressed | stated(value, grade, supportingCaptureIds) - | absent(absence, captureId) # populatable pending envelope pressure #2 - | conflicted(openIssueIds) - | diverged(prescribed, practiced) # only if the plugin declares a variant dimension - -_foldRuleDerivation: # foldTable holds overrides only; both example fold tables are empty - - cardinality one -> unique (grade order, if declared, gates promotion only) - - cardinality set -> set-union (member removal = supersede that member) - - cardinality ordered, positions stated in-utterance -> ordered-append - - order stated pairwise -> graph-union, order derived at read - - identity-bearing proposal -> union-find - - scope-bearing proposal -> shared-support - # closure: fold behavior = (cardinality) x (graded y/n) + two specials. Finite product - # space; the rule enum is its image. -``` - -One active value occupies a unique slot. Competing active, un-superseded values — at equal or -different grades — fold to `conflicted` with a typed issue, never a silent pick. Values that -are genuine alternatives must be declared as such and handled by the plugin's explicit -alternative rule; they do not authorize the fold to choose one. A higher-grade value replaces -a lower-grade value only through an explicit supersession, after which the lower capture is no -longer active. Regime divergence is per-slot; a regime-split _existence_ (the off-shift wash -that's prescribed-possible, practiced-never) is the degenerate node-level case. The present -`diverged` shorthand exposes neither side's grade nor support captures. Completion therefore fails -a demanded diverged slot with `unevaluable-divergence`; FE-1431 owns an evaluable constituent -shape and the later author-selectable all-sides/either-side rule. - -### Demand, scope, and firing conditions - -Requiredness is question-relative: a static floor (≥1 objective; entity coverage; a happy-path -flow) plus objective-demanded grades (a capacity objective demands quantile-grade durations on -the activities in its support). `DemandClause` is the smallest sufficient union: a presence clause -declares minimum cardinality over `ScopeExpr`; a slot clause declares scope, slot, -minimum grade, accepted epistemic statuses, and accepted absences. Presence clauses express -objective, entity, and path existence. Slot clauses require every selected slot to pass and an -empty selection fails. All matched clauses are conjunctive. Every active completion anchor must -match at least one row. Plugin and demand-table immutable versions are bound with target-document -revision into every evaluation and report. The complete evaluation and report shape live in the -provisional [target-document completion contract](elicitation-completion.md). -`ScopeExpr` has three constructors — `kind`, `where`-filter, `inSupport(anchor)` — and -**September ships kind-only**; the other two are the named growth path. `support(anchor)` is -defined as reference closure over a plugin-declared list of support-bearing proposal types -(proposal interiors carrying SubjectRefs _are_ the edges; the proposal type is the edge type, -per Layer-A property 4). - -`firesWhen` is a closed 7-value enum, every value a predicate over slot states — -`slot-unaddressed | below-demanded-grade | unspecified-marker-present | conflicted-open | -absence-uncorroborated | uniformity-unprobed | identity-ambiguous` — so the affordance column -is mechanically checkable. - -### Identity and supersession - -Identity assertions (`same-as`) are schema citizens: `inferred`, supersedable; reconciliation -is deterministic union-find over them; renames are new identity assertions, never edits. -Retraction residue: a retraction that replaces a claim with a guess-plus-known-unknown is a -`tentative` superseding capture now, plus an absence capture once the envelope's absence -locator lands (see Open strains). - -### The envelope is untouched - -Nothing here amends the capture envelope. The one confirmed pressure — absence captures carry -no payload, so a field-specific absence cannot name its slot — is recorded at the FE-1383 -seam with three concrete C1 cases; the locator an absence needs is precisely a fold-table -`target` coordinate. Adjudication happens at the seam, not by forking. - -## Testing Decisions - -A good test here asserts external behavior at a seam — captures in, slot states out — never -fold internals or intermediate representations. Three seams, one primary: - -1. **The fold** (primary, new, the highest seam available): `fold(pluginContract, -activeCaptures) → model` is pure by construction, so the whole contract is golden-testable - at one seam — hand-worked capture sets in, asserted slot states out. The seed gold set is - the FE-1405 worked-instance set from C1: the anchor changeover utterance (~9 captures → - four activities with graded durations and one rationale; the asymmetry claim derivable, not - captured), the quantile promotion pair (point supersedes to quantiles; slot promotes; no - refusal at any step), the regime-split off-shift wash, and the first-pass-yield absence - (blocked, and _asserted_ blocked, pending the envelope locator). This gold set doubles as - the sweep-accuracy rubric material for FE-1407 (the evaluation/gold-set effort). -2. **The acceptance oracle, executable**: a second projection consumes the folded model with - no transcript access and no generic-field interpretation — enforced structurally by the - projection's input type being register 2 only. For a code-bearing target, this proves the - scaffold and obligation plan; separate realization gates prove that the resulting artifact - compiles and runs. If the scaffold or obligations cannot be written from register 2 alone, - the failure is the finding. -3. **Contract validation**: plugin contract documents validate against the harness - meta-schema; the fold-rule derivation is tested as a table-driven pure function over its - finite product space (cardinality × gradedness + the two specials). - -Everything downstream — demand runner, slot-state deriver, promotion guard, affordance cuer — -reads off the fold's output, so seam 1 covers them without new seams. Modules under test are -FE-1393's fold engine and plugin SDK, with the CPS and Gherkin contracts as fixtures. Prior -art in this repo: the docs-index gate (`test/docs-index.test.ts`) for the table-driven, -fail-loud style, and the capture-store suite (FE-1390 side) for envelope-adjacent fixtures. -Test-fit order stands: smallest honest plugin and Gherkin before CPS. +# Spec: the plugin contract — one file per target formalism + +Status: **provisional**, reshaped 2026-08-25 by +[ADR-0006](../adr/0006-plugins-per-target-formalism.md). Ratification condition (inherited from +[ADR-0003](../adr/0003-three-register-ir.md)): a worked pass across at least three plugin +targets on a real fold. Decided on: FE-1405 (registers), FE-1480 (ADR-0005 outputs), and the +2026-08-25 design-convergence review (per-formalism plugin file). The normative exemplar for +every row and column shape named here is [`sdcpn-plugin.md`](sdcpn-plugin.md); where this +document and that file disagree about shape, the file wins and this document is amended. +The retired declarative draft is archived at +[`plugin-contract-2026-08-25-declarative-draft.md`](../archive/specs/plugin-contract-2026-08-25-declarative-draft.md). + +## What a plugin is + +A plugin is **per target formalism** — Gherkin, SDCPN — never per domain. It is one authored +Markdown file with fixed section headings, plus a small amount of code for `project` and +`validate`. The harness parses three tables from the file into the model vocabulary, the demand +list, and the pattern index; every other section is concatenated, in order, into the +interviewer's instructions. The end user never edits the file. + +Fixed headings, in this order: `## Purpose` · `## Kinds` · `## Must know` · `## Patterns` · +`## Moves` · `## Deliverable`. Subsections under a heading belong to that section. A plugin file +with a missing, renamed, or reordered contract heading does not load. + +Domain-neutrality rule: nothing in the file may name a domain. A new case that seems to need a +new row is a finding about the abstraction, decided by review, never content added to a plugin. + +## Relation to the three registers + +[ADR-0003](../adr/0003-three-register-ir.md) is unchanged. Register 1 is the capture store: +envelope-wrapped assertions carrying verbatim forms, hedges, absences, provenance. Register 2 is +the elicited model — a graph of nodes, each of exactly one **kind** from the `Kinds` table, each +with the slots the `Must know` table names for that kind — derived by a pure fold over active +captures and never stored. Register 3 is the projections. Write-time-only semantics governs +assembly: the fold is forbidden to interpret, so every bridge from user language into a slot is a +capture, and the model is a pure function of the store. + +## The three machine-read tables + +Column sets are fixed; the exemplar is normative for their names, order, and value vocabularies. + +| table | columns | read as | +| -------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `## Kinds` | `#`, `kind`, `what it is`, `projects to` | the closed node-kind catalog (Layer-A property 1); `projects to` is documentation for the loss report, not code | +| `## Must know` | `kind`, `slot`, `precision`, `"not applicable" allowed`, `why the model needs it` | one demand row per (kind, slot); `precision` is a word from the file's `Precision words` table | +| `## Patterns` | `id`, `when`, `ask` | discretionary, kind-indexed interviewing patterns; surfaced when a node matches `when` and a slot is unsatisfied | + +Rules the tables carry: + +- Every `Must know` row names a kind present in `Kinds`; every kind has at least one row. +- `precision` maps to an IR grade through the plugin's own `Precision words` table (`named`, + `number`, `range`, `spread`, `spelled out`, `at least N`). Grade means narrowing of + interpretation space, never claim strength. +- The static floor and the completion rule are stated in prose under `## Must know`; the harness + reads the floor's counts, and the rule itself is fixed by + [`elicitation-completion.md`](elicitation-completion.md). +- Cross-kind attributes (`quantity`, `source-regime`, `rationale`) are declared in prose under + `## Kinds` and apply to every kind; a plugin may not scope them to some kinds. +- Patterns are never mandates. The harness surfaces; the interviewer decides. + +## Version binding + +The plugin header declares an immutable version string (`sdcpn/2026-08-25.1`). Every completion +evaluation, projection output, and delivered report carries that version together with the +target-document revision it read. A report for one plugin version is not comparable with a model +folded under another; the caller retries rather than mixing them. + +## Code operations (ADR-0005 unchanged) + +`project` and `validate` remain plugin **code**, pure and snapshot-in/deltas-out (kernel §6.1, +adjudication C2). For a code-bearing target, `project` emits three outputs from register 2: + +1. a versioned scaffold with deterministic structure and field-local comments; +2. a sidecar of typed code obligations — target element and field, semantic intent, available + symbols, supporting capture ids, acceptance checks; +3. the typed loss report (`mapped-exactly / normalized / approximate / collapsed / omitted / + defaulted / unrepresentable`, per capture). + +The sidecar is the machine contract; comments are its readable projection. Artifact realization +is downstream application work +([ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md)); realized code is never a capture, +IR slot, fourth register, or plugin operation. `reconcile` remains optional. + +## Invariants that survive + +- **Acceptance oracle.** A second projection consumes register 2 without rereading the + transcript or interpreting generic capture fields; if it cannot, we have a capture ledger, not + an IR. +- **Promotion, never refusal.** Low-precision statements are captured honestly and never promote + to a demanded precision without a higher-precision capture superseding them. +- **Typed conflict, never a silent pick.** Competing active values on one slot fold to + `conflicted`; a divergence between `prescribed` and `practiced` is recorded on the same node + as an ordinary typed conflict — one model, never two. +- **Status ≠ precision ≠ confidence.** Epistemic status says how content relates to its source; + precision says how narrow the value is; confidence (`firm | hedged | speculative`) says claim + strength. None substitutes for another. +- **The envelope is untouched.** The absence-locator pressure (a field-specific absence cannot + name its slot) is adjudicated at the FE-1383 seam, not forked around here. +- **Smallest honest plugin.** A file whose `Kinds` table has one row and whose `Must know` table + demands one `named` slot must load and run (kernel §11.3). +- **Readability oracle.** Someone who has read `sdcpn-plugin.md` can write the Gherkin plugin + file by analogy in a sitting. A harness change that breaks this is a regression even if all + tests pass. + +## Testing + +The primary seam is still the fold: `fold(pluginFile, activeCaptures) → model`, golden-tested +with hand-worked capture sets in and slot states out. Two gates replace the retired meta-schema +validation: the **plugin-file parse gate** (headings fixed and complete, three tables parse, every +`Must know` kind exists, every precision word is declared) and the **completion fixtures** of +`evaluateCompletion` described in [`elicitation-completion.md`](elicitation-completion.md). +Test-fit order stands: smallest honest plugin, then Gherkin, then SDCPN. ## Open strains (first-class, with owners) -Adjudicated in the FE-1405 session; recorded here so settlement doesn't launder them into -decidedness. - -- **Strain 4 — grade sources (resolved in shape, worked pass pending).** Grade = narrowing; - form grades library-owned, composition grades plugin-declared. The shape is ratified; no - real fold has read a grade yet. Owner: FE-1393 (fold engine), against the gold set. -- **Strain 5 — support closure (downgraded, worked pass pending).** Typed references already - exist (interiors carrying SubjectRefs are edges); `support(anchor)` needs one declared - column and a closure rule, and a hand-worked pass over C1 before `inSupport` ships. - Owner: FE-1393, with FE-1402 (the completion contract) as consumer. -- **Strain 6 — temporal patterns (roped off).** Scheduling is out of scope as a modelling - area; `TemporalPattern` stays verbatim-heavy, coarse window only, no calendar algebra - claimed or planned. Reopening it is a deliberate act, not drift. -- **Strain 7 — sweep-time concentration (recorded; mitigations must travel).** Write-time-only - semantics makes the sweep's utterance→proposal mapping the single point of semantic failure. - Mitigations, none yet built: (a) the family index as a decision-tree sweep skill — route by - statement form, then fold target (FE-1403 authoring, FE-1392 consumes); (b) the proposal - catalog compiled to JSON Schema on the sweep tool input (FE-1392); (c) graceful - under-mapping via honest defaults, with over-mapping — silent hardening — as the real - failure, targeted by the verbatim-containment validators (FE-1392/FE-1393); (d) the - worked-instance gold set as the accuracy rubric (FE-1407). -- **Envelope pressure #2 — absence locator (seam, unresolved).** Absence captures carry no - payload; a field-specific absence cannot name its slot. Three C1 cases logged. The needed - locator is a fold-table coordinate. Current authority: the active soft edge in - [STEERING](../control/STEERING.md#active-soft-edges), for - adjudication — the register-2 `absent` state is representable but not populatable until it - lands. - -## Out of Scope - -- Amending the capture envelope. Pressure #2 is recorded for adjudication at the seam. -- Scheduling and temporal-pattern modelling (strain 6). -- The `where` and `inSupport` ScopeExpr constructors shipping in September (kind-only ships; - the others are the named growth path). -- Authoring the sweep skill, technique cards, and prompt-mechanism manifest keys - (FE-1392/FE-1403/FE-1406 territory — this spec fixes only the `firesWhen` and `technique` - hook points). -- Implementing artifact realization or the Petrinaut client-tool round trip (FE-1480/FE-1438). -- The full FE-1397-style ratification pass — it is this spec's _condition_, owed before the - provisional marker comes off, not part of its build scope. -- Loss-category content and projection implementations beyond the oracle projection. -- Any UI. - -## Further Notes - -- The provisional CPS technique-card IDs and clarification fragments are now defined in the - [FE-1403 CPS interview guidance](cps-interview-guidance.md). Its desk replay is design evidence, - not proof that `affordanceCuer` activates those cards correctly at runtime. FE-1403 also exposes - a declarative-authoring seam: several supported cards need one technique under multiple - `firesWhen` predicates, while `ProposalType.affordance` is singular. FE-1431 must decide binding - multiplicity or an evidence-preserving split; the current hook must not silently discard the - disjunction. -- This document is the settled form of the FE-1405 session's working draft and its - ds-pseudo YAML rendering — untracked session artifacts (`drafts/`, per the documentation - protocol) that collapsed into this spec and are not load-bearing anywhere. -- The decision this spec builds on is [ADR-0003](../adr/0003-three-register-ir.md); the - worked cross-plugin thumbnails (Gherkin thin, formal verification mid, CPS thick) live in - the working draft and discharge the ratification bar only provisionally. -- The authoring story is the product claim to protect: _declare your kinds and slots; declare - the proposals that fill them; annotate how to elicit each; say what your anchors demand._ - Any harness change that breaks the readability oracle — a Gherkin reader can write a third - plugin by analogy — is a regression even if all tests pass. +- **Dependency-slice closure (was strain 5).** "The nodes it depends on" is a `Must know` slot on + `objective`; the closure rule over reference-bearing captures still needs one hand-worked pass + before it is machine-read. Owner: FE-1393, with the completion fixtures as consumer. +- **Temporal patterns (strain 6, roped off).** Scheduling stays out of scope; calendar algebra is + neither claimed nor planned. +- **Sweep-time concentration (strain 7).** Write-time-only semantics makes the sweep the single + point of semantic failure; mitigations travel with FE-1392/FE-1393/FE-1407. +- **Absence locator (envelope pressure #2).** Authority remains the active soft edge in + [STEERING](../control/STEERING.md#active-soft-edges). + +## Retired 2026-08-25 + +Retired by [ADR-0006](../adr/0006-plugins-per-target-formalism.md); the full text survives in +the [archive copy](../archive/specs/plugin-contract-2026-08-25-declarative-draft.md). + +- **Domain-keyed CPS `DemandTable`** (`where(kind, role=…)` scopes, `ROW-BREAKDOWN` and kin): + it keyed demands to one baseline case's domain, so every new case needed new rows. +- **Typed `ScopeExpr` / `where` / `inSupport` algebra:** demands are now per (kind, slot), and + the objective's dependency slice replaces `inSupport`; the algebra had nothing left to select. +- **`ProposalType.affordance.firesWhen` (closed 7-value enum):** patterns are surfaced by a + node matching `when` with an unsatisfied slot, which needs no per-proposal predicate. +- **`NodeKind.completionAnchor`:** `objective` is the anchor kind by rule, not by flag. +- **Typed `foldTable` / `demandTable` / `variantDimension` / `lossCategories` declaration:** the + fold derives from the `Must know` rows, the demand list *is* that table, `source-regime` is a + fixed cross-kind attribute, and loss categories are fixed by kernel §6.1. +- **Interview cards as separate artifacts:** they became kind-indexed patterns P01–P13 and + `Moves` steps in the plugin file (mapping recorded on the + [archived guidance](../archive/specs/cps-interview-guidance-2026-08-25.md)). +- **The `ProposalType` catalog and standard-interiors library as plugin-authored declarations:** + utterance-shaped proposal interiors remain a harness concern (FE-1392/FE-1393); the plugin + file does not declare them. diff --git a/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md b/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md index 0ecc15b7aa2..5fd2f24f342 100644 --- a/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md +++ b/libs/@hashintel/brunch-agent/docs/specs/sdcpn-plugin.md @@ -165,9 +165,15 @@ decides whether and how to use it. ## Moves -Moves are the mandates: the shape every conversation follows regardless of domain. The harness -enforces what it can (completion, the sweep list, the ledger); the interviewer is responsible for -the rest. +Moves are the mandates: the shape a conversation follows regardless of domain. A plugin carries one +runbook per **job** it supports; every runbook works over the same `Kinds` and `Must know` tables +and differs only in kickoff, trajectory, checks, and stopping. This plugin supports two jobs. The +harness enforces what it can (completion, the sweep list, the ledger, the affected slice); the +interviewer is responsible for the rest. + +### Job: construct + +Kickoff: no model exists. The user knows the system; the interviewer knows the kinds. 1. **Open with objectives.** Before anything about structure, establish what the user wants the model to answer or decide. Capture each as an `objective` node. Expect to co-construct: these @@ -202,6 +208,52 @@ the rest. meets its demands. Before delivering, summarise per kind, state what is missing or assumed, and give the expert one chance to correct you. +### Job: review and revise + +Kickoff: a model already exists, with its captures and a projected net. The reviewer may not be +the original source. They arrive with an element of the net or a region of the model in view and +one of three intents: understand why it is modelled as it is, correct it, or extend it. The +engagement brief is the selected element, the intent, and nothing else; the interviewer does not +reopen the interview. + +1. **Orient on the artifact, not the conversation.** State which model node and slot the selected + net element projects from, and which captures support that slot — turn, speaker, quote, grade, + source-regime. This is the only admissible answer to "why is X modelled like Y": provenance, + never domain plausibility. If no capture supports the element, say so plainly: it is an + assumption in the ledger or a projection default, and the reviewer is looking at a gap, not at + knowledge. + +2. **Scope before eliciting.** The harness computes the affected slice: the node, its slots, every + `objective` whose dependency slice contains it, and every projected element those produce. State + the scope to the reviewer in one sentence. Nothing outside it will change; if the reviewer's + intent reaches outside it, say so and let them widen the scope explicitly. + +3. **Elicit the correction in three to five turns.** Apply the node's `Must know` rows and any + pattern its state triggers — P12 first: what is known, what precision is needed, the smallest + delta. The reviewer's statement is evidence at the precision actually given. A correction is a + new capture that **supersedes** the old one — single hop, active head — never an edit of it. If + the reviewer contradicts the original source rather than refining it, that is a conflict: record + both, name it, and ask the reviewer to resolve it explicitly before anything supersedes. + +4. **Re-evaluate the slice only.** Completion is recomputed over the affected objectives. Report + what moved: a slot that gained or lost precision, an objective newly supported or newly + unsupported, a conflict opened or closed. Do not report the rest of the model. + +5. **Project the delta.** Projection re-runs over the whole model, deterministically. The expected + delta is confined to the scope; show the reviewer which net elements changed, which are + unchanged, and which code obligations the change reopened. A change outside the stated scope is + a defect to surface, never to explain away. + +6. **Hand off.** State what changed, what each change traces to, which obligations remain open, + and that unrelated regions are unchanged. Stop when the reviewer's stated correction is captured + and projected, or when five turns pass without a superseding capture — say which, and do not + loop. Stopping outcomes are distinct and named: `corrected-and-projected`, + `corrected-obligation-open`, `conflict-unresolved`, `scope-exceeded`, `reviewer-stopped`. + +Checks the harness owns for this job: every changed net element traces to a superseding capture +made in this session; no capture outside the scope changed; the projection outside the scope is +identical before and after; the ledger records any default the correction displaced. + ## Deliverable When the interview ends — complete or not — produce: @@ -212,6 +264,10 @@ When the interview ends — complete or not — produce: 3. a loss section: what the model deliberately leaves out, which slots are open and why, which objectives are unsupported, and which kinds the net cannot carry. +For the review-and-revise job the deliverable is the **delta report** in place of the whole model: +the superseding captures made, the slots and objectives whose state moved, the net elements +changed and the elements confirmed unchanged, the obligations reopened, and the stopping outcome. + The SDCPN scaffold, the code-obligation sidecar, and the typed loss report are derived from the model by the plugin's projection; the interviewer does not write them and must not claim the model is loadable, compiled, or simulated. diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts index 699e82db006..9dbad1d2d4a 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-instrument.ts @@ -37,7 +37,8 @@ export const CONDITION_3_LOCKED_PATHS = [ "evaluations/cases/process-model-elicitation/baseline/opening-message.md", "evaluations/cases/process-model-elicitation/baseline/situation-pack.md", "docs/specs/elicitation-completion.md", - "docs/specs/cps-interview-guidance.md", + // Archived 2026-08-25 (ADR-0006): the cards became patterns in docs/specs/sdcpn-plugin.md. + "docs/archive/specs/cps-interview-guidance-2026-08-25.md", "docs/reference/research/elicitation/frontier-model-elicitor-failure-catalogue.md", "docs/evidence/evaluations/process-model-elicitation/baseline/readout.md", "docs/evidence/evaluations/process-model-elicitation/baseline/transcripts/condition-1.md", diff --git a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json index 4a9c95350a9..4d2ee783d30 100644 --- a/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json +++ b/libs/@hashintel/brunch-agent/evaluations/protocols/process-model-elicitation/baseline/condition-3-preregistration.lock.json @@ -39,7 +39,7 @@ "sha256": "f076dd6f50b7a901cecbe7310f62bcaefc6987de4667c928711d26b861ddf67d" }, { - "path": "docs/specs/cps-interview-guidance.md", + "path": "docs/archive/specs/cps-interview-guidance-2026-08-25.md", "sha256": "f434eb101a8087d5227589e6fed7505cdaa5a70e828681ae9f12341fc1a6dcf6" }, { diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts index c98d41a7b32..4ad2c27b8a1 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts @@ -321,6 +321,7 @@ async function createBaselineCopy(): Promise { "evaluations/cases/process-model-elicitation/baseline", ); const completionSpecDirectory = join(testDirectory, "docs/specs"); + const archivedSpecDirectory = join(testDirectory, "docs/archive/specs"); const researchDirectory = join( testDirectory, "docs/reference/research/elicitation", @@ -333,6 +334,7 @@ async function createBaselineCopy(): Promise { mkdir(protocolDirectory, { recursive: true }), mkdir(caseDirectory, { recursive: true }), mkdir(completionSpecDirectory, { recursive: true }), + mkdir(archivedSpecDirectory, { recursive: true }), mkdir(researchDirectory, { recursive: true }), mkdir(evidenceDirectory, { recursive: true }), ]); @@ -345,8 +347,11 @@ async function createBaselineCopy(): Promise { join(completionSpecDirectory, "elicitation-completion.md"), ), cp( - join(CONTEXT_ROOT, "docs/specs/cps-interview-guidance.md"), - join(completionSpecDirectory, "cps-interview-guidance.md"), + join( + CONTEXT_ROOT, + "docs/archive/specs/cps-interview-guidance-2026-08-25.md", + ), + join(archivedSpecDirectory, "cps-interview-guidance-2026-08-25.md"), ), cp( join( diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts index a79d22a4cb4..e311cfa8f52 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/condition-3-instrument.test.ts @@ -260,7 +260,7 @@ describe("condition 3 frozen envelopes", () => { }, ], }), - ).toThrow(); + ).toThrow('Expected "split-run" but received "changeover-accounting"'); }); test("preserves multiple objective anchors that project to one active row", () => { @@ -749,7 +749,7 @@ describe("condition 3 frozen envelopes", () => { condition1: { ...result.comparisons.condition1, comparison: "" }, }, }), - ).toThrow(); + ).toThrow("Invalid length: Expected >=1 but received 0"); expect(() => v.parse(Condition3ResultSchema, { ...result, @@ -761,6 +761,6 @@ describe("condition 3 frozen envelopes", () => { }, }, }), - ).toThrow(); + ).toThrow(/Expected "[0-9a-f]{64}" but received "0{64}"/); }); }); diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/control-surfaces.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/control-surfaces.test.ts index 6224ab98cf0..7ed3ba9f3ff 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/control-surfaces.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/control-surfaces.test.ts @@ -39,6 +39,13 @@ const strategyEntries = (markdown: string): StrategyEntry[] => ), })); +/** + * A strategy entry may supersede several earlier entries; the field lists them + * comma-separated, or reads `none`. + */ +const supersededTargets = (field: string): string[] => + field === "none" ? [] : field.split(",").map((target) => target.trim()); + describe.skipIf(!contextRootPresent)("strategic control surfaces", () => { const steering = readFileSync(STEERING_PATH, "utf8"); const strategyLog = readFileSync(STRATEGY_LOG_PATH, "utf8"); @@ -76,19 +83,18 @@ describe.skipIf(!contextRootPresent)("strategic control surfaces", () => { reason: "malformed" | "not-earlier"; }> = []; for (const entry of entries) { - const supersedes = entry.fields.get("Supersedes")!; - if (supersedes !== "none") { - if (!/^S-\d{3}$/.test(supersedes)) { + for (const target of supersededTargets(entry.fields.get("Supersedes")!)) { + if (!/^S-\d{3}$/.test(target)) { invalidTargets.push({ entry: entry.id, - target: supersedes, + target, reason: "malformed", }); } - if (!seen.has(supersedes)) { + if (!seen.has(target)) { invalidTargets.push({ entry: entry.id, - target: supersedes, + target, reason: "not-earlier", }); } @@ -107,9 +113,9 @@ describe.skipIf(!contextRootPresent)("strategic control surfaces", () => { const referencedIds = [...steering.matchAll(/S-\d{3}/g)].map(([id]) => id); const knownIds = new Set(entries.map(({ id }) => id)); const supersededIds = new Set( - entries - .map(({ fields }) => fields.get("Supersedes")!) - .filter((id) => id !== "none"), + entries.flatMap(({ fields }) => + supersededTargets(fields.get("Supersedes")!), + ), ); for (const id of referencedIds) { expect(knownIds.has(id)).toBe(true); From 90b860855093a9032502ad507bbae6a9a0c59c98 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 13:53:30 +0200 Subject: [PATCH 4/7] Give the controller read path an owner FE-1497 now holds epicentre E1, so STEERING drops the no-owner gate and the exceptional-root entry, names the issue in the frontier table and work order, and counts the completion invariants as the rewritten spec does. Co-Authored-By: Claude Fable 5 --- libs/@hashintel/brunch-agent/docs/control/STEERING.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/libs/@hashintel/brunch-agent/docs/control/STEERING.md b/libs/@hashintel/brunch-agent/docs/control/STEERING.md index 488105da4a1..14a2cb5b71f 100644 --- a/libs/@hashintel/brunch-agent/docs/control/STEERING.md +++ b/libs/@hashintel/brunch-agent/docs/control/STEERING.md @@ -35,14 +35,14 @@ evaluation apparatus) are not worked until an epicentre needs them. | Epicentre | Gap | Issue | | --- | --- | --- | -| **E1 — controller read path** | The harness writes captures and never reads them back: no fold to a model, no completion over objective slices, no sweep list, no cue to the next turn. The hollow centre between "captured facts" and "conducted an elicitation". | none yet — proposed new issue in `packages/core` | +| **E1 — controller read path** | The harness writes captures and never reads them back: no fold to a model, no completion over objective slices, no sweep list, no cue to the next turn. The hollow centre between "captured facts" and "conducted an elicitation". | FE-1497 (gist: harness controller read path) | | **E2 — the SDCPN plugin in code** | The plugin file exists as a spec; nothing parses its three tables, folds captures onto its kinds, or projects from them. | FE-1482 (gist: CPS plugin, redefined as the skeleton epicentre) | | **E3 — targeted correction** | `supersedes` is unreachable from extraction; no affected-slice computation; no delta; the target-document is still identified with the conversation. | FE-1479 (targeted re-elicitation), FE-1478 (provenance read), FE-1439 (durable session / document boundary) | | **E4 — the real entry** | Client-tool results do not return to the elicitor; retry/abandonment semantics unproven; realization gated. | FE-1438, FE-1420, FE-1480 | ```text skeleton (construct job; proves the loop, produces fixtures) -E1 controller read path -> FE-1482 plugin file + parser + fold +FE-1497 controller read path -> FE-1482 plugin file + parser + fold -> FE-1404 skeleton run against the baseline simulated expert reviewer lane (review-and-revise job; the acceptance proof) @@ -83,7 +83,7 @@ No hard blocker chain remains from the retired design queue. ### Active soft edges -- E1 precedes FE-1482 only by the width of an interface: the fold and completion functions are +- FE-1497 precedes FE-1482 only by the width of an interface: the fold and completion functions are harness code that the plugin's tables parameterise. Build them together on one branch if that is faster; do not design the interface before the first plugin exercises it. - FE-1420's idempotency and abandonment semantics precede FE-1438's external-tool protocol; FE-1439 @@ -102,7 +102,6 @@ The read-only Linear graph supplies mechanical availability, never priority. | Gate | Owner / source | Watch trigger | Last checked | Consequence | | --- | --- | --- | --- | --- | -| Controller read path has no owner | E1; no Linear issue exists | An issue is created and a branch opened. | 2026-08-25 | Neither proof can start; the plugin file has nothing to run in. | | FE-1480 executable realization unavailable | FE-1438; [ADR-0005](../adr/0005-model-assisted-sdcpn-realization.md) | Client tools return code diagnostics to the elicitor. | 2026-08-25 | Scaffold work may proceed; no runnable FE-1480 proof until the gate opens. | | Final use case outstanding | Dora; FE-1476 / September Plan | Dora confirms or changes it. | 2026-08-25 | If creation is required, Proof 1 becomes acceptance-relevant rather than a harness proof; reconcile ADR-0004/proof. | | Truck-fleet dossier missing from the repository | FE-1382 is Done but its promised `docs/reference/research/` artifact is absent. | Artifact path/branch is supplied or a reviewed replacement is selected. | 2026-08-25 | The generality half of Proof 1 uses a fixture derived from the inbox truck SDCPN and Layer B's worked example; claim no dossier-backed domain provenance. | @@ -113,7 +112,7 @@ The read-only Linear graph supplies mechanical availability, never priority. | --- | --- | --- | | Kind-level rows express the coatings case. | Medium-high; the twenty domain-keyed rows of the FE-1402 rehearsal collapse onto eight kind rows on paper. | Proof 1's first half. | | The truck-fleet case adds zero headings and zero rows. | Medium; Layer B was validated against it, but never through this file. | Proof 1's second half. | -| The controller read path is small. | Medium; `evaluateCompletion` is ~10 invariants over a fold the store already supports. | Build E1; if it exceeds the plugin file in size, stop and look. | +| The controller read path is small. | Medium; `evaluateCompletion` is nineteen invariants over a fold the store already supports. | Build E1; if it exceeds the plugin file in size, stop and look. | | Field-local code obligations support localized realization and repair. | Low-medium; the corpus and Petrinaut diagnostics are field-addressed, but no Brunch run exists. | Realize one stochastic transition without rewriting an unrelated field. | | Five turns yield a scoped correction. | Low; unrehearsed. The review-and-revise runbook in the plugin file is the first concrete trajectory. | Run two bounded rehearsals against a fixture model. | | Ask carries durable client-tool results. | Medium-low; machine results refused today. | Run one correlated FE-1438 round trip. | @@ -160,4 +159,3 @@ The read-only Linear graph supplies mechanical availability, never priority. - **FE-1476** — September delivery root; intended parent is FE-1357. - **FE-1477–FE-1482** — PM-authored outcome roots; intended parent is FE-1476 after overlap review and separately approved Linear mutation. -- **E1 controller read path** — no issue yet; the one new issue this replan proposes. From 6efd87577b2b41314ffde824569c324e2768c43b Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 14:03:03 +0200 Subject: [PATCH 5/7] Parent the two skeleton-adjacent roots FE-1482 (plugin walking skeleton) now sits under FE-1476 and FE-1406 (pattern lift) under FE-1357, so both leave the exceptional-roots list. Co-Authored-By: Claude Fable 5 --- libs/@hashintel/brunch-agent/docs/control/STEERING.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/@hashintel/brunch-agent/docs/control/STEERING.md b/libs/@hashintel/brunch-agent/docs/control/STEERING.md index 14a2cb5b71f..80591ff2921 100644 --- a/libs/@hashintel/brunch-agent/docs/control/STEERING.md +++ b/libs/@hashintel/brunch-agent/docs/control/STEERING.md @@ -153,9 +153,7 @@ The read-only Linear graph supplies mechanical availability, never priority. whether it joins the cold-start lane or receives a parent. - **FE-1334** — surprising-scenario validation gesture; parent or cancel when its owning outcome is chosen. -- **FE-1406** — harness strategy repertoire; shrunk to the post-skeleton lift of harness-generic - patterns out of the plugin file. - **FE-1472** — unrelated SDK-pin triage; assign an owning map or remove from the project. - **FE-1476** — September delivery root; intended parent is FE-1357. -- **FE-1477–FE-1482** — PM-authored outcome roots; intended parent is FE-1476 after overlap review - and separately approved Linear mutation. +- **FE-1477–FE-1481** — PM-authored outcome roots; intended parent is FE-1476 after overlap review + and separately approved Linear mutation. FE-1482 was parented to FE-1476 on 2026-08-25. From ff00464881127867bba95e3dc79087ec61c0d089 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 14:03:52 +0200 Subject: [PATCH 6/7] State the branch-to-issue rule as at-least-one A branch must answer to an issue but may carry several built together; work with no issue behind it rides the branch of the issue that forced it. Co-Authored-By: Claude Fable 5 --- .../brunch-agent/docs/agents/git-workflow.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libs/@hashintel/brunch-agent/docs/agents/git-workflow.md b/libs/@hashintel/brunch-agent/docs/agents/git-workflow.md index 759fd26bf8d..c1fb9390bdf 100644 --- a/libs/@hashintel/brunch-agent/docs/agents/git-workflow.md +++ b/libs/@hashintel/brunch-agent/docs/agents/git-workflow.md @@ -2,11 +2,12 @@ Branches are managed with **Graphite** (`gt`), matching HASH's repository-wide use of Graphite (its CI runs the Graphite optimizer). The standalone repository's late `gh stack` convention does -not carry over. The unit of branching is the **Linear issue**: one stacked branch per issue tackled, -created when work on that issue starts. Work discovered while -resolving the issue (slices, refinements, side-fixes it requires) stays on its branch; only a -different issue gets a new branch. Branches predating this convention (and the trunk) may mix -multiple issues. +not carry over. The unit of branching is the **Linear issue**: every stacked branch answers to at +least one issue, created when work on that issue starts, and a branch may carry several issues +when they are built together (two epicentres that settle one interface, for instance). Work +discovered while resolving an issue (slices, refinements, side-fixes, a pivot it forces) stays on +that issue's branch rather than getting a branch of its own; work with no issue behind it never +gets a branch. Branches predating this convention (and the trunk) may mix multiple issues. ## git vs gt boundary From 9879c16f734a0ee9660c85570eed4534da767114 Mon Sep 17 00:00:00 2001 From: Lu Nelson Date: Tue, 25 Aug 2026 14:53:17 +0200 Subject: [PATCH 7/7] Stabilize baseline runner test fixtures Make preregistration chronology deterministic across filesystems and move large model reply fixtures out of environment variables so Linux can spawn the test runner reliably. Co-authored-by: Cursor --- .../test/architecture/baseline-runner.test.ts | 30 ++++++++++++------- .../fixtures/baseline-anthropic-stub.ts | 10 ++++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts index 4ad2c27b8a1..c010acfaa00 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/baseline-runner.test.ts @@ -9,6 +9,7 @@ import { readdir, rm, symlink, + utimes, writeFile, } from "node:fs/promises"; import { createServer } from "node:http"; @@ -365,21 +366,26 @@ async function createBaselineCopy(): Promise { join(CONTEXT_ROOT, "../../../node_modules"), join(testDirectory, "node_modules"), ); - await new Promise((resolve) => setTimeout(resolve, 10)); + const sealedAt = new Date(); + const lockedFileTimestamp = new Date(sealedAt.getTime() - 1_000); const files = await Promise.all( - CONDITION_3_LOCKED_PATHS.map(async (path) => ({ - path, - sha256: createHash("sha256") - .update(await readFile(join(testDirectory, path), "utf8")) - .digest("hex"), - })), + CONDITION_3_LOCKED_PATHS.map(async (path) => { + const lockedFilePath = join(testDirectory, path); + await utimes(lockedFilePath, lockedFileTimestamp, lockedFileTimestamp); + return { + path, + sha256: createHash("sha256") + .update(await readFile(lockedFilePath, "utf8")) + .digest("hex"), + }; + }), ); await writeFile( join(protocolDirectory, "condition-3-preregistration.lock.json"), JSON.stringify( { version: CONDITION_3_INSTRUMENT_VERSION, - sealedAt: new Date().toISOString(), + sealedAt: sealedAt.toISOString(), files, }, null, @@ -404,6 +410,8 @@ async function runBaseline( requests: Array>; }> { const requestsPath = join(baselineCopy.testDirectory, "requests.jsonl"); + const repliesPath = join(baselineCopy.testDirectory, "replies.json"); + await writeFile(repliesPath, JSON.stringify(replies)); const subprocess = spawn( process.execPath, [ @@ -418,7 +426,7 @@ async function runBaseline( ...process.env, BRUNCH_BASELINE_ANTHROPIC_MODULE: STUB_MODULE, BRUNCH_BASELINE_TEST_OUTPUT_DIR: baselineCopy.outputDirectory, - BASELINE_STUB_REPLIES: JSON.stringify(replies), + BASELINE_STUB_REPLIES_PATH: repliesPath, BASELINE_STUB_REQUESTS_PATH: requestsPath, }, stdio: ["ignore", "ignore", "pipe"], @@ -471,6 +479,8 @@ async function runBaselineFailure( requests: Array>; }> { const requestsPath = join(baselineCopy.testDirectory, "requests.jsonl"); + const repliesPath = join(baselineCopy.testDirectory, "replies.json"); + await writeFile(repliesPath, JSON.stringify(replies)); const subprocess = spawn( process.execPath, [ @@ -485,7 +495,7 @@ async function runBaselineFailure( ...process.env, BRUNCH_BASELINE_ANTHROPIC_MODULE: STUB_MODULE, BRUNCH_BASELINE_TEST_OUTPUT_DIR: baselineCopy.outputDirectory, - BASELINE_STUB_REPLIES: JSON.stringify(replies), + BASELINE_STUB_REPLIES_PATH: repliesPath, BASELINE_STUB_REQUESTS_PATH: requestsPath, }, stdio: ["ignore", "ignore", "pipe"], diff --git a/libs/@hashintel/brunch-agent/packages/core/test/architecture/fixtures/baseline-anthropic-stub.ts b/libs/@hashintel/brunch-agent/packages/core/test/architecture/fixtures/baseline-anthropic-stub.ts index cac95247f9e..6af6a4f3f03 100644 --- a/libs/@hashintel/brunch-agent/packages/core/test/architecture/fixtures/baseline-anthropic-stub.ts +++ b/libs/@hashintel/brunch-agent/packages/core/test/architecture/fixtures/baseline-anthropic-stub.ts @@ -1,4 +1,4 @@ -import { appendFile } from "node:fs/promises"; +import { appendFile, readFile } from "node:fs/promises"; import type Anthropic from "@anthropic-ai/sdk"; @@ -7,9 +7,11 @@ export interface StubReply { truncated?: boolean; } -const replies = JSON.parse( - process.env["BASELINE_STUB_REPLIES"] ?? "[]", -) as StubReply[]; +const repliesPath = process.env["BASELINE_STUB_REPLIES_PATH"]; +if (!repliesPath) { + throw new Error("BASELINE_STUB_REPLIES_PATH is required"); +} +const replies = JSON.parse(await readFile(repliesPath, "utf8")) as StubReply[]; const requestsPath = process.env["BASELINE_STUB_REQUESTS_PATH"]; let requestCount = 0;