diff --git a/.gitignore b/.gitignore index eaed6fd..dc1fcb4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,15 @@ eval/labels/heldout/* !eval/labels/heldout/seal-manifest.json # the pre-M2X-030 path, kept ignored so an older checkout cannot leak a case through it eval/heldout/ +# the Phase 2 gate's answer key (M2X-045), sealed on the same terms as the held-out +# labels. Questions are public -- the system has to be asked them -- but a builder who can +# read the expected segments can tune retrieval until they come back, and a retrieval +# metric tuned against its own answer key measures nothing. +eval/rag/expected/* +!eval/rag/expected/*.gpg +!eval/rag/expected/seal-manifest.json +!eval/rag/expected/.gitkeep +!eval/rag/expected/README.md # labels in progress, before the dev/held-out split has been drawn. Every case is # plaintext here and ten of them are about to become the sealed set -- committing this # directory would put the held-out cases in git history, and no later encryption can diff --git a/eval/rag/README.md b/eval/rag/README.md new file mode 100644 index 0000000..3b66993 --- /dev/null +++ b/eval/rag/README.md @@ -0,0 +1,121 @@ +# The Phase 2 gate instrument (M2X-045) + +Thirty questions — **20 single-meeting, 5 cross-meeting, 5 that must abstain** — written +blind against the corpus, never against system output. The Friday gate (M2X-050) scores +context precision ≥0.75, faithfulness ≥0.80 and citation accuracy ≥0.90 on this set; +M2X-046 is the harness that computes them. + +## Two halves, held differently + +| | in git? | why | +|---|---|---| +| `questions.jsonl` | **yes**, plaintext | the system has to be asked the questions; hiding them would only stop the harness running | +| `expected/.json` | **never** — git-ignored | a builder who can read the expected segments can tune retrieval until they come back | +| `expected/.json.gpg` | yes | recoverability, so a fresh clone can score the gate | +| `expected/seal-manifest.json` | yes | integrity — digests, so an edit is visible | + +Same tooling and the same reasoning as the held-out labels; see +[`../labels/heldout/README.md`](../labels/heldout/README.md) for why both artefacts are +needed and why the manifest, not the ciphertext, is the one that proves anything. + +```bash +uv run python scripts/seal_heldout.py verify --dir eval/rag/expected # no passphrase +uv run python scripts/seal_heldout.py unseal --dir eval/rag/expected # gate day +uv run python scripts/validate_rag_questions.py # needs it unsealed +``` + +**The same caveat as the labels applies.** One operator writes the questions, the expected +answers and the system under test, so the seal is a discipline rather than an enforced +boundary — see [`../labels/README.md`](../labels/README.md) §"these labels are not +independent". Unlike the extraction labels, though, these questions were written against +*human-annotated reference transcripts nobody in this project authored*, so what a question +asks about is at least not downstream of the system's own vocabulary. + +## The corpus is `eval/tiron/`, and that is a decision + +`data/` is git-ignored, so a fresh clone has no meeting transcripts. Every citation `m2x +ask` has ever produced is a `§ heading` against this repository's own markdown — the +`[meeting · speaker · mm:ss–mm:ss]` path, which is the entire product promise, has **never +been exercised against a real meeting** (`docs/design/day4-ask.md` §"What is not verified +here"). + +The tiron reference transcripts are committed, carry human speaker turns and real +timestamps, and already ground the Phase 1B labels. A gate written against them runs on +every clone with no audio and no transcription provider, and exercises the meeting citation +path on every question. + +Four meetings are covered — a mall aquarium project, a marketing meeting for outdoor sports +equipment, a company retreat to Mexico, and a robotic-nurse feature with a contrarian Q&A. +They share a speaker pool, which is deliberate: it makes the cross-meeting questions hard +in the way cross-meeting questions are supposed to be hard. + +## Ground truth is turn ranges, never rendered citations + +An expected answer names `(meeting_id, first_turn, last_turn)` into the reference +transcript. + +- **Not an `mm:ss` string** — that scores the renderer, not the retrieval. +- **Not a chunk id** — that is a function of the chunking parameters and changes the moment + anyone tunes them, silently invalidating the whole set. + +Turn indices are positions in a committed file. A retrieved chunk records the segment range +it covers, so citation accuracy is an **overlap** test between the two — which is the +ticket's rule ("the cited segment id is among the ground-truth segments, or overlaps its +time range"). Overlap rather than containment, because chunks are packed to a character +budget and a correct citation routinely covers the answer plus its neighbours; demanding +containment would fail a chunk for being the size the indexer chose. + +## How the questions were chosen + +Written by reading the four transcripts end to end and picking facts that are **stated**, +not inferable. Recurring shapes, all deliberate: + +- **Answers split across interrupted turns.** `q05` — *"I'm gonna come back on Tuesday + with a"* / *(Sarah interrupts)* / *"follow up on this"*. A retrieval that returns only the + first turn has a day and no object. +- **A claim and its qualifier in one span.** `q13` asks whether family can come; spouses + yes, children explicitly no, eight turns apart. Returning half is a confidently wrong + answer, which is worse than an abstention. +- **Speaker distractors.** `q10` — Ron proposes Kim Kardashian, Rachel asks Sarah a + *different* Kim Kardashian question, and only then does Sarah answer. Matching on the + name alone lands on the wrong turn. + +### The must-abstain five are near misses on purpose + +Four of the five sit right on top of a strongly matching passage: + +| | why it is unanswerable | +|---|---| +| `q26` total aquarium cost | Rachel **asks for exactly this** and is never answered. Highly relevant context retrieves; the system still has to decline. | +| `q27` name of the equipment company | The marketing meeting says "our products" throughout and never names the company. | +| `q28` Nursing Bot's annual revenue | The company is named and described at length; the number is absent. | +| `q29` which hotel the retreat is booked with | The retreat centre's studio, restaurant and pool are all described; it is never named. | +| `q30` 2026 cricket world cup | The control — no lexical overlap with anything. If this one does not abstain, nothing else in the bucket is interpretable. | + +A must-abstain set of `q30`s would be trivially passed by the distance threshold alone and +would measure nothing about the model's judgement. These four force the *model* to abstain +on passages the retriever was right to return. + +## The re-read pass + +The ticket asks for a second pass hours later: *can each answerable question really be +answered from the recorded segment?* Both halves were done. + +**Mechanical** — `scripts/validate_rag_questions.py`. Ids unique and paired across both +halves, the 20/5/5 mix, answerable questions carrying both evidence and a gist, unanswerable +ones carrying neither, cross-meeting questions genuinely spanning ≥2 meetings, and every +cited turn resolving against the reference. Exits non-zero on any problem. + +**By reading** — every cited span printed back beside its question and checked to contain +the answer. One correction resulted: `q10`'s span ended one turn past Sarah's reply. + +The judgement itself is recorded per question in the sealed `notes` field, so at the gate an +adjudicator can see *why* a span was chosen and where the call was close. + +## Known limitation + +**`gist` is prose and is graded by a judge, not by string equality.** Two correct answers to +the same question rarely share wording — the same finding that replaced token-set F1 with +embedding cosine in M2X-036. That means the answer-correctness half of this set inherits +whatever the judge model's biases are, and only citation accuracy and abstention are +mechanically checkable. Read the gate numbers with that split in mind. diff --git a/eval/rag/expected/.gitkeep b/eval/rag/expected/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/eval/rag/expected/seal-manifest.json b/eval/rag/expected/seal-manifest.json new file mode 100644 index 0000000..b1d51ce --- /dev/null +++ b/eval/rag/expected/seal-manifest.json @@ -0,0 +1,158 @@ +{ + "schema": 1, + "sealed_at": "2026-08-13", + "cipher": "AES256", + "case_count": 30, + "cases": [ + { + "case_id": "q01", + "file": "q01.json", + "sha256": "003b54d21c4b0b6bbf530395d13e39555d0c4fc69270689945494610226e02a8" + }, + { + "case_id": "q02", + "file": "q02.json", + "sha256": "ca5791a898534fd921a13d48ed6f7507a78e36ba62574446231e5d29c07924dc" + }, + { + "case_id": "q03", + "file": "q03.json", + "sha256": "2924e9c8530759855128b6437ee1bcb35c40a8a5ea2ea4150dad6c3251d1ddca" + }, + { + "case_id": "q04", + "file": "q04.json", + "sha256": "ae969b53081cede6e389256f3373e703545cab4980fb3062f4c57f38a1398c16" + }, + { + "case_id": "q05", + "file": "q05.json", + "sha256": "4c4548a9fd2a3566f97be828ffeff90ab9120feddcc9eba800e7c9ca192bdfde" + }, + { + "case_id": "q06", + "file": "q06.json", + "sha256": "30d3bce2c4334c2a569e05b7d369a6cad23738e8f29aef6c439599139b992e32" + }, + { + "case_id": "q07", + "file": "q07.json", + "sha256": "269b445389eee860d4502e2e72a5c6613c1e85ea2e1500cf8799f4b908320521" + }, + { + "case_id": "q08", + "file": "q08.json", + "sha256": "2f5910533e00dddc13e7d34c98bfab62406f6c2f2a7c2cf4247cc462c38311b0" + }, + { + "case_id": "q09", + "file": "q09.json", + "sha256": "d09de7791c37dca654f8059abdb68f9494fea86eee78625bdaaafeb9a535e11b" + }, + { + "case_id": "q10", + "file": "q10.json", + "sha256": "b0a88ee664f3d5f7a77f5598c0681f384a7add914bfec91ef5131d2fdfd00f9f" + }, + { + "case_id": "q11", + "file": "q11.json", + "sha256": "95ddfccc1981419a9953bc4484adde81f5320f947b0ad9eda014afef15322ce3" + }, + { + "case_id": "q12", + "file": "q12.json", + "sha256": "29773d15c0f6ca5a5c02ac43830547b20927f0bfe689163e5a4b2077f57bd8ed" + }, + { + "case_id": "q13", + "file": "q13.json", + "sha256": "7e897183a0dcdba2b999b95eadf6e012cf594782c2339419e0122982ebabd3b5" + }, + { + "case_id": "q14", + "file": "q14.json", + "sha256": "e359bb77c72e59c9430bcc066804c6acbf2464ee5645ebd9267763f8819c7336" + }, + { + "case_id": "q15", + "file": "q15.json", + "sha256": "3ca1fb198e33c7ad109e9c03fd9d91eb078ffd788eed8eec1ccb5a878a9fc6db" + }, + { + "case_id": "q16", + "file": "q16.json", + "sha256": "e8e682b5403373b8eb771f0b8bb3880fd35054df6f5014339d30cfebb0fe2641" + }, + { + "case_id": "q17", + "file": "q17.json", + "sha256": "1d0e08e8659c2e4f8f95f128e27314a2f7be85b57fc847ef0f57fa1fedfd2c7c" + }, + { + "case_id": "q18", + "file": "q18.json", + "sha256": "74a1c9c330854d1281bf7381262c6c9f5505469df86de342a687d4856e5f7e86" + }, + { + "case_id": "q19", + "file": "q19.json", + "sha256": "713f521a15abf8dc89bc0fc1e296c8d95919508b13abf66166f28238556d49dc" + }, + { + "case_id": "q20", + "file": "q20.json", + "sha256": "a29f59c2509a26ec503fe32901fb62e62529663de04b26a7773fc12d7aca85f1" + }, + { + "case_id": "q21", + "file": "q21.json", + "sha256": "00dcacb09a03241e857dbe6f554c3ae6a94aff8e112c63b8f8d2cc85ac90da45" + }, + { + "case_id": "q22", + "file": "q22.json", + "sha256": "00146cd6d216d1732ef4b6ec6e23b8fd09f755ae17300bc45ce14e82266c03f4" + }, + { + "case_id": "q23", + "file": "q23.json", + "sha256": "0351f9425bfbe8e6e4fa20b8796788cfe8dbb34bfe486391cddcc358faf39828" + }, + { + "case_id": "q24", + "file": "q24.json", + "sha256": "b4ecea86c4eebeb4f06b84c5db9605bdce21f19fb22eb76b4da0f1fda70907bb" + }, + { + "case_id": "q25", + "file": "q25.json", + "sha256": "e9adc19ba1b4b5b58ed5791c9153bd799a202c178974d164dbb22fb53e759449" + }, + { + "case_id": "q26", + "file": "q26.json", + "sha256": "a0610d73c5b9f610c111c4a7e77baf6c2f4a87901c859c754193f6cdb6b36950" + }, + { + "case_id": "q27", + "file": "q27.json", + "sha256": "226d9de8209d77c6e6140e34b326c5cf450ecf8b28c9cf12df829765ac9fd186" + }, + { + "case_id": "q28", + "file": "q28.json", + "sha256": "93d9f6ca1728f6952d45e74e9d3063c1490a5a7cfd3588e5c11907a5d79da100" + }, + { + "case_id": "q29", + "file": "q29.json", + "sha256": "50f398b8819e440b237e2a78766165a6d43044eb439d94fe6c1173d5c0303713" + }, + { + "case_id": "q30", + "file": "q30.json", + "sha256": "d1aa4ba18a17676ee872c2f2f1b773adfd33dd6936ebed99795186022cc1ee91" + } + ] +} diff --git a/eval/rag/questions.jsonl b/eval/rag/questions.jsonl new file mode 100644 index 0000000..a4af186 --- /dev/null +++ b/eval/rag/questions.jsonl @@ -0,0 +1,30 @@ +{"question_id": "q01", "kind": "single_meeting", "question": "What is holding up the aquarium according to the legal department?"} +{"question_id": "q02", "kind": "single_meeting", "question": "What did the marketing department buy to draw shoppers up to the third floor?"} +{"question_id": "q03", "kind": "single_meeting", "question": "Which role has the mall still not hired for the third floor?"} +{"question_id": "q04", "kind": "single_meeting", "question": "What did Beth report about the numbers this week?"} +{"question_id": "q05", "kind": "single_meeting", "question": "When did John say he would come back about the legal problem?"} +{"question_id": "q06", "kind": "single_meeting", "question": "Which overseas market did Beth want to target, and through what channel?"} +{"question_id": "q07", "kind": "single_meeting", "question": "Why did Sarah argue against focusing on Japan?"} +{"question_id": "q08", "kind": "single_meeting", "question": "What did Sarah ask Donald to produce, and by when?"} +{"question_id": "q09", "kind": "single_meeting", "question": "Which part of the US did Donald name as the best target, and why?"} +{"question_id": "q10", "kind": "single_meeting", "question": "Which celebrity did Ron suggest as an influencer, and how did Sarah respond?"} +{"question_id": "q11", "kind": "single_meeting", "question": "Where is the company retreat being held?"} +{"question_id": "q12", "kind": "single_meeting", "question": "What did Sally ask to add to the retreat programme?"} +{"question_id": "q13", "kind": "single_meeting", "question": "Can employees bring family to the retreat?"} +{"question_id": "q14", "kind": "single_meeting", "question": "Where does the trip stop on the way to Mexico, and for how long?"} +{"question_id": "q15", "kind": "single_meeting", "question": "What did Jenny say when Donald suggested visiting a theme park in Florida?"} +{"question_id": "q16", "kind": "single_meeting", "question": "Who developed the robotic nurse and how tall is it?"} +{"question_id": "q17", "kind": "single_meeting", "question": "What does the piece claim about the robotic nurse's availability compared with human nurses?"} +{"question_id": "q18", "kind": "single_meeting", "question": "How does Nursing Bot Incorporated address the safety and privacy concerns?"} +{"question_id": "q19", "kind": "single_meeting", "question": "What is the main objection raised in the contrarian Q&A about robots in healthcare?"} +{"question_id": "q20", "kind": "single_meeting", "question": "What does the contrarian say about the cost savings argument for robots?"} +{"question_id": "q21", "kind": "cross_meeting", "question": "What does Donald offer or ask for in the marketing meeting and in the retreat meeting?"} +{"question_id": "q22", "kind": "cross_meeting", "question": "What food plans are discussed in the mall meeting and in the retreat meeting?"} +{"question_id": "q23", "kind": "cross_meeting", "question": "What does Sarah ask the group to do in each of the two meetings she chairs?"} +{"question_id": "q24", "kind": "cross_meeting", "question": "Which cost or budget concerns are raised across the mall meeting and the marketing meeting?"} +{"question_id": "q25", "kind": "cross_meeting", "question": "What physical or outdoor activities come up in the marketing meeting and the retreat meeting?"} +{"question_id": "q26", "kind": "unanswerable", "question": "How much did the third-floor aquarium project cost in total?"} +{"question_id": "q27", "kind": "unanswerable", "question": "What is the name of the company whose sports equipment is being marketed?"} +{"question_id": "q28", "kind": "unanswerable", "question": "What is Nursing Bot Incorporated's annual revenue?"} +{"question_id": "q29", "kind": "unanswerable", "question": "Which hotel is the Mexico retreat booked with?"} +{"question_id": "q30", "kind": "unanswerable", "question": "Who won the 2026 cricket world cup?"} diff --git a/scripts/validate_rag_questions.py b/scripts/validate_rag_questions.py new file mode 100644 index 0000000..8b6b937 --- /dev/null +++ b/scripts/validate_rag_questions.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Check the RAG question set before it is trusted to grade anything. + +The ticket asks for a second pass a few hours later: *can each answerable question really +be answered from the recorded segment?* Half of that is a judgement no script can make, and +it is recorded per question in the sealed `notes` field. The other half is mechanical and +belongs here — the judgement has to point at something real, and a pointer into a turn that +does not exist corrupts the gate exactly as badly as a wrong judgement does. + +What it checks: ids unique and paired across both halves, the 20/5/5 mix the ticket +specifies, every answerable question carrying evidence and a gist, every unanswerable one +carrying neither, cross-meeting questions genuinely spanning two meetings, and every cited +turn range resolving against the committed reference transcripts. + +Needs the expected answers **unsealed**, so it is an Evaluator command. Exits non-zero on +any problem, so a set cannot be recorded sound by reading past the output. + +Usage:: + + uv run python scripts/validate_rag_questions.py +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from m2x.errors import ConfigError +from m2x.rag_questions import ( + DEFAULT_RAG_EVAL_DIR, + EXPECTED_DIRNAME, + QUESTIONS_FILENAME, + QuestionKind, + load_expected, + load_questions, + validate_question_set, +) +from m2x.reference_transcript import DEFAULT_REFERENCE_DIR + + +def main(argv: list[str] | None = None) -> int: + """Validate the set. + + Args: + argv: Arguments, defaulting to ``sys.argv[1:]``. + + Returns: + Process exit code: 0 sound, 1 problems found, 2 the set could not be read. + """ + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--dir", type=Path, default=DEFAULT_RAG_EVAL_DIR, help="root of the eval set") + parser.add_argument( + "--reference-dir", + type=Path, + default=DEFAULT_REFERENCE_DIR, + help="where the reference transcripts live", + ) + args = parser.parse_args(argv) + + try: + questions = load_questions(args.dir / QUESTIONS_FILENAME) + expected = load_expected(args.dir / EXPECTED_DIRNAME) + except (ConfigError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + problems = validate_question_set(questions, expected, reference_dir=args.reference_dir) + if problems: + for problem in problems: + print(f"FAIL {problem}", file=sys.stderr) + return 1 + + spans = sum(len(expected[question.question_id].evidence) for question in questions) + meetings = sorted( + {span.meeting_id for answer in expected.values() for span in answer.evidence} + ) + by_kind = {kind: sum(1 for q in questions if q.kind is kind) for kind in QuestionKind} + + print(f"OK {len(questions)} questions, {spans} evidence spans, every turn resolves") + for kind, count in by_kind.items(): + print(f" {kind.value:<16} {count}") + print(f" meetings covered: {len(meetings)} — {', '.join(meetings)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/m2x/rag_questions.py b/src/m2x/rag_questions.py new file mode 100644 index 0000000..62ec4be --- /dev/null +++ b/src/m2x/rag_questions.py @@ -0,0 +1,341 @@ +"""The Phase 2 gate instrument: thirty questions and what counts as answering them. + +Same discipline as the Phase 1B ground truth (M2X-033), and the same split of concerns. +A question is **public** — the system has to be asked it, so hiding it would only stop the +harness running. An expected answer is **sealed**, because a builder who can read the +expected segments can tune retrieval until they come back, and a retrieval metric tuned +against its own answer key measures nothing. + +## The corpus is `eval/tiron/`, and that is a decision + +`data/` is git-ignored, so a fresh clone has no meeting transcripts and the only corpus +`m2x ask` has ever been verified against is this repository's own markdown — every citation +in `docs/design/day4-ask.md` is a `§ heading`, not an `mm:ss`. Writing the gate set against +documents would leave the meeting citation path, which is the whole product promise, +unmeasured at the gate. + +The tiron reference transcripts are committed, carry human speaker turns and real +timestamps, and are already the substrate for the extraction labels. Questions written +against them exercise `[meeting · speaker · mm:ss–mm:ss]` on every clone, with no audio and +no provider. + +## Ground truth is turn ranges, never rendered citations + +An expected answer names ``(meeting_id, first_turn, last_turn)`` into the reference +transcript. Not an `mm:ss` string — that would score the renderer rather than the +retrieval — and not a chunk id, which is a function of the chunking parameters and changes +the moment anyone tunes them. + +Turn indices are stable: they are positions in a committed file. A retrieved chunk records +the segment range it covers, so citation accuracy is an **overlap** test between the two, +exactly as the ticket specifies ("the cited segment id is among the ground-truth segments, +or overlaps its time range"). +""" + +from __future__ import annotations + +import json +from collections import Counter +from enum import Enum +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from m2x.errors import ConfigError +from m2x.reference_transcript import DEFAULT_REFERENCE_DIR, load_reference_segments + +DEFAULT_RAG_EVAL_DIR = Path("eval/rag") +"""Root of the Phase 2 gate instrument.""" + +QUESTIONS_FILENAME = "questions.jsonl" +"""Public half: the questions, committed in plaintext because the system must be asked them.""" + +EXPECTED_DIRNAME = "expected" +"""Sealed half: one file per question. Plaintext git-ignored, ciphertext and digests committed.""" + +SINGLE_MEETING_COUNT = 20 +CROSS_MEETING_COUNT = 5 +UNANSWERABLE_COUNT = 5 +"""The ticket's mix: 20 single-meeting, 5 cross-meeting, 5 that must abstain. + +Pinned as constants so the set cannot quietly drift toward whichever kind the system +happens to be good at. :func:`validate_question_set` fails on any other distribution. +""" + + +class QuestionKind(str, Enum): + """What a question is testing. + + Kept as three values rather than an ``answerable`` boolean because the three score + differently: a cross-meeting question that retrieves one meeting well is a *failure* + the single-meeting bucket cannot express, and an unanswerable question is scored on + abstention rather than on content at all. + """ + + SINGLE_MEETING = "single_meeting" + """Answerable from one meeting.""" + + CROSS_MEETING = "cross_meeting" + """Needs facts from two or more meetings; retrieving only one is a miss.""" + + UNANSWERABLE = "unanswerable" + """Not in the corpus. Correct only if the system abstains.""" + + +class EvidenceSpan(BaseModel): + """A range of reference turns that contains the answer. + + Attributes: + meeting_id: Reference meeting, e.g. ``tiron-MTG_32185``. + first_turn: First turn, 0-based inclusive. + last_turn: Last turn, 0-based inclusive. + """ + + model_config = ConfigDict(frozen=True) + + meeting_id: str = Field(min_length=1) + first_turn: int = Field(ge=0) + last_turn: int = Field(ge=0) + + @model_validator(mode="after") + def _bounds_are_ordered(self) -> EvidenceSpan: + """Reject an inverted range. + + Returns: + The validated span. + + Raises: + ValueError: ``last_turn`` precedes ``first_turn``. + """ + if self.last_turn < self.first_turn: + raise ValueError(f"turn range inverted: {self.first_turn} > {self.last_turn}") + return self + + def overlaps(self, first: int, last: int) -> bool: + """Whether a retrieved chunk's turn range intersects this span. + + Overlap rather than containment: chunks are packed to a character budget and a + correct citation routinely covers the answer plus its neighbours. Demanding + containment would fail a chunk for being the size the indexer chose. + + Args: + first: Chunk's first turn, 0-based inclusive. + last: Chunk's last turn, 0-based inclusive. + + Returns: + True when the ranges intersect. + """ + return first <= self.last_turn and last >= self.first_turn + + +class RagQuestion(BaseModel): + """The public half of one eval case. + + Attributes: + question_id: Stable id, ``q01``…``q30``. + kind: What the question tests. + question: The question as it is asked of the system, verbatim. + """ + + model_config = ConfigDict(frozen=True) + + question_id: str = Field(min_length=1) + kind: QuestionKind + question: str = Field(min_length=1) + + +class ExpectedAnswer(BaseModel): + """The sealed half of one eval case. + + Attributes: + question_id: The question this answers. + gist: What a correct answer must convey, in one sentence. Graded by a judge, never + by string equality — two correct answers to the same question rarely share + wording, which is the same finding that replaced token-set F1 in M2X-036. + evidence: Turn ranges containing the answer. Empty exactly when the question is + unanswerable. + notes: Why this is the answer, and where it was nearly something else. Written for + the adjudicator at the gate, not for the harness. + """ + + model_config = ConfigDict(frozen=True) + + question_id: str = Field(min_length=1) + gist: str = "" + evidence: list[EvidenceSpan] = Field(default_factory=list) + notes: str = "" + + @property + def must_abstain(self) -> bool: + """Whether the only correct behaviour is abstention. + + Derived from ``evidence`` being empty rather than stored as its own flag: two + fields encoding one fact drift, and the drift would silently turn a must-abstain + case into a scored one. + + Returns: + True when no passage in the corpus answers the question. + """ + return not self.evidence + + def meeting_ids(self) -> set[str]: + """Meetings this answer draws on. + + Returns: + The distinct meeting ids across every evidence span. + """ + return {span.meeting_id for span in self.evidence} + + +def load_questions(path: Path) -> list[RagQuestion]: + """Read the public question file. + + Args: + path: ``questions.jsonl``. + + Returns: + Questions in file order. + + Raises: + OSError: The file could not be read. + pydantic.ValidationError: A line is not a valid question. + """ + return [ + RagQuestion.model_validate_json(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def load_expected(directory: Path) -> dict[str, ExpectedAnswer]: + """Read the unsealed expected answers. + + Args: + directory: ``eval/rag/expected``, after unsealing. + + Returns: + ``{question_id: ExpectedAnswer}``. + + Raises: + ConfigError: The directory holds no expected answers, which would let a gate run + score every question as unanswerable and report a perfect abstention rate. + OSError: A file could not be read. + pydantic.ValidationError: A file is not a valid expected answer. + """ + files = sorted(path for path in directory.glob("*.json") if path.name != "seal-manifest.json") + if not files: + raise ConfigError( + f"no expected answers in {directory} — the set is sealed, or was never " + "written. Unseal with `uv run python scripts/seal_heldout.py unseal --dir " + f"{directory}` before scoring." + ) + answers = [ExpectedAnswer.model_validate_json(path.read_text(encoding="utf-8")) for path in files] + return {answer.question_id: answer for answer in answers} + + +def save_expected(answer: ExpectedAnswer, directory: Path) -> Path: + """Write one expected answer. + + Args: + answer: Answer to persist. + directory: Destination, created if absent. + + Returns: + The path written. + + Raises: + OSError: The file could not be written. + """ + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{answer.question_id}.json" + path.write_text( + json.dumps(answer.model_dump(mode="json"), indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + return path + + +def validate_question_set( + questions: list[RagQuestion], + expected: dict[str, ExpectedAnswer], + *, + reference_dir: Path = DEFAULT_REFERENCE_DIR, +) -> list[str]: + """Check the set is well formed and every citation resolves. + + This is the ticket's "second pass a few hours later", made mechanical. The manual + version — *can each answerable question really be answered from the recorded + segment?* — is a judgement no code can make, and it is recorded per question in + ``notes``. What code **can** check is that the judgement points somewhere real, and a + broken pointer corrupts the gate exactly as badly as a broken judgement. + + Args: + questions: The public questions. + expected: Expected answers by question id. + reference_dir: Where the reference transcripts live. + + Returns: + Human-readable problems, empty when the set is sound. + + Raises: + OSError: A reference transcript could not be read. + """ + problems: list[str] = [] + ids = [question.question_id for question in questions] + + problems += [f"{qid}: duplicate question id" for qid, n in Counter(ids).items() if n > 1] + problems += [f"{qid}: question has no expected answer" for qid in ids if qid not in expected] + problems += [f"{qid}: expected answer has no question" for qid in expected if qid not in set(ids)] + + counts = Counter(question.kind for question in questions) + for kind, wanted in ( + (QuestionKind.SINGLE_MEETING, SINGLE_MEETING_COUNT), + (QuestionKind.CROSS_MEETING, CROSS_MEETING_COUNT), + (QuestionKind.UNANSWERABLE, UNANSWERABLE_COUNT), + ): + if counts.get(kind, 0) != wanted: + problems.append(f"{kind.value}: {counts.get(kind, 0)} questions, the ticket asks for {wanted}") + + # Cached per meeting: fifteen questions can name the same transcript, and re-reading it + # per span would make validation slower than the eval it guards. + lengths: dict[str, int] = {} + for question in questions: + answer = expected.get(question.question_id) + if answer is None: + continue + + if question.kind is QuestionKind.UNANSWERABLE and not answer.must_abstain: + problems.append(f"{question.question_id}: marked unanswerable but carries evidence") + if question.kind is not QuestionKind.UNANSWERABLE and answer.must_abstain: + problems.append(f"{question.question_id}: answerable but names no evidence") + if question.kind is not QuestionKind.UNANSWERABLE and not answer.gist: + problems.append(f"{question.question_id}: answerable but records no expected gist") + if question.kind is QuestionKind.CROSS_MEETING and len(answer.meeting_ids()) < 2: + problems.append( + f"{question.question_id}: marked cross-meeting but its evidence is all in " + f"{answer.meeting_ids() or 'no meeting'} — a question one meeting answers " + "measures single-meeting retrieval under a cross-meeting label" + ) + if question.kind is QuestionKind.SINGLE_MEETING and len(answer.meeting_ids()) > 1: + problems.append( + f"{question.question_id}: marked single-meeting but draws on " + f"{len(answer.meeting_ids())} meetings" + ) + + for span in answer.evidence: + if span.meeting_id not in lengths: + try: + lengths[span.meeting_id] = len( + load_reference_segments(span.meeting_id, reference_dir=reference_dir) + ) + except ConfigError as error: + lengths[span.meeting_id] = -1 + problems.append(f"{question.question_id}: {error}") + turns = lengths[span.meeting_id] + if turns >= 0 and span.last_turn >= turns: + problems.append( + f"{question.question_id}: cites turn {span.last_turn} of " + f"{span.meeting_id}, which has {turns} — the reference changed under " + "this question and its evidence can no longer be trusted" + ) + return problems diff --git a/tests/test_rag_questions.py b/tests/test_rag_questions.py new file mode 100644 index 0000000..590419a --- /dev/null +++ b/tests/test_rag_questions.py @@ -0,0 +1,269 @@ +"""Tests for the Phase 2 question set and its validator. + +The validator is the only thing standing between a broken question and a gate number that +looks fine, so what is under test is its **refusals**. A validator that passes a +cross-meeting question answerable from one meeting, or a citation into a turn that does not +exist, is worse than no validator: it converts an unchecked set into a checked-looking one. + +The committed set is exercised too. Those tests fail if anyone edits `questions.jsonl` +without re-running the generator, which is exactly when the two halves drift apart. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from m2x.rag_questions import ( + CROSS_MEETING_COUNT, + DEFAULT_RAG_EVAL_DIR, + EXPECTED_DIRNAME, + QUESTIONS_FILENAME, + SINGLE_MEETING_COUNT, + UNANSWERABLE_COUNT, + EvidenceSpan, + ExpectedAnswer, + QuestionKind, + RagQuestion, + load_questions, + save_expected, + validate_question_set, +) + +MEETING = "ref-001" +OTHER = "ref-002" + + +def _write_reference(directory: Path, meeting_id: str = MEETING, count: int = 40) -> None: + """Write a reference pair the spans can resolve against. + + Args: + directory: Reference directory, created if absent. + meeting_id: Meeting the pair describes. + count: How many turns. + """ + directory.mkdir(parents=True, exist_ok=True) + turns = [ + {"t_start": float(i), "t_end": float(i) + 0.5, "speaker": f"spk-{i % 2}"} + for i in range(count) + ] + (directory / f"{meeting_id}.speakers.json").write_text( + json.dumps({"meeting_id": meeting_id, "segments": turns}), encoding="utf-8" + ) + (directory / f"{meeting_id}.txt").write_text( + "\n".join(f"turn {i}" for i in range(count)) + "\n", encoding="utf-8" + ) + + +def _span(meeting_id: str = MEETING, first: int = 1, last: int = 3) -> EvidenceSpan: + """Build one evidence span.""" + return EvidenceSpan(meeting_id=meeting_id, first_turn=first, last_turn=last) + + +def _set(*, cross_spans: list[EvidenceSpan] | None = None) -> tuple[list, dict]: + """Build a well-formed set of the ticket's exact shape. + + Args: + cross_spans: Override the evidence on the first cross-meeting question, which is + how the cross-meeting refusal is exercised. + + Returns: + ``(questions, expected)``. + """ + questions: list[RagQuestion] = [] + expected: dict[str, ExpectedAnswer] = {} + index = 0 + + def add(kind: QuestionKind, spans: list[EvidenceSpan]) -> None: + nonlocal index + index += 1 + qid = f"q{index:02d}" + questions.append(RagQuestion(question_id=qid, kind=kind, question=f"question {qid}?")) + expected[qid] = ExpectedAnswer( + question_id=qid, gist="the answer" if spans else "", evidence=spans + ) + + for _ in range(SINGLE_MEETING_COUNT): + add(QuestionKind.SINGLE_MEETING, [_span()]) + for position in range(CROSS_MEETING_COUNT): + spans = ( + cross_spans + if position == 0 and cross_spans is not None + else [_span(), _span(OTHER, 2, 4)] + ) + add(QuestionKind.CROSS_MEETING, spans) + for _ in range(UNANSWERABLE_COUNT): + add(QuestionKind.UNANSWERABLE, []) + return questions, expected + + +@pytest.fixture +def reference(tmp_path: Path) -> Path: + """A reference directory holding both test meetings.""" + _write_reference(tmp_path, MEETING) + _write_reference(tmp_path, OTHER) + return tmp_path + + +def test_a_well_formed_set_validates(reference: Path) -> None: + """The baseline, so the refusals below mean something.""" + questions, expected = _set() + + assert validate_question_set(questions, expected, reference_dir=reference) == [] + + +def test_a_cross_meeting_question_answerable_from_one_meeting_is_rejected(reference: Path) -> None: + """The refusal that matters most. + + A cross-meeting question whose evidence sits in one meeting measures single-meeting + retrieval under a cross-meeting label, and it would inflate the hardest bucket in the + set without anything in the file looking wrong. + """ + questions, expected = _set(cross_spans=[_span(MEETING, 1, 3), _span(MEETING, 10, 12)]) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("cross-meeting" in problem for problem in problems) + + +def test_a_span_past_the_end_of_the_reference_is_rejected(reference: Path) -> None: + """If the reference changed, the question's evidence points at different words. + + Same failure the labelled cases guard against: turn indices are positions in a file, + and a file that shrank silently re-aims every question after the edit. + """ + questions, expected = _set() + expected["q01"] = ExpectedAnswer( + question_id="q01", gist="the answer", evidence=[_span(MEETING, 900, 950)] + ) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("the reference changed" in problem for problem in problems) + + +def test_an_unknown_meeting_is_rejected(reference: Path) -> None: + """A typo in a meeting id must fail loudly, not silently score zero citations.""" + questions, expected = _set() + expected["q01"] = ExpectedAnswer( + question_id="q01", gist="the answer", evidence=[_span("ref-does-not-exist")] + ) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("does not exist" in problem for problem in problems) + + +def test_an_answerable_question_with_no_evidence_is_rejected(reference: Path) -> None: + """Empty evidence means must-abstain, so this would flip a scored case silently.""" + questions, expected = _set() + expected["q01"] = ExpectedAnswer(question_id="q01", gist="the answer", evidence=[]) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("names no evidence" in problem for problem in problems) + + +def test_an_unanswerable_question_carrying_evidence_is_rejected(reference: Path) -> None: + """The inverse: a must-abstain case that is secretly answerable grades abstention wrong.""" + questions, expected = _set() + expected["q30"] = ExpectedAnswer(question_id="q30", evidence=[_span()]) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("marked unanswerable but carries evidence" in problem for problem in problems) + + +def test_the_wrong_mix_is_rejected(reference: Path) -> None: + """20/5/5 is the ticket's number, not a preference. + + Without this the set could drift toward whichever kind the system handles well, and + the gate figure would move without any question changing. + """ + questions, expected = _set() + dropped = questions.pop() + expected.pop(dropped.question_id) + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("the ticket asks for" in problem for problem in problems) + + +def test_a_question_without_an_expected_answer_is_rejected(reference: Path) -> None: + """The two halves live in different files and one is sealed, so they can drift.""" + questions, expected = _set() + expected.pop("q01") + + problems = validate_question_set(questions, expected, reference_dir=reference) + + assert any("no expected answer" in problem for problem in problems) + + +def test_must_abstain_is_derived_not_stored() -> None: + """One fact, one field. + + A separate boolean beside the evidence list would let the two disagree, and the + disagreement would turn a must-abstain case into a scored one without looking wrong. + """ + assert ExpectedAnswer(question_id="q01").must_abstain + assert not ExpectedAnswer(question_id="q01", gist="x", evidence=[_span()]).must_abstain + + +def test_an_inverted_span_is_rejected() -> None: + """Bounds are checked at construction, so a bad span cannot reach the scorer.""" + with pytest.raises(ValueError, match="inverted"): + EvidenceSpan(meeting_id=MEETING, first_turn=9, last_turn=2) + + +@pytest.mark.parametrize( + ("first", "last", "expected_overlap"), + [(0, 0, False), (0, 1, True), (2, 2, True), (3, 9, True), (4, 9, False), (0, 99, True)], +) +def test_overlap_is_inclusive_at_both_ends(first: int, last: int, expected_overlap: bool) -> None: + """Citation accuracy is an overlap test, so its boundaries are the metric. + + An off-by-one here shifts every citation score in the gate by up to one chunk. + """ + assert _span(MEETING, 1, 3).overlaps(first, last) is expected_overlap + + +def test_the_committed_set_is_the_shape_the_ticket_asks_for() -> None: + """Guards the real file, not a fixture. + + The expected answers are sealed, so this half is what a reviewer without the + passphrase can still check. + """ + questions = load_questions(DEFAULT_RAG_EVAL_DIR / QUESTIONS_FILENAME) + counts = {kind: sum(1 for q in questions if q.kind is kind) for kind in QuestionKind} + + assert len(questions) == SINGLE_MEETING_COUNT + CROSS_MEETING_COUNT + UNANSWERABLE_COUNT + assert counts[QuestionKind.SINGLE_MEETING] == SINGLE_MEETING_COUNT + assert counts[QuestionKind.CROSS_MEETING] == CROSS_MEETING_COUNT + assert counts[QuestionKind.UNANSWERABLE] == UNANSWERABLE_COUNT + assert len({q.question_id for q in questions}) == len(questions) + + +def test_the_committed_questions_never_leak_their_answers() -> None: + """The public half must stay public-safe. + + `questions.jsonl` is committed in plaintext; if a gist or an evidence span ever + appeared in it the seal on `expected/` would be decorative. + """ + text = (DEFAULT_RAG_EVAL_DIR / QUESTIONS_FILENAME).read_text(encoding="utf-8") + + assert "gist" not in text + assert "evidence" not in text + assert "first_turn" not in text + + +def test_round_trip_preserves_an_expected_answer(tmp_path: Path) -> None: + """A sealed answer must read back identical, or the digest check is meaningless.""" + answer = ExpectedAnswer( + question_id="q01", gist="the answer", evidence=[_span()], notes="why" + ) + + written = save_expected(answer, tmp_path) + + assert ExpectedAnswer.model_validate_json(written.read_text(encoding="utf-8")) == answer