From f6150a0bc53587ef5590524f9cc7a0829594cf60 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 17:38:16 -0700 Subject: [PATCH 1/8] feat: add a spoken-punctuation eval and freeze its dataset A dictation user who wants a comma often says "comma", and the transcript comes back with the word in it. Nothing in the harness posed that: the disfluency injector only inserts hesitation, and both sides of every real paired corpus here are already punctuated. `--spoken-punctuation RATE` layers the task onto the real corpus rather than replacing it, so nyra's hand-annotated disfluencies stay and only the punctuation is synthetic: input : Um we shipped it today period monday was really, quiet comma so nobody noticed period target: We shipped it today. Monday was quiet, so nobody noticed. A mark is spoken only where the reference licenses it -- the same word carries the same mark in the target -- so the correct answer never stops being the corpus's own target and the target is never edited. `really,` above survives as a comma because the annotator deleted the span it introduced, and asking for a mark that would then be scored as an error teaches an instruction that commands are sometimes to be ignored. ALL CAPS is the one exception, and has to be: a target with no uppercase in it cannot pose the task. Verified over 2,631 commands on 1,200 rows -- none mis-scored, and a perfect cleanup still scores exactly 1.0. It selects on `format`, not `blend`. normalize() casefolds and strips marks, so a restored comma and a missed one are the same string to the content axis and a casing command is invisible to it. Alongside the axes the run reports what became of the commands themselves -- converted / left as words / dropped -- which is the number WER cannot give and only a synthetic operator can be asked. Four candidate instructions come with it. `punct-appended` is the cheapest possible change to what ships: PRIOR_WINNER plus one clause, at 2006 of the 2048 characters. That leaves 42 characters of headroom, so GEPA cannot grow from it -- `punct-explicit` at 1430 is the searchable seed. BASELINE stays `prior-winner`, which has never heard of the task and is therefore the right bar: the held-out comparison answers what teaching it this buys. Three things this turned up. The false-start classifier read "question mark" as an abandoned two-word phrase -- rule 2 is "two or more non-hesitation words that echo nothing", which is exactly the shape of a dictation command -- and charged it FALSE_START_WEIGHT per word. That made a leftover "question mark" ten errors and a leftover "period" one, an asymmetry nobody chose and one that would have pointed the search at the multi-word commands for arithmetic reasons. metrics.score now takes a `not_abandoned` set that Utterance fills from the commands it planted; it defaults to empty, and the false-start fraction over 300 nyra rows is 0.207 with spoken punctuation and 0.207 without, so every number measured before it still holds. A proper-noun heuristic stood in the injector and was removed as harmful. The word after a spoken terminal mark is sentence-initial in the reference by construction, so restoring its capital is the rule every candidate already states -- name or not. Protecting names would have handed the answer to 30% of the commands on nyra. Only the pronoun I is exempt, because it is capitalized mid-sentence too. `--help` has been broken since a help string first said "~42%": argparse %-expands help text, so a bare % raises TypeError and the whole thing fails. Invisible to every test, because nothing had asked for the help. Now pinned. Also freezes a dataset. `--dump-corpus PATH` writes the loaded corpus as JSONL and `--jsonl` reads it back exactly, commands included, so a corpus otherwise assembled at load time from a download plus two seeded injectors becomes something diffable. data/spoken-punctuation.jsonl is generated from `--source builtin`, whose sentences are written for this repo and so carry no third-party terms; nyra derives from LDC-licensed Switchboard and stays out of the tree. test_eval.py regenerates the committed file and asserts it is byte-identical, so an injector change fails the suite instead of leaving a stale dataset behind. That corpus grew from 12 sentences to 76 to give the task something to bite on -- internal commas, questions, exclamations, colons and semicolons that nyra never supplies. It also does the one thing nyra cannot: 15% of its references use a command word as ordinary content ("one grace period, so plan accordingly"), against ~0-1% of nyra's, so it is the only corpus here that can charge an instruction for converting a word the speaker meant literally. Everything downstream keys off what the corpus contains rather than what flags were passed, so the saved dataset is scored the same way as the corpus it came from. Before that it selected on `blend` and reported no command outcomes. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 199 ++++++- evals/dictation-prompt/candidates.py | 83 +++ evals/dictation-prompt/corpus.py | 301 +++++++++- evals/dictation-prompt/data/README.md | 78 +++ .../data/spoken-punctuation.jsonl | 76 +++ evals/dictation-prompt/metrics.py | 48 +- .../optimize_cleanup_prompt.py | 202 ++++++- evals/dictation-prompt/program.py | 74 ++- evals/dictation-prompt/spoken_punctuation.py | 547 ++++++++++++++++++ evals/dictation-prompt/test_eval.py | 537 +++++++++++++++++ 10 files changed, 2061 insertions(+), 84 deletions(-) create mode 100644 evals/dictation-prompt/data/README.md create mode 100644 evals/dictation-prompt/data/spoken-punctuation.jsonl create mode 100644 evals/dictation-prompt/spoken_punctuation.py diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 2663180..1e64076 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -59,6 +59,160 @@ reparanda, and rewrites the verbatim side into its own conventions (`[UH]`, `[la It does not pose punctuation _restoration_, since both sides are already punctuated. Only `--source builtin --strip-formatting` does, on a dozen bundled sentences. +### Spoken punctuation — "period, comma, question mark, ALL CAPS" + +A dictation user who wants a comma often _says_ "comma", and the transcript comes back +with the word in it. `--spoken-punctuation RATE` poses that task, on top of whatever the +input side already is: + +```bash +# Rank the punctuation instructions against the shipped one. No search, ~5 candidates. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py \ + --spoken-punctuation 0.8 --optimizer none --limit 600 + +# Then search from the best of them. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py \ + --spoken-punctuation 0.8 --start best-candidate +``` + +Unlike `--severity`, this applies to a **paired** source too, and that is the point: the +disfluencies stay the ones annotators marked by hand and only the punctuation is +synthetic. A row comes out carrying both, which is the task a real user poses — + +``` +input : Um we shipped it today period monday was really, quiet comma so nobody noticed period +target: We shipped it today. Monday was quiet, so nobody noticed. +``` + +Two operators, and only one of them touches a target: + +**Marks** are spoken on the **input side only**, and only where the reference +**licenses** them — the same word carries the same mark in the target. So the correct +answer never stops being the corpus's own target, and the target is not edited at all. +A mark the reference lacks is never spoken: `really,` above stays a comma, because the +annotator deleted the span it introduced, and asking for a mark that would then be +scored as an error teaches an instruction that commands are sometimes to be ignored. +`.` `,` `?` `!` `:` `;` are all in the vocabulary with their common aliases (`period` / +`full stop`); on `nyra` only the first three ever fire, because its references contain +483 periods, 460 commas, 43 question marks and none of the other three. + +**ALL CAPS** is the exception, and the only place in the harness that edits a reference. +You cannot pose "uppercase this word" against a target with no uppercase in it, so the +chosen run is uppercased in the target and prefixed with the command, lowercased, in the +input. `--spoken-caps-rate` (default 0.25) sets how often a row gets one, and at most one +does. + +The word after a spoken sentence-ending mark is **lowercased**, which is what makes the +task a task: leave the capital in and an instruction can restore the period from the +casing alone. Proper nouns are lowercased along with everything else — that word is +sentence-initial in the reference by construction, so `Monday` and `today` ask for the +same rule and the same answer, and protecting names would have handed the answer to 30% +of the commands (the share whose following word appears capitalized mid-sentence +somewhere in `nyra`). The pronoun _I_ is the one exception: it is capitalized +mid-sentence too, so `politics full stop i'm not sure` is a transcript no service +returns and would show the model one token cased two ways in one utterance. + +**It selects on `format` by default**, not `blend`. `metrics.normalize` casefolds and +strips punctuation, so a restored comma and a missed one are the same string to the +content axis and an ALL CAPS command is invisible to it. `format` sees the whole task — +and still sees the expensive failure, a command left in the output as a literal word, +because that is an inserted token on either axis. + +Alongside the axes the run reports what became of the commands themselves: + +| column | what it means | +| --------------- | --------------------------------------------------------- | +| `converted` | the mark or the uppercasing landed on the right word | +| `left as words` | the command reached the output as text — visible nonsense | +| `dropped` | the command words are gone and nothing replaced them | + +That is the number WER cannot give you, and it exists only because the operator is +synthetic: nothing else knows what was planted. `left as words` is counted apart from +`dropped` because it is the failure that reaches the user's document. + +#### The saved dataset + +`--dump-corpus PATH` writes the loaded corpus as JSONL and carries on; `--jsonl PATH` +reads it back **exactly**, commands included, so scoring the file is scoring the corpus it +came from rather than an approximation of it. Both work under `--dry-run`, so generating a +dataset needs no API key. + +That matters because the corpus this harness scores against is otherwise assembled at load +time out of a dataset download, a seeded disfluency injector and a seeded punctuation +injector. Reproducible in principle; unreviewable in practice — nobody reads a generator to +find out whether the examples are any good, and a change to an injector silently changes +what every past number was measured on. A file in the tree is diffable. + +[`data/spoken-punctuation.jsonl`](./data/spoken-punctuation.jsonl) is that file: 76 rows +generated from `--source builtin`, whose sentences are written for this repo and so carry +no third-party terms. `nyra` derives from LDC-licensed Switchboard transcripts and is +deliberately **not** committed — dump it locally if you want it frozen. `test_eval.py` +regenerates the committed file and asserts it is byte-identical, so an injector change +fails the suite instead of leaving a stale dataset in the tree. + +Everything downstream keys off **what the corpus contains, not what flags were passed** — +`Corpus.has_commands` decides the axis, the candidate table and the command report. Before +that, reading the saved dataset back selected on `blend` and printed no command outcomes: +the same corpus scored two different ways depending on how it was reached. + +Read the committed file as a **fixture**, not a benchmark. 50 dev / 25 test rows is far +below the resolution this README argues for everywhere else. What it is uniquely good for +is the one thing `nyra` cannot do: **15% of its references use a command word as ordinary +content** ("one grace period, so plan accordingly", "add a comma after the second +clause"), against ~0-1% of `nyra`'s. It is the only corpus here that can charge an +instruction for converting a word the speaker meant literally — and those rows compose +with the injector into the sharpest case on the task, +`one grace period comma so plan accordingly`, where the word and the command sit side by +side. + +#### What it changed elsewhere, and what it deliberately did not + +One interaction was worth a change in `metrics.py`. `_is_abandoned`'s rule 2 — "two or +more non-hesitation words that echo nothing" — is exactly the shape of `question mark`, +so the false-start classifier read a dictation command as an abandoned phrase and charged +it `FALSE_START_WEIGHT` per word. That made a leftover `question mark` ten errors and a +leftover `period` one, an asymmetry nobody chose and one that would have pointed the +search at the multi-word commands for arithmetic reasons. `metrics.score` now takes a +`not_abandoned` set, which `corpus.Utterance` fills from the commands it actually +planted. It defaults to empty, and over 300 `nyra` rows the false-start fraction is +**0.207 with spoken punctuation and 0.207 without** — so every number measured before it +existed still holds. + +Three things this cannot tell you, all of them reported rather than assumed away: + +- **Over-conversion is nearly unmeasured.** An instruction that rewrites "the Jurassic + period was long" into "the Jurassic. was long" is only punished on rows whose reference + uses a command phrase as content, and on `nyra` that is ~0–1% of them. The run prints + the figure. The clause forbidding it is in every candidate on product grounds, not + scored ones — the same shape of blind spot as `REQUIRED_SAFEGUARDS`, and unlike those + it is not gated, so a search is free to delete it. +- **The input is the clean form of the task.** A real speech-to-text pass applies its own + casing and punctuation model on top of the command, so what it actually returns is + probably nearer `Children period.` than `children period`. What it does exactly is + unknown without measuring the service, and `--verify-live` is the only thing here that + speaks to the real pipeline. +- **`BASELINE` stays `prior-winner`.** It has never heard of spoken punctuation, which is + what makes it the right bar: the held-out comparison against it answers the product + question — what does teaching the shipped instruction this task buy? The bar a _search_ + has to clear is the best of the punctuation candidates on dev, which is what + `--start best-candidate` seeds from. + +A spoken-punctuation run scores **`BASELINE` plus the four punctuation candidates**, not +the whole table. `BASELINE` alone is not enough — a search seeded from an instruction that +has never heard of the task starts outside the region worth exploring — but the six terse +contrast instructions are worse than useless here: they exist to rank framings of +_disfluency_ cleanup, none of them mentions punctuation, and at the default 900-row dev +split that ordering costs 5,400 model calls to re-discover. `--candidates all` buys them +if you want to watch it happen. + +The candidates are in `candidates.SPOKEN_PUNCTUATION_CANDIDATES`, merged into the table +only when the corpus poses the task — a punctuation clause cannot help against a corpus that poses +no commands, and each one costs a full dev sweep. `punct-appended` is the cheapest +possible change to what Blurt ships: `PRIOR_WINNER` plus `SPOKEN_PUNCTUATION_CLAUSE`, at +2006 of the 2048 characters. That leaves 42 characters of headroom, so GEPA cannot grow +from it — `punct-explicit` at 1430 is the searchable seed. Which one wins on dev and +which one a search can improve are different questions. + [nyra]: https://huggingface.co/datasets/nyralabs/disfluency_speech_english [ds]: https://huggingface.co/datasets/amaai-lab/DisfluencySpeech [dq]: https://huggingface.co/datasets/google-research-datasets/disfl_qa @@ -259,26 +413,29 @@ is `--model`. Its proposal prompts are multi-field, so they keep DSPy's marker p ## The knobs that matter -| Flag | Default | What it changes | -| -------------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------ | -| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | -| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | -| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | -| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | -| `--metric` | `blend` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | -| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | -| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | -| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | -| `--start` | `prior-winner` | Which instruction GEPA evolves from — the compressed prior winner, or the best hand-written candidate. | -| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | -| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | -| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | -| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | -| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | -| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | -| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | -| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | -| `--seed` | `7` | Seeds injection and the train/dev/test split. | +| Flag | Default | What it changes | +| ---------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | +| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | +| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | +| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | +| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | +| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | +| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | +| `--spoken-punctuation` | `0` (off) | Speak this share of the marks the reference licenses. Applies to paired sources too; selects on `format`. | +| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | +| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | +| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | +| `--start` | `prior-winner` | Which instruction GEPA evolves from — the compressed prior winner, or the best hand-written candidate. | +| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | +| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | +| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | +| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | +| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | +| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | +| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | +| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | +| `--seed` | `7` | Seeds injection and the train/dev/test split. | Both optimizers run with few-shot demos disabled. `config.llm.instruction` is a single string the service applies in one pass, so an optimized program that depended on bundled examples @@ -576,6 +733,8 @@ between them hold. macOS only, and off by default — it costs real transcriptio | `candidates.py` | The instructions under test, the character cap, and the GEPA seed. | | `corpus.py` | Sources, loading, de-tagging, splitting, the echo floor. | | `disfluency.py` | The seeded, additive disfluency injector. | +| `spoken_punctuation.py` | The spoken-punctuation injector, its command scorer, and the reflector's note on it. | +| `data/` | The committed dataset and its provenance — see [`data/README.md`](./data/README.md). | | `metrics.py` | Token alignment, the two word-error-rate axes, the false-start surcharge, GEPA feedback text. | | `live.py` | Synthesis + the real `/transcribe` round trip, for `--verify-live`. | | `program.py` | Everything that imports DSPy — the program, metrics adapters, optimizers. | diff --git a/evals/dictation-prompt/candidates.py b/evals/dictation-prompt/candidates.py index cb1d447..6ec6cda 100644 --- a/evals/dictation-prompt/candidates.py +++ b/evals/dictation-prompt/candidates.py @@ -482,6 +482,89 @@ def revision_directive(proposal: str, notes: list[Objection]) -> str: CANDIDATES["prior-winner"] = PRIOR_WINNER +#: The clause that turns a cleanup instruction into a spoken-punctuation one, in the +#: least room it can be said in. Sized to fit **after** `PRIOR_WINNER`, which leaves 519 +#: characters under the cap — so this is what the shipped instruction can be taught +#: without giving anything up, and `punct-appended` is that experiment exactly. +#: +#: It omits "punctuation already in the transcript is correct", which `PRIOR_WINNER` +#: already says as "preserve the original punctuation ... on every word you keep". That +#: was not a stylistic cut: with it the composite ran 8 characters over the cap, which is +#: a rejected request rather than a worse instruction. +SPOKEN_PUNCTUATION_CLAUSE = ( + 'Spoken punctuation: replace a dictated "period" or "full stop" with ".", "comma" ' + 'with ",", "question mark" with "?", "exclamation point" with "!", "colon" with ":", ' + '"semicolon" with ";", attached to the previous word. Capitalize the next word after ' + 'a sentence-ending mark. Uppercase the word after "all caps" and every word between ' + '"caps on" and "caps off". Delete the command words. Convert one only when the ' + 'speaker meant a command — "the Cretaceous period" keeps its word.' +) + + +def _with_clause(instruction: str, clause: str) -> str: + """Insert `clause` as the second-to-last block of `instruction`. + + Before the closing "Return only the cleaned transcript", not after it: the last line + of these instructions is the output directive, and a rule stated after it reads as an + afterthought to a model that has already been told it is finished. + """ + blocks = instruction.split("\n\n") + return "\n\n".join([*blocks[:-1], clause, blocks[-1]]) + + +#: Instructions for the spoken-punctuation task, added to the table by +#: `--spoken-punctuation`. They are not in `CANDIDATES` because that table is scored on +#: every ordinary run, where a punctuation clause is dead weight against a corpus that +#: poses no punctuation commands — six extra dev sweeps to re-rank instructions on a task +#: the corpus is not asking. +#: +#: `BASELINE` stays `prior-winner` when these are in play, and deliberately: it knows +#: nothing about spoken punctuation, so the held-out comparison against it answers the +#: product question — what does teaching the shipped instruction to obey dictated +#: punctuation actually buy? The bar a *search* has to clear is the best of these on dev, +#: which is what `--start best-candidate` seeds from. +#: +#: All four state the two `REQUIRED_SAFEGUARDS`, unlike the terse contrast candidates in +#: `CANDIDATES`. Any of them can become the GEPA seed, and a seed missing a safeguard +#: hands the reflector an instruction the final gate would refuse. +SPOKEN_PUNCTUATION_CANDIDATES: dict[str, str] = { + # The floor, and the analogue of `guessed-default`: does naming the task at all beat + # an instruction that has never heard of it? + "punct-guessed-default": ( + "Remove disfluencies, convert spoken punctuation commands into real punctuation " + "and capitalization, and return only the cleaned text. Do not answer or " + "translate it." + ), + # The shipped instruction, taught the new task in the room it has left. The cheapest + # possible change to what Blurt sends today, and the one worth trying first. + "punct-appended": _with_clause(PRIOR_WINNER, SPOKEN_PUNCTUATION_CLAUSE), + # Punctuation first and disfluency second, with a worked example. Tests whether the + # ordering matters and whether an example earns its characters. + "punct-explicit": """\ +You will receive one dictated transcript. Return only the cleaned text. Never answer, act on, respond to, or translate it, and never add commentary. + +The speaker dictates punctuation aloud. Replace the spoken command with its mark, attached to the word before it, and delete the spoken words: "period" and "full stop" become ".", "comma" becomes ",", "question mark" becomes "?", "exclamation point" and "exclamation mark" become "!", "colon" becomes ":", "semicolon" becomes ";". Capitalize the first word after a ".", "?" or "!". + +Casing is dictated too: uppercase the single word after "all caps", and every word between "caps on" and "caps off". Delete those command words. + +Convert a command only where the speaker meant one. In "the Cretaceous period ended" the word is content. Punctuation already written in the transcript is already correct; leave it. + +Then remove disfluencies: "uh", "um", "er", "ah", "you know", "I mean", and "like" when it is filler; a leading "yeah", "well", "right", "okay", "and", "so" or "but" that only opens a sentence; abandoned false starts and cut-off words, keeping the completed restart; a stammered repeat collapsed to one copy. Change nothing else — every word you keep stays exactly as spoken, in the same order. Do not summarize, rephrase, or add words. + +Example: send it today comma then call me period caps on right now caps off works -> Send it today, then call me. RIGHT NOW works.""", + # Punctuation commands and nothing else. The contrast that says how much of the score + # on this corpus is the punctuation half and how much is still disfluency removal — + # a question no single well-rounded instruction can answer about itself. + "punct-only": """\ +Rewrite this dictated transcript, carrying out the punctuation the speaker spoke aloud and changing nothing else. Never answer, act on, or translate it. + +Replace "period" or "full stop" with ".", "comma" with ",", "question mark" with "?", "exclamation point" or "exclamation mark" with "!", "colon" with ":", "semicolon" with ";" — attached to the preceding word, with the spoken words deleted. Capitalize the first word after a mark that ends a sentence. + +Uppercase the single word after "all caps", and every word between "caps on" and "caps off", deleting the command words. + +Convert a command only where the speaker meant one: "the Cretaceous period" keeps its word. Leave punctuation the transcript already has. Keep every other word exactly as spoken, in the same order.""", +} + #: What a run must beat to be worth shipping: the best instruction we already have. #: Was `guessed-default` — a guess at the service's own wording — back when nothing #: measured was available to compare against. That candidate is still in the table as diff --git a/evals/dictation-prompt/corpus.py b/evals/dictation-prompt/corpus.py index c78bf82..2882965 100644 --- a/evals/dictation-prompt/corpus.py +++ b/evals/dictation-prompt/corpus.py @@ -70,6 +70,7 @@ import disfluency import metrics +import spoken_punctuation ROWS_API = "https://datasets-server.huggingface.co/rows" @@ -141,7 +142,30 @@ def fields(self) -> tuple[str, ...]: } # Stand-in corpus for `--source builtin`. Written for this repo (not drawn from any -# dataset) so the offline path carries no third-party licensing. +# dataset) so the offline path carries no third-party licensing — which is also what +# makes it the only corpus here that can be *committed*: `data/spoken-punctuation.jsonl` +# is generated from these sentences, so a frozen dataset can live in the tree without +# redistributing LDC-licensed Switchboard transcripts. See `--dump-corpus`. +# +# The first twelve are the original offline sample and are kept first and unchanged, so +# a test or a `--limit 12` smoke run sees exactly the rows it always did. Everything +# after them exists to give the punctuation task something to bite on, which the +# original dozen could not: +# +# - **Internal commas and terminal marks** on most rows, since `spoken_punctuation` +# speaks only marks the reference licenses. A corpus of bare declaratives licenses one +# period each and nothing else. +# - **Questions and exclamations**, so `?` and `!` fire at all. `nyra` supplies 43 +# question marks per 400 rows and no exclamation points whatsoever. +# - **Colons and semicolons**, which `nyra` never supplies, so those entries in +# `spoken_punctuation.SPOKEN_FORMS` are exercised by something. +# - **Literal-use traps** — references that use "period", "comma", "question mark", +# "colon", "dash" or "all caps" as ordinary content. These are the blind spot `nyra` +# cannot cover (~0-1% of its rows), and they compose with the injector into the +# sharpest case on the task: "one grace period comma so plan accordingly" has to come +# back as "one grace period, so plan accordingly", keeping the word and obeying the +# command that follows it. An instruction that pattern-matches the vocabulary rather +# than reading the sentence loses a content word here, which the score charges. BUILTIN_SAMPLE: tuple[str, ...] = ( "The build failed because the signing certificate expired over the weekend.", "Can you send me the latest numbers before the review meeting tomorrow morning?", @@ -155,6 +179,77 @@ def fields(self) -> tuple[str, ...]: "It turns out the regression was introduced by the change to the clipboard restore path.", "Please double check the sample rate before you send the audio to the transcription endpoint.", "The design review is blocked on whether we keep the menu bar item at all.", + # Prose with internal commas, which is what licenses anything other than a period. + "If the upload stalls, retry it once, and then fall back to the smaller chunk size.", + "We shipped the change on Tuesday, and by Thursday the error rate had halved.", + "The onboarding flow works, but the second screen still asks for a permission we never use.", + "Before you merge, rebase on main, run the whole suite, and check the coverage gate.", + "I read the incident report, and the root cause was a stale cache in the edge layer.", + "Once the lease expires, the worker stops accepting jobs, which is what we wanted.", + "The vendor confirmed the outage, apologised, and promised a postmortem by Friday.", + "Send the draft to Priya, loop in the design team, and we can review it together.", + "When the mic is muted, the waveform freezes, and users read that as a crash.", + "The migration touched four tables, two indexes, and one view nobody remembered.", + "After the retry budget runs out, the request fails, and the overlay says try again.", + "We looked at three vendors, and only one of them will sign a data processing agreement.", + "The cache warms in about a minute, so the first few requests are always slower.", + "If you cannot reproduce it locally, attach the sysdiagnose, and I will look tonight.", + "The estimate assumed two engineers, and we have one, so the date needs to move.", + "Their API returns a 202, then polls, then hands back a URL that expires in an hour.", + "I moved the standup to nine, cancelled the Thursday sync, and blocked out Friday afternoon.", + "The feature is behind a flag, off by default, and only enabled for the internal team.", + # Questions, so the question-mark command has something to attach to. + "Did anyone check whether the new entitlement survives a clean install?", + "Should we hold the release until the notarization queue clears, or ship it now?", + "Can you remind me what the retry budget is on the streaming endpoint?", + "Do you know why the waveform stops animating when the window loses focus?", + "Is there a reason we still ship the old audio unit alongside the new one?", + "What happens to a partial transcript if the socket closes before the final message?", + "Would it be easier to gate this on the account tier instead of a flag?", + "Have we ever measured how long the first paste takes on a cold launch?", + "Are the crash reports symbolicated, or do I need to upload the archive myself?", + "Who owns the dashboard now that the analytics team has been folded into platform?", + # Exclamations, which no corpus here otherwise supplies. + "That fixed it, and the latency dropped by half!", + "Please do not ship this on a Friday afternoon again!", + "The whole suite passed on the first try for once!", + "Watch out, the staging database is still pointed at production!", + # Colons and semicolons, so those commands are exercised by something. + "Here is the plan: land the fix, cut a build, and hand it to QA tomorrow.", + "Two things are still open: the entitlement review and the App Store description.", + "The cause was simple: we were reading the sample rate from the wrong device.", + "It builds cleanly on my machine; it fails on the runner every single time.", + "Ship the smaller change first; the refactor can wait until after the release.", + "The tradeoff is straightforward: more accuracy for about eighty milliseconds of latency.", + "Keep the interface as it is; only the storage layer needs to change.", + "One caveat: the migration is not reversible once the first write lands.", + # Literal-use traps. The reference uses a command word as content, so an instruction + # that converts on sight loses a real word here. + "We only support one grace period, so plan accordingly before the trial ends.", + "Add a comma after the second clause and the sentence reads much better.", + "Every question mark in that survey was ambiguous, so we rewrote the whole form.", + "The billing period rolls over at midnight UTC, not at midnight local time.", + "Put a colon after the heading and leave the rest of the line alone.", + "She used a full stop where the style guide clearly asks for a semicolon.", + "The legal team wants the warning in all caps, which our design system forbids.", + "That exclamation point in the release notes reads as sarcasm, so please remove it.", + "There is a dash missing from the second bullet on the pricing page.", + "I said period, and it typed the word instead of the punctuation mark.", + "The Cretaceous period ended with an impact, which is roughly how the demo went.", + "Use a semicolon there; a comma is not strong enough to join those two clauses.", + # More ordinary dictation, to keep the traps from dominating a small corpus. + "Remind me to follow up with the accessibility team about the focus ring.", + "The transcript came back empty, which usually means the audio was all silence.", + "I will draft the announcement tonight and send it round for comments in the morning.", + "We should probably stop supporting the beta channel now that nobody is on it.", + "The keyboard shortcut conflicts with the system dictation shortcut on a fresh install.", + "Let me know if the new model handles background noise any better than the old one.", + "Nothing in the logs explains why the first request after a sleep always times out.", + "The onboarding video is four minutes long and most people quit after thirty seconds.", + "I would rather fix the flake than mark the test as skipped and forget about it.", + "Our smallest customer files more bug reports than the other twenty combined.", + "The release notes need a line about the new permission before we can publish.", + "Check whether the trial expiry is stored in the keychain or in user defaults.", ) @@ -166,6 +261,23 @@ class Utterance: disfluent: str # Injector operators, empty for a real paired corpus — nobody annotated those. operations: tuple[str, ...] = field(default=()) + #: Spoken punctuation commands planted in `disfluent` by + #: `spoken_punctuation.inject`, empty unless `--spoken-punctuation` is on. Kept on + #: the utterance because the only thing that can say whether a command was obeyed + #: is the record of what was planted — WER sees a missing comma and a missing word + #: as the same kind of error, and cannot see a *casing* command at all on the + #: content axis. Travels with the pair for the same reason both sides do. + commands: tuple[spoken_punctuation.Command, ...] = field(default=()) + #: True when the input side came from the source rather than from an injector here — + #: a paired dataset column, or a `--jsonl` row that carried a `disfluent` key. + #: + #: Recorded rather than inferred from `is_disfluent`, which was the proxy before and + #: is wrong in exactly the case a dumped corpus produces: a row whose input happens + #: to equal its reference (no command drawn, no disfluency drawn) reads as "needs + #: injecting", so re-loading a dumped file would silently hand that row a *different* + #: input than the file records. The documented contract is "objects with both + #: disfluent and reference are used as-is", and this is what makes it true. + input_supplied: bool = False @property def is_disfluent(self) -> bool: @@ -182,7 +294,20 @@ def scored(self, hypothesis: str) -> metrics.Score: entirely reasonable. An utterance holds both sides already, so let it be the thing that remembers. """ - return metrics.score(self.reference, hypothesis, self.disfluent) + return metrics.score(self.reference, hypothesis, self.disfluent, self.not_abandoned) + + @property + def not_abandoned(self) -> frozenset[str]: + """Words in `disfluent` that are surplus by construction, not by abandonment. + + Only the spoken punctuation commands, and empty for every corpus that has none — + so this changes no number measured before it existed. `metrics._is_abandoned` + says why it has to exist: "question mark" is two non-hesitation words echoing + nothing, which is exactly the shape of an abandoned phrase, and left unnamed it + would be charged `FALSE_START_WEIGHT` per word while a leftover "period" was + charged one. + """ + return frozenset(word for command in self.commands for word in command.spoken_words) @dataclass(frozen=True) @@ -197,6 +322,24 @@ class Corpus: def __len__(self) -> int: return len(self.utterances) + @property + def commands_planted(self) -> int: + """Spoken-punctuation commands across the whole corpus. + + Read instead of `--spoken-punctuation` wherever the run has to decide whether it + is posing the punctuation task — which axis to select on, which candidates to + score, whether to print the command table. A corpus loaded from a + `--dump-corpus` file via `--jsonl` carries the commands and not the flag, and + keying on the flag meant that dataset silently selected on `blend` and reported no + command outcomes: the same corpus, scored two different ways depending on how it + was reached. + """ + return sum(len(u.commands) for u in self.utterances) + + @property + def has_commands(self) -> bool: + return self.commands_planted > 0 + @property def disfluent_fraction(self) -> float: """Share of examples whose input actually differs from its target.""" @@ -221,16 +364,21 @@ def _usable_reference(text: str, *, for_injection: bool) -> bool: return not for_injection or text[-1] in ".?!" -def _collect(pairs, limit: int, *, for_injection: bool) -> list[Utterance]: +def _collect(utterances, limit: int, *, for_injection: bool) -> list[Utterance]: """Filter, de-duplicate by reference, and cap — the one copy of that policy. Consumes lazily, so a paging loader stops fetching as soon as `limit` usable pairs have been found. + + Takes whole `Utterance`s rather than `(disfluent, reference)` pairs so that a source + can carry more than the two strings through this filter: a `--jsonl` row restores the + commands and operators a `--dump-corpus` run recorded, and dropping them here would + make a saved dataset score differently from the corpus it was saved from. """ seen: set[str] = set() kept: list[Utterance] = [] - for disfluent, reference in pairs: - reference, disfluent = _tidy(reference), _tidy(disfluent) + for utterance in utterances: + reference, disfluent = _tidy(utterance.reference), _tidy(utterance.disfluent) if not reference or not disfluent: continue if not _usable_reference(reference, for_injection=for_injection): @@ -238,7 +386,7 @@ def _collect(pairs, limit: int, *, for_injection: bool) -> list[Utterance]: if reference in seen: continue seen.add(reference) - kept.append(Utterance(reference=reference, disfluent=disfluent)) + kept.append(replace(utterance, reference=reference, disfluent=disfluent)) if len(kept) >= limit: break return kept @@ -371,31 +519,79 @@ def _fetch_page(url: str, headers: dict, source: Source, token: str | None) -> d raise AssertionError("unreachable: the loop either returns or raises") -def _pairs_from_rows(rows, source: Source, where: str): - """Project raw rows into (disfluent, reference), de-tagging where needed.""" +def _utterances_from_rows(rows, source: Source, where: str): + """Project raw dataset rows into utterances, de-tagging where needed.""" detag = source.detag or (lambda text: text) for row in rows: reference = detag(_tidy(_field(row, source.target_field, where))) if source.is_paired: - yield detag(_tidy(_field(row, source.input_field, where))), reference + yield Utterance( + reference=reference, + disfluent=detag(_tidy(_field(row, str(source.input_field), where))), + input_supplied=True, + ) else: # Reference-only: the injector fills the input side in `load`. - yield reference, reference + yield Utterance(reference=reference, disfluent=reference) -def _pairs_from_jsonl(path: str): - """Read pairs or bare references from a local file.""" +def _utterances_from_jsonl(path: str): + """Read a local file: pairs, bare references, or a `--dump-corpus` dataset. + + A dumped row round-trips exactly — its `commands` and `operations` come back, so the + saved dataset scores identically to the corpus it was saved from and `load` knows not + to inject over it. A hand-written row with only a reference still goes through the + injector, and a bare line is read as a reference. + """ with open(path, encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue if not line.startswith("{"): - yield line, line + yield Utterance(reference=line, disfluent=line) continue row = json.loads(line) reference = row.get("reference") or row.get("text") or row.get("transcript") or "" - yield row.get("disfluent") or reference, reference + yield Utterance( + reference=reference, + disfluent=row.get("disfluent") or reference, + operations=tuple(row.get("operations", ())), + commands=tuple( + spoken_punctuation.Command.from_json(c) for c in row.get("commands", ()) + ), + input_supplied="disfluent" in row, + ) + + +def dump_jsonl(loaded: Corpus, path: str) -> int: + """Write a loaded corpus as JSONL, exactly as `_utterances_from_jsonl` reads it back. + + Why a dataset is worth freezing at all: the corpus this harness scores against is + assembled at load time from a dataset download, a seeded disfluency injector and a + seeded punctuation injector. That is reproducible in principle and unreviewable in + practice — nobody reads a generator to find out whether the examples are any good, and + a change to an injector silently changes what every past number was measured on. A + file in the tree is diffable. + + What can be committed is limited by licensing, not by size. `--source builtin` is + written for this repo (see `BUILTIN_SAMPLE`), so a dataset generated from it carries + no third-party terms; `nyra` derives from LDC-licensed Switchboard transcripts, so + dump it locally and leave it out of the tree. + """ + lines = [] + for utterance in loaded.utterances: + row: dict[str, object] = { + "reference": utterance.reference, + "disfluent": utterance.disfluent, + } + if utterance.operations: + row["operations"] = list(utterance.operations) + if utterance.commands: + row["commands"] = [command.to_json() for command in utterance.commands] + lines.append(json.dumps(row, ensure_ascii=False)) + pathlib.Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8") + return len(lines) def load( @@ -407,13 +603,15 @@ def load( seed: int = 7, severity: float = 0.35, strip_formatting: bool = False, + spoken_punctuation_rate: float = 0.0, + spoken_caps_rate: float = 0.25, ) -> Corpus: """Load up to `limit` pairs, injecting disfluencies for reference-only sources.""" if jsonl: - pairs = _pairs_from_jsonl(jsonl) + rows = _utterances_from_jsonl(jsonl) spec, detail, formatted = None, {"path": jsonl}, True elif source == "builtin": - pairs = ((text, text) for text in BUILTIN_SAMPLE) + rows = (Utterance(reference=text, disfluent=text) for text in BUILTIN_SAMPLE) spec, formatted = None, True detail = {"note": "bundled sample; no dataset was downloaded"} elif source in SOURCES: @@ -425,8 +623,7 @@ def load( if split: spec = replace(spec, split=split) where = f"{spec.dataset}/{spec.split}" - rows = _rows_via_api(spec, limit) - pairs = _pairs_from_rows(rows, spec, where) + rows = _utterances_from_rows(_rows_via_api(spec, limit), spec, where) formatted = spec.formatting_is_measurable detail = { "dataset": spec.dataset, @@ -444,16 +641,19 @@ def load( # needs a reference long and well-punctuated enough to inject into. A jsonl # file is the user's own text, so it is filtered leniently either way. reference_only = spec is None or not spec.is_paired - utterances = _collect(pairs, limit, for_injection=reference_only and jsonl is None) + utterances = _collect(rows, limit, for_injection=reference_only and jsonl is None) if reference_only: injected: list[Utterance] = [] for index, utterance in enumerate(utterances): - # A jsonl row that already carried a disfluent side keeps it. Per-example - # seeds (rather than one shared generator) keep an utterance's - # disfluencies stable when the corpus around it changes, so re-running - # with a larger --limit doesn't reshuffle what you already looked at. - if utterance.is_disfluent: + # A jsonl row that already carried a disfluent side keeps it — read off + # `input_supplied` rather than `is_disfluent`, so a dumped row whose input + # happens to equal its reference is not quietly re-injected into something + # else. Per-example seeds (rather than one shared generator) keep an + # utterance's disfluencies stable when the corpus around it changes, so + # re-running with a larger --limit doesn't reshuffle what you already + # looked at. + if utterance.input_supplied: injected.append(utterance) continue disfluent, operations = disfluency.inject( @@ -468,6 +668,48 @@ def load( utterances = injected detail |= {"injected": True, "severity": severity, "strip_formatting": strip_formatting} + # Spoken punctuation goes on last, over whatever the input side already is — real + # annotated disfluencies from a paired source, or the injector's. That ordering is + # the point: the task under test is a dictation user saying "comma" *while* also + # hesitating, not either in isolation, and layering it keeps the disfluencies the + # ones a corpus actually recorded rather than ones this repo wrote. Per-example + # seeds for the same reason the disfluency injector uses them — a row's commands + # stay put when the corpus around it grows. + if spoken_punctuation_rate: + spoken: list[Utterance] = [] + for index, utterance in enumerate(utterances): + # A row that already carries commands has been through this injector — a + # `--dump-corpus` dataset read back with the flag still set. Injecting again + # would speak marks that are no longer there and score the result against a + # reference whose ALL CAPS run has been uppercased twice. + if utterance.commands: + spoken.append(utterance) + continue + reference, disfluent, commands = spoken_punctuation.inject( + utterance.reference, + utterance.disfluent, + seed=seed + index, + rate=spoken_punctuation_rate, + caps_rate=spoken_caps_rate, + ) + spoken.append( + replace( + utterance, + reference=reference, + disfluent=disfluent, + operations=utterance.operations + tuple(c.label for c in commands), + commands=commands, + ) + ) + utterances = spoken + detail |= { + "spoken_punctuation_rate": spoken_punctuation_rate, + "spoken_caps_rate": spoken_caps_rate, + "commands_per_row": sum(len(u.commands) for u in utterances) / len(utterances) + if utterances + else 0.0, + } + if not utterances: raise RuntimeError(f"no usable pairs from {source!r} (needs {MIN_WORDS}-{MAX_WORDS} words)") @@ -540,10 +782,19 @@ def false_start_fraction(utterances: list[Utterance]) -> float: Printed next to the floor because `metrics.FALSE_START_WEIGHT` only bites on these rows, so it is the number that says whether the weighting is shaping the search or is a rounding error on this corpus. Pure arithmetic; no model is involved. + + Reads each utterance's own `not_abandoned`, so the figure printed is the figure + charged. Without it a spoken-punctuation corpus reported 55% of rows carrying an + abandoned span against 21% for the same rows unmodified — the difference being + "question mark" counted as a false start in the report while `Utterance.scored` + correctly declined to charge it. """ if not utterances: return 0.0 - carrying = sum(bool(metrics.false_start_tokens(u.disfluent, u.reference)) for u in utterances) + carrying = sum( + bool(metrics.false_start_tokens(u.disfluent, u.reference, u.not_abandoned)) + for u in utterances + ) return carrying / len(utterances) diff --git a/evals/dictation-prompt/data/README.md b/evals/dictation-prompt/data/README.md new file mode 100644 index 0000000..83193e2 --- /dev/null +++ b/evals/dictation-prompt/data/README.md @@ -0,0 +1,78 @@ +# Frozen eval datasets + +## `spoken-punctuation.jsonl` + +76 dictated utterances that speak their own punctuation — `period`, `comma`, +`question mark`, `all caps` — on top of injected disfluency. The input side is what a +speech-to-text pass would hand the rewrite model; the `reference` is what the speaker +meant to write. + +```json +{ + "reference": "Can you send me the LATEST NUMBERS before the review meeting tomorrow morning?", + "disfluent": "well can you sort of send me the caps on latest numbers caps off before the review um meeting tomorrow morning question mark", + "operations": [ + "opener", + "filler", + "filler", + "spoken:caps-on", + "spoken:question-mark" + ], + "commands": [ + { + "spoken": "caps on", + "mark": "", + "kind": "caps", + "anchor": "", + "words": ["latest", "numbers"] + }, + { + "spoken": "question mark", + "mark": "?", + "kind": "mark", + "anchor": "morning", + "words": [] + } + ] +} +``` + +`commands` records what was planted, which is the only reason the run can report how many +commands were **obeyed** rather than just how close the text came. `--jsonl` reads it back +exactly — see `corpus._utterances_from_jsonl` — so scoring this file is scoring the corpus +it was generated from, not an approximation of it. + +### Regenerating it + +```bash +python3 evals/dictation-prompt/optimize_cleanup_prompt.py \ + --source builtin --limit 200 --spoken-punctuation 0.8 --severity 0.35 --seed 7 \ + --dump-corpus evals/dictation-prompt/data/spoken-punctuation.jsonl \ + --dry-run --show-samples 0 +``` + +No network and no API key: `builtin` is bundled and both injectors are seeded. +`test_eval.py` runs exactly this and asserts the result is byte-identical to the committed +file, so a change to either injector fails the suite instead of silently invalidating a +dataset that is still in the tree. + +### Why this corpus and not the real one + +The measurement corpus is `nyra` — ~5k Switchboard utterances whose disfluencies trained +annotators marked by hand — and it is **not** committed here. It derives from +LDC-licensed transcripts, and `corpus.BUILTIN_SAMPLE` exists precisely so the offline path +carries no third-party licensing. Dump it locally if you want it frozen: + +```bash +python3 evals/dictation-prompt/optimize_cleanup_prompt.py --spoken-punctuation 0.8 \ + --limit 4000 --dump-corpus /tmp/nyra-spoken.jsonl --dry-run --show-samples 0 +``` + +So read this file as a **fixture**, not a benchmark. 76 rows split into 50 dev / 25 test is +far below the resolution the parent README argues for — differences between good +instructions on this task are a few hundredths, and 25 test rows cannot see them. What it +is good for: reading the examples, reviewing a diff when an injector changes, and the one +thing `nyra` genuinely cannot do — **literal-use traps**. 15% of these references use a +command word as ordinary content ("one grace period, so plan accordingly", "add a comma +after the second clause"), against ~0–1% of `nyra`'s, so this is the only corpus here that +can charge an instruction for converting a word the speaker meant literally. diff --git a/evals/dictation-prompt/data/spoken-punctuation.jsonl b/evals/dictation-prompt/data/spoken-punctuation.jsonl new file mode 100644 index 0000000..faa44b1 --- /dev/null +++ b/evals/dictation-prompt/data/spoken-punctuation.jsonl @@ -0,0 +1,76 @@ +{"reference": "The build failed because the signing certificate expired over the weekend.", "disfluent": "okay so the failed— or rather build failed because the ah signing kind of certificate expired over the weekend period", "operations": ["opener", "false_start", "filler", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "weekend", "words": []}]} +{"reference": "Can you send me the LATEST NUMBERS before the review meeting tomorrow morning?", "disfluent": "well can you sort of send me the caps on latest numbers caps off before the review um meeting tomorrow morning question mark", "operations": ["opener", "filler", "filler", "spoken:caps-on", "spoken:question-mark"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["latest", "numbers"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "morning", "words": []}]} +{"reference": "I think we should ship the fix behind a flag and watch the crash rate for a day.", "disfluent": "I think we should should ship the the fix actually behind a flag and watch kind of the c- crash rate for a day period", "operations": ["repeat", "repeat", "filler", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "day", "words": []}]} +{"reference": "The microphone permission dialog never appears when the app runs from a temporary directory.", "disfluent": "The microphone permission dialog never appears when the app runs from a directory— or rather temporary directory period", "operations": ["false_start", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "directory", "words": []}]} +{"reference": "Let's move the retrospective to Thursday so everyone in Berlin can attend.", "disfluent": "Let's move the retrospective to Thursday so everyone in uh Berlin can attend full stop", "operations": ["filler", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "attend", "words": []}]} +{"reference": "She pointed out that the transcript is already punctuated when it comes back from the service.", "disfluent": "She pointed out hmm that the transcript is already punctuated punctuated when it comes back from the service period", "operations": ["filler", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "service", "words": []}]} +{"reference": "We measured about a hundred and seventy milliseconds of connection setup on a cold start.", "disfluent": "I mean we measured about a hundred and seventy milliseconds milliseconds— I mean of connection setup on er a co- cold start period", "operations": ["opener", "false_start", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "start", "words": []}]} +{"reference": "The OVERLAY SHOULD STAY on screen until the paste actually lands in the target application.", "disfluent": "I mean the caps on overlay should stay caps off on I mean screen until the paste like actually lands in the target application.", "operations": ["opener", "filler", "filler", "spoken:caps-on"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["overlay", "should", "stay"]}]} +{"reference": "Nobody has looked at the notarization logs since the last release went out.", "disfluent": "n- Nobody has looked notarization— or rather at the notarization logs since the last release went out period", "operations": ["stutter", "false_start", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "out", "words": []}]} +{"reference": "It turns out the regression was introduced by the change to the clipboard restore path.", "disfluent": "well it turns restore— or rather out the regression was introduced by the change kind of to the clipboard restore path period", "operations": ["opener", "false_start", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "path", "words": []}]} +{"reference": "Please double check the sample rate before you send the audio to the transcription endpoint.", "disfluent": "Please double check the sample rate you know before you send the audio to the transcription endpoint.", "operations": ["filler"]} +{"reference": "The design review is blocked on whether we keep the menu bar ITEM at all.", "disfluent": "I mean the design review is blocked on whether we keep the the menu bar all caps item at all period", "operations": ["opener", "repeat", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["item"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "all", "words": []}]} +{"reference": "If the upload stalls, retry it once, and then fall back to the smaller chunk size.", "disfluent": "If the upload stalls comma retry it once comma and then fall back hmm to the the smaller chunk actually size.", "operations": ["filler", "repeat", "filler", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "stalls", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "once", "words": []}]} +{"reference": "We shipped the change on Tuesday, and by Thursday the error rate had halved.", "disfluent": "We shipped the change on Tuesday comma and by Thursday the error kind of rate had halved.", "operations": ["filler", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tuesday", "words": []}]} +{"reference": "The onboarding flow works, but the second screen STILL ASKS for a permission we never use.", "disfluent": "I mean the onboarding flow works comma but the second screen still second screen caps on still asks caps off for a permission for a permission we never I mean use period", "operations": ["opener", "restart", "restart", "filler", "spoken:comma", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "works", "words": []}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["still", "asks"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "use", "words": []}]} +{"reference": "Before you merge, rebase on main, run the whole suite, and check the coverage gate.", "disfluent": "Before before you merge comma rebase on main, um run the whole suite comma and check the whole— or rather coverage gate period", "operations": ["restart", "filler", "false_start", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "merge", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "suite", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "gate", "words": []}]} +{"reference": "I read the incident report, and the root cause was a stale cache in the edge layer.", "disfluent": "I read the the incident report, and the root cause was actually a stale cache cache in the edge er layer.", "operations": ["repeat", "filler", "repeat", "filler"]} +{"reference": "Once the lease expires, the worker stops accepting jobs, which is what we wanted.", "disfluent": "Once the lease expires, the w- worker stops accepting jobs comma which which is what I mean we wanted period", "operations": ["stutter", "repeat", "filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "jobs", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "wanted", "words": []}]} +{"reference": "The vendor confirmed the outage, apologised, and promised a postmortem by Friday.", "disfluent": "let's see the vendor confirmed the vendor confirmed the outage, apologised apologised, promised— sorry and promised a postmortem by Friday full stop", "operations": ["opener", "restart", "repeat", "false_start", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "friday", "words": []}]} +{"reference": "Send the draft to Priya, loop in the design team, and we can review it together.", "disfluent": "Send the draft to Priya comma l- loop in the design team comma and we can we can review it together period", "operations": ["stutter", "restart", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "priya", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "team", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "together", "words": []}]} +{"reference": "When the mic is muted, the waveform freezes, and users read that as a crash.", "disfluent": "When the mic read— or rather is muted comma the waveform freezes comma and freezes and users read that as like a as like a crash.", "operations": ["false_start", "restart", "filler", "restart", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "muted", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "freezes", "words": []}]} +{"reference": "The migration touched four tables, two indexes, and one view NOBODY remembered.", "disfluent": "okay so the migration touched four tables comma two indexes comma and one view all caps nobody remembered full stop", "operations": ["opener", "spoken:comma", "spoken:comma", "spoken:all-caps", "spoken:full-stop"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tables", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "indexes", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["nobody"]}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "remembered", "words": []}]} +{"reference": "After the retry budget runs out, the request fails, and the overlay says try again.", "disfluent": "After the retry budget runs out comma the ah request fails comma and and the overlay says try again period", "operations": ["filler", "repeat", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "out", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fails", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "again", "words": []}]} +{"reference": "We looked at three vendors, and only one of them will sign a data processing agreement.", "disfluent": "We looked looked at three vendors comma and only one of them will kind of sign a data processing agreement period", "operations": ["repeat", "filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "vendors", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "agreement", "words": []}]} +{"reference": "The cache warms in about a minute, so the FIRST few requests are always slower.", "disfluent": "so the cache the cache warms in about a ah minute comma so the all caps first few you know requests are always slower.", "operations": ["opener", "restart", "filler", "filler", "spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "minute", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["first"]}]} +{"reference": "If you cannot REPRODUCE it locally, attach the sysdiagnose, and I will look tonight.", "disfluent": "okay so if you cannot like all caps reproduce it locally comma attach the sysdiagnose comma and I will l- look tonight.", "operations": ["opener", "filler", "stutter", "spoken:all-caps", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["reproduce"]}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "locally", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sysdiagnose", "words": []}]} +{"reference": "The estimate assumed two engineers, and we have one, so the date needs to move.", "disfluent": "The estimate assumed two engineers comma and we have one comma so the date needs to move.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "engineers", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "one", "words": []}]} +{"reference": "Their API returns a 202, then polls, then hands back a URL that expires in an hour.", "disfluent": "Their API returns a 202 comma then polls, then hands hands back a URL that expires hmm in an hour.", "operations": ["repeat", "filler", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "202", "words": []}]} +{"reference": "I moved the standup to nine, cancelled the Thursday sync, and blocked out Friday afternoon.", "disfluent": "I moved the standup to nine comma cancelled the Thursday sync, and blocked out Friday afternoon full stop", "operations": ["spoken:comma", "spoken:full-stop"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "nine", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "afternoon", "words": []}]} +{"reference": "The feature is behind a flag, off by default, and only enabled for the internal team.", "disfluent": "so the feature is behind a a flag, off by default, and only enabled for the internal team.", "operations": ["opener", "repeat"]} +{"reference": "Did anyone check whether the new entitlement survives a clean install?", "disfluent": "Did Did anyone check whether the new entitlement su- survives a clean install question mark", "operations": ["repeat", "stutter", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "install", "words": []}]} +{"reference": "Should we hold the release until the notarization queue clears, or ship it now?", "disfluent": "Should we ho- hold the release until the notarization queue clears comma or ship it now now question mark", "operations": ["stutter", "repeat", "spoken:comma", "spoken:question-mark"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "clears", "words": []}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "now", "words": []}]} +{"reference": "Can you remind me what the RETRY budget is on the streaming endpoint?", "disfluent": "well actually can you remind me what the all caps retry budget is you know on the streaming kind of endpoint question mark", "operations": ["opener", "filler", "filler", "filler", "spoken:all-caps", "spoken:question-mark"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["retry"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "endpoint", "words": []}]} +{"reference": "Do you know why the waveform stops animating when the window loses focus?", "disfluent": "Do basically you know why the waveform stops stops— I mean animating when the window loses focus?", "operations": ["filler", "false_start"]} +{"reference": "Is there a reason we still ship the old audio unit alongside the new one?", "disfluent": "okay so is there a reason we still ship the um old audio unit alongside the new one question mark", "operations": ["opener", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "one", "words": []}]} +{"reference": "What happens to a partial transcript if the socket closes before the final message?", "disfluent": "ah What happens to a partial ah transcript if basically the socket closes before the final message question mark", "operations": ["filler", "filler", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "message", "words": []}]} +{"reference": "Would it be easier to gate this on the account TIER instead of a flag?", "disfluent": "I mean would it be easier to to gate this on the account all caps tier instead of uh a flag question mark", "operations": ["opener", "repeat", "filler", "spoken:all-caps", "spoken:question-mark"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["tier"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "flag", "words": []}]} +{"reference": "Have we ever measured how long the first paste takes on a cold launch?", "disfluent": "right so have hmm we um ever me- measured how long the first paste measured— I mean takes on a cold launch question mark", "operations": ["opener", "filler", "filler", "stutter", "false_start", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "launch", "words": []}]} +{"reference": "Are the crash reports symbolicated, or do I need to upload the archive myself?", "disfluent": "well are the crash crash symbolicated— I mean reports symbolicated comma actually or do I need to upload the like archive myself question mark", "operations": ["opener", "repeat", "false_start", "filler", "filler", "spoken:comma", "spoken:question-mark"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "symbolicated", "words": []}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "myself", "words": []}]} +{"reference": "Who owns the dashboard now that the analytics team has been folded into platform?", "disfluent": "Who owns the dashboard now that the like analytics team sort of has been folded into platform question mark", "operations": ["filler", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "platform", "words": []}]} +{"reference": "That fixed it, and the latency dropped by half!", "disfluent": "well that fixed it comma and the latency dropped by latency dropped by half exclamation point", "operations": ["opener", "restart", "spoken:comma", "spoken:exclamation-point"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "it", "words": []}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "half", "words": []}]} +{"reference": "Please do not ship this on a Friday afternoon again!", "disfluent": "Please do not ship this on a Friday afternoon again again exclamation mark", "operations": ["repeat", "spoken:exclamation-mark"], "commands": [{"spoken": "exclamation mark", "mark": "!", "kind": "mark", "anchor": "again", "words": []}]} +{"reference": "The WHOLE SUITE passed on the first try for once!", "disfluent": "well the caps on whole suite caps off passed on the first try for for once exclamation point", "operations": ["opener", "repeat", "spoken:caps-on", "spoken:exclamation-point"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["whole", "suite"]}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "once", "words": []}]} +{"reference": "Watch out, the staging database is still pointed at production!", "disfluent": "Watch out comma the staging database basically is still pointed at ah production exclamation point", "operations": ["filler", "filler", "spoken:comma", "spoken:exclamation-point"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "out", "words": []}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "production", "words": []}]} +{"reference": "Here is the plan: land the fix, cut a build, and HAND it to QA tomorrow.", "disfluent": "right so here is the plan colon land the fix comma cut a build, and all caps hand it to QA sort of tomorrow period", "operations": ["opener", "filler", "spoken:colon", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "plan", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fix", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["hand"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "tomorrow", "words": []}]} +{"reference": "Two things are still open: the entitlement review and the App Store description.", "disfluent": "Two things are still open colon the entitlement review and the App um Store description full stop", "operations": ["filler", "spoken:colon", "spoken:full-stop"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "open", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "description", "words": []}]} +{"reference": "The cause was simple: we were reading the sample rate from the wrong device.", "disfluent": "The cause was simple: we were reading the like sample rate from the wrong device period", "operations": ["filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "device", "words": []}]} +{"reference": "It builds cleanly on my machine; it fails on the runner every single time.", "disfluent": "It builds cleanly on my machine semicolon it fails on the runner on the runner every single time full stop", "operations": ["restart", "spoken:semicolon", "spoken:full-stop"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "machine", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "time", "words": []}]} +{"reference": "Ship the smaller change first; the refactor can WAIT UNTIL AFTER the release.", "disfluent": "okay so ship the smaller change first; the refactor can caps on wait until after caps off the release period", "operations": ["opener", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["wait", "until", "after"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "release", "words": []}]} +{"reference": "The tradeoff is straightforward: more accuracy for about eighty milliseconds of latency.", "disfluent": "The tradeoff is straightforward colon more accuracy for about eighty eighty milliseconds of latency period", "operations": ["repeat", "spoken:colon", "spoken:period"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "straightforward", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "latency", "words": []}]} +{"reference": "Keep the interface as it is; only the storage LAYER NEEDS to change.", "disfluent": "right so keep the interface as it is; only the storage caps on layer needs caps off to change period", "operations": ["opener", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["layer", "needs"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "change", "words": []}]} +{"reference": "One caveat: the migration is not reversible once the first write lands.", "disfluent": "One caveat colon the like migration is not reversible once the first write right lands.", "operations": ["filler", "filler", "spoken:colon"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "caveat", "words": []}]} +{"reference": "We only support one grace period, so plan accordingly before the TRIAL ends.", "disfluent": "I mean we only— I mean only support one grace period, so plan accordingly before the all caps trial ends period", "operations": ["opener", "false_start", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["trial"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "ends", "words": []}]} +{"reference": "Add a comma after the second clause and the sentence reads much better.", "disfluent": "right so add a comma after the after the second clause and the sentence reads m- much better period", "operations": ["opener", "restart", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "better", "words": []}]} +{"reference": "Every question mark in that survey was ambiguous, so we rewrote the whole form.", "disfluent": "Every question mark in that survey was ambiguous comma so we rewrote the whole form period", "operations": ["spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "ambiguous", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "form", "words": []}]} +{"reference": "The billing period rolls over at midnight UTC, not at midnight local time.", "disfluent": "The billing period rolls over at midnight UTC comma not at midnight local time period", "operations": ["spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "utc", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "time", "words": []}]} +{"reference": "Put a colon after the heading and leave the rest of the line alone.", "disfluent": "Put a colon a- after the heading and leave the r- rest of the line alone full stop", "operations": ["stutter", "stutter", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "alone", "words": []}]} +{"reference": "She used a full stop where the style guide clearly asks for a semicolon.", "disfluent": "She used a used a full stop where the ah style guide c- clearly asks for a semicolon period", "operations": ["restart", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "semicolon", "words": []}]} +{"reference": "The legal team wants the warning in all caps, which our design system forbids.", "disfluent": "yeah so the legal team wants the warning in all caps comma which our design system forbids period", "operations": ["opener", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "caps", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "forbids", "words": []}]} +{"reference": "That exclamation point in the release NOTES READS as sarcasm, so please remove it.", "disfluent": "well that exclamation point in the please— sorry release caps on notes reads caps off as er sarcasm comma so please remove you know it.", "operations": ["opener", "false_start", "filler", "filler", "spoken:caps-on", "spoken:comma"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["notes", "reads"]}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sarcasm", "words": []}]} +{"reference": "There is a dash missing from the second BULLET on the pricing page.", "disfluent": "let's see there is a dash missing from the second all caps bullet on the pricing page period", "operations": ["opener", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["bullet"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "page", "words": []}]} +{"reference": "I said period, and it typed the word instead of the punctuation mark.", "disfluent": "I said period, and and it typed the word instead of the punctuation mark full stop", "operations": ["repeat", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "mark", "words": []}]} +{"reference": "The Cretaceous period ended with an impact, which is roughly how the demo went.", "disfluent": "kind of The Cretaceous period ended with an impact comma which is roughly how the demo went period", "operations": ["filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "impact", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "went", "words": []}]} +{"reference": "Use a semicolon there; a comma is not strong enough to join those two clauses.", "disfluent": "Use a semicolon there semicolon a comma is not strong enough to hmm join those two clauses full stop", "operations": ["filler", "spoken:semicolon", "spoken:full-stop"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "there", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "clauses", "words": []}]} +{"reference": "Remind me to follow up with the accessibility team about the focus ring.", "disfluent": "right so remind me to follow up with with the accessibility team about the focus ring period", "operations": ["opener", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "ring", "words": []}]} +{"reference": "The transcript came back empty, which usually MEANS the audio was all silence.", "disfluent": "right so the transcript came back empty comma which usually all caps means the audio was all audio— I mean silence period", "operations": ["opener", "false_start", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "empty", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["means"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "silence", "words": []}]} +{"reference": "I will draft the announcement tonight and send it round for comments in the morning.", "disfluent": "right so i will draft sort of the announcement right tonight and send it round for comments kind of in the morning period", "operations": ["opener", "filler", "filler", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "morning", "words": []}]} +{"reference": "We should probably stop supporting the beta channel now that nobody is on it.", "disfluent": "We hmm should probably stop supporting beta— no wait the beta channel now that now that nobody is on it period", "operations": ["filler", "false_start", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} +{"reference": "The keyboard shortcut conflicts with the system dictation shortcut on a fresh install.", "disfluent": "The ke- keyboard shortcut conflicts with the s- system dictation shortcut on a fresh install period", "operations": ["stutter", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "install", "words": []}]} +{"reference": "Let me know if the new model handles background noise any better than the old one.", "disfluent": "well let me know if the new model model— or rather handles background noise noise any better than the old one.", "operations": ["opener", "false_start", "repeat"]} +{"reference": "Nothing in the logs explains why the first request after a sleep always times out.", "disfluent": "Nothing in the logs logs explains why the first request after request after a sleep always times out period", "operations": ["repeat", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "out", "words": []}]} +{"reference": "The onboarding video is four minutes long and most people quit after thirty seconds.", "disfluent": "The onboarding video is four minutes long and most people most people quit after thirty seconds full stop", "operations": ["restart", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "seconds", "words": []}]} +{"reference": "I would rather fix the FLAKE THAN mark the test as skipped and forget about it.", "disfluent": "yeah so i would rather fix uh the caps on flake than caps off mark the test as skipped and forget about and forget about it full stop", "operations": ["opener", "filler", "restart", "spoken:caps-on", "spoken:full-stop"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["flake", "than"]}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} +{"reference": "Our smallest customer files more bug reports than the other twenty combined.", "disfluent": "right so our smallest customer files more bug reports than the other twenty combined period", "operations": ["opener", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "combined", "words": []}]} +{"reference": "The release notes need a line about the new permission before we can publish.", "disfluent": "The release notes need a line about the new permission before we can publish publish period", "operations": ["repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "publish", "words": []}]} +{"reference": "Check whether the trial EXPIRY is stored in the keychain or in user defaults.", "disfluent": "well check whether the trial all caps expiry is stored in in the keychain or in user defaults period", "operations": ["opener", "repeat", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["expiry"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "defaults", "words": []}]} diff --git a/evals/dictation-prompt/metrics.py b/evals/dictation-prompt/metrics.py index 68f7b2b..8fb4dac 100644 --- a/evals/dictation-prompt/metrics.py +++ b/evals/dictation-prompt/metrics.py @@ -261,7 +261,9 @@ def _echoes_its_neighbour(run: list[int], spoken: list[tuple[str, str]]) -> bool return words in (before, after) -def _is_abandoned(run: list[int], spoken: list[tuple[str, str]]) -> bool: +def _is_abandoned( + run: list[int], spoken: list[tuple[str, str]], not_abandoned: frozenset[str] = frozenset() +) -> bool: """Whether one surplus run is an abandoned span rather than a stumble. Two ways to qualify, in order of how much they can be trusted: @@ -272,16 +274,27 @@ def _is_abandoned(run: list[int], spoken: list[tuple[str, str]]) -> bool: beside them — a phrase the speaker started and replaced. One word is not enough: a lone surplus content word is a repetition or a slip, and charging triple for it would sweep in most of what the old flat weight already handled. + + `not_abandoned` is extra vocabulary the *caller's corpus* knows is surplus for some + other reason, counted alongside `HESITATIONS` in rule 2. It exists for spoken + punctuation: "question mark" is two non-hesitation words that echo nothing, so + without it the classifier reads a dictation command as an abandoned phrase and + charges it `FALSE_START_WEIGHT` per word — which makes a leftover "question mark" + ten errors and a leftover "period" one, an asymmetry nobody chose. It defaults to + empty and is passed only by an utterance that planted commands, so every number + measured before it existed is unchanged. """ if any(_is_cut_off(spoken[index][0]) for index in run): return True words = [spoken[index][1] for index in run] - if sum(word not in HESITATIONS for word in words) < 2: + if sum(word not in HESITATIONS and word not in not_abandoned for word in words) < 2: return False return not _echoes_its_neighbour(run, spoken) -def false_start_tokens(disfluent: str, reference: str) -> tuple[str, ...]: +def false_start_tokens( + disfluent: str, reference: str, not_abandoned: frozenset[str] = frozenset() +) -> tuple[str, ...]: """The words of `disfluent` the speaker abandoned, normalized, in order. A multiset, not a set: "we wouldn't ha-, we wouldn't have them" abandons three @@ -295,18 +308,24 @@ def false_start_tokens(disfluent: str, reference: str) -> tuple[str, ...]: only recognizable as abandoned by the `ha-` sitting in front of it. A hedge caught inside such a run ("um we walked-") is charged with it, which is right: the whole region was abandoned, and the model has to delete all of it or none. + + `not_abandoned` passes through to `_is_abandoned` — see there for why a corpus that + plants dictation commands has to name them. """ spoken = [(raw, word) for raw in disfluent.split() if (word := normalize_text(raw))] alignment = align(normalize(reference), [word for _, word in spoken]) abandoned: list[str] = [] for run in _runs(sorted(set(alignment.inserted_positions))): - if _is_abandoned(run, spoken): + if _is_abandoned(run, spoken, not_abandoned): abandoned.extend(spoken[index][1] for index in run) return tuple(abandoned) def uncorrected_false_starts( - disfluent: str, reference: str, alignment: Alignment + disfluent: str, + reference: str, + alignment: Alignment, + not_abandoned: frozenset[str] = frozenset(), ) -> tuple[str, ...]: """Abandoned words the cleanup left in — the multiset `FALSE_START_WEIGHT` charges. @@ -316,7 +335,8 @@ def uncorrected_false_starts( hypothesis; both sides are already normalized there, which is what lets the raw `store—` in the input match the `store` the model echoed back. """ - left_in = Counter(alignment.inserted) & Counter(false_start_tokens(disfluent, reference)) + spoken = false_start_tokens(disfluent, reference, not_abandoned) + left_in = Counter(alignment.inserted) & Counter(spoken) return tuple(sorted(left_in.elements())) @@ -396,7 +416,12 @@ def _surcharged(alignment: Alignment, extra_errors: float) -> float: return alignment.error_rate + extra_errors / alignment.reference_length -def score(reference: str, hypothesis: str, disfluent: str | None = None) -> Score: +def score( + reference: str, + hypothesis: str, + disfluent: str | None = None, + not_abandoned: frozenset[str] = frozenset(), +) -> Score: """Score a cleanup against its reference on both axes — see `from_error_rate`. `disfluent` is what the model was given. Pass it and leftover false starts are @@ -409,10 +434,17 @@ def score(reference: str, hypothesis: str, disfluent: str | None = None) -> Scor The same surcharge lands on both axes. A word left in is a word left in whichever way you tokenize, and exempting the formatting axis would quietly dilute the weighting by 30% under the default `blend`. + + `not_abandoned` exempts vocabulary the corpus planted deliberately — see + `_is_abandoned`. Empty by default, so nothing that does not pass it changes. """ content_alignment = align(normalize(reference), normalize(hypothesis)) format_alignment = align(surface(reference), surface(hypothesis)) - left_in = uncorrected_false_starts(disfluent, reference, content_alignment) if disfluent else () + left_in = ( + uncorrected_false_starts(disfluent, reference, content_alignment, not_abandoned) + if disfluent + else () + ) surcharge = (FALSE_START_WEIGHT - 1.0) * len(left_in) return Score( content=from_error_rate(_surcharged(content_alignment, surcharge)), diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index 9ff9087..622bb99 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -69,10 +69,12 @@ import corpus # noqa: E402 import metrics # noqa: E402 +import spoken_punctuation # noqa: E402 from candidates import ( # noqa: E402 BASELINE, CANDIDATES, INSTRUCTION_CHARACTER_CAP, + SPOKEN_PUNCTUATION_CANDIDATES, missing_safeguards, objections, overage, @@ -90,6 +92,31 @@ def print_table(rows: list[tuple[str, dict[str, float]]], title: str, axis: str) print(f" {name:<22} " + " ".join(f"{scores[a]:>9.4f}" for a in metrics.AXES)) +def print_command_table(rows: list[tuple[str, dict[str, float]]], axis: str) -> None: + """What became of the planted commands, per candidate. + + Printed apart from the axis table because it answers a different question. WER says + how close the text came; this says how many of the commands actually planted were + obeyed, left in the output as words, or silently dropped — and `literal` is the one + that reaches the user's document as visible nonsense, so it earns its own column + rather than being folded into "did not convert". + + Ordered by the selecting axis, not by conversion rate, so a candidate that converts + more commands and still loses is visible as exactly that. + """ + scored = [row for row in rows if row[1].get("commands_total")] + if not scored: + return + print("\nSpoken punctuation commands, per candidate") + print(f" {'candidate':<22} {'converted':>10} {'left as words':>14} {'dropped':>9}") + print(f" {'-' * 22} {'-' * 10} {'-' * 14} {'-' * 9}") + for name, scores in sorted(scored, key=lambda row: row[1][axis], reverse=True): + print( + f" {name:<22} {scores['commands_converted']:>10.1%} " + f"{scores['commands_literal']:>14.1%} {scores['commands_missing']:>9.1%}" + ) + + def print_samples(loaded: corpus.Corpus, count: int) -> None: print(f"\nExamples (showing {min(count, len(loaded))} of {len(loaded)})") for utterance in loaded.utterances[:count]: @@ -153,12 +180,42 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="also lowercase and unpunctuate, making formatting restoration part of the task", ) + spoken = parser.add_argument_group("spoken punctuation (any source)") + spoken.add_argument( + "--spoken-punctuation", + type=float, + default=0.0, + metavar="RATE", + help="0..1; speak this share of the punctuation the reference licenses, turning " + 'each mark into the words a dictation user says out loud ("comma", "question ' + 'mark", "all caps"). 0 (default) is off. Unlike --severity this applies to a ' + "PAIRED source too: the marks come from the corpus's own clean side, so the " + "disfluencies stay the ones annotators marked by hand and only the punctuation " + "task is synthetic. Below 1 on purpose — a corpus with no real marks left in the " + "input teaches an instruction to punctuate by guesswork instead of by command", + ) + spoken.add_argument( + "--spoken-caps-rate", + type=float, + default=0.25, + metavar="RATE", + help="0..1; chance a row also gets one ALL CAPS command (default: 0.25). At most " + "one per row — two in a 15-word utterance would make the operator most of the " + "corpus. Only this operator edits the reference, because a target with no " + "uppercase in it cannot pose the task. Ignored unless --spoken-punctuation is on", + ) + evaluation = parser.add_argument_group("evaluation") evaluation.add_argument( "--metric", - default="blend", + default=None, choices=metrics.AXES, - help="which axis selects the winner (default: blend, 0.7 content / 0.3 format)", + help="which axis selects the winner. Default: blend (0.7 content / 0.3 format), " + "or format under --spoken-punctuation. Spoken punctuation is a formatting task " + "that content cannot see — normalize() casefolds and strips marks, so a restored " + "comma and a missed one are the same string to it, and an ALL CAPS command is " + "invisible. format is the axis that sees the whole task, and it still sees the " + "expensive failure (a command left in as a word) as an inserted token", ) evaluation.add_argument( "--dev-fraction", @@ -185,7 +242,10 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "own advice is the smallest set that still matches the task distribution, and 50 " "was that; it was raised to 150 because at 50 the Pareto front was ranking candidates " "on differences it could not resolve, and picking a winner that dev then rejected, and " - "tripled again alongside dev. Noise falls as the square root, so 3x the rows is ~42% " + # `%%` because argparse %-expands help strings against a dict of params, so a bare + # `%` raises TypeError and `--help` fails outright — which it did, from the day + # this figure was written until someone tried to read the help. + "tripled again alongside dev. Noise falls as the square root, so 3x the rows is ~42%% " "less of it — nothing cheaper buys that", ) evaluation.add_argument( @@ -334,10 +394,44 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "at that wording, and this uses the wording itself", ) + parser.add_argument( + "--dump-corpus", + default=None, + metavar="PATH", + help="write the loaded corpus to PATH as JSONL and carry on. Reading it back with " + "--jsonl reproduces it exactly, commands and all, so a dataset can be frozen in " + "the tree and diffed instead of re-derived from a download plus two seeded " + "injectors. Works under --dry-run, so generating one needs no API key. Mind the " + "licensing: --source builtin is written for this repo, while nyra derives from " + "LDC-licensed Switchboard transcripts and should stay out of version control", + ) parser.add_argument("--out", default=None, help="write the full results as JSON to this path") return parser.parse_args(argv) +#: Axis each corpus shape selects on when `--metric` is not given. +DEFAULT_AXIS = "blend" +SPOKEN_PUNCTUATION_AXIS = "format" + + +def instruction_table(loaded: corpus.Corpus) -> dict[str, str]: + """The instructions this run scores — the shipped set, plus the task's own. + + `SPOKEN_PUNCTUATION_CANDIDATES` are merged in rather than living in `CANDIDATES` + because they are dead weight on every ordinary run: a punctuation clause cannot help + against a corpus that poses no punctuation commands, and each one costs a full dev + sweep to re-rank. `BASELINE` is untouched either way, so the held-out comparison is + still against what Blurt ships. + + Keyed on the loaded corpus rather than on `--spoken-punctuation`, so a frozen dataset + read back with `--jsonl` is scored the same way as the corpus it was dumped from: it + carries the commands and not the flag. + """ + if not loaded.has_commands: + return dict(CANDIDATES) + return dict(CANDIDATES) | SPOKEN_PUNCTUATION_CANDIDATES + + def describe_length(instruction: str) -> str: """`2048 chars` plus how that sits against the cap — the line every report ends on.""" over = overage(instruction) @@ -347,7 +441,7 @@ def describe_length(instruction: str) -> str: return f"{len(instruction)} chars, {headroom} under the {INSTRUCTION_CHARACTER_CAP} cap" -def check_candidates() -> None: +def check_candidates(table: dict[str, str] | None = None) -> None: """Refuse to start if a hand-written candidate could never be shipped. Before any model call, because these are typo-class mistakes and paying for a full @@ -359,9 +453,15 @@ def check_candidates() -> None: deliberately terse one-liners whose job is to be *contrast* — `guessed-default` is a floor, not something anyone would ship — and demanding the full safeguard set of them would turn the comparison set into six copies of the same careful paragraph. + + Takes the table rather than reading `CANDIDATES` so that whatever + `instruction_table` merged in is checked too; omitting it checks the shipped set. A `--spoken-punctuation` candidate is + exactly as capable of being 8 characters over the cap as any other — `punct-appended` + lands at 2006 of 2048 — and discovering that after paying for a dev sweep is the + waste this function exists to prevent. """ problems: list[str] = [] - for name, text in CANDIDATES.items(): + for name, text in (CANDIDATES if table is None else table).items(): if overage(text): problems.append(f" {name}: {describe_length(text)}") if name == BASELINE and (absent := missing_safeguards(text)): @@ -374,7 +474,11 @@ def check_candidates() -> None: ) -def resolve_candidates(args: argparse.Namespace) -> list[str]: +def resolve_candidates( + args: argparse.Namespace, + table: dict[str, str] | None = None, + spoken: bool = False, +) -> list[str]: """Which candidates get scored on dev before the search. Sweeping all of them costs `len(CANDIDATES) × dev` calls to re-establish an @@ -388,14 +492,29 @@ def resolve_candidates(args: argparse.Namespace) -> list[str]: A run that isn't searching gets the full table, because ranking one candidate is not a ranking. That is the shape of `--optimizer none`: the cheap pass you make when you have changed the corpus or the model and want the ordering itself. + + A `spoken` corpus gets a **third** answer: the punctuation candidates plus + `BASELINE`, and not the six terse contrast instructions. `BASELINE` alone is not + enough — it has never heard of the task, so a search seeded from it starts outside + the region worth exploring — but the six one-liners are worse than useless here. They + exist to rank framings of *disfluency* cleanup ("verbatim-preserving" against + "dictation-intent"), none of them mentions punctuation, and every one of them will + land near the floor for the same reason. At the default 900-row dev split that + ordering costs 5,400 model calls to re-discover. `--candidates all` still buys them + if you want to see it happen. """ + if args.candidates == "all": + return list(CANDIDATES if table is None else table) + if spoken: + # BASELINE first so the table reads as "the bar, then the challengers". + return [BASELINE, *SPOKEN_PUNCTUATION_CANDIDATES] if ( - args.candidates == "all" - or (args.candidates is None and args.optimizer == "none") + args.candidates is None + and args.optimizer == "none" # Nothing has been scored yet, so "best" is unknowable without the sweep. or args.start == "best-candidate" ): - return list(CANDIDATES) + return list(CANDIDATES if table is None else table) return [BASELINE] @@ -463,7 +582,7 @@ def run_live_verification( return summaries -def resolve_axis(requested: str, loaded: corpus.Corpus) -> str: +def resolve_axis(requested: str | None, loaded: corpus.Corpus) -> str: """Refuse to select on an axis the corpus cannot measure. Some corpora carry casing or punctuation their targets did not intend — most @@ -471,7 +590,15 @@ def resolve_axis(requested: str, loaded: corpus.Corpus) -> str: strands the following word in lowercase. Scoring formatting against that penalizes a *correct* cleanup, so `blend` degrades to `content` with a note and an explicit `--metric format` is an error rather than a meaningless number. + + `requested` is None when `--metric` was not given, which is where the default lives + rather than in `argparse`: it depends on whether the corpus poses punctuation + commands. A `--spoken-punctuation` corpus that also cannot be scored on formatting + fails below rather than quietly selecting on content, because content cannot see an + ALL CAPS command at all and half the task would go unmeasured. """ + if requested is None: + requested = SPOKEN_PUNCTUATION_AXIS if loaded.has_commands else DEFAULT_AXIS if loaded.formatting_is_measurable or requested == "content": return requested if requested == "format": @@ -479,6 +606,12 @@ def resolve_axis(requested: str, loaded: corpus.Corpus) -> str: f"--metric format needs a corpus with trustworthy target formatting; " f"{loaded.source} does not have it. Use --source nyra for repaired casing, or " "--source builtin --strip-formatting to pose formatting restoration as a task." + + ( + " Spoken punctuation needs it too: the commands it plants are answered " + "in capitalization and marks, which the content axis cannot see." + if loaded.has_commands + else "" + ) ) print( f"\nNote: {loaded.source} targets carry formatting artifacts from mechanical " @@ -489,8 +622,6 @@ def resolve_axis(requested: str, loaded: corpus.Corpus) -> str: def main(argv: list[str] | None = None) -> int: args = parse_args(argv) - check_candidates() - loaded = corpus.load( source=args.source, limit=args.limit, @@ -499,10 +630,19 @@ def main(argv: list[str] | None = None) -> int: seed=args.seed, severity=args.severity, strip_formatting=args.strip_formatting, + spoken_punctuation_rate=args.spoken_punctuation, + spoken_caps_rate=args.spoken_caps_rate, ) print(f"Loaded {len(loaded)} pairs from {loaded.source}: {loaded.detail}") print(f"{loaded.disfluent_fraction:.0%} of pairs differ from their target") + # After the load, because which instructions are in play depends on whether the + # corpus poses the punctuation task — and a `--jsonl` dataset answers that by what it + # contains, not by a flag. Still before any model call, which is what the check is + # for: a typo in candidates.py should not cost a paid sweep to discover. + table = instruction_table(loaded) + check_candidates(table) + train, dev, test = corpus.split( list(loaded.utterances), args.seed, args.dev_fraction, args.test_fraction ) @@ -518,10 +658,17 @@ def main(argv: list[str] | None = None) -> int: f"{len(dev)} dev / {len(test)} test" ) + if args.dump_corpus: + written = corpus.dump_jsonl(loaded, args.dump_corpus) + print(f"\nWrote {written} rows to {args.dump_corpus}") + if args.show_samples: print_samples(loaded, args.show_samples) axis = resolve_axis(args.metric, loaded) + # Written back so the results file records the axis the run actually selected on + # rather than the `None` that means "let the corpus decide". + args.metric = axis floors = {"dev": corpus.no_cleanup_floor(dev), "test": corpus.no_cleanup_floor(test)} print( f"\nNo-cleanup floor ({axis}) — the corpus's disfluent side scored against its " @@ -533,6 +680,22 @@ def main(argv: list[str] | None = None) -> int: f"rather than one; {corpus.false_start_fraction(dev):.0%} of dev rows and " f"{corpus.false_start_fraction(test):.0%} of test rows contain one" ) + if loaded.has_commands: + planted = sum(len(u.commands) for u in dev + test) + print( + f"\nSpoken punctuation: {planted} commands planted across {len(dev) + len(test)} " + f"dev+test rows ({planted / max(1, len(dev) + len(test)):.2f} per row); " + f"{sum(bool(u.commands) for u in dev + test) / max(1, len(dev) + len(test)):.0%} " + "of rows carry at least one" + ) + # Bounds what the run can say about over-conversion, so it is printed rather + # than left to be assumed away. See spoken_punctuation.literal_use_fraction. + literal = spoken_punctuation.literal_use_fraction(u.reference for u in dev + test) + print( + f"{literal:.1%} of those references use a command phrase as ordinary content, so " + "the score is nearly blind to an instruction that converts 'the Cretaceous " + "period'. That clause is in the candidates on product grounds, not scored ones" + ) if args.dry_run: print("\n--dry-run: corpus and scoring verified; no model was called.") @@ -549,22 +712,23 @@ def main(argv: list[str] | None = None) -> int: ) program.configure(spec) - scoring = resolve_candidates(args) + scoring = resolve_candidates(args, table, spoken=loaded.has_commands) dev_rows: list[tuple[str, dict[str, float]]] = [] with Progress(len(scoring) * len(dev), "Scoring candidates on dev") as meter: for index, name in enumerate(scoring, start=1): note = f"{name} ({index}/{len(scoring)})" scores = program.evaluate( - program.build(CANDIDATES[name]), + program.build(table[name]), dev, args.num_threads, on_example=lambda n=note: meter.tick(n), ) dev_rows.append((name, scores)) print_table(dev_rows, f"Candidate instructions on dev, selecting on {axis}", axis) + print_command_table(dev_rows, axis) winner_name, winner_scores = max(dev_rows, key=lambda row: row[1][axis]) - winner_instruction = CANDIDATES[winner_name] + winner_instruction = table[winner_name] if args.optimizer != "none": # The seed is an ordinary candidate — it fits the cap, so it was scored in the @@ -572,7 +736,7 @@ def main(argv: list[str] | None = None) -> int: # need to keep it out of the selection: unlike the over-cap instruction this # replaced, shipping it is a real option. seed_name = BASELINE if args.start == "prior-winner" else winner_name - seed_instruction = CANDIDATES[seed_name] + seed_instruction = table[seed_name] seed_scores = dict(dev_rows)[seed_name] proposer = program.CappedInstructionProposer(INSTRUCTION_CHARACTER_CAP) @@ -608,6 +772,7 @@ def main(argv: list[str] | None = None) -> int: optimized_name = f"{args.optimizer}-optimized" dev_rows.append((optimized_name, optimized_scores)) print_table(dev_rows, f"With the optimized instruction, on dev ({axis})", axis) + print_command_table(dev_rows, axis) print(f"\nThe optimized instruction is {describe_length(optimized_instruction)}.") delta = optimized_scores[axis] - seed_scores[axis] print( @@ -644,7 +809,7 @@ def main(argv: list[str] | None = None) -> int: # reported improvement is measured on data no selection decision saw. scored_on_test = [(winner_name, winner_instruction)] if winner_name != BASELINE: - scored_on_test.append((BASELINE, CANDIDATES[BASELINE])) + scored_on_test.append((BASELINE, table[BASELINE])) test_rows = [] with Progress(len(scored_on_test) * len(test), "Scoring on held-out test") as meter: for name, instruction in scored_on_test: @@ -660,6 +825,7 @@ def main(argv: list[str] | None = None) -> int: ) ) print_table(test_rows, f"Held-out test ({axis})", axis) + print_command_table(test_rows, axis) live_summary = run_live_verification(args, winner_name, winner_instruction, test) @@ -695,7 +861,7 @@ def main(argv: list[str] | None = None) -> int: "length": len(winner_instruction), }, "live": live_summary, - "candidates": CANDIDATES, + "candidates": table, } Path(args.out).write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") print(f"\nWrote {args.out}") diff --git a/evals/dictation-prompt/program.py b/evals/dictation-prompt/program.py index 5f7d186..e53fc81 100644 --- a/evals/dictation-prompt/program.py +++ b/evals/dictation-prompt/program.py @@ -16,6 +16,7 @@ from gepa.strategies.instruction_proposal import InstructionProposalSignature import metrics +import spoken_punctuation from candidates import constraint_preamble, objections, revision_directive, trim_to_fit from corpus import Utterance @@ -146,7 +147,32 @@ def _cleaned(prediction) -> str: return getattr(prediction, OUTPUT_FIELD, "") or "" -def make_feedback_metric(axis: str): +def commands_by_input(utterances: list[Utterance]) -> dict[str, tuple]: + """`{input text: planted spoken-punctuation commands}` for the metric to look up. + + GEPA hands the metric a `dspy.Example`, so anything the metric needs has to be + reachable from one. Keying on the input text rather than adding a field to the + Example is deliberate: `Example.labels()` returns every non-input key, and GEPA + builds its reflective dataset out of those, so a `commands` field would put a list of + dataclass reprs into the reflection prompt the model reads. A side table costs a dict + and changes nothing GEPA sees. + + Raises on a duplicate input rather than letting one row silently answer for another. + Two identical transcripts would be a corpus problem — `corpus._collect` de-duplicates + on the *reference* side, so this is the check that does not exist yet. + """ + table: dict[str, tuple] = {} + for utterance in utterances: + if utterance.disfluent in table and table[utterance.disfluent] != utterance.commands: + raise RuntimeError( + "two rows share an input side but carry different planted commands: " + f"{utterance.disfluent!r}" + ) + table[utterance.disfluent] = utterance.commands + return table + + +def make_feedback_metric(axis: str, commands: dict[str, tuple] | None = None): """Metric for GEPA: the same score plus a diff its reflector can act on. `axis` decides which score the reflector is shown, and it is shown only that one. @@ -154,22 +180,34 @@ def make_feedback_metric(axis: str): `metrics.feedback` for why the formatting score in particular is misleading on the default corpus. The length budget is not here at all; it belongs in the proposer's preamble, said once with the real target rather than eight times with the cap. + + `commands` adds the spoken-punctuation note when the corpus planted any. Without it + the reflector sees a leftover "comma" only as `metrics.feedback`'s "left disfluencies + in the output: comma" — true, and pointing at the wrong rule — and sees a *missing* + mark or a missed ALL CAPS as nothing more specific than "capitalization or + punctuation differs from the reference". On a search whose whole subject is + punctuation commands, that is most of the signal thrown away. """ + commands = commands or {} def scorer(gold, pred, trace=None, pred_name=None, pred_trace=None, **_): hypothesis = _cleaned(pred) + target, spoken = getattr(gold, OUTPUT_FIELD), getattr(gold, INPUT_FIELD) + planted = commands.get(spoken, ()) # The input as well as the target: `metrics.score` charges a leftover false # start `FALSE_START_WEIGHT` errors, and only the spoken side says which - # leftovers were abandoned. - scored = metrics.score(getattr(gold, OUTPUT_FIELD), hypothesis, getattr(gold, INPUT_FIELD)) + # leftovers were abandoned. `not_abandoned` keeps a two-word dictation command + # from being read as one — see `metrics._is_abandoned`. + scored = metrics.score( + target, + hypothesis, + spoken, + frozenset(word for command in planted for word in command.spoken_words), + ) return dspy.Prediction( score=scored.value(axis), - feedback=metrics.feedback( - getattr(gold, OUTPUT_FIELD), - getattr(gold, INPUT_FIELD), - scored, - axis=axis, - ), + feedback=metrics.feedback(target, spoken, scored, axis=axis) + + spoken_punctuation.feedback_note(planted, hypothesis), ) return scorer @@ -213,12 +251,22 @@ def evaluate( ) counted = program if on_example is None else _Ticking(program, on_example) predictions = runner([(counted, {INPUT_FIELD: u.disfluent}) for u in utterances]) - return metrics.mean( + cleaned = [_cleaned(prediction) for prediction in predictions] + axes = metrics.mean( [ - utterance.scored(_cleaned(prediction)) - for utterance, prediction in zip(utterances, predictions, strict=True) + utterance.scored(hypothesis) + for utterance, hypothesis in zip(utterances, cleaned, strict=True) ] ) + # Reported next to the axes rather than instead of them. WER answers "how close is + # the text", which mixes the punctuation task into every other kind of error; the + # tally answers "of the commands planted, how many were obeyed", which is the + # question `--spoken-punctuation` is asking and which only a synthetic operator can + # be asked. Zeros with `commands_total` at 0 mean "not asked", not "failed". + return axes | spoken_punctuation.tally( + (utterance.commands, hypothesis) + for utterance, hypothesis in zip(utterances, cleaned, strict=True) + ) class CappedInstructionProposer: @@ -443,7 +491,7 @@ def optimize( """ trainset, valset = to_examples(train), to_examples(validation) return dspy.GEPA( - metric=make_feedback_metric(axis), + metric=make_feedback_metric(axis, commands_by_input(train + validation)), auto=auto, # Never below the task model's ceiling: the reflector writes whole # instructions and thinks at length first, so it is the call most likely diff --git a/evals/dictation-prompt/spoken_punctuation.py b/evals/dictation-prompt/spoken_punctuation.py new file mode 100644 index 0000000..3292b37 --- /dev/null +++ b/evals/dictation-prompt/spoken_punctuation.py @@ -0,0 +1,547 @@ +"""Spoken punctuation injection — the "period, comma, question mark, ALL CAPS" task. + +A dictation user who wants a comma often *says* "comma". The speech-to-text pass +transcribes that as the word, so the rewrite model is handed `send it today comma +then call me` and has to produce `send it today, then call me`. Nothing in +`disfluency.py` poses that: it only inserts hesitation, and every real paired corpus +in `corpus.py` has punctuation on both sides already. + +This module layers the task onto a **real** pair rather than inventing one. It reads +the corpus's clean side to decide what a spoken mark would have to become, and then +speaks marks on the disfluent side only. So the target is still the corpus's own +target, the disfluencies are still the ones annotators marked by hand, and the only +synthetic part is the one thing being measured. + +Two operators, and they are not the same kind of thing: + +**Marks** (`period`, `comma`, `question mark`, …) are injected *subtractively on the +input side only*. A mark is spoken only when the reference **licenses** it — the same +word carries the same mark in the target — so the correct output is always the +reference exactly as it already stands, and the reference is never edited. A mark the +reference does not have is never spoken, because "say it and then delete it" is not a +rule any instruction should learn. + +**ALL CAPS** is the exception, and the only place here that edits a target. You cannot +pose "uppercase this word" against a target with no uppercase in it, so the chosen run +is uppercased in the reference and prefixed with the command, lowercased, in the input. +That is a deliberate departure from `disfluency.py`'s additive-only invariant, and it +is confined to this one operator. + +Both are visible almost entirely on the **format** axis. `metrics.normalize` casefolds +and strips punctuation, so a restored comma and a missed one are the same string to the +content axis; what content *does* see is the expensive failure — a command left in the +output as a literal word, which is an inserted token on both axes. That split is why +`--spoken-punctuation` selects on `format` by default: it is the axis that can see the +whole task, and it sees the leftover word too. + +What this does **not** emulate is how a real speech-to-text pass renders a spoken +command in the first place. It probably applies its own casing and punctuation model on +top — returning something nearer `Children period.` than `children period` — and what it +does exactly is unknown without measuring the service. So the input here is the clean +form of the task: the command as a bare word, and no casing left to give it away. Read a +result as a ranking of instructions on that task, and `--verify-live` (README) as the +only thing that speaks to the real pipeline. + +What the corpus cannot tell you: whether an instruction converts the word "period" when +the speaker meant the era. Only ~1% of `nyra` references use any command word literally +(the run reports the figure), so the score barely moves either way — the same shape of +blind spot as `candidates.REQUIRED_SAFEGUARDS`, and handled the same way, by putting the +clause in the instruction and saying here that nothing measures it. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass + +import metrics + +#: Marks a dictation user says out loud, and the words they say. Weighted because the +#: aliases are not equally common — "period" far outruns "full stop" in US dictation — +#: and present at all because an instruction tuned against a single spelling of each +#: command is an instruction that fails on the other one. The eval cannot see that +#: fragility if the injector only ever emits one form. +#: +#: `!`, `:` and `;` are here and effectively never fire on `nyra`: over 400 rows its +#: references carry 483 periods, 460 commas, 43 question marks and none of the other +#: three. They cost nothing to keep, they are what a `--jsonl` corpus of written prose +#: would exercise, and leaving them out would make the vocabulary a claim about +#: Switchboard rather than about dictation. +#: +#: Quotes, parentheses and dashes are deliberately absent. They are *paired* or +#: *span-scoped* commands ("open quote" … "close quote"), so injecting one means +#: deciding where the span ends — and Switchboard telephone transcripts contain almost +#: none of them, so the decision would be exercised by nothing. +SPOKEN_FORMS: dict[str, tuple[tuple[str, int], ...]] = { + ".": (("period", 6), ("full stop", 1)), + ",": (("comma", 1),), + "?": (("question mark", 1),), + "!": (("exclamation point", 2), ("exclamation mark", 1)), + ":": (("colon", 1),), + ";": (("semicolon", 1),), +} + +#: Marks that end a sentence, so the word after them was capitalized by the transcriber +#: rather than by the speaker. Spoken, that capital has to go — see `inject`. +TERMINAL_MARKS = ".?!" + +#: What the input says to uppercase a single word, and to open/close a longer run. +#: Both conventions are in real use; a run needs the bracketing form because "all caps" +#: alone says nothing about where the shouting stops. +CAPS_WORD = "all caps" +CAPS_ON = "caps on" +CAPS_OFF = "caps off" + +#: Shortest word worth uppercasing. Below this the run is a function word, and "all caps +#: the" is not a thing anyone dictates — while `metrics.HESITATIONS` covers the openers +#: `candidates.PRIOR_WINNER` is told to delete aggressively, which would otherwise let a +#: caps command and a deletion rule fight over the same token. +MIN_CAPS_WORD = 4 + +#: Length of an uppercased run, and how often each length is chosen. Single words +#: dominate because that is what people dictate; the longer runs exist so the +#: bracketing form (`caps on` … `caps off`) is exercised at all. +CAPS_RUN_LENGTHS: tuple[tuple[int, int], ...] = ((1, 7), (2, 2), (3, 1)) + + +@dataclass(frozen=True) +class Command: + """One spoken instruction planted in the input, and how to tell if it landed. + + Carries the *anchor* — the word the mark attaches to — rather than only the mark, + because "did this command get converted" is otherwise unanswerable: a hypothesis + containing a comma somewhere says nothing about whether it is the comma that was + asked for. With the anchor it is a local question about one word. + """ + + #: The words inserted into the input: `"comma"`, `"all caps"`, `"caps on"`. + spoken: str + #: The punctuation the target carries, or `""` for a casing command. + mark: str + #: `"mark"` or `"caps"` — which of the two operators produced this. + kind: str + #: Normalized word the mark attaches to; empty for a casing command. + anchor: str = "" + #: Normalized words a casing command uppercases; empty for a mark. + words: tuple[str, ...] = () + + def to_json(self) -> dict: + """The command as plain JSON, for `corpus.dump_jsonl`. + + Explicit rather than `dataclasses.asdict` so the on-disk shape is a decision + rather than a consequence of field order, and a field added here has to be given + a name on disk deliberately. + """ + return { + "spoken": self.spoken, + "mark": self.mark, + "kind": self.kind, + "anchor": self.anchor, + "words": list(self.words), + } + + @classmethod + def from_json(cls, row: dict) -> Command: + """Read one back. The round trip has to be exact — a dumped corpus that scores + differently from the corpus it was dumped from is worse than no dumped corpus.""" + return cls( + spoken=row["spoken"], + mark=row.get("mark", ""), + kind=row["kind"], + anchor=row.get("anchor", ""), + words=tuple(row.get("words", ())), + ) + + @property + def label(self) -> str: + """Short name for `Utterance.operations` and the sample dump.""" + return f"spoken:{self.spoken.replace(' ', '-')}" + + @property + def spoken_words(self) -> tuple[str, ...]: + """The command's own words, normalized — what it added to the input. + + Handed to `metrics.score` as `not_abandoned` so the false-start classifier does + not read "question mark" as an abandoned phrase and charge it five errors a word. + See `metrics._is_abandoned`. + """ + return tuple(metrics.normalize_text(self.spoken).split()) + + @property + def literal(self) -> tuple[str, ...]: + """The normalized token run that means the command was left in as words. + + Anchored for a mark (`("today", "comma")`) and prefixed for a casing command + (`("all", "caps", "urgent")`), so an instruction that happened to use the word + elsewhere is not charged for it. + """ + spoken = self.spoken_words + return (self.anchor, *spoken) if self.kind == "mark" else (*spoken, *self.words) + + +def _tokens(text: str) -> list[tuple[int, str]]: + """`[(index into text.split(), normalized word)]`, skipping bare punctuation. + + The corpus emits standalone `.` tokens (`"Yeah, I do . Yes uh"`), which normalize to + the empty string. Dropping them here keeps every word-index computation below + working on words, while the raw index stays available for editing the text. + """ + return [(i, w) for i, raw in enumerate(text.split()) if (w := metrics.normalize_text(raw))] + + +def _split_mark(raw: str) -> tuple[str, str] | None: + """`("today", ",")` for `"today,"`; None when there is no single trailing mark. + + Requires the mark to be the token's **last** character rather than searching a + trailing run, so `it?"` is skipped rather than turned into `it question mark"`. + Requires a non-empty stem for the same reason `_tokens` drops bare punctuation. + """ + if len(raw) < 2 or raw[-1] not in SPOKEN_FORMS: + return None + stem = raw[:-1] + return (stem, raw[-1]) if metrics.normalize_text(stem) else None + + +def licensed_marks(reference: str) -> set[tuple[str, str]]: + """`(anchor, mark)` pairs the reference itself carries — the only ones speakable. + + This is the whole reason the reference never needs editing for a mark. `nyra`'s two + sides are punctuated independently, so the disfluent side has marks the target does + not: a comma sitting in front of a span the annotator deleted. Speaking *that* comma + would ask the instruction to produce a mark and then have it scored as an error, + teaching it that commands are sometimes to be ignored. Matching on `(anchor, mark)` + keeps a spoken command one whose answer is already in the target. + + Word-and-mark rather than position, so nothing has to align the two sides. A + repeated word can license a mark on a different copy of itself; that is a + false *accept*, which costs nothing — the correct output still contains that + mark on that word. + """ + pairs: set[tuple[str, str]] = set() + for raw in reference.split(): + if split := _split_mark(raw): + pairs.add((metrics.normalize_text(split[0]), split[1])) + return pairs + + +def _keeps_its_capital(raw: str) -> bool: + """Whether this token's capital survives having its sentence's mark spoken. + + Only the pronoun *I*, and the reasoning is worth recording because a corpus-derived + proper-noun rule stood here first and was removed as actively harmful. + + Speaking a terminal mark takes the following capital with it: the transcriber wrote + `children. Schroeder's` and a speaker saying "period" produced no capital, so leaving + one lets an instruction restore the period from the casing alone. The objection was + that lowercasing `Schroeder's` charges a correct cleanup for damage the injector did — + and measurement says it does not. That word is **sentence-initial in the reference by + construction**, because a terminal mark is the only kind spoken here, so the required + output is `Schroeder's` and the required action is the rule every candidate already + states: capitalize the first word after a sentence-ending mark. Name or ordinary word, + the task and the answer are identical. + + Protecting names, meanwhile, costs something real. Over 1000 `nyra` rows, 30% of the + words following a licensed terminal mark appear capitalized mid-sentence somewhere in + the corpus — so protecting them would hand the answer to a third of the commands, and + a third chosen by which rows happen to mention a place name. Every spoken terminal + mark now poses the same sub-task, which is the cleaner experiment. + + *I* is the exception because it is capitalized **everywhere**, mid-sentence included. + Lowercasing it is the one case that makes the input a transcript no speech-to-text + service would return — `politics full stop i'm not sure` — and it would show the model + the same token cased two ways in one utterance. Matched on the raw token (`I`, `I'm`, + `I'll`) rather than normalized, since `normalize_text` drops the apostrophe and `ill` + is then indistinguishable from the adjective. + """ + return raw == "I" or raw.startswith("I'") + + +def _occurrences(words: list[str], run: tuple[str, ...]) -> int: + """How many times `run` appears contiguously in `words`.""" + return sum(1 for i in range(len(words) - len(run) + 1) if tuple(words[i : i + len(run)]) == run) + + +def _pick_caps_run( + reference: str, disfluent: str, rng: random.Random +) -> tuple[tuple[int, ...], tuple[int, ...], tuple[str, ...]] | None: + """A word run to uppercase, as `(reference indices, input indices, words)`. + + Three conditions, each closing off a way the example would be unanswerable: + + - **No punctuation on any token in the run.** `caps off` has to go after the run's + last word, and a run ending in `important,` would put it after the comma. This + also guarantees a caps run never overlaps a converted mark, so the two operators + cannot collide. + - **Long enough, and not a hesitation.** `all caps the` is not dictation, and + `metrics.HESITATIONS` holds the openers `candidates.PRIOR_WINNER` deletes + aggressively — a caps command on one of those pits two rules against each other. + - **Exactly one occurrence in each side.** Uppercasing is applied by matching words, + so a second copy makes "which one" undecidable for both the edit and the scoring. + """ + reference_words, disfluent_words = _tokens(reference), _tokens(disfluent) + plain = [w for _, w in disfluent_words] + raw_reference = reference.split() + lengths = [length for length, weight in CAPS_RUN_LENGTHS for _ in range(weight)] + rng.shuffle(lengths) + + for length in lengths: + starts = list(range(len(reference_words) - length + 1)) + rng.shuffle(starts) + for start in starts: + window = reference_words[start : start + length] + run = tuple(w for _, w in window) + if any(len(w) < MIN_CAPS_WORD or w in metrics.HESITATIONS for w in run): + continue + if any(raw_reference[i] != metrics.normalize_text(raw_reference[i]) for i, _ in window): + continue + if _occurrences([w for _, w in reference_words], run) != 1: + continue + if _occurrences(plain, run) != 1: + continue + first = next( + i for i in range(len(plain) - length + 1) if tuple(plain[i : i + length]) == run + ) + return ( + tuple(i for i, _ in window), + tuple(disfluent_words[i][0] for i in range(first, first + length)), + run, + ) + return None + + +def _speak(mark: str, rng: random.Random) -> str: + """One of `mark`'s spoken forms, drawn at its measured frequency.""" + forms, weights = zip(*SPOKEN_FORMS[mark], strict=True) + return rng.choices(forms, weights=weights)[0] + + +def inject( + reference: str, + disfluent: str, + *, + seed: int, + rate: float = 0.8, + caps_rate: float = 0.25, +) -> tuple[str, str, tuple[Command, ...]]: + """Speak some of this pair's punctuation. Returns `(reference, input, commands)`. + + `rate` is the chance each **licensed** mark is spoken rather than left as a mark. + Deliberately below 1: a corpus where every mark is a word teaches "there is no + punctuation in the input", and the shortest instruction satisfying that is one that + inserts marks by guesswork. Leaving some real marks in place keeps the instruction + responsible for telling a command from a mark that is already correct. + + `caps_rate` is the chance the pair also gets one casing command — at most one, + because a user shouting twice in one dictated sentence is not the common case and + two commands in a 15-word utterance would make the operator most of the corpus. + + The reference comes back **unchanged** unless a casing command was planted, which is + the invariant that lets this run over a real paired corpus at all. + """ + if not 0.0 <= rate <= 1.0: + raise ValueError(f"rate must be in 0..1, got {rate}") + if not 0.0 <= caps_rate <= 1.0: + raise ValueError(f"caps_rate must be in 0..1, got {caps_rate}") + + rng = random.Random(seed) + licensed = licensed_marks(reference) + commands: list[Command] = [] + + caps = _pick_caps_run(reference, disfluent, rng) if rng.random() < caps_rate else None + caps_indices = frozenset(caps[1]) if caps else frozenset() + + out: list[str] = [] + lowercase_next = False + for index, raw in enumerate(disfluent.split()): + if caps and index == caps[1][0]: + out.append(CAPS_WORD if len(caps[2]) == 1 else CAPS_ON) + commands.append( + Command( + spoken=CAPS_WORD if len(caps[2]) == 1 else CAPS_ON, + mark="", + kind="caps", + words=caps[2], + ) + ) + + token = raw + # A word the transcriber capitalized because a sentence started there, whose + # mark is now a spoken word, loses the capital with it — restoring it is the task. + # Only the pronoun "I" is exempt; see `_keeps_its_capital`. + if lowercase_next and not _keeps_its_capital(token): + token = token[:1].lower() + token[1:] + lowercase_next = False + + # Inside a caps run the input carries the plain word; the reference carries the + # shout. Applied before mark conversion, which `_pick_caps_run` has already + # guaranteed cannot apply to these tokens. + if index in caps_indices: + out.append(token.lower()) + elif ( + (split := _split_mark(token)) + and (metrics.normalize_text(split[0]), split[1]) in licensed + and rng.random() < rate + ): + stem, mark = split + spoken = _speak(mark, rng) + out.extend((stem, spoken)) + commands.append( + Command( + spoken=spoken, + mark=mark, + kind="mark", + anchor=metrics.normalize_text(stem), + ) + ) + lowercase_next = mark in TERMINAL_MARKS + else: + out.append(token) + + if caps and index == caps[1][-1]: + if len(caps[2]) > 1: + out.append(CAPS_OFF) + + if caps: + raw_reference = reference.split() + for i in caps[0]: + raw_reference[i] = raw_reference[i].upper() + reference = " ".join(raw_reference) + + return reference, " ".join(out), tuple(commands) + + +#: How a planted command turned out in a cleanup. `literal` is the one that reaches the +#: user's document as visible nonsense, which is why it is counted apart from `missing`. +OUTCOMES = ("converted", "literal", "missing") + + +def _contains(words: list[str], run: tuple[str, ...]) -> bool: + return _occurrences(words, run) > 0 + + +def outcome(command: Command, hypothesis: str) -> str: + """Whether `hypothesis` obeyed `command`, left it as words, or did neither. + + Answered against the command's anchor rather than by counting marks, so a cleanup + that happens to punctuate elsewhere is not credited. `converted` accepts the mark on + any copy of the anchor word — the same false accept `licensed_marks` tolerates, and + for the same reason: on a repeated word the correct output carries the mark on that + word either way. + """ + plain = [w for _, w in _tokens(hypothesis)] + if _contains(plain, command.literal): + return "literal" + + if command.kind == "mark": + for raw in hypothesis.split(): + # Trailing brackets and quotes sit outside the mark a speaker asked for. + trimmed = raw.rstrip("\"')]}»") + if trimmed.endswith(command.mark) and metrics.normalize_text(trimmed) == command.anchor: + return "converted" + return "missing" + + # Located as a contiguous run, the way `_pick_caps_run` chose it, and not by looking + # each word up on its own. A word-keyed lookup took the *first* copy of each word, + # which on "All the people signed confessions ... trying THESE PEOPLE now" found the + # lowercase "people" from eight words earlier and reported a correctly shouted run as + # missed. The run is unique in the reference by construction; it need not be in a + # hypothesis, so any fully uppercased occurrence counts. + raw = hypothesis.split() + words = _tokens(hypothesis) + length = len(command.words) + for start in range(len(words) - length + 1): + window = words[start : start + length] + if tuple(word for _, word in window) != command.words: + continue + stems = [raw[index].strip(metrics.PUNCTUATION) for index, _ in window] + if all(stem and stem.isupper() for stem in stems): + return "converted" + return "missing" + + +def tally(pairs) -> dict[str, float]: + """Command outcomes over `(commands, hypothesis)` pairs, as shares of the total. + + The number this whole module exists to produce. WER answers "how close is the text", + which mixes the punctuation task into every other kind of error; this answers "of the + commands actually planted, how many were obeyed" — and it can only be asked of a + synthetic operator, because nothing else knows what was planted. + + `commands_total` rides along so a share of nothing is distinguishable from a share of + everything: a corpus loaded without `--spoken-punctuation` reports zeros here and + they mean "not asked", not "failed". + """ + counts: dict[str, int] = dict.fromkeys(OUTCOMES, 0) + for commands, hypothesis in pairs: + for command in commands: + counts[outcome(command, hypothesis)] += 1 + total = sum(counts.values()) + stats = {f"commands_{name}": (counts[name] / total if total else 0.0) for name in OUTCOMES} + return stats | {"commands_total": float(total)} + + +def feedback_note(commands: tuple[Command, ...], hypothesis: str) -> str: + """What the reflector needs to hear about the punctuation commands, or "". + + Separate from `metrics.feedback` because that module is the bottom of the dependency + chain — it imports nothing else in the harness, so that nothing about how a corpus + was built can leak into how it is scored. The command vocabulary is corpus + construction, so it composes on top rather than moving down. + + Names the failure in the two shapes that matter and nothing else. A leftover literal + is reported first: `metrics.feedback` sees it only as "left disfluencies in the + output: comma", which is true, actionable in the wrong direction, and the worst + outcome of the three. + """ + if not commands: + return "" + outcomes = [(command, outcome(command, hypothesis)) for command in commands] + literal = [c.spoken for c, o in outcomes if o == "literal"] + missing = [c for c, o in outcomes if o == "missing"] + notes: list[str] = [] + if literal: + notes.append( + f"left spoken punctuation commands in the output as words: {', '.join(literal)} — " + "the speaker was asking for the punctuation mark, not dictating the word, so the " + "words must be replaced by the mark and never pasted" + ) + if missing: + asked = ", ".join( + f"{c.spoken!r} -> {c.mark!r}" if c.kind == "mark" else f"{c.spoken!r} -> uppercase" + for c in missing + ) + notes.append(f"did not carry out spoken punctuation commands: {asked}") + return f" It also {'; '.join(notes)}." if notes else "" + + +def phrases() -> tuple[tuple[str, ...], ...]: + """Every command as a normalized word run — the vocabulary, in matchable form.""" + spoken = [form for forms in SPOKEN_FORMS.values() for form, _ in forms] + spoken += [CAPS_WORD, CAPS_ON, CAPS_OFF] + return tuple(tuple(metrics.normalize_text(form).split()) for form in spoken) + + +def literal_use_fraction(references) -> float: + """Share of references that use a command *phrase* as ordinary content. + + Reported next to the floor because it bounds what the corpus can say about + over-conversion. An instruction that rewrites "the Jurassic period was long" into + "the Jurassic. was long" is only punished on these rows, and on `nyra` there are + about 1% of them — so the clause forbidding it belongs in the instruction on the + strength of the product, not on the strength of a score. Same shape of blind spot as + `candidates.REQUIRED_SAFEGUARDS`, named rather than hidden. + + Whole phrases, contiguously, not the union of their words. Matching words was the + first cut and it reported 22% of `nyra` rows as traps, which was ordinary English + prose containing "all", "on", "off", "point" and "stop" — a diagnostic that says the + blind spot is a fifth of the corpus when it is a fiftieth is worse than none, because + it argues the gap has already been closed. + """ + references = list(references) + if not references: + return 0.0 + vocabulary = phrases() + return sum( + any(_contains(metrics.normalize(text), phrase) for phrase in vocabulary) + for text in references + ) / len(references) diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index c3b0a67..7eec33d 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -25,6 +25,7 @@ import metrics import optimize_cleanup_prompt as cli import progress +import spoken_punctuation from disfluency import inject CAP = candidates.INSTRUCTION_CHARACTER_CAP @@ -1070,6 +1071,21 @@ def test_the_default_corpus_can_measure_formatting(): assert corpus.SOURCES[cli.parse_args([]).source].formatting_is_measurable +def test_help_renders(capsys): + """`--help` was broken from the day a help string first said "~42%". + + argparse %-expands help text against a dict of params, so a bare `%` raises + TypeError and the whole thing fails — silently invisible to every test, because + nothing here had ever asked for the help. + """ + with pytest.raises(SystemExit) as raised: + cli.parse_args(["--help"]) + assert raised.value.code == 0 + printed = capsys.readouterr().out + assert "--spoken-punctuation" in printed + assert "--dump-corpus" in printed + + def test_a_bare_invocation_is_the_recommended_gepa_run(): """Running with no flags now spends money — pin what it spends it on.""" args = cli.parse_args([]) @@ -1431,6 +1447,527 @@ def test_dry_run_never_reaches_the_dspy_module(): assert "program" not in sys.modules +# -------------------------------------------------------------------------- +# Spoken punctuation +# -------------------------------------------------------------------------- + +# One pair in the shape `nyra` supplies: both sides punctuated, the disfluent side +# carrying a comma (`really,`) the reference does not, and a proper noun sitting +# where a sentence starts. +SPOKEN_REFERENCE = "We shipped it today. Monday was quiet, so nobody noticed." +SPOKEN_DISFLUENT = "Um we shipped it today. Monday was really, quiet, so nobody noticed." + + +def spoken_pair(**kwargs): + """`(reference, input, commands)` for the pair above.""" + return spoken_punctuation.inject(SPOKEN_REFERENCE, SPOKEN_DISFLUENT, **kwargs) + + +def test_a_perfect_cleanup_is_still_the_reference_exactly(): + """The invariant the whole module rests on: the planted task is achievable. + + Marks are spoken only where the reference licenses them, so the correct answer never + stops being the corpus's own target. If this fails, some row is asking for a mark its + target does not contain and no instruction can score 1.0 on it. + """ + reference, disfluent, commands = spoken_pair(seed=3, rate=1.0, caps_rate=1.0) + assert commands + scored = metrics.score(reference, reference, disfluent) + assert (scored.content, scored.format) == (1.0, 1.0) + assert spoken_punctuation.tally([(commands, reference)])["commands_converted"] == 1.0 + + +def test_leaving_every_command_in_is_scored_as_leaving_every_command_in(): + reference, disfluent, commands = spoken_pair(seed=3, rate=1.0, caps_rate=1.0) + assert spoken_punctuation.tally([(commands, disfluent)])["commands_literal"] == 1.0 + + +def test_spoken_injection_is_deterministic_for_a_seed(): + assert spoken_pair(seed=5) == spoken_pair(seed=5) + + +def test_rate_zero_plants_nothing_and_leaves_both_sides_alone(): + reference, disfluent, commands = spoken_pair(seed=5, rate=0.0, caps_rate=0.0) + assert (reference, disfluent, commands) == (SPOKEN_REFERENCE, SPOKEN_DISFLUENT, ()) + + +def test_rates_out_of_range_are_rejected(): + with pytest.raises(ValueError): + spoken_pair(seed=1, rate=1.5) + with pytest.raises(ValueError): + spoken_pair(seed=1, caps_rate=-0.1) + + +def test_only_marks_the_reference_licenses_are_ever_spoken(): + """A mark the target lacks would ask for punctuation and then score it as an error. + + `really,` is in the input and not in the reference, which is ordinary for `nyra` — + the annotator deleted the span it introduced. Speaking it would teach an instruction + that commands are sometimes to be ignored. + """ + assert ("really", ",") not in spoken_punctuation.licensed_marks(SPOKEN_REFERENCE) + _, disfluent, _ = spoken_pair(seed=5, rate=1.0, caps_rate=0.0) + assert "really," in disfluent + + +def test_a_spoken_terminal_mark_takes_its_sentence_capital_with_it(): + """Otherwise the casing alone restores the period and the eval measures nothing.""" + reference, disfluent, commands = spoken_punctuation.inject( + "We shipped it today. Nobody noticed.", + "We shipped it today. Nobody noticed.", + seed=2, + rate=1.0, + caps_rate=0.0, + ) + assert "today period nobody" in disfluent + assert reference == "We shipped it today. Nobody noticed." + assert [c.mark for c in commands] == [".", "."] + + +def test_a_name_after_a_spoken_mark_is_lowercased_like_any_other_word(): + """It is sentence-initial in the reference, so restoring its capital *is* the task. + + A corpus-derived proper-noun rule stood here first, on the theory that lowercasing + `Monday` charges a correct cleanup for the injector's damage. It does not: the + required output is `Monday` and the required action is "capitalize the first word + after a sentence-ending mark", which is the same for a name and an ordinary word. What + the rule did do was hand the answer to 30% of the commands on `nyra` — the share whose + following word appears capitalized mid-sentence somewhere in the corpus. + """ + _, disfluent, _ = spoken_pair(seed=5, rate=1.0, caps_rate=0.0) + assert "today period monday" in disfluent + # And the target still asks for the capital, so nothing about the task got easier. + assert "today. Monday" in SPOKEN_REFERENCE + + +def test_the_pronoun_i_is_the_one_capital_that_survives(): + """It is capitalized everywhere, mid-sentence included, so lowercasing it is unlike + every other case. + + `politics full stop i'm not sure` is a transcript no speech-to-text service returns, + and it would show the model the same token cased two ways in one utterance. + """ + text = "I like politics. I'm not sure why." + _, disfluent, _ = spoken_punctuation.inject(text, text, seed=1, rate=1.0, caps_rate=0.0) + assert "I'm" in disfluent + assert "i'm" not in disfluent + + +def test_all_caps_is_the_only_operator_that_edits_the_reference(): + """And it has to: a target with no uppercase in it cannot pose the task.""" + reference, _, _ = spoken_pair(seed=5, rate=1.0, caps_rate=0.0) + assert reference == SPOKEN_REFERENCE + shouted, disfluent, commands = spoken_pair(seed=5, rate=0.0, caps_rate=1.0) + caps = [c for c in commands if c.kind == "caps"] + assert caps + assert shouted != SPOKEN_REFERENCE + assert any(word.strip(metrics.PUNCTUATION).isupper() for word in shouted.split()) + # The input carries the command plus the plain lowercase word, so nothing in it + # leaks the answer. + assert caps[0].spoken in disfluent + for word in caps[0].words: + assert word in metrics.normalize(disfluent) + assert word.upper() not in disfluent.split() + + +def test_a_caps_run_carries_no_punctuation_so_the_two_operators_cannot_collide(): + """`caps off` goes after the run's last word; a run ending in `today.` would strand it.""" + for seed in range(40): + _, _, commands = spoken_pair(seed=seed, rate=1.0, caps_rate=1.0) + for command in (c for c in commands if c.kind == "caps"): + assert all(word == metrics.normalize_text(word) for word in command.words) + + +def test_a_caps_command_is_invisible_to_the_content_axis_and_visible_to_format(): + """Which is why `--spoken-punctuation` selects on format.""" + shouted, disfluent, commands = spoken_pair(seed=5, rate=0.0, caps_rate=1.0) + assert [c.kind for c in commands] == ["caps"] + # A cleanup that did everything but the shouting. + unshouted = " ".join( + word.lower() if word.strip(metrics.PUNCTUATION).isupper() else word + for word in shouted.split() + ) + scored = metrics.score(shouted, unshouted, disfluent) + assert scored.content == 1.0 + assert scored.format < 1.0 + assert spoken_punctuation.outcome(commands[0], unshouted) == "missing" + + +def test_a_two_word_command_is_not_charged_as_an_abandoned_false_start(): + """Without the exemption a leftover "question mark" costs ten errors and "period" one. + + `_is_abandoned`'s rule 2 is "two or more non-hesitation words that echo nothing", + which is exactly the shape of a dictation command. The asymmetry it produces is one + nobody chose, and it would push the search at the multi-word commands for arithmetic + reasons. + """ + reference = "Is it ready?" + disfluent = "is it ready question mark" + exempt = frozenset({"question", "mark"}) + assert metrics.false_start_tokens(disfluent, reference) == ("question", "mark") + assert metrics.false_start_tokens(disfluent, reference, exempt) == () + charged = metrics.score(reference, disfluent, disfluent) + exempted = metrics.score(reference, disfluent, disfluent, exempt) + assert exempted.content > charged.content + + +def test_the_exemption_is_neutral_on_a_corpus_that_plants_nothing(): + """Every number measured before it existed has to still hold.""" + reference = "We would have them." + disfluent = "we wouldn't ha- we would have them" + assert metrics.score(reference, disfluent, disfluent) == metrics.score( + reference, disfluent, disfluent, frozenset() + ) + + +def test_the_utterance_hands_its_own_planted_commands_to_the_scorer(): + """`Utterance.scored` is the one place that knows both sides and what was planted.""" + reference, disfluent, commands = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + utterance = corpus.Utterance(reference=reference, disfluent=disfluent, commands=commands) + assert utterance.not_abandoned == frozenset({"question", "mark"}) + assert utterance.scored(disfluent) == metrics.score( + reference, disfluent, disfluent, utterance.not_abandoned + ) + + +def test_the_reported_false_start_fraction_is_the_charged_one(): + """The figure printed and the figure charged came from different code paths once.""" + reference, disfluent, commands = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + utterances = [corpus.Utterance(reference=reference, disfluent=disfluent, commands=commands)] + assert corpus.false_start_fraction(utterances) == 0.0 + + +def test_a_command_left_in_is_told_apart_from_a_command_dropped(): + reference, disfluent, commands = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + (command,) = commands + assert spoken_punctuation.outcome(command, "Is it ready?") == "converted" + assert spoken_punctuation.outcome(command, "Is it ready question mark") == "literal" + assert spoken_punctuation.outcome(command, "Is it ready") == "missing" + + +def test_conversion_is_credited_on_the_anchor_not_on_any_mark_anywhere(): + """Otherwise punctuating something else entirely reads as obeying the command.""" + _, _, (command,) = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + assert spoken_punctuation.outcome(command, "Is it? ready") == "missing" + + +def test_the_literal_match_is_anchored_so_an_ordinary_word_is_not_charged(): + """ "the Cretaceous period was long" is content, not a command left in.""" + _, _, (command,) = spoken_punctuation.inject( + "It ended then.", "it ended then.", seed=1, rate=1.0, caps_rate=0.0 + ) + assert command.anchor == "then" + assert ( + spoken_punctuation.outcome(command, "The Cretaceous period was long. It ended then.") + == "converted" + ) + + +def test_a_shouted_run_is_found_as_a_run_not_word_by_word(): + """A word-keyed lookup found the wrong copy and marked a correct answer missed. + + On "All the people signed confessions ... trying THESE PEOPLE now" it took the first + "people" — lowercase, eight words earlier — and reported the run as not shouted. + """ + command = spoken_punctuation.Command( + spoken="caps on", mark="", kind="caps", words=("these", "people") + ) + hypothesis = "All the people signed confessions. They been trying THESE PEOPLE now." + assert spoken_punctuation.outcome(command, hypothesis) == "converted" + assert ( + spoken_punctuation.outcome(command, "All the people signed. Trying these people now.") + == "missing" + ) + + +def test_every_planted_command_is_answered_by_its_own_reference(): + """The corpus-wide version of the achievability invariant, over enough rows to bite. + + A one-row check passes on a construction that is wrong on some shape appearing once + in four hundred — which is how the run-as-a-run bug above got in. + """ + reference = "All the people signed confessions, they went to a trial by jury." + disfluent = "All the people signed confessions, they went to a trial by jury." + for seed in range(60): + target, spoken_side, commands = spoken_punctuation.inject( + reference, disfluent, seed=seed, rate=1.0, caps_rate=1.0 + ) + assert metrics.score(target, target, spoken_side).format == 1.0 + for command in commands: + assert spoken_punctuation.outcome(command, target) == "converted", command + + +def test_the_tally_of_nothing_says_nothing_rather_than_zero_percent(): + assert spoken_punctuation.tally([])["commands_total"] == 0.0 + assert spoken_punctuation.tally([((), "anything")])["commands_converted"] == 0.0 + + +def test_the_literal_use_fraction_matches_phrases_not_their_words(): + """Matching words reported a fifth of `nyra` as traps; it was the words "all" and "on". + + A diagnostic that overstates the blind spot by 10x is worse than none, because it + argues the gap has already been closed. + """ + assert spoken_punctuation.literal_use_fraction(["The Cretaceous period was long."]) == 1.0 + assert ( + spoken_punctuation.literal_use_fraction(["We looked at all of it and turned it on."]) == 0.0 + ) + assert spoken_punctuation.literal_use_fraction([]) == 0.0 + + +def test_the_command_vocabulary_covers_the_marks_the_task_is_named_for(): + """period, comma, question mark, ALL CAPS — plus the aliases people actually say.""" + assert {".", ",", "?"} <= set(spoken_punctuation.SPOKEN_FORMS) + assert "full stop" in {form for form, _ in spoken_punctuation.SPOKEN_FORMS["."]} + assert spoken_punctuation.CAPS_WORD == "all caps" + + +def test_spoken_punctuation_layers_onto_the_real_disfluencies_rather_than_replacing_them(): + """The task under test is saying "comma" *while* hesitating, not either alone.""" + loaded = corpus.load(source="builtin", limit=12, spoken_punctuation_rate=1.0) + operations = {op for u in loaded.utterances for op in u.operations} + assert any(op.startswith("spoken:") for op in operations) + assert any(not op.startswith("spoken:") for op in operations) + assert loaded.detail["spoken_punctuation_rate"] == 1.0 + + +def test_a_corpus_without_the_flag_plants_nothing(): + loaded = corpus.load(source="builtin", limit=12) + assert all(u.commands == () for u in loaded.utterances) + assert all(u.not_abandoned == frozenset() for u in loaded.utterances) + + +# The committed dataset, and the exact arguments that produce it. Both live here rather +# than only in `data/README.md` so the file is self-verifying: a change to either injector +# fails this suite instead of quietly invalidating a dataset still sitting in the tree. +FROZEN_DATASET = pathlib.Path(__file__).parent / "data" / "spoken-punctuation.jsonl" +FROZEN_ARGV = ( + "--source builtin --limit 200 --spoken-punctuation 0.8 --severity 0.35 --seed 7" +).split() + + +def frozen_corpus(): + return corpus.load( + source="builtin", limit=200, spoken_punctuation_rate=0.8, severity=0.35, seed=7 + ) + + +def test_the_committed_dataset_is_what_the_generator_still_produces(tmp_path): + """A frozen dataset nobody re-derives is a dataset that silently goes stale. + + The alternative is a file in the tree that was correct when it was written and is now + a different corpus from the one the code builds — and every number measured on it + unattributable to either. Regenerate it with the command in `data/README.md`. + """ + regenerated = tmp_path / "spoken-punctuation.jsonl" + corpus.dump_jsonl(frozen_corpus(), str(regenerated)) + assert regenerated.read_text(encoding="utf-8") == FROZEN_DATASET.read_text(encoding="utf-8") + + +def test_the_frozen_dataset_round_trips_through_the_loader(): + """Reading it back has to reproduce the corpus, commands included. + + Scoring a dumped dataset is only scoring the corpus it came from if the commands + survive: without them the tally reports nothing and the false-start classifier goes + back to charging "question mark" as an abandoned phrase. + """ + generated = frozen_corpus() + reloaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + assert len(reloaded) == len(generated) + for a, b in zip(generated.utterances, reloaded.utterances, strict=True): + assert (a.reference, a.disfluent) == (b.reference, b.disfluent) + assert a.commands == b.commands + assert a.not_abandoned == b.not_abandoned + assert a.scored(a.reference) == b.scored(b.reference) + + +def test_a_reloaded_dataset_is_not_injected_over_again(): + """The flag is often still on the command line; injecting twice would speak marks that + are no longer there and uppercase an ALL CAPS run a second time.""" + reloaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200, spoken_punctuation_rate=0.8) + generated = frozen_corpus() + assert [u.disfluent for u in reloaded.utterances] == [u.disfluent for u in generated.utterances] + + +def test_a_dumped_row_whose_input_matches_its_reference_is_still_left_alone(tmp_path): + """The case `is_disfluent` got wrong, and the reason `input_supplied` is recorded. + + A row where no command and no disfluency were drawn reads as "needs injecting", so + reloading would hand it a different input than the file records — a dumped corpus that + is not the corpus it was dumped from, with nothing to say so. + """ + path = tmp_path / "clean.jsonl" + text = "The build failed because the certificate expired over the weekend." + path.write_text(json.dumps({"reference": text, "disfluent": text}) + "\n", encoding="utf-8") + (utterance,) = corpus.load(jsonl=str(path), limit=1).utterances + assert utterance.disfluent == text + + +def test_a_jsonl_row_with_only_a_reference_still_goes_through_the_injector(tmp_path): + path = tmp_path / "bare.jsonl" + text = "The build failed because the certificate expired over the weekend." + path.write_text(json.dumps({"reference": text}) + "\n", encoding="utf-8") + (utterance,) = corpus.load(jsonl=str(path), limit=1).utterances + assert utterance.disfluent != text + assert utterance.operations + + +def test_the_frozen_dataset_can_charge_over_conversion_where_nyra_cannot(): + """Its whole reason for existing beyond being a fixture. + + `nyra` uses a command word as content on ~0-1% of rows, so nothing there punishes an + instruction that rewrites "the Jurassic period was long" into "the Jurassic. was + long". These sentences were written to. + """ + references = [u.reference for u in corpus.load(source="builtin", limit=200).utterances] + assert spoken_punctuation.literal_use_fraction(references) > 0.1 + + +def test_the_dump_flag_works_without_a_key_or_a_network(tmp_path): + """Generating a dataset must not require paying for a run.""" + path = tmp_path / "dump.jsonl" + assert ( + cli.main([*FROZEN_ARGV, "--dump-corpus", str(path), "--dry-run", "--show-samples", "0"]) + == 0 + ) + assert path.read_text(encoding="utf-8") == FROZEN_DATASET.read_text(encoding="utf-8") + + +def test_the_original_offline_sample_is_still_the_first_twelve_rows(): + """`--limit 12` and the tests that use it have to see what they always saw.""" + assert len(corpus.BUILTIN_SAMPLE) > 12 + assert corpus.BUILTIN_SAMPLE[0].startswith("The build failed because the signing") + assert corpus.BUILTIN_SAMPLE[11].endswith("menu bar item at all.") + + +def test_every_bundled_sentence_survives_the_injection_filter(): + """A fixture that silently drops the rows it was written to contain is a bad fixture.""" + kept = {u.reference for u in corpus.load(source="builtin", limit=500).utterances} + assert len(kept) == len(corpus.BUILTIN_SAMPLE) + + +def test_the_bundled_sample_exercises_every_mark_in_the_vocabulary(): + """`nyra` supplies no exclamation point, colon or semicolon at all, so if these + entries in SPOKEN_FORMS are to mean anything, this corpus has to license them.""" + licensed = set() + for text in corpus.BUILTIN_SAMPLE: + licensed |= {mark for _, mark in spoken_punctuation.licensed_marks(text)} + assert set(spoken_punctuation.SPOKEN_FORMS) <= licensed + + +def test_the_punctuation_candidates_join_the_table_only_when_the_corpus_asks(): + """Keyed on the corpus, not the flag, so a frozen dataset read with --jsonl behaves + the same as the corpus it was dumped from.""" + plain = corpus.load(source="builtin", limit=12) + assert cli.instruction_table(plain) == dict(candidates.CANDIDATES) + table = cli.instruction_table(corpus.load(jsonl=str(FROZEN_DATASET), limit=200)) + assert set(candidates.SPOKEN_PUNCTUATION_CANDIDATES) <= set(table) + # The bar stays what Blurt ships, so the held-out comparison answers the product + # question: what does teaching it the new task buy? + assert candidates.BASELINE in table + + +def test_every_punctuation_candidate_is_shippable_and_keeps_the_safeguards(): + """Any of them can become the GEPA seed, and a seed the final gate would refuse + wastes the whole search.""" + for name, text in candidates.SPOKEN_PUNCTUATION_CANDIDATES.items(): + assert candidates.overage(text) == 0, name + assert candidates.missing_safeguards(text) == [], name + + +def test_an_oversized_punctuation_candidate_stops_the_run_too(monkeypatch): + monkeypatch.setitem(candidates.SPOKEN_PUNCTUATION_CANDIDATES, "too-long", "x" * 5000) + loaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + with pytest.raises(SystemExit) as raised: + cli.check_candidates(cli.instruction_table(loaded)) + assert "too-long" in str(raised.value) + + +def test_the_clause_fits_after_the_shipped_instruction(): + """That is the experiment `punct-appended` is: what can be taught for free?""" + composite = candidates.SPOKEN_PUNCTUATION_CANDIDATES["punct-appended"] + assert candidates.PRIOR_WINNER.split("\n\n")[0] in composite + assert composite.endswith("Return only the cleaned transcript.") + assert candidates.SPOKEN_PUNCTUATION_CLAUSE in composite + assert candidates.overage(composite) == 0 + + +def test_a_spoken_punctuation_run_selects_on_format_by_default(): + """Content casefolds and strips marks, so it cannot see most of this task.""" + plain = corpus.load(source="builtin", limit=12) + spoken = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + assert cli.resolve_axis(None, plain) == "blend" + assert cli.resolve_axis(None, spoken) == "format" + # An explicit choice still wins. + assert cli.resolve_axis("content", spoken) == "content" + + +def test_a_spoken_punctuation_run_scores_the_bar_and_the_challengers(): + """Not `BASELINE` alone, and not the whole table. + + Alone it would search from an instruction that has never heard of the task; the whole + table adds six terse instructions that rank framings of *disfluency* cleanup, none of + which mentions punctuation, at 900 dev calls each. + """ + loaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + table = cli.instruction_table(loaded) + scoring = cli.resolve_candidates(cli.parse_args([]), table, spoken=True) + assert scoring[0] == candidates.BASELINE + assert set(scoring) == {candidates.BASELINE, *candidates.SPOKEN_PUNCTUATION_CANDIDATES} + assert set(scoring) < set(table) + + +def test_the_full_table_is_still_available_on_a_spoken_run(): + scoring = cli.resolve_candidates( + cli.parse_args(["--candidates", "all"]), + cli.instruction_table(corpus.load(jsonl=str(FROZEN_DATASET), limit=200)), + spoken=True, + ) + assert set(candidates.CANDIDATES) <= set(scoring) + + +def test_the_command_table_is_ordered_by_the_selecting_axis(capsys): + """A candidate that converts more commands and still loses should read as that.""" + rows = [ + ( + "worse-text", + dict.fromkeys(metrics.AXES, 0.1) + | { + "commands_converted": 0.9, + "commands_literal": 0.05, + "commands_missing": 0.05, + "commands_total": 10.0, + }, + ), + ( + "better-text", + dict.fromkeys(metrics.AXES, 0.9) + | { + "commands_converted": 0.2, + "commands_literal": 0.4, + "commands_missing": 0.4, + "commands_total": 10.0, + }, + ), + ] + cli.print_command_table(rows, "format") + printed = capsys.readouterr().out + assert printed.index("better-text") < printed.index("worse-text") + + +def test_the_command_table_is_silent_when_nothing_was_planted(capsys): + cli.print_command_table([("x", dict.fromkeys(metrics.AXES, 0.5))], "format") + assert capsys.readouterr().out == "" + + def test_program_is_the_only_module_that_imports_dspy(): """Keeps the offline guarantee from eroding one convenience import at a time.""" here = pathlib.Path(__file__).parent From 46e8275d9b7e4d5d1dd6bbaba74361a82057cb51 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 18:06:21 -0700 Subject: [PATCH 2/8] feat: weight leftover punctuation commands, and exercise the capital after one Two gaps the "how is this weighted" question exposed. **A leftover command word cost 2, and nobody decided 2.** Plain WER charges one twice on the format axis -- a substitution for the mark that never appeared plus an insertion for the word that did -- so it fell out of the arithmetic. That is only twice what *dropping* the command costs, and dropping it is the cosmetic version: `children` where the reference wants `children.` is a missing mark, while `children period` puts a word in the user's document they never meant to write. Those are not two grades of the same mistake. metrics.COMMAND_WEIGHT charges it 3. Swept over 1114 command-carrying nyra rows, the gradient in the region rival instructions differ in (0% to 25% residue) is 0.056 unweighted and 0.104 at 3, so a weight is worth having -- and it keeps growing, so the gradient alone picks nothing. Two bounds meet at 3. FALSE_START_WEIGHT is the ceiling and it binds at 4, not 5: on the format axis a leftover command carries the mark's substitution too, so its total is the weight plus one, and at 3 that is 4 -- between a leftover filler (1) and a leftover abandoned word (5). At 4 it ties the abandoned word, which is the wrong ordering, since an abandoned span fabricates a clause the speaker never said while a command is one stray word that failed to disappear. The decaying tail bounds it from the same side: a row whose residue alone clears WER 2 stops being rankable, which is 1.0% of rows at 3, 4.6% at 5 and 20% at 8. Measured per occurrence, that leaves filler 1, command dropped 1, capital missed 1, ALL CAPS missed 1, command left in 3.97, abandoned word 5. `not_abandoned` is renamed `command_words`, because it now does two jobs and both follow from one fact -- that these words are commands rather than speech. A token charged as abandoned is never charged again as a command: rule 1 of _is_abandoned fires on a cut-off word whatever the vocabulary, so a command caught inside an abandoned run is charged at the higher weight only. Still empty for every corpus that plants none, so nothing measured before it exists moves: plain nyra's floor is unchanged at 0.6452/0.6011/0.6320 and its false-start fraction at 0.207. metrics.feedback now names leftover commands apart from disfluencies and says what they cost, and spoken_punctuation.feedback_note drops that half. Said in both places it was the same complaint twice in every reflection prompt; the split follows what each module can see -- metrics owns the leftover because that is where the weight lives, and the note owns a *dropped* command, which is one substitution among many on the format axis and nothing at all on content. **The fixture never exercised the capital after a spoken mark.** 0 of 61 terminal marks had a following word, because every bundled sentence ended at its own period -- and restoring the next word's capital is half of what a period command asks for. On nyra, where utterances run to several sentences, it is 26% of terminal marks. Fifteen multi-sentence rows put it back, at 13% and 14 concrete cases, including the two shapes the injector treats specially: a proper noun after the mark, and the pronoun I. Pinned by a test, as is the every-mark-in-the-vocabulary property. The bundled corpus is now 91 sentences and the frozen dataset 91 rows; the literal-use trap rate is 12%, still an order of magnitude above nyra's. Also fixes `--help`, which has been broken since a help string first said "~42%": argparse %-expands help text, so a bare % raises TypeError and the whole thing fails. Invisible to every test, because nothing had asked for the help. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 61 +++++++- evals/dictation-prompt/corpus.py | 47 ++++-- evals/dictation-prompt/data/README.md | 8 +- .../data/spoken-punctuation.jsonl | 15 ++ evals/dictation-prompt/metrics.py | 144 +++++++++++++++--- evals/dictation-prompt/program.py | 2 +- evals/dictation-prompt/spoken_punctuation.py | 22 +-- evals/dictation-prompt/test_eval.py | 128 +++++++++++++++- 8 files changed, 367 insertions(+), 60 deletions(-) diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 1e64076..1e30b13 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -23,7 +23,7 @@ ones we thought to write down. | `--source` | What it is | Measures | | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | | `nyra` (default) | [`nyralabs/disfluency_speech_english`][nyra] — ~5k Switchboard utterances whose disfluencies trained annotators marked **by hand**, repackaged with casing repaired so the formatting axis is live. Costs some fidelity for that. | content + format | -| `builtin` | A dozen bundled sentences plus injection. No network, no key — for smoke-testing. | content + format | +| `builtin` | 91 bundled sentences plus injection. No network, no key. The only corpus that can be committed, and the only one with literal-use traps. | content + format | `--jsonl` reads a local file instead: objects with both `disfluent` and `reference` are used as-is; anything with only a reference goes through the injector. @@ -32,7 +32,7 @@ as-is; anything with only a reference goes through the injector. path** — `--source builtin` needs no network and no key, which is what lets the test suite and `--dry-run` verify the pipeline end to end. It also keeps a **severity dial** (`--severity`) and a **punctuation-restoration** task (`--strip-formatting`) that the real corpus cannot pose, -though on a dozen bundled sentences those are smoke tests rather than measurements. +though on 91 bundled sentences those are smoke tests rather than measurements. ### How the hand annotation reaches the eval @@ -57,7 +57,7 @@ reparanda, and rewrites the verbatim side into its own conventions (`[UH]`, `[la `th*`) that the loader has to undo. Prefer it when formatting matters more than exact recall. It does not pose punctuation _restoration_, since both sides are already punctuated. Only -`--source builtin --strip-formatting` does, on a dozen bundled sentences. +`--source builtin --strip-formatting` does, on the bundled sentences. ### Spoken punctuation — "period, comma, question mark, ALL CAPS" @@ -143,7 +143,7 @@ injector. Reproducible in principle; unreviewable in practice — nobody reads a find out whether the examples are any good, and a change to an injector silently changes what every past number was measured on. A file in the tree is diffable. -[`data/spoken-punctuation.jsonl`](./data/spoken-punctuation.jsonl) is that file: 76 rows +[`data/spoken-punctuation.jsonl`](./data/spoken-punctuation.jsonl) is that file: 91 rows generated from `--source builtin`, whose sentences are written for this repo and so carry no third-party terms. `nyra` derives from LDC-licensed Switchboard transcripts and is deliberately **not** committed — dump it locally if you want it frozen. `test_eval.py` @@ -155,9 +155,9 @@ Everything downstream keys off **what the corpus contains, not what flags were p that, reading the saved dataset back selected on `blend` and printed no command outcomes: the same corpus scored two different ways depending on how it was reached. -Read the committed file as a **fixture**, not a benchmark. 50 dev / 25 test rows is far +Read the committed file as a **fixture**, not a benchmark. 60 dev / 30 test rows is far below the resolution this README argues for everywhere else. What it is uniquely good for -is the one thing `nyra` cannot do: **15% of its references use a command word as ordinary +is the one thing `nyra` cannot do: **12% of its references use a command word as ordinary content** ("one grace period, so plan accordingly", "add a comma after the second clause"), against ~0-1% of `nyra`'s. It is the only corpus here that can charge an instruction for converting a word the speaker meant literally — and those rows compose @@ -511,6 +511,55 @@ target, since nothing about a leftover word says whether it was abandoned — `c carries both sides for exactly that reason. And numbers either side of a change to `FALSE_START_WEIGHT` are not comparable; 1.0 restores plain WER. +### Nor are dictated commands + +The second departure, and it only bites on a `--spoken-punctuation` corpus. **A word of a +dictated punctuation command left in the output costs `metrics.COMMAND_WEIGHT` errors (3) +rather than one.** + +Plain WER already charged one twice on the format axis — a substitution for the mark that +never appeared, plus an insertion for the word that did — so it came to 2 without anyone +deciding 2. That is only twice what _dropping_ the command costs, and dropping it is the +cosmetic version of the mistake: `children` where the reference wants `children.` is a +missing mark, while `children period` puts a word in the user's document they never meant +to write. + +Three is where two bounds meet. Swept over 1114 command-carrying `nyra` rows, the gradient +in the region rival instructions actually differ in (0% to 25% residue) is 0.056 unweighted +and 0.104 at 3 — so a weight is worth having — and it keeps growing, so the gradient alone +picks nothing. `FALSE_START_WEIGHT` is the ceiling, and it binds at 4 rather than 5: on the +format axis a leftover command carries the mark's substitution as well, so its total is the +weight plus one, and at 3 that is 4 — between a leftover filler (1) and a leftover abandoned +word (5). At 4 it would tie the abandoned word, which is the wrong ordering, because an +abandoned span fabricates a clause the speaker never said while a command is one stray word +that failed to disappear. The decaying tail bounds it from the same side: a row whose +residue alone clears WER 2 stops being rankable against its neighbours, which is 1.0% of +rows at 3, 4.6% at 5 and 20% at 8. + +Measured per occurrence on the committed fixture, that leaves this ordering: + +| failure, per occurrence | content | format | +| ---------------------------------------- | ------: | -----: | +| filler left in (`um`) | 1.00 | 1.00 | +| **abandoned span word left in** | 5.00 | 5.00 | +| command left in as words (per word) | 3.03 | 3.97 | +| command dropped, no mark produced | 0.00 | 1.09 | +| mark restored, next word not capitalised | 0.00 | 1.00 | +| ALL CAPS missed (per word) | 0.00 | 1.00 | + +The same set of words does both jobs, because both follow from one fact — that these words +are commands rather than speech. `_is_abandoned` must not read `question mark` as an +abandoned phrase, which is exactly its shape (two non-hesitation words echoing nothing), and +left unnamed it charged a leftover `question mark` ten errors against a leftover `period`'s +one. A token charged as abandoned is never charged again as a command: rule 1 fires on a +cut-off word whatever the vocabulary, so a command caught inside an abandoned run is charged +at the higher weight only. + +`corpus.Utterance.command_words` is empty for every corpus that plants none, so this changes +nothing measured before it existed — the false-start fraction over 300 `nyra` rows is 0.207 +with spoken punctuation and 0.207 without, and plain `nyra`'s floor is unmoved at +0.6452 / 0.6011 / 0.6320. + All three axes are printed for every candidate, with the selecting one starred, so you can see whether a winner gained on wording or only on punctuation. The winner is chosen on a dev split and re-scored on a held-out test split alongside `BASELINE` — `prior-winner`, the best diff --git a/evals/dictation-prompt/corpus.py b/evals/dictation-prompt/corpus.py index 2882965..1bae952 100644 --- a/evals/dictation-prompt/corpus.py +++ b/evals/dictation-prompt/corpus.py @@ -250,6 +250,28 @@ def fields(self) -> tuple[str, ...]: "Our smallest customer files more bug reports than the other twenty combined.", "The release notes need a line about the new permission before we can publish.", "Check whether the trial expiry is stored in the keychain or in user defaults.", + # Multi-sentence rows. Every other sentence here ends at the terminal mark, so the + # word *after* a spoken period was never exercised: 0 of 61 terminal marks on the + # generated corpus had a following word, and restoring its capital is half of what + # a period command asks for. On `nyra` that case is 26% of terminal marks, because + # Switchboard utterances run to several sentences. These put it back — including the + # two shapes the injector treats specially, a proper noun after the mark and the + # pronoun "I", which keep their capitals where an ordinary word loses it. + "The build is green. Ship it before the release window closes tonight.", + "I read the whole thread. Nobody actually answered the question that was asked.", + "We tried that last quarter. It made the cold start worse, so we reverted it.", + "The fix is small. The test that proves it is not, and that is the whole delay.", + "Priya reviewed the diff. She wants the retry logic split into its own function.", + "London is three hours ahead. Move the sync earlier or half the team misses it.", + "Something changed upstream. I cannot reproduce yesterday's numbers at all now.", + "Do not merge this yet. The staging run has not finished and I want to see it.", + "It works on my machine. It does not work on the runner, which is the usual story.", + "Check the entitlement first. Everything else follows from whether that survived.", + "That was the last blocker! We can cut the build as soon as CI comes back green.", + "Are we still shipping Thursday? The notarization queue was two hours this morning.", + "The overlay flickers once on launch. Nobody has been able to catch it on video.", + "Thursday is a holiday in Berlin. Let us push the retrospective out by a week.", + "I filed it as a P2. Honestly it should be a P1 given how many users hit it.", ) @@ -294,18 +316,19 @@ def scored(self, hypothesis: str) -> metrics.Score: entirely reasonable. An utterance holds both sides already, so let it be the thing that remembers. """ - return metrics.score(self.reference, hypothesis, self.disfluent, self.not_abandoned) + return metrics.score(self.reference, hypothesis, self.disfluent, self.command_words) @property - def not_abandoned(self) -> frozenset[str]: - """Words in `disfluent` that are surplus by construction, not by abandonment. - - Only the spoken punctuation commands, and empty for every corpus that has none — - so this changes no number measured before it existed. `metrics._is_abandoned` - says why it has to exist: "question mark" is two non-hesitation words echoing - nothing, which is exactly the shape of an abandoned phrase, and left unnamed it - would be charged `FALSE_START_WEIGHT` per word while a leftover "period" was - charged one. + def command_words(self) -> frozenset[str]: + """The dictation commands planted in `disfluent`, as normalized words. + + Empty for every corpus that plants none, so this changes no number measured + before it existed. Scoring needs it for two things, both consequences of the same + fact — that these words are commands rather than speech. `metrics._is_abandoned` + must not read "question mark" as an abandoned phrase, which is exactly its shape: + two non-hesitation words echoing nothing. And `metrics.COMMAND_WEIGHT` charges + one left in the output more than an ordinary surplus word, because it reaches the + user's document as a word they never meant to write. """ return frozenset(word for command in self.commands for word in command.spoken_words) @@ -783,7 +806,7 @@ def false_start_fraction(utterances: list[Utterance]) -> float: rows, so it is the number that says whether the weighting is shaping the search or is a rounding error on this corpus. Pure arithmetic; no model is involved. - Reads each utterance's own `not_abandoned`, so the figure printed is the figure + Reads each utterance's own `command_words`, so the figure printed is the figure charged. Without it a spoken-punctuation corpus reported 55% of rows carrying an abandoned span against 21% for the same rows unmodified — the difference being "question mark" counted as a false start in the report while `Utterance.scored` @@ -792,7 +815,7 @@ def false_start_fraction(utterances: list[Utterance]) -> float: if not utterances: return 0.0 carrying = sum( - bool(metrics.false_start_tokens(u.disfluent, u.reference, u.not_abandoned)) + bool(metrics.false_start_tokens(u.disfluent, u.reference, u.command_words)) for u in utterances ) return carrying / len(utterances) diff --git a/evals/dictation-prompt/data/README.md b/evals/dictation-prompt/data/README.md index 83193e2..b91dcd8 100644 --- a/evals/dictation-prompt/data/README.md +++ b/evals/dictation-prompt/data/README.md @@ -2,7 +2,7 @@ ## `spoken-punctuation.jsonl` -76 dictated utterances that speak their own punctuation — `period`, `comma`, +91 dictated utterances that speak their own punctuation — `period`, `comma`, `question mark`, `all caps` — on top of injected disfluency. The input side is what a speech-to-text pass would hand the rewrite model; the `reference` is what the speaker meant to write. @@ -68,11 +68,11 @@ python3 evals/dictation-prompt/optimize_cleanup_prompt.py --spoken-punctuation 0 --limit 4000 --dump-corpus /tmp/nyra-spoken.jsonl --dry-run --show-samples 0 ``` -So read this file as a **fixture**, not a benchmark. 76 rows split into 50 dev / 25 test is +So read this file as a **fixture**, not a benchmark. 91 rows split into 60 dev / 30 test is far below the resolution the parent README argues for — differences between good -instructions on this task are a few hundredths, and 25 test rows cannot see them. What it +instructions on this task are a few hundredths, and 30 test rows cannot see them. What it is good for: reading the examples, reviewing a diff when an injector changes, and the one -thing `nyra` genuinely cannot do — **literal-use traps**. 15% of these references use a +thing `nyra` genuinely cannot do — **literal-use traps**. 12% of these references use a command word as ordinary content ("one grace period, so plan accordingly", "add a comma after the second clause"), against ~0–1% of `nyra`'s, so this is the only corpus here that can charge an instruction for converting a word the speaker meant literally. diff --git a/evals/dictation-prompt/data/spoken-punctuation.jsonl b/evals/dictation-prompt/data/spoken-punctuation.jsonl index faa44b1..f7301bd 100644 --- a/evals/dictation-prompt/data/spoken-punctuation.jsonl +++ b/evals/dictation-prompt/data/spoken-punctuation.jsonl @@ -74,3 +74,18 @@ {"reference": "Our smallest customer files more bug reports than the other twenty combined.", "disfluent": "right so our smallest customer files more bug reports than the other twenty combined period", "operations": ["opener", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "combined", "words": []}]} {"reference": "The release notes need a line about the new permission before we can publish.", "disfluent": "The release notes need a line about the new permission before we can publish publish period", "operations": ["repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "publish", "words": []}]} {"reference": "Check whether the trial EXPIRY is stored in the keychain or in user defaults.", "disfluent": "well check whether the trial all caps expiry is stored in in the keychain or in user defaults period", "operations": ["opener", "repeat", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["expiry"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "defaults", "words": []}]} +{"reference": "The build is green. Ship it before the release window closes tonight.", "disfluent": "The um build is green. Ship is green ship it before the release window closes tonight period", "operations": ["filler", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "tonight", "words": []}]} +{"reference": "I read the whole thread. Nobody actually answered the question that was asked.", "disfluent": "I basically read the whole thread. Nobody actually answered the question that was asked period", "operations": ["filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "asked", "words": []}]} +{"reference": "We tried THAT LAST quarter. It made the cold start worse, so we reverted it.", "disfluent": "right so we tried caps on that last caps off quarter period it made the cold start worse, so we reverted it period", "operations": ["opener", "spoken:caps-on", "spoken:period", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["that", "last"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "quarter", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} +{"reference": "The fix is small. The test that proves it is not, and that is the whole delay.", "disfluent": "that— I mean The fix is small period the test that proves it is not comma and that is the whole delay period", "operations": ["false_start", "spoken:period", "spoken:comma", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "small", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "not", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "delay", "words": []}]} +{"reference": "Priya reviewed the diff. She wants the retry LOGIC SPLIT into its own function.", "disfluent": "okay so priya reviewed the diff full stop she wants the retry uh caps on logic split caps off into its own function period", "operations": ["opener", "filler", "spoken:full-stop", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "diff", "words": []}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["logic", "split"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "function", "words": []}]} +{"reference": "London is three hours ahead. Move the sync earlier or half the team misses it.", "disfluent": "okay so london is t- three hours ahead full stop move the er sync earlier or half the sync— I mean team misses it period", "operations": ["opener", "stutter", "filler", "false_start", "spoken:full-stop", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "ahead", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} +{"reference": "Something changed upstream. I cannot reproduce yesterday's NUMBERS at all now.", "disfluent": "right so something changed upstream. upstream— sorry I cannot reproduce cannot— I mean yesterday's all caps numbers at all actually now period", "operations": ["opener", "false_start", "false_start", "filler", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["numbers"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "now", "words": []}]} +{"reference": "Do not merge THIS yet. The staging run has not finished and I want to see it.", "disfluent": "so do not merge all caps this yet period the staging run has not finished and I want to see it.", "operations": ["opener", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["this"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "yet", "words": []}]} +{"reference": "It works on my machine. It does not work on the runner, WHICH is the usual story.", "disfluent": "okay so it works on my machine full stop it does not work on the runner comma all caps which is story— or rather the usual story period", "operations": ["opener", "false_start", "spoken:full-stop", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "machine", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "runner", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["which"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "story", "words": []}]} +{"reference": "Check the entitlement first. Everything else follows from whether that survived.", "disfluent": "Check the entitlement first period everything else follows from whether that survived period", "operations": ["spoken:period", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "first", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "survived", "words": []}]} +{"reference": "That was the last blocker! We can cut the build as soon as CI comes back green.", "disfluent": "That was the er last blocker exclamation point we can cut the build as soon as soon as CI comes back green period", "operations": ["filler", "restart", "spoken:exclamation-point", "spoken:period"], "commands": [{"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "blocker", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "green", "words": []}]} +{"reference": "Are we still shipping Thursday? The notarization queue was two hours this morning.", "disfluent": "Are we still shipping shipping Thursday question mark the notarization queue you know was two hours this morning period", "operations": ["repeat", "filler", "spoken:question-mark", "spoken:period"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "thursday", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "morning", "words": []}]} +{"reference": "The overlay flickers once on launch. Nobody has been able to catch it on video.", "disfluent": "The overlay flickers once on launch full stop nobody hmm has been a- able to right catch it on video full stop", "operations": ["filler", "stutter", "filler", "spoken:full-stop", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "launch", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "video", "words": []}]} +{"reference": "Thursday is a holiday in Berlin. Let us push the retrospective out by a week.", "disfluent": "I mean thursday is a holiday in like Berlin. in like berlin Let us push the the retrospective out by a week period", "operations": ["opener", "filler", "restart", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "week", "words": []}]} +{"reference": "I FILED it as a P2. Honestly it should be a P1 given how many users hit it.", "disfluent": "let's see i all caps filed it right as should— or rather a p2 P2 period honestly hmm it should be a P1 given given how many users hit it period", "operations": ["opener", "filler", "false_start", "repeat", "filler", "repeat", "spoken:all-caps", "spoken:period", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["filed"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "p2", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} diff --git a/evals/dictation-prompt/metrics.py b/evals/dictation-prompt/metrics.py index 8fb4dac..8ff0b42 100644 --- a/evals/dictation-prompt/metrics.py +++ b/evals/dictation-prompt/metrics.py @@ -110,6 +110,65 @@ #: side of a change to it are not comparable. FALSE_START_WEIGHT = 5.0 +#: What one word of a dictated punctuation command left in the output costs, in errors. +#: 1.0 charges it like any other surplus word. +#: +#: Its own weight because it is its own failure, and because the number it had before was +#: an accident. A command left in is charged twice by plain WER already — a substitution +#: for the mark that never appeared, plus an insertion for the word that did — so it came +#: to 2 on the format axis without anyone deciding 2. That is only twice what *dropping* +#: the command costs, and dropping it is the cosmetic version: `children` where the +#: reference wants `children.` is a missing mark, while `children period` pastes into the +#: user's document a word they never meant to write. Those are not two grades of the same +#: mistake. +#: +#: The argument for a weight is `FALSE_START_WEIGHT`'s, almost verbatim: a leftover "um" +#: reads as a typo, and a leftover "period" reads as the software malfunctioning. The +#: argument for it being *lower* than the false-start weight is that an abandoned span is +#: usually several words (4.1 on nyra) and fabricates a clause the speaker never said, +#: while a command is one or two words and merely fails to disappear. +#: +#: **Why 3.** Swept over 1114 command-carrying `nyra` rows (2.36 commands each), against +#: hypotheses that leave a fixed share of each row's commands unconverted in front of an +#: otherwise perfect cleanup. Format axis, since that is what a spoken run selects on: +#: +#: weight 0% 25% 50% 75% 100% gap 0->25% unrankable +#: 1 1.000 0.944 0.842 0.725 0.677 0.056 1/1114 +#: 2 1.000 0.920 0.770 0.594 0.520 0.080 3/1114 +#: 3 1.000 0.896 0.697 0.464 0.370 0.104 11/1114 +#: 4 1.000 0.873 0.626 0.339 0.231 0.127 25/1114 +#: 5 1.000 0.849 0.556 0.221 0.104 0.151 51/1114 +#: 8 1.000 0.778 0.365 -0.079 -0.196 0.222 225/1114 +#: +#: The gradient in the region candidates actually differ in (0% to 25% residue) argues +#: for any weight at all — 3 nearly doubles the unweighted 0.056 — and keeps growing, so +#: it does not pick a number. Two things bound it. +#: +#: **`FALSE_START_WEIGHT` is the ceiling, and it binds at 4, not 5.** On the format axis a +#: leftover command is already charged the mark's substitution as well as the word's +#: insertion, so its total is this weight plus one: 4 at a weight of 3, sitting between a +#: leftover filler (1) and a leftover abandoned word (5). At a weight of 4 it ties the +#: abandoned word, which is the wrong ordering — an abandoned span fabricates a clause the +#: speaker never said, while a command is one stray word that failed to disappear. +#: +#: **The tail bounds it too.** `from_error_rate` decays past WER 1 rather than clipping, +#: and that decay flattens, so a row whose residue alone clears WER 2 stops being rankable +#: against its neighbours. That is 1.0% of rows at 3, 4.6% at 5, and 20% at 8. +#: +#: It also lands 4x what *dropping* the command costs, which is the distinction the old +#: incidental 2 could barely make: `children` for `children.` is a missing mark, and +#: `children period` is a word in the user's document they never meant to write. +#: +#: Charged per command **word**, so "question mark" costs twice "comma" — the same +#: per-word convention as the false-start surcharge, and for the same reason: two spurious +#: words in the user's text is twice the mess of one. +#: +#: A token already charged as an abandoned false start is not charged again here. Rule 1 +#: of `_is_abandoned` fires on a cut-off word regardless of vocabulary, so a command word +#: caught inside an abandoned run is charged at the higher weight and excluded from this +#: one. +COMMAND_WEIGHT = 3.0 + def normalize_text(text: str) -> str: """Casefold, drop punctuation, collapse whitespace — still a string.""" @@ -262,7 +321,7 @@ def _echoes_its_neighbour(run: list[int], spoken: list[tuple[str, str]]) -> bool def _is_abandoned( - run: list[int], spoken: list[tuple[str, str]], not_abandoned: frozenset[str] = frozenset() + run: list[int], spoken: list[tuple[str, str]], command_words: frozenset[str] = frozenset() ) -> bool: """Whether one surplus run is an abandoned span rather than a stumble. @@ -275,25 +334,24 @@ def _is_abandoned( enough: a lone surplus content word is a repetition or a slip, and charging triple for it would sweep in most of what the old flat weight already handled. - `not_abandoned` is extra vocabulary the *caller's corpus* knows is surplus for some - other reason, counted alongside `HESITATIONS` in rule 2. It exists for spoken - punctuation: "question mark" is two non-hesitation words that echo nothing, so - without it the classifier reads a dictation command as an abandoned phrase and - charges it `FALSE_START_WEIGHT` per word — which makes a leftover "question mark" - ten errors and a leftover "period" one, an asymmetry nobody chose. It defaults to - empty and is passed only by an utterance that planted commands, so every number - measured before it existed is unchanged. + `command_words` is vocabulary the *caller's corpus* planted as dictation commands, + counted alongside `HESITATIONS` in rule 2. Without it the classifier reads "question + mark" — two non-hesitation words that echo nothing — as an abandoned phrase and + charges it `FALSE_START_WEIGHT` per word, making a leftover "question mark" ten errors + and a leftover "period" one, an asymmetry nobody chose. Those words are charged + `COMMAND_WEIGHT` instead. It defaults to empty and is passed only by an utterance that + planted commands, so every number measured before it existed is unchanged. """ if any(_is_cut_off(spoken[index][0]) for index in run): return True words = [spoken[index][1] for index in run] - if sum(word not in HESITATIONS and word not in not_abandoned for word in words) < 2: + if sum(word not in HESITATIONS and word not in command_words for word in words) < 2: return False return not _echoes_its_neighbour(run, spoken) def false_start_tokens( - disfluent: str, reference: str, not_abandoned: frozenset[str] = frozenset() + disfluent: str, reference: str, command_words: frozenset[str] = frozenset() ) -> tuple[str, ...]: """The words of `disfluent` the speaker abandoned, normalized, in order. @@ -309,14 +367,14 @@ def false_start_tokens( inside such a run ("um we walked-") is charged with it, which is right: the whole region was abandoned, and the model has to delete all of it or none. - `not_abandoned` passes through to `_is_abandoned` — see there for why a corpus that + `command_words` passes through to `_is_abandoned` — see there for why a corpus that plants dictation commands has to name them. """ spoken = [(raw, word) for raw in disfluent.split() if (word := normalize_text(raw))] alignment = align(normalize(reference), [word for _, word in spoken]) abandoned: list[str] = [] for run in _runs(sorted(set(alignment.inserted_positions))): - if _is_abandoned(run, spoken, not_abandoned): + if _is_abandoned(run, spoken, command_words): abandoned.extend(spoken[index][1] for index in run) return tuple(abandoned) @@ -325,7 +383,7 @@ def uncorrected_false_starts( disfluent: str, reference: str, alignment: Alignment, - not_abandoned: frozenset[str] = frozenset(), + command_words: frozenset[str] = frozenset(), ) -> tuple[str, ...]: """Abandoned words the cleanup left in — the multiset `FALSE_START_WEIGHT` charges. @@ -335,11 +393,33 @@ def uncorrected_false_starts( hypothesis; both sides are already normalized there, which is what lets the raw `store—` in the input match the `store` the model echoed back. """ - spoken = false_start_tokens(disfluent, reference, not_abandoned) + spoken = false_start_tokens(disfluent, reference, command_words) left_in = Counter(alignment.inserted) & Counter(spoken) return tuple(sorted(left_in.elements())) +def uncorrected_commands( + alignment: Alignment, command_words: frozenset[str], abandoned: tuple[str, ...] = () +) -> tuple[str, ...]: + """Dictated command words the cleanup left in — what `COMMAND_WEIGHT` charges. + + A multiset read straight off the diff: every word the hypothesis added over the + reference that the input contained as a command. No membership test beyond that is + needed, because `command_words` is already scoped to this one pair — the corpus + planted those words in this input, so an added occurrence of one is a command that + failed to disappear. + + `abandoned` is subtracted so no token is charged twice. `_is_abandoned`'s first rule + fires on a cut-off word whatever the vocabulary, so a command word caught inside an + abandoned run is charged at `FALSE_START_WEIGHT` and must not also be charged here. + The higher weight wins, which is right: that whole region has to go. + """ + if not command_words: + return () + left_in = Counter(word for word in alignment.inserted if word in command_words) + return tuple(sorted((left_in - Counter(abandoned)).elements())) + + @dataclass(frozen=True) class Score: """One cleanup attempt, scored on both axes.""" @@ -352,6 +432,10 @@ class Score: #: Kept alongside them so `feedback` can name the failure instead of the reflector #: having to guess which of its leftovers cost triple. uncorrected_false_starts: tuple[str, ...] = () + #: Dictated command words the cleanup left in, priced at `COMMAND_WEIGHT`. Same + #: reason as above: a reflector told only "left disfluencies in: comma" cannot tell + #: that this leftover is neither a disfluency nor charged like one. + uncorrected_commands: tuple[str, ...] = () @property def blend(self) -> float: @@ -420,7 +504,7 @@ def score( reference: str, hypothesis: str, disfluent: str | None = None, - not_abandoned: frozenset[str] = frozenset(), + command_words: frozenset[str] = frozenset(), ) -> Score: """Score a cleanup against its reference on both axes — see `from_error_rate`. @@ -435,23 +519,26 @@ def score( way you tokenize, and exempting the formatting axis would quietly dilute the weighting by 30% under the default `blend`. - `not_abandoned` exempts vocabulary the corpus planted deliberately — see - `_is_abandoned`. Empty by default, so nothing that does not pass it changes. + `command_words` names the dictation commands the corpus planted in `disfluent`. They + are exempt from the false-start classifier and charged `COMMAND_WEIGHT` when left in — + see both constants. Empty by default, so nothing that does not pass it changes. """ content_alignment = align(normalize(reference), normalize(hypothesis)) format_alignment = align(surface(reference), surface(hypothesis)) left_in = ( - uncorrected_false_starts(disfluent, reference, content_alignment, not_abandoned) + uncorrected_false_starts(disfluent, reference, content_alignment, command_words) if disfluent else () ) - surcharge = (FALSE_START_WEIGHT - 1.0) * len(left_in) + commands = uncorrected_commands(content_alignment, command_words, left_in) + surcharge = (FALSE_START_WEIGHT - 1.0) * len(left_in) + (COMMAND_WEIGHT - 1.0) * len(commands) return Score( content=from_error_rate(_surcharged(content_alignment, surcharge)), format=from_error_rate(_surcharged(format_alignment, surcharge)), content_alignment=content_alignment, format_alignment=format_alignment, uncorrected_false_starts=left_in, + uncorrected_commands=commands, ) @@ -482,9 +569,12 @@ def feedback( on real transcripts whose disfluencies nobody enumerated in advance. Abandoned spans are reported apart from ordinary leftovers, and told what they - cost. They are the failure the score now weights heaviest (`FALSE_START_WEIGHT`), + cost. They are the failure the score weights heaviest (`FALSE_START_WEIGHT`), and a reflector shown "left disfluencies in: um, ha-, we, wouldn't" has no way to - tell which four of those words moved the number. + tell which four of those words moved the number. Dictated commands + (`COMMAND_WEIGHT`) are split out for the same reason, and because calling one a + disfluency points the reflector at the wrong rule: "comma" is not a hesitation the + speaker made, it is an instruction they gave. Three things are deliberately **not** here, because the reflector already has them or is misled by them: @@ -515,11 +605,14 @@ def feedback( # ordinary ones, so the reflector reads two distinct failures rather than one list # in which the expensive words are indistinguishable from the cheap ones. abandoned = Counter(scored.uncorrected_false_starts) + commands = Counter(scored.uncorrected_commands) leftover: list[str] = [] invented: list[str] = [] for word in content.inserted: if abandoned[word]: abandoned[word] -= 1 + elif commands[word]: + commands[word] -= 1 elif word in spoken: leftover.append(word) else: @@ -533,6 +626,13 @@ def feedback( f"and each one is scored as {FALSE_START_WEIGHT:g} errors rather than one. This is " "the costliest mistake on this task" ) + if scored.uncorrected_commands: + kept = ", ".join(sorted(set(scored.uncorrected_commands))) + notes.append( + f"left dictated punctuation commands in the output as words: {kept} — each is " + f"scored as {COMMAND_WEIGHT:g} errors rather than one, because it reaches the " + "reader as a word the speaker never meant to write" + ) if leftover: notes.append(f"left disfluencies in the output: {', '.join(sorted(set(leftover)))}") if invented: diff --git a/evals/dictation-prompt/program.py b/evals/dictation-prompt/program.py index e53fc81..6aeb554 100644 --- a/evals/dictation-prompt/program.py +++ b/evals/dictation-prompt/program.py @@ -196,7 +196,7 @@ def scorer(gold, pred, trace=None, pred_name=None, pred_trace=None, **_): planted = commands.get(spoken, ()) # The input as well as the target: `metrics.score` charges a leftover false # start `FALSE_START_WEIGHT` errors, and only the spoken side says which - # leftovers were abandoned. `not_abandoned` keeps a two-word dictation command + # leftovers were abandoned. `command_words` keeps a two-word dictation command # from being read as one — see `metrics._is_abandoned`. scored = metrics.score( target, diff --git a/evals/dictation-prompt/spoken_punctuation.py b/evals/dictation-prompt/spoken_punctuation.py index 3292b37..1628326 100644 --- a/evals/dictation-prompt/spoken_punctuation.py +++ b/evals/dictation-prompt/spoken_punctuation.py @@ -488,23 +488,23 @@ def feedback_note(commands: tuple[Command, ...], hypothesis: str) -> str: was built can leak into how it is scored. The command vocabulary is corpus construction, so it composes on top rather than moving down. - Names the failure in the two shapes that matter and nothing else. A leftover literal - is reported first: `metrics.feedback` sees it only as "left disfluencies in the - output: comma", which is true, actionable in the wrong direction, and the worst - outcome of the three. + Reports only the commands that were **dropped** — the ones whose words are gone and + whose mark never appeared. A command left in as words is `metrics.feedback`'s to + report, because that is where its weight lives (`metrics.COMMAND_WEIGHT`) and a + failure named without its cost is half a fact. Saying it in both places was the first + cut and it put the same complaint twice in every reflection prompt. + + Dropped commands are here because nothing in `metrics` can see them. A mark that was + never produced is one substitution among many on the format axis and *nothing at all* + on the content axis, and a missed ALL CAPS is invisible to content too — so without + this the reflector reads "capitalization or punctuation differs from the reference" + and has to guess that a command was the reason. """ if not commands: return "" outcomes = [(command, outcome(command, hypothesis)) for command in commands] - literal = [c.spoken for c, o in outcomes if o == "literal"] missing = [c for c, o in outcomes if o == "missing"] notes: list[str] = [] - if literal: - notes.append( - f"left spoken punctuation commands in the output as words: {', '.join(literal)} — " - "the speaker was asking for the punctuation mark, not dictating the word, so the " - "words must be replaced by the mark and never pasted" - ) if missing: asked = ", ".join( f"{c.spoken!r} -> {c.mark!r}" if c.kind == "mark" else f"{c.spoken!r} -> uppercase" diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index 7eec33d..ff8f2e6 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -1611,6 +1611,106 @@ def test_a_two_word_command_is_not_charged_as_an_abandoned_false_start(): assert exempted.content > charged.content +def test_a_leftover_command_word_costs_its_own_weight(): + """Not one, and not the incidental two it used to cost. + + Plain WER already charged a leftover command twice on the format axis — a + substitution for the mark that never appeared plus an insertion for the word that + did — so it came to 2 without anyone deciding 2. + """ + # Long enough that the charge stays inside WER 1, where `1 - WER` is linear and the + # error count can be read back off the score. Past that `from_error_rate` decays. + clean = "Can you send me the numbers before the review meeting tomorrow?" + reference, disfluent, (command,) = spoken_punctuation.inject( + clean, clean, seed=1, rate=1.0, caps_rate=0.0 + ) + words = frozenset(command.spoken_words) + left_in = metrics.score(reference, clean[:-1] + " question mark", disfluent, words) + charged = (1.0 - left_in.content) * left_in.content_alignment.reference_length + assert charged == pytest.approx(metrics.COMMAND_WEIGHT * len(command.spoken_words)) + assert left_in.uncorrected_commands == ("mark", "question") + + +def test_leaving_a_command_in_is_clearly_worse_than_dropping_it(): + """The distinction the old incidental 2 could barely make. + + `Is it ready` for `Is it ready?` is a missing mark. `Is it ready question mark` puts + two words in the reader's document that the speaker never meant to write. + """ + reference, disfluent, (command,) = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + words = frozenset(command.spoken_words) + dropped = metrics.score(reference, "Is it ready", disfluent, words) + left_in = metrics.score(reference, "Is it ready question mark", disfluent, words) + assert left_in.format < dropped.format + lost = dropped.format - left_in.format + assert lost > (1.0 - dropped.format), "leaving it in must cost more than the mark alone" + + +def test_a_command_word_is_charged_once_even_inside_an_abandoned_span(): + """Rule 1 of `_is_abandoned` fires on a cut-off word whatever the vocabulary, so a + command caught in that run would otherwise be charged at both weights.""" + reference = "We would have them." + disfluent = "we wouldn't ha- period we would have them" + words = frozenset({"period"}) + scored = metrics.score(reference, disfluent, disfluent, words) + charged_as_abandoned = set(scored.uncorrected_false_starts) + assert "period" in charged_as_abandoned + assert "period" not in scored.uncorrected_commands + + +def test_a_command_costs_less_than_an_abandoned_word_and_more_than_a_filler(): + """The ordering the two weights exist to express. + + An abandoned span fabricates a clause the speaker never said; a command is one stray + word that failed to disappear; a filler reads as a typo. + """ + assert 1.0 < metrics.COMMAND_WEIGHT < metrics.FALSE_START_WEIGHT + + +def test_the_command_weight_is_neutral_on_a_corpus_that_plants_nothing(): + reference = "We would have them." + disfluent = "we wouldn't ha- we would have them" + assert metrics.score(reference, disfluent, disfluent) == metrics.score( + reference, disfluent, disfluent, frozenset() + ) + + +def test_feedback_names_a_leftover_command_apart_from_a_disfluency_and_says_what_it_cost(): + """Calling "comma" a disfluency points the reflector at the wrong rule: it is not a + hesitation the speaker made, it is an instruction they gave.""" + reference, disfluent, (command,) = spoken_punctuation.inject( + "Is it ready?", "um is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + words = frozenset(command.spoken_words) + scored = metrics.score(reference, "Um is it ready question mark", disfluent, words) + text = metrics.feedback(reference, disfluent, scored) + assert "dictated punctuation commands" in text + assert f"{metrics.COMMAND_WEIGHT:g} errors" in text + # The filler was in the input, so it is a leftover — reported, and separately. + assert "left disfluencies in the output: um" in text + + +def test_the_note_and_the_feedback_do_not_both_report_a_leftover_command(): + """Said in both places it was the same complaint twice in every reflection prompt. + + `metrics.feedback` owns it, because that is where the weight lives and a failure named + without its cost is half a fact. The note owns what metrics cannot see: a command + whose words are gone and whose mark never appeared. + """ + reference, disfluent, (command,) = spoken_punctuation.inject( + "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + ) + hypothesis = "Is it ready question mark" + note = spoken_punctuation.feedback_note((command,), hypothesis) + assert note == "" + assert "left in" not in note + # Dropped, on the other hand, is invisible to metrics on the content axis. + dropped = spoken_punctuation.feedback_note((command,), "Is it ready") + assert "did not carry out" in dropped + + def test_the_exemption_is_neutral_on_a_corpus_that_plants_nothing(): """Every number measured before it existed has to still hold.""" reference = "We would have them." @@ -1626,9 +1726,9 @@ def test_the_utterance_hands_its_own_planted_commands_to_the_scorer(): "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 ) utterance = corpus.Utterance(reference=reference, disfluent=disfluent, commands=commands) - assert utterance.not_abandoned == frozenset({"question", "mark"}) + assert utterance.command_words == frozenset({"question", "mark"}) assert utterance.scored(disfluent) == metrics.score( - reference, disfluent, disfluent, utterance.not_abandoned + reference, disfluent, disfluent, utterance.command_words ) @@ -1742,7 +1842,7 @@ def test_spoken_punctuation_layers_onto_the_real_disfluencies_rather_than_replac def test_a_corpus_without_the_flag_plants_nothing(): loaded = corpus.load(source="builtin", limit=12) assert all(u.commands == () for u in loaded.utterances) - assert all(u.not_abandoned == frozenset() for u in loaded.utterances) + assert all(u.command_words == frozenset() for u in loaded.utterances) # The committed dataset, and the exact arguments that produce it. Both live here rather @@ -1785,7 +1885,7 @@ def test_the_frozen_dataset_round_trips_through_the_loader(): for a, b in zip(generated.utterances, reloaded.utterances, strict=True): assert (a.reference, a.disfluent) == (b.reference, b.disfluent) assert a.commands == b.commands - assert a.not_abandoned == b.not_abandoned + assert a.command_words == b.command_words assert a.scored(a.reference) == b.scored(b.reference) @@ -1854,6 +1954,26 @@ def test_every_bundled_sentence_survives_the_injection_filter(): assert len(kept) == len(corpus.BUILTIN_SAMPLE) +def test_the_bundled_sample_exercises_the_capital_after_a_spoken_mark(): + """It did not, for 0 of 61 terminal marks: every sentence ended at its own period, so + there was no following word whose capital a period command has to restore. + + That is half of what a terminal command asks for, and on `nyra` — where utterances run + to several sentences — it is 26% of terminal marks. + """ + eligible = terminal = 0 + for utterance in corpus.load(source="builtin", limit=500).utterances: + toks = utterance.reference.split() + for i, raw in enumerate(toks): + if raw and raw[-1] in spoken_punctuation.TERMINAL_MARKS: + if not spoken_punctuation._split_mark(raw): + continue + terminal += 1 + if i + 1 < len(toks) and not spoken_punctuation._keeps_its_capital(toks[i + 1]): + eligible += 1 + assert eligible >= 10, f"only {eligible} of {terminal} terminal marks have a next word" + + def test_the_bundled_sample_exercises_every_mark_in_the_vocabulary(): """`nyra` supplies no exclamation point, colon or semicolon at all, so if these entries in SPOKEN_FORMS are to mean anything, this corpus has to license them.""" From 21851ee1d481e2699be1e1a105e17289774fa64b Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 18:40:28 -0700 Subject: [PATCH 3/8] feat: score the punctuation commands and nothing else Two changes, both from the same objection: a command has to actually do something, and nothing but the commands should be able to move the score. **--punctuation-only.** The input now differs from the target by the commands alone. No disfluencies are injected, and a paired source's verbatim side is discarded in favour of its clean one -- so on nyra the eval scores against the intended transcripts, real conversational English, and nothing a cleanup does about hesitation can move the number. Every row is guaranteed at least one command (spoken_punctuation.inject gained `require`), rows the injector cannot plant one in are dropped, and the rate defaults to 1.0 because the mode is meaningless without it. The property is asserted on the alignment rather than by reconstruction: on the content axis the input is the target plus command words, no word substituted and none dropped. **The utterance-final mark is never spoken.** This is the half that makes a command a test of anything. A dictated "period" on the last word asks for a mark any instruction produces unprompted -- every candidate here says "restore punctuation" -- so obeying it and ignoring it score identically and the row is padding. Those were 55% of everything BUILTIN_SAMPLE licensed and 37% of nyra's. What survives is discriminative by construction, and spoken_punctuation.effect prices it against the reference with that one command's effect undone: a mid-utterance terminal mark is worth 2 (the mark, and the capital behind it), an internal mark 1, ALL CAPS one per word. A test asserts nothing is worth 0 over the whole corpus -- a zero means an invariant broke, not that the row is unusable. That exclusion is why PUNCTUATION_SAMPLE exists as a second bundled source. A disfluency smoke corpus wants plain declaratives with room to inject hesitation into; a punctuation corpus wants internal marks, and 37 of BUILTIN_SAMPLE's 91 rows have none. All 80 of these carry at least one, most several, and 14% use a command word as ordinary content -- against ~0% of nyra's, which is what makes this the only corpus here that can charge an instruction for converting a word the speaker meant literally. The committed dataset is regenerated from it, punctuation-only. Selection stays on format WER rather than the raw conversion rate, and deliberately. "Fraction of commands obeyed" is the number the task is named for and it is blind to the damage done getting there: an instruction that converts every command and deletes half the sentence scores 1.0 on it, and GEPA would find that. On a punctuation-only corpus format WER is not a compromise either -- every error it can charge is a command failure or collateral damage to text the instruction was told to leave alone, and both belong in the objective. Conversion rate is reported. Two pure candidates join the table. punct-mapping and punct-literal-guard carry no disfluency rules at all, which matters here: there are no disfluencies to remove, so every deletion those rules invite is damage. The composites stay, because whether carrying them costs anything measurable is one of the things a run answers. Also fixes a real inconsistency the new test caught. `caps on ... caps off` emits the word "off", but the Command only knew ("caps", "on"), so "off" sat in the input outside command_words and a cleanup that left `caps off` standing was charged one ordinary insertion for it instead of COMMAND_WEIGHT -- the same command priced two ways depending on which half survived. Command.closing records it. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 165 ++++++----- evals/dictation-prompt/candidates.py | 22 ++ evals/dictation-prompt/corpus.py | 149 +++++++++- evals/dictation-prompt/data/README.md | 25 +- .../data/spoken-punctuation.jsonl | 171 ++++++------ .../optimize_cleanup_prompt.py | 32 ++- evals/dictation-prompt/spoken_punctuation.py | 115 +++++++- evals/dictation-prompt/test_eval.py | 262 ++++++++++++++---- 8 files changed, 708 insertions(+), 233 deletions(-) diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 1e30b13..51afdda 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -23,7 +23,8 @@ ones we thought to write down. | `--source` | What it is | Measures | | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | | `nyra` (default) | [`nyralabs/disfluency_speech_english`][nyra] — ~5k Switchboard utterances whose disfluencies trained annotators marked **by hand**, repackaged with casing repaired so the formatting axis is live. Costs some fidelity for that. | content + format | -| `builtin` | 91 bundled sentences plus injection. No network, no key. The only corpus that can be committed, and the only one with literal-use traps. | content + format | +| `builtin` | 91 bundled sentences plus disfluency injection. No network, no key — the offline smoke path. | content + format | +| `punctuation` | 80 bundled sentences written for `--punctuation-only`: every row carries an internal mark. Committable, and the only source with literal-use traps. | format | `--jsonl` reads a local file instead: objects with both `disfluent` and `reference` are used as-is; anything with only a reference goes through the injector. @@ -62,39 +63,65 @@ It does not pose punctuation _restoration_, since both sides are already punctua ### Spoken punctuation — "period, comma, question mark, ALL CAPS" A dictation user who wants a comma often _says_ "comma", and the transcript comes back -with the word in it. `--spoken-punctuation RATE` poses that task, on top of whatever the -input side already is: +with the word in it. `--spoken-punctuation RATE` poses that task; `--punctuation-only` +poses **nothing else**. ```bash -# Rank the punctuation instructions against the shipped one. No search, ~5 candidates. +# The focused run: the input differs from the target by nothing but the commands. uv run evals/dictation-prompt/optimize_cleanup_prompt.py \ - --spoken-punctuation 0.8 --optimizer none --limit 600 + --source punctuation --punctuation-only --optimizer none -# Then search from the best of them. -uv run evals/dictation-prompt/optimize_cleanup_prompt.py \ - --spoken-punctuation 0.8 --start best-candidate +# The same task layered on top of real annotated disfluency, for the mixed question. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py --spoken-punctuation 0.8 ``` -Unlike `--severity`, this applies to a **paired** source too, and that is the point: the -disfluencies stay the ones annotators marked by hand and only the punctuation is -synthetic. A row comes out carrying both, which is the task a real user poses — +`--punctuation-only` makes every source reference-only: no disfluencies are injected, and +a paired source's **verbatim side is discarded** in favour of its clean one. So on `nyra` +it scores against the intended transcripts — real conversational English — and nothing a +cleanup does about hesitation can move the number. Every row is guaranteed at least one +command, rows the injector cannot plant one in are dropped, and the rate defaults to 1.0. +Stated on the alignment, and pinned by a test: on the content axis the input is the target +plus command words, with **no word substituted and none dropped**. ``` -input : Um we shipped it today period monday was really, quiet comma so nobody noticed period -target: We shipped it today. Monday was quiet, so nobody noticed. +input : I read the whole thread period nobody actually answered the caps on question that caps off was asked. +target: I read the whole thread. Nobody actually answered the QUESTION THAT was asked. ``` -Two operators, and only one of them touches a target: +#### A command has to actually do something + +**The utterance-final mark is never spoken.** A dictated "period" on the last word asks +for a mark any instruction produces unprompted — every candidate here says "restore +punctuation" — so obeying that command and ignoring it score identically, and the row is +padding. Those marks were **55%** of everything `BUILTIN_SAMPLE` licensed and 37% of +`nyra`'s. + +What survives is discriminative by construction, and +`spoken_punctuation.effect` prices it — the reference against the reference with that one +command's effect undone: + +| what is spoken | worth | what obeying it forces | +| ----------------------------------------- | -----: | --------------------------------------------- | +| a **mid-utterance terminal mark** | 2 | the mark, and the capital behind it | +| an **internal comma, colon or semicolon** | 1 | a placement inside the clause | +| **ALL CAPS** | 1/word | uppercase, which nothing produces by accident | + +Nothing can be worth 0, and a test asserts it over the whole corpus rather than filtering +on it: a zero means an invariant broke, not that the row is unusable. -**Marks** are spoken on the **input side only**, and only where the reference -**licenses** them — the same word carries the same mark in the target. So the correct -answer never stops being the corpus's own target, and the target is not edited at all. -A mark the reference lacks is never spoken: `really,` above stays a comma, because the -annotator deleted the span it introduced, and asking for a mark that would then be -scored as an error teaches an instruction that commands are sometimes to be ignored. -`.` `,` `?` `!` `:` `;` are all in the vocabulary with their common aliases (`period` / -`full stop`); on `nyra` only the first three ever fire, because its references contain -483 periods, 460 commas, 43 question marks and none of the other three. +That exclusion is why `--source punctuation` exists as a second bundled sample. A +disfluency smoke corpus wants plain declaratives with room to inject hesitation into; a +punctuation corpus wants **internal** marks, and 37 of `BUILTIN_SAMPLE`'s 91 rows have +none. Every row of `PUNCTUATION_SAMPLE` carries at least one, most carry several. + +#### The two operators + +**Marks** are spoken on the **input side only**, and only where the reference **licenses** +them — the same word carries the same mark in the target. So the correct answer never +stops being the corpus's own target, and the target is not edited at all. A mark the +reference lacks is never spoken: asking for one that would then be scored as an error +teaches an instruction that commands are sometimes to be ignored. `.` `,` `?` `!` `:` `;` +are all in the vocabulary with their common aliases (`period` / `full stop`). **ALL CAPS** is the exception, and the only place in the harness that edits a reference. You cannot pose "uppercase this word" against a target with no uppercase in it, so the @@ -106,19 +133,24 @@ The word after a spoken sentence-ending mark is **lowercased**, which is what ma task a task: leave the capital in and an instruction can restore the period from the casing alone. Proper nouns are lowercased along with everything else — that word is sentence-initial in the reference by construction, so `Monday` and `today` ask for the -same rule and the same answer, and protecting names would have handed the answer to 30% -of the commands (the share whose following word appears capitalized mid-sentence -somewhere in `nyra`). The pronoun _I_ is the one exception: it is capitalized -mid-sentence too, so `politics full stop i'm not sure` is a transcript no service -returns and would show the model one token cased two ways in one utterance. +same rule and the same answer, and protecting names would have handed the answer to 30% of +the commands. The pronoun _I_ is the one exception: it is capitalized mid-sentence too, so +`politics full stop i'm not sure` is a transcript no service returns. -**It selects on `format` by default**, not `blend`. `metrics.normalize` casefolds and -strips punctuation, so a restored comma and a missed one are the same string to the -content axis and an ALL CAPS command is invisible to it. `format` sees the whole task — -and still sees the expensive failure, a command left in the output as a literal word, -because that is an inserted token on either axis. +#### What it selects on -Alongside the axes the run reports what became of the commands themselves: +**`format`, not `blend`.** `metrics.normalize` casefolds and strips punctuation, so a +restored comma and a missed one are the same string to the content axis and an ALL CAPS +command is invisible to it. On a `--punctuation-only` corpus that is not a compromise: +since the input differs from the target by nothing but the commands, every error the +format axis can charge is either a command failure or collateral damage to text the +instruction was told to leave alone. Both belong in the objective. + +Which is also why the selecting axis is **not** the raw conversion rate. "Fraction of +commands obeyed" is the number this task is named for, and it is blind to the damage done +in getting there: an instruction that converts every command and deletes half the sentence +scores 1.0 on it. GEPA would find that. So conversion rate is **reported** and WER +**selects**. | column | what it means | | --------------- | --------------------------------------------------------- | @@ -143,9 +175,9 @@ injector. Reproducible in principle; unreviewable in practice — nobody reads a find out whether the examples are any good, and a change to an injector silently changes what every past number was measured on. A file in the tree is diffable. -[`data/spoken-punctuation.jsonl`](./data/spoken-punctuation.jsonl) is that file: 91 rows -generated from `--source builtin`, whose sentences are written for this repo and so carry -no third-party terms. `nyra` derives from LDC-licensed Switchboard transcripts and is +[`data/spoken-punctuation.jsonl`](./data/spoken-punctuation.jsonl) is that file: 80 +`--punctuation-only` rows generated from `--source punctuation`, whose sentences are +written for this repo and so carry no third-party terms. `nyra` derives from LDC-licensed Switchboard transcripts and is deliberately **not** committed — dump it locally if you want it frozen. `test_eval.py` regenerates the committed file and asserts it is byte-identical, so an injector change fails the suite instead of leaving a stale dataset in the tree. @@ -155,9 +187,9 @@ Everything downstream keys off **what the corpus contains, not what flags were p that, reading the saved dataset back selected on `blend` and printed no command outcomes: the same corpus scored two different ways depending on how it was reached. -Read the committed file as a **fixture**, not a benchmark. 60 dev / 30 test rows is far +Read the committed file as a **fixture**, not a benchmark. 53 dev / 26 test rows is far below the resolution this README argues for everywhere else. What it is uniquely good for -is the one thing `nyra` cannot do: **12% of its references use a command word as ordinary +is the one thing `nyra` cannot do: **14% of its references use a command word as ordinary content** ("one grace period, so plan accordingly", "add a comma after the second clause"), against ~0-1% of `nyra`'s. It is the only corpus here that can charge an instruction for converting a word the speaker meant literally — and those rows compose @@ -197,7 +229,7 @@ Three things this cannot tell you, all of them reported rather than assumed away has to clear is the best of the punctuation candidates on dev, which is what `--start best-candidate` seeds from. -A spoken-punctuation run scores **`BASELINE` plus the four punctuation candidates**, not +A spoken-punctuation run scores **`BASELINE` plus the six punctuation candidates**, not the whole table. `BASELINE` alone is not enough — a search seeded from an instruction that has never heard of the task starts outside the region worth exploring — but the six terse contrast instructions are worse than useless here: they exist to rank framings of @@ -213,6 +245,12 @@ possible change to what Blurt ships: `PRIOR_WINNER` plus `SPOKEN_PUNCTUATION_CLA from it — `punct-explicit` at 1430 is the searchable seed. Which one wins on dev and which one a search can improve are different questions. +`punct-mapping` and `punct-literal-guard` carry **no disfluency rules at all**, which +matters on a `--punctuation-only` corpus: there are no disfluencies to remove there, so +every deletion those rules invite is damage to text the instruction was told to leave +alone. Whether carrying them costs anything measurable is one of the things the run +answers, which is why the composites stay in the table rather than being swapped out. + [nyra]: https://huggingface.co/datasets/nyralabs/disfluency_speech_english [ds]: https://huggingface.co/datasets/amaai-lab/DisfluencySpeech [dq]: https://huggingface.co/datasets/google-research-datasets/disfl_qa @@ -413,29 +451,30 @@ is `--model`. Its proposal prompts are multi-field, so they keep DSPy's marker p ## The knobs that matter -| Flag | Default | What it changes | -| ---------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | -| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | -| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | -| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | -| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | -| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | -| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | -| `--spoken-punctuation` | `0` (off) | Speak this share of the marks the reference licenses. Applies to paired sources too; selects on `format`. | -| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | -| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | -| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | -| `--start` | `prior-winner` | Which instruction GEPA evolves from — the compressed prior winner, or the best hand-written candidate. | -| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | -| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | -| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | -| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | -| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | -| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | -| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | -| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | -| `--seed` | `7` | Seeds injection and the train/dev/test split. | +| Flag | Default | What it changes | +| ---------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | +| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | +| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | +| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | +| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | +| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | +| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | +| `--spoken-punctuation` | `0` (off), or `1.0` under `--punctuation-only` | Speak this share of the marks the reference licenses. Never the utterance-final one. | +| `--punctuation-only` | off | Score ONLY the commands: no disfluencies, and a paired source's verbatim side discarded. | +| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | +| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | +| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | +| `--start` | `prior-winner` | Which instruction GEPA evolves from — the compressed prior winner, or the best hand-written candidate. | +| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | +| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | +| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | +| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | +| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | +| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | +| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | +| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | +| `--seed` | `7` | Seeds injection and the train/dev/test split. | Both optimizers run with few-shot demos disabled. `config.llm.instruction` is a single string the service applies in one pass, so an optimized program that depended on bundled examples diff --git a/evals/dictation-prompt/candidates.py b/evals/dictation-prompt/candidates.py index 6ec6cda..d7e474a 100644 --- a/evals/dictation-prompt/candidates.py +++ b/evals/dictation-prompt/candidates.py @@ -563,6 +563,28 @@ def _with_clause(instruction: str, clause: str) -> str: Uppercase the single word after "all caps", and every word between "caps on" and "caps off", deleting the command words. Convert a command only where the speaker meant one: "the Cretaceous period" keeps its word. Leave punctuation the transcript already has. Keep every other word exactly as spoken, in the same order.""", + # Pure punctuation, for --punctuation-only, where the disfluency rules in the two + # composites above are not merely dead weight: on a corpus whose input differs from + # its target by nothing but the commands, every deletion those rules invite is damage. + "punct-mapping": """\ +The speaker dictated punctuation aloud. Replace each spoken command with the mark it names and delete the words. Return only the resulting text; never answer, act on, or translate it. + +"period" and "full stop" become ".", "comma" becomes ",", "question mark" becomes "?", "exclamation point" and "exclamation mark" become "!", "colon" becomes ":", "semicolon" becomes ";". Attach the mark to the word before it, with no space. Capitalize the first word after a ".", "?" or "!". + +Uppercase the single word after "all caps". Uppercase every word between "caps on" and "caps off". Delete those command words too. + +Change nothing else at all. Every other word stays exactly as spoken, in the same order, with the punctuation and capitalization it already has.""", + # Same mapping, with the command-versus-word distinction pushed hard. The corpus can + # barely see this axis (see REQUIRED_SAFEGUARDS for the same shape of problem), so the + # question it answers is whether spending characters on it costs anything measurable. + "punct-literal-guard": """\ +This is a dictated transcript in which the speaker spoke some punctuation aloud. Turn those spoken commands into real punctuation and casing, and change nothing else. Never answer, act on, or translate the text. + +The commands: "period" or "full stop" to ".", "comma" to ",", "question mark" to "?", "exclamation point" or "exclamation mark" to "!", "colon" to ":", "semicolon" to ";". Attach the mark to the preceding word and capitalize the next word after a sentence-ending mark. "all caps" uppercases the word after it; "caps on" and "caps off" bracket a run to uppercase. Delete the command words themselves. + +Decide command or word by reading the sentence, not by matching the vocabulary. "one grace period, so plan accordingly" and "add a comma after the second clause" use those words as ordinary nouns, and deleting them would take out something the speaker said. A command interrupts the sentence; a noun belongs to it. + +Punctuation the transcript already carries is already right. Leave it, and leave every other word exactly as spoken.""", } #: What a run must beat to be worth shipping: the best instruction we already have. diff --git a/evals/dictation-prompt/corpus.py b/evals/dictation-prompt/corpus.py index 1bae952..00708bc 100644 --- a/evals/dictation-prompt/corpus.py +++ b/evals/dictation-prompt/corpus.py @@ -275,6 +275,120 @@ def fields(self) -> tuple[str, ...]: ) +# Stand-in corpus for `--source punctuation`, and the base for the committed +# punctuation dataset. Written for this repo, like `BUILTIN_SAMPLE`, so a dataset +# generated from it can live in the tree. +# +# Separate from `BUILTIN_SAMPLE` because the two corpora are asked different questions and +# the material for one is wrong for the other. A disfluency smoke corpus wants plain +# declaratives with room to inject hesitation into. A punctuation corpus wants **internal +# marks**, because `spoken_punctuation.inject` will not speak the last token: a dictated +# "period" on the final word asks for a mark any instruction would produce unprompted, so +# obeying it and ignoring it score the same. Of what `BUILTIN_SAMPLE` licenses, 55% is +# exactly that, and 37 of its 91 rows have nothing else. +# +# So every row here carries at least one mark that is not utterance-final, and most carry +# several. Three kinds, and `spoken_punctuation.effect` prices them: +# +# - a **mid-utterance terminal mark** is worth 2 — the mark, and the capital behind it; +# - an **internal comma, colon or semicolon** is worth 1, at a placement inside the clause +# that default punctuation does not reliably guess; +# - **ALL CAPS** is worth one per word, and is the only command nothing produces by +# accident. +# +# The last group are literal-use traps, and they still carry internal marks so they test +# both things at once: "one grace period comma so plan accordingly" has to come back with +# the noun kept and the command obeyed. +PUNCTUATION_SAMPLE: tuple[str, ...] = ( + # Two sentences: the internal mark forces a split and the capital behind it. + "The build is green. Ship it before the release window closes tonight.", + "I read the whole thread. Nobody actually answered the question that was asked.", + "We tried that last quarter. It made the cold start worse, so we reverted it.", + "The fix is small. The test that proves it is not, and that is the whole delay.", + "Priya reviewed the diff. She wants the retry logic split into its own function.", + "London is three hours ahead. Move the sync earlier or half the team misses it.", + "Something changed upstream. I cannot reproduce yesterday's numbers at all now.", + "Do not merge this yet. The staging run has not finished and I want to see it.", + "It works on my machine. It does not work on the runner, which is the usual story.", + "Check the entitlement first. Everything else follows from whether that survived.", + "That was the last blocker! We can cut the build as soon as CI comes back green.", + "Are we still shipping Thursday? The notarization queue was two hours this morning.", + "The overlay flickers once on launch. Nobody has caught it on video yet.", + "Thursday is a holiday in Berlin. Let us push the retrospective out by a week.", + "I filed it as a P2. Honestly it should be a P1 given how many users hit it.", + "The vendor confirmed the outage. They promised a postmortem by Friday afternoon.", + "Stop the rollout. The error rate tripled in the last twenty minutes of traffic.", + "Nobody owns this dashboard. That is why it has been broken since February.", + "The estimate assumed two engineers. We have one, so the date has to move out.", + "I will draft the announcement tonight. Send me any corrections before nine.", + # Comma-rich, so the placement inside the clause is what is being asked for. + "If the upload stalls, retry it once, and then fall back to the smaller chunk size.", + "We shipped the change on Tuesday, and by Thursday the error rate had halved.", + "The onboarding flow works, but the second screen asks for a permission we never use.", + "Before you merge, rebase on main, run the whole suite, and check the coverage gate.", + "I read the incident report, and the root cause was a stale cache in the edge layer.", + "Once the lease expires, the worker stops accepting jobs, which is what we wanted.", + "Send the draft to Priya, loop in the design team, and we can review it together.", + "When the mic is muted, the waveform freezes, and users read that as a crash.", + "The migration touched four tables, two indexes, and one view nobody remembered.", + "After the retry budget runs out, the request fails, and the overlay says try again.", + "We looked at three vendors, and only one will sign a data processing agreement.", + "The cache warms in about a minute, so the first few requests are always slower.", + "If you cannot reproduce it locally, attach the sysdiagnose, and I will look tonight.", + "Their API returns a 202, then polls, then hands back a URL that expires in an hour.", + "I moved the standup to nine, cancelled the Thursday sync, and blocked out Friday.", + "The feature is behind a flag, off by default, and enabled only for the internal team.", + "Given the timing, the risk, and how little we know, I would rather wait a week.", + "The parser is fine, the writer is fine, and the thing between them loses a byte.", + "Whatever we do, do it before the freeze, because after that nothing lands.", + "She asked for the numbers, the caveats, and a recommendation on one page.", + # A mark mid-utterance and a question or exclamation, so those commands fire too. + "The queue drained overnight. Are we confident it will hold under Monday traffic?", + "I looked at the trace. Why is the second request slower than the first one?", + "We fixed the leak. Did anyone check whether the fix survives a clean install?", + "The demo went well. Can you send the recording to the whole team this afternoon?", + "It finally reproduced! The trick was launching from a read-only directory.", + "That is the third regression this week! We need the gate back on before Friday.", + "The numbers came in. Honestly, they are better than anything we projected.", + "Everything is symbolicated now. You should be able to read the crash directly.", + # Colons and semicolons, internal, so those commands are exercised at all. + "Here is the plan: land the fix, cut a build, and hand it to QA tomorrow morning.", + "Two things are still open: the entitlement review and the store description.", + "The cause was simple: we were reading the sample rate from the wrong device.", + "It builds cleanly on my machine; it fails on the runner every single time.", + "Ship the smaller change first; the refactor can wait until after the release.", + "The tradeoff is straightforward: more accuracy for eighty milliseconds of latency.", + "Keep the interface as it is; only the storage layer actually needs to change.", + "One caveat: the migration is not reversible once the first write has landed.", + "Three people asked for this: two customers and one person on the support rota.", + "The rule is simple; if the test is flaky, fix it or delete it the same day.", + # Content worth shouting, so the casing command has somewhere natural to land. + "This is urgent, and the deadline is Friday, not the following Wednesday.", + "Do not deploy this today. The database migration has not been reviewed yet.", + "The answer is no, and it will stay no until the security review is finished.", + "Never commit the signing key. Rotate it immediately if it ever reaches a log.", + "Everything downstream depends on this table, so treat the schema as frozen.", + "The important part is the ordering, not the individual steps in the pipeline.", + "Read the whole thread before replying, because the decision changed twice.", + "That number is wrong, and it has been wrong in every deck since November.", + "Only the release manager can approve this, and only after the gate is green.", + "The critical path is the notarization, not the build, so start it early.", + # Literal-use traps, each with an internal mark so both things are tested at once. + "We only support one grace period, so plan accordingly before the trial ends.", + "Add a comma after the second clause, and the sentence reads much better.", + "Every question mark in that survey was ambiguous, so we rewrote the whole form.", + "The billing period rolls over at midnight UTC, not at midnight local time.", + "Put a colon after the heading, and leave the rest of the line exactly as it is.", + "She used a full stop where the style guide asks for a semicolon, which is minor.", + "The legal team wants that warning in all caps, which our design system forbids.", + "That exclamation point reads as sarcasm, so please take it out of the release notes.", + "There is a dash missing from the second bullet, and a typo in the third one.", + "I said period, and it typed the word instead of the punctuation mark I wanted.", + "The Cretaceous period ended with an impact, which is roughly how the demo went.", + "Use a semicolon there; a comma is not strong enough to join those two clauses.", +) + + @dataclass(frozen=True) class Utterance: """One eval example: what the model is given, and what it should produce.""" @@ -329,6 +443,14 @@ def command_words(self) -> frozenset[str]: two non-hesitation words echoing nothing. And `metrics.COMMAND_WEIGHT` charges one left in the output more than an ordinary surplus word, because it reaches the user's document as a word they never meant to write. + + Bare words, not positions, so on a row that uses one as content — "Stop the + rollout" against a spoken "full stop" — the exemption covers the speaker's own + noun too. It costs almost nothing: the exemption only bites on words the + *hypothesis added*, so a cleanup that keeps the content word correctly, or drops + it, is unaffected either way, and only one that duplicates it is over-charged. + Positions would fix it and would have to survive the round trip through + `--dump-corpus` to be worth having. """ return frozenset(word for command in self.commands for word in command.spoken_words) @@ -628,6 +750,7 @@ def load( strip_formatting: bool = False, spoken_punctuation_rate: float = 0.0, spoken_caps_rate: float = 0.25, + punctuation_only: bool = False, ) -> Corpus: """Load up to `limit` pairs, injecting disfluencies for reference-only sources.""" if jsonl: @@ -637,6 +760,10 @@ def load( rows = (Utterance(reference=text, disfluent=text) for text in BUILTIN_SAMPLE) spec, formatted = None, True detail = {"note": "bundled sample; no dataset was downloaded"} + elif source == "punctuation": + rows = (Utterance(reference=text, disfluent=text) for text in PUNCTUATION_SAMPLE) + spec, formatted = None, True + detail = {"note": "bundled punctuation sample; no dataset was downloaded"} elif source in SOURCES: spec = SOURCES[source] # Each source defaults to its held-out split, which is small (250 rows for @@ -663,10 +790,20 @@ def load( # A reference-only source needs the injector to produce an input side, and # needs a reference long and well-punctuated enough to inject into. A jsonl # file is the user's own text, so it is filtered leniently either way. - reference_only = spec is None or not spec.is_paired + # `punctuation_only` makes any source reference-only: the target becomes the input + # base and the verbatim side is discarded. That is the whole point of the mode — the + # input must differ from the target by nothing but the spoken commands, so a score + # cannot be moved by disfluency removal. On `nyra` it means scoring against the + # intended side, which is real conversational English and no longer a paired corpus. + reference_only = spec is None or not spec.is_paired or punctuation_only utterances = _collect(rows, limit, for_injection=reference_only and jsonl is None) + if punctuation_only: + utterances = [ + replace(u, disfluent=u.reference, input_supplied=False, operations=()) + for u in utterances + ] - if reference_only: + if reference_only and not punctuation_only: injected: list[Utterance] = [] for index, utterance in enumerate(utterances): # A jsonl row that already carried a disfluent side keeps it — read off @@ -714,7 +851,14 @@ def load( seed=seed + index, rate=spoken_punctuation_rate, caps_rate=spoken_caps_rate, + require=punctuation_only, ) + # A row the injector could not plant anything in measures nothing on a corpus + # whose only subject is commands, and dilutes every mean it appears in. It + # happens when the reference's only mark is its last one, which `inject` + # refuses to speak. + if punctuation_only and not commands: + continue spoken.append( replace( utterance, @@ -728,6 +872,7 @@ def load( detail |= { "spoken_punctuation_rate": spoken_punctuation_rate, "spoken_caps_rate": spoken_caps_rate, + "punctuation_only": punctuation_only, "commands_per_row": sum(len(u.commands) for u in utterances) / len(utterances) if utterances else 0.0, diff --git a/evals/dictation-prompt/data/README.md b/evals/dictation-prompt/data/README.md index b91dcd8..bc9c04e 100644 --- a/evals/dictation-prompt/data/README.md +++ b/evals/dictation-prompt/data/README.md @@ -2,10 +2,10 @@ ## `spoken-punctuation.jsonl` -91 dictated utterances that speak their own punctuation — `period`, `comma`, -`question mark`, `all caps` — on top of injected disfluency. The input side is what a -speech-to-text pass would hand the rewrite model; the `reference` is what the speaker -meant to write. +80 dictated utterances that speak their own punctuation — `period`, `comma`, +`question mark`, `all caps` — and **nothing else**. The input differs from the target by +the commands alone: no disfluencies are injected, so no score here can be moved by +anything but the punctuation task. ```json { @@ -37,6 +37,11 @@ meant to write. } ``` +Every command is guaranteed to change something: a mid-utterance terminal mark forces the +mark _and_ the capital behind it, an internal comma forces a placement, and ALL CAPS forces +uppercase. A mark on the final word is never spoken, because that is the one an instruction +would produce unprompted. + `commands` records what was planted, which is the only reason the run can report how many commands were **obeyed** rather than just how close the text came. `--jsonl` reads it back exactly — see `corpus._utterances_from_jsonl` — so scoring this file is scoring the corpus @@ -46,7 +51,7 @@ it was generated from, not an approximation of it. ```bash python3 evals/dictation-prompt/optimize_cleanup_prompt.py \ - --source builtin --limit 200 --spoken-punctuation 0.8 --severity 0.35 --seed 7 \ + --source punctuation --punctuation-only --limit 200 --spoken-caps-rate 0.35 --seed 7 \ --dump-corpus evals/dictation-prompt/data/spoken-punctuation.jsonl \ --dry-run --show-samples 0 ``` @@ -64,15 +69,15 @@ LDC-licensed transcripts, and `corpus.BUILTIN_SAMPLE` exists precisely so the of carries no third-party licensing. Dump it locally if you want it frozen: ```bash -python3 evals/dictation-prompt/optimize_cleanup_prompt.py --spoken-punctuation 0.8 \ - --limit 4000 --dump-corpus /tmp/nyra-spoken.jsonl --dry-run --show-samples 0 +python3 evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ + --limit 4000 --dump-corpus /tmp/nyra-punctuation.jsonl --dry-run --show-samples 0 ``` -So read this file as a **fixture**, not a benchmark. 91 rows split into 60 dev / 30 test is +So read this file as a **fixture**, not a benchmark. 80 rows split into 53 dev / 26 test is far below the resolution the parent README argues for — differences between good -instructions on this task are a few hundredths, and 30 test rows cannot see them. What it +instructions on this task are a few hundredths, and 26 test rows cannot see them. What it is good for: reading the examples, reviewing a diff when an injector changes, and the one -thing `nyra` genuinely cannot do — **literal-use traps**. 12% of these references use a +thing `nyra` genuinely cannot do — **literal-use traps**. 14% of these references use a command word as ordinary content ("one grace period, so plan accordingly", "add a comma after the second clause"), against ~0–1% of `nyra`'s, so this is the only corpus here that can charge an instruction for converting a word the speaker meant literally. diff --git a/evals/dictation-prompt/data/spoken-punctuation.jsonl b/evals/dictation-prompt/data/spoken-punctuation.jsonl index f7301bd..5373f1d 100644 --- a/evals/dictation-prompt/data/spoken-punctuation.jsonl +++ b/evals/dictation-prompt/data/spoken-punctuation.jsonl @@ -1,91 +1,80 @@ -{"reference": "The build failed because the signing certificate expired over the weekend.", "disfluent": "okay so the failed— or rather build failed because the ah signing kind of certificate expired over the weekend period", "operations": ["opener", "false_start", "filler", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "weekend", "words": []}]} -{"reference": "Can you send me the LATEST NUMBERS before the review meeting tomorrow morning?", "disfluent": "well can you sort of send me the caps on latest numbers caps off before the review um meeting tomorrow morning question mark", "operations": ["opener", "filler", "filler", "spoken:caps-on", "spoken:question-mark"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["latest", "numbers"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "morning", "words": []}]} -{"reference": "I think we should ship the fix behind a flag and watch the crash rate for a day.", "disfluent": "I think we should should ship the the fix actually behind a flag and watch kind of the c- crash rate for a day period", "operations": ["repeat", "repeat", "filler", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "day", "words": []}]} -{"reference": "The microphone permission dialog never appears when the app runs from a temporary directory.", "disfluent": "The microphone permission dialog never appears when the app runs from a directory— or rather temporary directory period", "operations": ["false_start", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "directory", "words": []}]} -{"reference": "Let's move the retrospective to Thursday so everyone in Berlin can attend.", "disfluent": "Let's move the retrospective to Thursday so everyone in uh Berlin can attend full stop", "operations": ["filler", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "attend", "words": []}]} -{"reference": "She pointed out that the transcript is already punctuated when it comes back from the service.", "disfluent": "She pointed out hmm that the transcript is already punctuated punctuated when it comes back from the service period", "operations": ["filler", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "service", "words": []}]} -{"reference": "We measured about a hundred and seventy milliseconds of connection setup on a cold start.", "disfluent": "I mean we measured about a hundred and seventy milliseconds milliseconds— I mean of connection setup on er a co- cold start period", "operations": ["opener", "false_start", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "start", "words": []}]} -{"reference": "The OVERLAY SHOULD STAY on screen until the paste actually lands in the target application.", "disfluent": "I mean the caps on overlay should stay caps off on I mean screen until the paste like actually lands in the target application.", "operations": ["opener", "filler", "filler", "spoken:caps-on"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["overlay", "should", "stay"]}]} -{"reference": "Nobody has looked at the notarization logs since the last release went out.", "disfluent": "n- Nobody has looked notarization— or rather at the notarization logs since the last release went out period", "operations": ["stutter", "false_start", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "out", "words": []}]} -{"reference": "It turns out the regression was introduced by the change to the clipboard restore path.", "disfluent": "well it turns restore— or rather out the regression was introduced by the change kind of to the clipboard restore path period", "operations": ["opener", "false_start", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "path", "words": []}]} -{"reference": "Please double check the sample rate before you send the audio to the transcription endpoint.", "disfluent": "Please double check the sample rate you know before you send the audio to the transcription endpoint.", "operations": ["filler"]} -{"reference": "The design review is blocked on whether we keep the menu bar ITEM at all.", "disfluent": "I mean the design review is blocked on whether we keep the the menu bar all caps item at all period", "operations": ["opener", "repeat", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["item"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "all", "words": []}]} -{"reference": "If the upload stalls, retry it once, and then fall back to the smaller chunk size.", "disfluent": "If the upload stalls comma retry it once comma and then fall back hmm to the the smaller chunk actually size.", "operations": ["filler", "repeat", "filler", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "stalls", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "once", "words": []}]} -{"reference": "We shipped the change on Tuesday, and by Thursday the error rate had halved.", "disfluent": "We shipped the change on Tuesday comma and by Thursday the error kind of rate had halved.", "operations": ["filler", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tuesday", "words": []}]} -{"reference": "The onboarding flow works, but the second screen STILL ASKS for a permission we never use.", "disfluent": "I mean the onboarding flow works comma but the second screen still second screen caps on still asks caps off for a permission for a permission we never I mean use period", "operations": ["opener", "restart", "restart", "filler", "spoken:comma", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "works", "words": []}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["still", "asks"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "use", "words": []}]} -{"reference": "Before you merge, rebase on main, run the whole suite, and check the coverage gate.", "disfluent": "Before before you merge comma rebase on main, um run the whole suite comma and check the whole— or rather coverage gate period", "operations": ["restart", "filler", "false_start", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "merge", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "suite", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "gate", "words": []}]} -{"reference": "I read the incident report, and the root cause was a stale cache in the edge layer.", "disfluent": "I read the the incident report, and the root cause was actually a stale cache cache in the edge er layer.", "operations": ["repeat", "filler", "repeat", "filler"]} -{"reference": "Once the lease expires, the worker stops accepting jobs, which is what we wanted.", "disfluent": "Once the lease expires, the w- worker stops accepting jobs comma which which is what I mean we wanted period", "operations": ["stutter", "repeat", "filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "jobs", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "wanted", "words": []}]} -{"reference": "The vendor confirmed the outage, apologised, and promised a postmortem by Friday.", "disfluent": "let's see the vendor confirmed the vendor confirmed the outage, apologised apologised, promised— sorry and promised a postmortem by Friday full stop", "operations": ["opener", "restart", "repeat", "false_start", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "friday", "words": []}]} -{"reference": "Send the draft to Priya, loop in the design team, and we can review it together.", "disfluent": "Send the draft to Priya comma l- loop in the design team comma and we can we can review it together period", "operations": ["stutter", "restart", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "priya", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "team", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "together", "words": []}]} -{"reference": "When the mic is muted, the waveform freezes, and users read that as a crash.", "disfluent": "When the mic read— or rather is muted comma the waveform freezes comma and freezes and users read that as like a as like a crash.", "operations": ["false_start", "restart", "filler", "restart", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "muted", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "freezes", "words": []}]} -{"reference": "The migration touched four tables, two indexes, and one view NOBODY remembered.", "disfluent": "okay so the migration touched four tables comma two indexes comma and one view all caps nobody remembered full stop", "operations": ["opener", "spoken:comma", "spoken:comma", "spoken:all-caps", "spoken:full-stop"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tables", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "indexes", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["nobody"]}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "remembered", "words": []}]} -{"reference": "After the retry budget runs out, the request fails, and the overlay says try again.", "disfluent": "After the retry budget runs out comma the ah request fails comma and and the overlay says try again period", "operations": ["filler", "repeat", "spoken:comma", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "out", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fails", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "again", "words": []}]} -{"reference": "We looked at three vendors, and only one of them will sign a data processing agreement.", "disfluent": "We looked looked at three vendors comma and only one of them will kind of sign a data processing agreement period", "operations": ["repeat", "filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "vendors", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "agreement", "words": []}]} -{"reference": "The cache warms in about a minute, so the FIRST few requests are always slower.", "disfluent": "so the cache the cache warms in about a ah minute comma so the all caps first few you know requests are always slower.", "operations": ["opener", "restart", "filler", "filler", "spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "minute", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["first"]}]} -{"reference": "If you cannot REPRODUCE it locally, attach the sysdiagnose, and I will look tonight.", "disfluent": "okay so if you cannot like all caps reproduce it locally comma attach the sysdiagnose comma and I will l- look tonight.", "operations": ["opener", "filler", "stutter", "spoken:all-caps", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["reproduce"]}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "locally", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sysdiagnose", "words": []}]} -{"reference": "The estimate assumed two engineers, and we have one, so the date needs to move.", "disfluent": "The estimate assumed two engineers comma and we have one comma so the date needs to move.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "engineers", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "one", "words": []}]} -{"reference": "Their API returns a 202, then polls, then hands back a URL that expires in an hour.", "disfluent": "Their API returns a 202 comma then polls, then hands hands back a URL that expires hmm in an hour.", "operations": ["repeat", "filler", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "202", "words": []}]} -{"reference": "I moved the standup to nine, cancelled the Thursday sync, and blocked out Friday afternoon.", "disfluent": "I moved the standup to nine comma cancelled the Thursday sync, and blocked out Friday afternoon full stop", "operations": ["spoken:comma", "spoken:full-stop"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "nine", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "afternoon", "words": []}]} -{"reference": "The feature is behind a flag, off by default, and only enabled for the internal team.", "disfluent": "so the feature is behind a a flag, off by default, and only enabled for the internal team.", "operations": ["opener", "repeat"]} -{"reference": "Did anyone check whether the new entitlement survives a clean install?", "disfluent": "Did Did anyone check whether the new entitlement su- survives a clean install question mark", "operations": ["repeat", "stutter", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "install", "words": []}]} -{"reference": "Should we hold the release until the notarization queue clears, or ship it now?", "disfluent": "Should we ho- hold the release until the notarization queue clears comma or ship it now now question mark", "operations": ["stutter", "repeat", "spoken:comma", "spoken:question-mark"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "clears", "words": []}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "now", "words": []}]} -{"reference": "Can you remind me what the RETRY budget is on the streaming endpoint?", "disfluent": "well actually can you remind me what the all caps retry budget is you know on the streaming kind of endpoint question mark", "operations": ["opener", "filler", "filler", "filler", "spoken:all-caps", "spoken:question-mark"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["retry"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "endpoint", "words": []}]} -{"reference": "Do you know why the waveform stops animating when the window loses focus?", "disfluent": "Do basically you know why the waveform stops stops— I mean animating when the window loses focus?", "operations": ["filler", "false_start"]} -{"reference": "Is there a reason we still ship the old audio unit alongside the new one?", "disfluent": "okay so is there a reason we still ship the um old audio unit alongside the new one question mark", "operations": ["opener", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "one", "words": []}]} -{"reference": "What happens to a partial transcript if the socket closes before the final message?", "disfluent": "ah What happens to a partial ah transcript if basically the socket closes before the final message question mark", "operations": ["filler", "filler", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "message", "words": []}]} -{"reference": "Would it be easier to gate this on the account TIER instead of a flag?", "disfluent": "I mean would it be easier to to gate this on the account all caps tier instead of uh a flag question mark", "operations": ["opener", "repeat", "filler", "spoken:all-caps", "spoken:question-mark"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["tier"]}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "flag", "words": []}]} -{"reference": "Have we ever measured how long the first paste takes on a cold launch?", "disfluent": "right so have hmm we um ever me- measured how long the first paste measured— I mean takes on a cold launch question mark", "operations": ["opener", "filler", "filler", "stutter", "false_start", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "launch", "words": []}]} -{"reference": "Are the crash reports symbolicated, or do I need to upload the archive myself?", "disfluent": "well are the crash crash symbolicated— I mean reports symbolicated comma actually or do I need to upload the like archive myself question mark", "operations": ["opener", "repeat", "false_start", "filler", "filler", "spoken:comma", "spoken:question-mark"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "symbolicated", "words": []}, {"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "myself", "words": []}]} -{"reference": "Who owns the dashboard now that the analytics team has been folded into platform?", "disfluent": "Who owns the dashboard now that the like analytics team sort of has been folded into platform question mark", "operations": ["filler", "filler", "spoken:question-mark"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "platform", "words": []}]} -{"reference": "That fixed it, and the latency dropped by half!", "disfluent": "well that fixed it comma and the latency dropped by latency dropped by half exclamation point", "operations": ["opener", "restart", "spoken:comma", "spoken:exclamation-point"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "it", "words": []}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "half", "words": []}]} -{"reference": "Please do not ship this on a Friday afternoon again!", "disfluent": "Please do not ship this on a Friday afternoon again again exclamation mark", "operations": ["repeat", "spoken:exclamation-mark"], "commands": [{"spoken": "exclamation mark", "mark": "!", "kind": "mark", "anchor": "again", "words": []}]} -{"reference": "The WHOLE SUITE passed on the first try for once!", "disfluent": "well the caps on whole suite caps off passed on the first try for for once exclamation point", "operations": ["opener", "repeat", "spoken:caps-on", "spoken:exclamation-point"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["whole", "suite"]}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "once", "words": []}]} -{"reference": "Watch out, the staging database is still pointed at production!", "disfluent": "Watch out comma the staging database basically is still pointed at ah production exclamation point", "operations": ["filler", "filler", "spoken:comma", "spoken:exclamation-point"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "out", "words": []}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "production", "words": []}]} -{"reference": "Here is the plan: land the fix, cut a build, and HAND it to QA tomorrow.", "disfluent": "right so here is the plan colon land the fix comma cut a build, and all caps hand it to QA sort of tomorrow period", "operations": ["opener", "filler", "spoken:colon", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "plan", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fix", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["hand"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "tomorrow", "words": []}]} -{"reference": "Two things are still open: the entitlement review and the App Store description.", "disfluent": "Two things are still open colon the entitlement review and the App um Store description full stop", "operations": ["filler", "spoken:colon", "spoken:full-stop"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "open", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "description", "words": []}]} -{"reference": "The cause was simple: we were reading the sample rate from the wrong device.", "disfluent": "The cause was simple: we were reading the like sample rate from the wrong device period", "operations": ["filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "device", "words": []}]} -{"reference": "It builds cleanly on my machine; it fails on the runner every single time.", "disfluent": "It builds cleanly on my machine semicolon it fails on the runner on the runner every single time full stop", "operations": ["restart", "spoken:semicolon", "spoken:full-stop"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "machine", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "time", "words": []}]} -{"reference": "Ship the smaller change first; the refactor can WAIT UNTIL AFTER the release.", "disfluent": "okay so ship the smaller change first; the refactor can caps on wait until after caps off the release period", "operations": ["opener", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["wait", "until", "after"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "release", "words": []}]} -{"reference": "The tradeoff is straightforward: more accuracy for about eighty milliseconds of latency.", "disfluent": "The tradeoff is straightforward colon more accuracy for about eighty eighty milliseconds of latency period", "operations": ["repeat", "spoken:colon", "spoken:period"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "straightforward", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "latency", "words": []}]} -{"reference": "Keep the interface as it is; only the storage LAYER NEEDS to change.", "disfluent": "right so keep the interface as it is; only the storage caps on layer needs caps off to change period", "operations": ["opener", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["layer", "needs"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "change", "words": []}]} -{"reference": "One caveat: the migration is not reversible once the first write lands.", "disfluent": "One caveat colon the like migration is not reversible once the first write right lands.", "operations": ["filler", "filler", "spoken:colon"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "caveat", "words": []}]} -{"reference": "We only support one grace period, so plan accordingly before the TRIAL ends.", "disfluent": "I mean we only— I mean only support one grace period, so plan accordingly before the all caps trial ends period", "operations": ["opener", "false_start", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["trial"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "ends", "words": []}]} -{"reference": "Add a comma after the second clause and the sentence reads much better.", "disfluent": "right so add a comma after the after the second clause and the sentence reads m- much better period", "operations": ["opener", "restart", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "better", "words": []}]} -{"reference": "Every question mark in that survey was ambiguous, so we rewrote the whole form.", "disfluent": "Every question mark in that survey was ambiguous comma so we rewrote the whole form period", "operations": ["spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "ambiguous", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "form", "words": []}]} -{"reference": "The billing period rolls over at midnight UTC, not at midnight local time.", "disfluent": "The billing period rolls over at midnight UTC comma not at midnight local time period", "operations": ["spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "utc", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "time", "words": []}]} -{"reference": "Put a colon after the heading and leave the rest of the line alone.", "disfluent": "Put a colon a- after the heading and leave the r- rest of the line alone full stop", "operations": ["stutter", "stutter", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "alone", "words": []}]} -{"reference": "She used a full stop where the style guide clearly asks for a semicolon.", "disfluent": "She used a used a full stop where the ah style guide c- clearly asks for a semicolon period", "operations": ["restart", "filler", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "semicolon", "words": []}]} -{"reference": "The legal team wants the warning in all caps, which our design system forbids.", "disfluent": "yeah so the legal team wants the warning in all caps comma which our design system forbids period", "operations": ["opener", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "caps", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "forbids", "words": []}]} -{"reference": "That exclamation point in the release NOTES READS as sarcasm, so please remove it.", "disfluent": "well that exclamation point in the please— sorry release caps on notes reads caps off as er sarcasm comma so please remove you know it.", "operations": ["opener", "false_start", "filler", "filler", "spoken:caps-on", "spoken:comma"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["notes", "reads"]}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sarcasm", "words": []}]} -{"reference": "There is a dash missing from the second BULLET on the pricing page.", "disfluent": "let's see there is a dash missing from the second all caps bullet on the pricing page period", "operations": ["opener", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["bullet"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "page", "words": []}]} -{"reference": "I said period, and it typed the word instead of the punctuation mark.", "disfluent": "I said period, and and it typed the word instead of the punctuation mark full stop", "operations": ["repeat", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "mark", "words": []}]} -{"reference": "The Cretaceous period ended with an impact, which is roughly how the demo went.", "disfluent": "kind of The Cretaceous period ended with an impact comma which is roughly how the demo went period", "operations": ["filler", "spoken:comma", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "impact", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "went", "words": []}]} -{"reference": "Use a semicolon there; a comma is not strong enough to join those two clauses.", "disfluent": "Use a semicolon there semicolon a comma is not strong enough to hmm join those two clauses full stop", "operations": ["filler", "spoken:semicolon", "spoken:full-stop"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "there", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "clauses", "words": []}]} -{"reference": "Remind me to follow up with the accessibility team about the focus ring.", "disfluent": "right so remind me to follow up with with the accessibility team about the focus ring period", "operations": ["opener", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "ring", "words": []}]} -{"reference": "The transcript came back empty, which usually MEANS the audio was all silence.", "disfluent": "right so the transcript came back empty comma which usually all caps means the audio was all audio— I mean silence period", "operations": ["opener", "false_start", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "empty", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["means"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "silence", "words": []}]} -{"reference": "I will draft the announcement tonight and send it round for comments in the morning.", "disfluent": "right so i will draft sort of the announcement right tonight and send it round for comments kind of in the morning period", "operations": ["opener", "filler", "filler", "filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "morning", "words": []}]} -{"reference": "We should probably stop supporting the beta channel now that nobody is on it.", "disfluent": "We hmm should probably stop supporting beta— no wait the beta channel now that now that nobody is on it period", "operations": ["filler", "false_start", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} -{"reference": "The keyboard shortcut conflicts with the system dictation shortcut on a fresh install.", "disfluent": "The ke- keyboard shortcut conflicts with the s- system dictation shortcut on a fresh install period", "operations": ["stutter", "stutter", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "install", "words": []}]} -{"reference": "Let me know if the new model handles background noise any better than the old one.", "disfluent": "well let me know if the new model model— or rather handles background noise noise any better than the old one.", "operations": ["opener", "false_start", "repeat"]} -{"reference": "Nothing in the logs explains why the first request after a sleep always times out.", "disfluent": "Nothing in the logs logs explains why the first request after request after a sleep always times out period", "operations": ["repeat", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "out", "words": []}]} -{"reference": "The onboarding video is four minutes long and most people quit after thirty seconds.", "disfluent": "The onboarding video is four minutes long and most people most people quit after thirty seconds full stop", "operations": ["restart", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "seconds", "words": []}]} -{"reference": "I would rather fix the FLAKE THAN mark the test as skipped and forget about it.", "disfluent": "yeah so i would rather fix uh the caps on flake than caps off mark the test as skipped and forget about and forget about it full stop", "operations": ["opener", "filler", "restart", "spoken:caps-on", "spoken:full-stop"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["flake", "than"]}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} -{"reference": "Our smallest customer files more bug reports than the other twenty combined.", "disfluent": "right so our smallest customer files more bug reports than the other twenty combined period", "operations": ["opener", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "combined", "words": []}]} -{"reference": "The release notes need a line about the new permission before we can publish.", "disfluent": "The release notes need a line about the new permission before we can publish publish period", "operations": ["repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "publish", "words": []}]} -{"reference": "Check whether the trial EXPIRY is stored in the keychain or in user defaults.", "disfluent": "well check whether the trial all caps expiry is stored in in the keychain or in user defaults period", "operations": ["opener", "repeat", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["expiry"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "defaults", "words": []}]} -{"reference": "The build is green. Ship it before the release window closes tonight.", "disfluent": "The um build is green. Ship is green ship it before the release window closes tonight period", "operations": ["filler", "restart", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "tonight", "words": []}]} -{"reference": "I read the whole thread. Nobody actually answered the question that was asked.", "disfluent": "I basically read the whole thread. Nobody actually answered the question that was asked period", "operations": ["filler", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "asked", "words": []}]} -{"reference": "We tried THAT LAST quarter. It made the cold start worse, so we reverted it.", "disfluent": "right so we tried caps on that last caps off quarter period it made the cold start worse, so we reverted it period", "operations": ["opener", "spoken:caps-on", "spoken:period", "spoken:period"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["that", "last"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "quarter", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} -{"reference": "The fix is small. The test that proves it is not, and that is the whole delay.", "disfluent": "that— I mean The fix is small period the test that proves it is not comma and that is the whole delay period", "operations": ["false_start", "spoken:period", "spoken:comma", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "small", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "not", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "delay", "words": []}]} -{"reference": "Priya reviewed the diff. She wants the retry LOGIC SPLIT into its own function.", "disfluent": "okay so priya reviewed the diff full stop she wants the retry uh caps on logic split caps off into its own function period", "operations": ["opener", "filler", "spoken:full-stop", "spoken:caps-on", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "diff", "words": []}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["logic", "split"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "function", "words": []}]} -{"reference": "London is three hours ahead. Move the sync earlier or half the team misses it.", "disfluent": "okay so london is t- three hours ahead full stop move the er sync earlier or half the sync— I mean team misses it period", "operations": ["opener", "stutter", "filler", "false_start", "spoken:full-stop", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "ahead", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} -{"reference": "Something changed upstream. I cannot reproduce yesterday's NUMBERS at all now.", "disfluent": "right so something changed upstream. upstream— sorry I cannot reproduce cannot— I mean yesterday's all caps numbers at all actually now period", "operations": ["opener", "false_start", "false_start", "filler", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["numbers"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "now", "words": []}]} -{"reference": "Do not merge THIS yet. The staging run has not finished and I want to see it.", "disfluent": "so do not merge all caps this yet period the staging run has not finished and I want to see it.", "operations": ["opener", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["this"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "yet", "words": []}]} -{"reference": "It works on my machine. It does not work on the runner, WHICH is the usual story.", "disfluent": "okay so it works on my machine full stop it does not work on the runner comma all caps which is story— or rather the usual story period", "operations": ["opener", "false_start", "spoken:full-stop", "spoken:comma", "spoken:all-caps", "spoken:period"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "machine", "words": []}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "runner", "words": []}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["which"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "story", "words": []}]} -{"reference": "Check the entitlement first. Everything else follows from whether that survived.", "disfluent": "Check the entitlement first period everything else follows from whether that survived period", "operations": ["spoken:period", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "first", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "survived", "words": []}]} -{"reference": "That was the last blocker! We can cut the build as soon as CI comes back green.", "disfluent": "That was the er last blocker exclamation point we can cut the build as soon as soon as CI comes back green period", "operations": ["filler", "restart", "spoken:exclamation-point", "spoken:period"], "commands": [{"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "blocker", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "green", "words": []}]} -{"reference": "Are we still shipping Thursday? The notarization queue was two hours this morning.", "disfluent": "Are we still shipping shipping Thursday question mark the notarization queue you know was two hours this morning period", "operations": ["repeat", "filler", "spoken:question-mark", "spoken:period"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "thursday", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "morning", "words": []}]} -{"reference": "The overlay flickers once on launch. Nobody has been able to catch it on video.", "disfluent": "The overlay flickers once on launch full stop nobody hmm has been a- able to right catch it on video full stop", "operations": ["filler", "stutter", "filler", "spoken:full-stop", "spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "launch", "words": []}, {"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "video", "words": []}]} -{"reference": "Thursday is a holiday in Berlin. Let us push the retrospective out by a week.", "disfluent": "I mean thursday is a holiday in like Berlin. in like berlin Let us push the the retrospective out by a week period", "operations": ["opener", "filler", "restart", "repeat", "spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "week", "words": []}]} -{"reference": "I FILED it as a P2. Honestly it should be a P1 given how many users hit it.", "disfluent": "let's see i all caps filed it right as should— or rather a p2 P2 period honestly hmm it should be a P1 given given how many users hit it period", "operations": ["opener", "filler", "false_start", "repeat", "filler", "repeat", "spoken:all-caps", "spoken:period", "spoken:period"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["filed"]}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "p2", "words": []}, {"spoken": "period", "mark": ".", "kind": "mark", "anchor": "it", "words": []}]} +{"reference": "The build is green. Ship it before the release WINDOW closes tonight.", "disfluent": "The build is green period ship it before the release all caps window closes tonight.", "operations": ["spoken:period", "spoken:all-caps"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "green", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["window"], "closing": ""}]} +{"reference": "I read the whole thread. Nobody actually answered the QUESTION THAT was asked.", "disfluent": "I read the whole thread period nobody actually answered the caps on question that caps off was asked.", "operations": ["spoken:period", "spoken:caps-on"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "thread", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["question", "that"], "closing": "caps off"}]} +{"reference": "We tried that last quarter. It made the cold start worse, so we reverted it.", "disfluent": "We tried that last quarter full stop it made the cold start worse comma so we reverted it.", "operations": ["spoken:full-stop", "spoken:comma"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "quarter", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "worse", "words": [], "closing": ""}]} +{"reference": "The fix is small. The test that proves it is not, and that is the whole delay.", "disfluent": "The fix is small period the test that proves it is not comma and that is the whole delay.", "operations": ["spoken:period", "spoken:comma"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "small", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "not", "words": [], "closing": ""}]} +{"reference": "Priya reviewed the diff. She wants the retry logic split into its own function.", "disfluent": "Priya reviewed the diff full stop she wants the retry logic split into its own function.", "operations": ["spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "diff", "words": [], "closing": ""}]} +{"reference": "London is three hours ahead. Move the sync earlier or half the team misses it.", "disfluent": "London is three hours ahead period move the sync earlier or half the team misses it.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "ahead", "words": [], "closing": ""}]} +{"reference": "Something changed upstream. I CANNOT REPRODUCE yesterday's numbers at all now.", "disfluent": "Something changed upstream period I caps on cannot reproduce caps off yesterday's numbers at all now.", "operations": ["spoken:period", "spoken:caps-on"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "upstream", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["cannot", "reproduce"], "closing": "caps off"}]} +{"reference": "Do not merge this yet. The STAGING run has not finished and I want to see it.", "disfluent": "Do not merge this yet period the all caps staging run has not finished and I want to see it.", "operations": ["spoken:period", "spoken:all-caps"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "yet", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["staging"], "closing": ""}]} +{"reference": "It works on my machine. It does not work on the runner, which is the usual story.", "disfluent": "It works on my machine period it does not work on the runner comma which is the usual story.", "operations": ["spoken:period", "spoken:comma"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "machine", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "runner", "words": [], "closing": ""}]} +{"reference": "Check the entitlement first. Everything else follows from whether that survived.", "disfluent": "Check the entitlement first period everything else follows from whether that survived.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "first", "words": [], "closing": ""}]} +{"reference": "That was the last blocker! We can cut the build as soon as CI comes back green.", "disfluent": "That was the last blocker exclamation mark we can cut the build as soon as CI comes back green.", "operations": ["spoken:exclamation-mark"], "commands": [{"spoken": "exclamation mark", "mark": "!", "kind": "mark", "anchor": "blocker", "words": [], "closing": ""}]} +{"reference": "Are we still shipping Thursday? The notarization queue was two HOURS this morning.", "disfluent": "Are we still shipping Thursday question mark the notarization queue was two all caps hours this morning.", "operations": ["spoken:question-mark", "spoken:all-caps"], "commands": [{"spoken": "question mark", "mark": "?", "kind": "mark", "anchor": "thursday", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["hours"], "closing": ""}]} +{"reference": "The overlay flickers once on launch. Nobody has caught it on video yet.", "disfluent": "The overlay flickers once on launch period nobody has caught it on video yet.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "launch", "words": [], "closing": ""}]} +{"reference": "Thursday is a holiday in Berlin. Let us push the retrospective out by a week.", "disfluent": "Thursday is a holiday in Berlin period let us push the retrospective out by a week.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "berlin", "words": [], "closing": ""}]} +{"reference": "I filed it as a P2. Honestly it should be a P1 given how MANY USERS hit it.", "disfluent": "I filed it as a P2 period honestly it should be a P1 given how caps on many users caps off hit it.", "operations": ["spoken:period", "spoken:caps-on"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "p2", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["many", "users"], "closing": "caps off"}]} +{"reference": "The vendor confirmed the outage. They promised a postmortem by Friday afternoon.", "disfluent": "The vendor confirmed the outage period they promised a postmortem by Friday afternoon.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "outage", "words": [], "closing": ""}]} +{"reference": "Stop the rollout. The error rate tripled in the last twenty minutes of traffic.", "disfluent": "Stop the rollout full stop the error rate tripled in the last twenty minutes of traffic.", "operations": ["spoken:full-stop"], "commands": [{"spoken": "full stop", "mark": ".", "kind": "mark", "anchor": "rollout", "words": [], "closing": ""}]} +{"reference": "Nobody owns this dashboard. That is why it has been broken since February.", "disfluent": "Nobody owns this dashboard period that is why it has been broken since February.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "dashboard", "words": [], "closing": ""}]} +{"reference": "The estimate assumed two engineers. We have one, so the date has to move out.", "disfluent": "The estimate assumed two engineers period we have one comma so the date has to move out.", "operations": ["spoken:period", "spoken:comma"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "engineers", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "one", "words": [], "closing": ""}]} +{"reference": "I will draft the announcement tonight. Send me any corrections before nine.", "disfluent": "I will draft the announcement tonight period send me any corrections before nine.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "tonight", "words": [], "closing": ""}]} +{"reference": "If the upload stalls, retry it once, and then fall back to the smaller chunk size.", "disfluent": "If the upload stalls comma retry it once comma and then fall back to the smaller chunk size.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "stalls", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "once", "words": [], "closing": ""}]} +{"reference": "We shipped the change on Tuesday, and by Thursday the ERROR rate had halved.", "disfluent": "We shipped the change on Tuesday comma and by Thursday the all caps error rate had halved.", "operations": ["spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tuesday", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["error"], "closing": ""}]} +{"reference": "The onboarding flow works, but the second screen asks for a permission we never use.", "disfluent": "The onboarding flow works comma but the second screen asks for a permission we never use.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "works", "words": [], "closing": ""}]} +{"reference": "Before you merge, rebase on main, run the whole suite, and check the coverage gate.", "disfluent": "Before you merge comma rebase on main comma run the whole suite comma and check the coverage gate.", "operations": ["spoken:comma", "spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "merge", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "main", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "suite", "words": [], "closing": ""}]} +{"reference": "I read the incident report, and the root cause was a stale CACHE in the edge layer.", "disfluent": "I read the incident report comma and the root cause was a stale all caps cache in the edge layer.", "operations": ["spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "report", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["cache"], "closing": ""}]} +{"reference": "Once the lease expires, the worker STOPS accepting jobs, which is what we wanted.", "disfluent": "Once the lease expires comma the worker all caps stops accepting jobs comma which is what we wanted.", "operations": ["spoken:comma", "spoken:all-caps", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "expires", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["stops"], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "jobs", "words": [], "closing": ""}]} +{"reference": "Send the draft to Priya, loop in the design team, and we can review it together.", "disfluent": "Send the draft to Priya comma loop in the design team comma and we can review it together.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "priya", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "team", "words": [], "closing": ""}]} +{"reference": "When the mic is muted, the waveform freezes, and users read that as a crash.", "disfluent": "When the mic is muted comma the waveform freezes comma and users read that as a crash.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "muted", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "freezes", "words": [], "closing": ""}]} +{"reference": "The migration touched four tables, two indexes, and one view nobody remembered.", "disfluent": "The migration touched four tables comma two indexes comma and one view nobody remembered.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "tables", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "indexes", "words": [], "closing": ""}]} +{"reference": "After the retry budget runs out, the request fails, and the overlay SAYS try again.", "disfluent": "After the retry budget runs out comma the request fails comma and the overlay all caps says try again.", "operations": ["spoken:comma", "spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "out", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fails", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["says"], "closing": ""}]} +{"reference": "We looked at three vendors, and only one will sign a data processing agreement.", "disfluent": "We looked at three vendors comma and only one will sign a data processing agreement.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "vendors", "words": [], "closing": ""}]} +{"reference": "The cache warms in about a minute, so the first few requests are always slower.", "disfluent": "The cache warms in about a minute comma so the first few requests are always slower.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "minute", "words": [], "closing": ""}]} +{"reference": "If you cannot reproduce it locally, ATTACH the sysdiagnose, and I will look tonight.", "disfluent": "If you cannot reproduce it locally comma all caps attach the sysdiagnose comma and I will look tonight.", "operations": ["spoken:comma", "spoken:all-caps", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "locally", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["attach"], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sysdiagnose", "words": [], "closing": ""}]} +{"reference": "Their API returns a 202, then polls, then hands back a URL that expires in an hour.", "disfluent": "Their API returns a 202 comma then polls comma then hands back a URL that expires in an hour.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "202", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "polls", "words": [], "closing": ""}]} +{"reference": "I moved the standup to nine, cancelled the Thursday sync, and blocked out Friday.", "disfluent": "I moved the standup to nine comma cancelled the Thursday sync comma and blocked out Friday.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "nine", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sync", "words": [], "closing": ""}]} +{"reference": "The feature is behind a flag, off by default, and enabled only for the internal team.", "disfluent": "The feature is behind a flag comma off by default comma and enabled only for the internal team.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "flag", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "default", "words": [], "closing": ""}]} +{"reference": "Given the timing, the risk, and how LITTLE we know, I would rather wait a week.", "disfluent": "Given the timing comma the risk comma and how all caps little we know comma I would rather wait a week.", "operations": ["spoken:comma", "spoken:comma", "spoken:all-caps", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "timing", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "risk", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["little"], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "know", "words": [], "closing": ""}]} +{"reference": "The parser is fine, the writer is fine, and the thing between them loses a byte.", "disfluent": "The parser is fine comma the writer is fine comma and the thing between them loses a byte.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fine", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fine", "words": [], "closing": ""}]} +{"reference": "Whatever we do, do it before the freeze, BECAUSE after that nothing lands.", "disfluent": "Whatever we do comma do it before the freeze comma all caps because after that nothing lands.", "operations": ["spoken:comma", "spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "do", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "freeze", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["because"], "closing": ""}]} +{"reference": "She asked for the numbers, the caveats, and a recommendation on one page.", "disfluent": "She asked for the numbers comma the caveats comma and a recommendation on one page.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "numbers", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "caveats", "words": [], "closing": ""}]} +{"reference": "The queue drained overnight. Are we confident it will hold under Monday traffic?", "disfluent": "The queue drained overnight period are we confident it will hold under Monday traffic?", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "overnight", "words": [], "closing": ""}]} +{"reference": "I looked at the trace. Why is the second request slower than the first one?", "disfluent": "I looked at the trace period why is the second request slower than the first one?", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "trace", "words": [], "closing": ""}]} +{"reference": "We fixed the leak. Did anyone CHECK WHETHER the fix survives a clean install?", "disfluent": "We fixed the leak period did anyone caps on check whether caps off the fix survives a clean install?", "operations": ["spoken:period", "spoken:caps-on"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "leak", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["check", "whether"], "closing": "caps off"}]} +{"reference": "The demo went well. Can you send the recording to the whole team this afternoon?", "disfluent": "The demo went well period can you send the recording to the whole team this afternoon?", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "well", "words": [], "closing": ""}]} +{"reference": "It FINALLY reproduced! The trick was launching from a read-only directory.", "disfluent": "It all caps finally reproduced exclamation point the trick was launching from a read-only directory.", "operations": ["spoken:all-caps", "spoken:exclamation-point"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["finally"], "closing": ""}, {"spoken": "exclamation point", "mark": "!", "kind": "mark", "anchor": "reproduced", "words": [], "closing": ""}]} +{"reference": "That is the third regression this week! We need the gate back on before Friday.", "disfluent": "That is the third regression this week exclamation mark we need the gate back on before Friday.", "operations": ["spoken:exclamation-mark"], "commands": [{"spoken": "exclamation mark", "mark": "!", "kind": "mark", "anchor": "week", "words": [], "closing": ""}]} +{"reference": "The numbers came in. Honestly, they are better than anything we projected.", "disfluent": "The numbers came in period honestly comma they are better than anything we projected.", "operations": ["spoken:period", "spoken:comma"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "in", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "honestly", "words": [], "closing": ""}]} +{"reference": "Everything is symbolicated now. You should be able to read the crash directly.", "disfluent": "Everything is symbolicated now period you should be able to read the crash directly.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "now", "words": [], "closing": ""}]} +{"reference": "Here is the plan: land the fix, cut a build, and HAND it to QA tomorrow morning.", "disfluent": "Here is the plan colon land the fix comma cut a build comma and all caps hand it to QA tomorrow morning.", "operations": ["spoken:colon", "spoken:comma", "spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "plan", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "fix", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "build", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["hand"], "closing": ""}]} +{"reference": "Two things are still open: the entitlement review and the store description.", "disfluent": "Two things are still open colon the entitlement review and the store description.", "operations": ["spoken:colon"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "open", "words": [], "closing": ""}]} +{"reference": "The cause was simple: we were reading the sample RATE FROM the wrong device.", "disfluent": "The cause was simple colon we were reading the sample caps on rate from caps off the wrong device.", "operations": ["spoken:colon", "spoken:caps-on"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "simple", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["rate", "from"], "closing": "caps off"}]} +{"reference": "It builds cleanly on my machine; it fails on the runner every single time.", "disfluent": "It builds cleanly on my machine semicolon it fails on the runner every single time.", "operations": ["spoken:semicolon"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "machine", "words": [], "closing": ""}]} +{"reference": "Ship the smaller change first; the REFACTOR can wait until after the release.", "disfluent": "Ship the smaller change first semicolon the all caps refactor can wait until after the release.", "operations": ["spoken:semicolon", "spoken:all-caps"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "first", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["refactor"], "closing": ""}]} +{"reference": "The tradeoff is straightforward: more ACCURACY for eighty milliseconds of latency.", "disfluent": "The tradeoff is straightforward colon more all caps accuracy for eighty milliseconds of latency.", "operations": ["spoken:colon", "spoken:all-caps"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "straightforward", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["accuracy"], "closing": ""}]} +{"reference": "Keep the interface as it is; only the storage layer actually needs to change.", "disfluent": "Keep the interface as it is semicolon only the storage layer actually needs to change.", "operations": ["spoken:semicolon"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "is", "words": [], "closing": ""}]} +{"reference": "One caveat: the migration is not reversible once the first write has landed.", "disfluent": "One caveat colon the migration is not reversible once the first write has landed.", "operations": ["spoken:colon"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "caveat", "words": [], "closing": ""}]} +{"reference": "Three people asked for this: two customers and one person on the support rota.", "disfluent": "Three people asked for this colon two customers and one person on the support rota.", "operations": ["spoken:colon"], "commands": [{"spoken": "colon", "mark": ":", "kind": "mark", "anchor": "this", "words": [], "closing": ""}]} +{"reference": "The rule is simple; if the test is flaky, fix it or delete it the same day.", "disfluent": "The rule is simple semicolon if the test is flaky comma fix it or delete it the same day.", "operations": ["spoken:semicolon", "spoken:comma"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "simple", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "flaky", "words": [], "closing": ""}]} +{"reference": "This is urgent, and the deadline is Friday, not the following Wednesday.", "disfluent": "This is urgent comma and the deadline is Friday comma not the following Wednesday.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "urgent", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "friday", "words": [], "closing": ""}]} +{"reference": "Do not deploy this today. The DATABASE MIGRATION has not been reviewed yet.", "disfluent": "Do not deploy this today period the caps on database migration caps off has not been reviewed yet.", "operations": ["spoken:period", "spoken:caps-on"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "today", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["database", "migration"], "closing": "caps off"}]} +{"reference": "The answer is no, and it will stay no UNTIL the security review is finished.", "disfluent": "The answer is no comma and it will stay no all caps until the security review is finished.", "operations": ["spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "no", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["until"], "closing": ""}]} +{"reference": "Never commit the signing key. Rotate it immediately if it ever reaches a log.", "disfluent": "Never commit the signing key period rotate it immediately if it ever reaches a log.", "operations": ["spoken:period"], "commands": [{"spoken": "period", "mark": ".", "kind": "mark", "anchor": "key", "words": [], "closing": ""}]} +{"reference": "Everything downstream depends on this table, so treat the schema as frozen.", "disfluent": "Everything downstream depends on this table comma so treat the schema as frozen.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "table", "words": [], "closing": ""}]} +{"reference": "The important part is the ordering, not the individual steps in the pipeline.", "disfluent": "The important part is the ordering comma not the individual steps in the pipeline.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "ordering", "words": [], "closing": ""}]} +{"reference": "Read the WHOLE thread before replying, because the decision changed twice.", "disfluent": "Read the all caps whole thread before replying comma because the decision changed twice.", "operations": ["spoken:all-caps", "spoken:comma"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["whole"], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "replying", "words": [], "closing": ""}]} +{"reference": "That number is wrong, and it has BEEN wrong in every deck since November.", "disfluent": "That number is wrong comma and it has all caps been wrong in every deck since November.", "operations": ["spoken:comma", "spoken:all-caps"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "wrong", "words": [], "closing": ""}, {"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["been"], "closing": ""}]} +{"reference": "Only the release manager can APPROVE this, and only after the gate is green.", "disfluent": "Only the release manager can all caps approve this comma and only after the gate is green.", "operations": ["spoken:all-caps", "spoken:comma"], "commands": [{"spoken": "all caps", "mark": "", "kind": "caps", "anchor": "", "words": ["approve"], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "this", "words": [], "closing": ""}]} +{"reference": "The critical path is the notarization, not the build, so start it early.", "disfluent": "The critical path is the notarization comma not the build comma so start it early.", "operations": ["spoken:comma", "spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "notarization", "words": [], "closing": ""}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "build", "words": [], "closing": ""}]} +{"reference": "We only support one grace period, so plan accordingly before the trial ends.", "disfluent": "We only support one grace period comma so plan accordingly before the trial ends.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "period", "words": [], "closing": ""}]} +{"reference": "Add a comma after the second clause, and the sentence reads much better.", "disfluent": "Add a comma after the second clause comma and the sentence reads much better.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "clause", "words": [], "closing": ""}]} +{"reference": "Every question mark in that survey was ambiguous, so we rewrote the whole form.", "disfluent": "Every question mark in that survey was ambiguous comma so we rewrote the whole form.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "ambiguous", "words": [], "closing": ""}]} +{"reference": "The billing period rolls over at midnight UTC, not at midnight local time.", "disfluent": "The billing period rolls over at midnight UTC comma not at midnight local time.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "utc", "words": [], "closing": ""}]} +{"reference": "Put a colon after the heading, and leave the rest of the LINE EXACTLY as it is.", "disfluent": "Put a colon after the heading comma and leave the rest of the caps on line exactly caps off as it is.", "operations": ["spoken:comma", "spoken:caps-on"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "heading", "words": [], "closing": ""}, {"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["line", "exactly"], "closing": "caps off"}]} +{"reference": "She used a full stop where the style GUIDE ASKS for a semicolon, which is minor.", "disfluent": "She used a full stop where the style caps on guide asks caps off for a semicolon comma which is minor.", "operations": ["spoken:caps-on", "spoken:comma"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["guide", "asks"], "closing": "caps off"}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "semicolon", "words": [], "closing": ""}]} +{"reference": "The legal team wants that warning in all caps, which our design system forbids.", "disfluent": "The legal team wants that warning in all caps comma which our design system forbids.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "caps", "words": [], "closing": ""}]} +{"reference": "That EXCLAMATION POINT READS as sarcasm, so please take it out of the release notes.", "disfluent": "That caps on exclamation point reads caps off as sarcasm comma so please take it out of the release notes.", "operations": ["spoken:caps-on", "spoken:comma"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["exclamation", "point", "reads"], "closing": "caps off"}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "sarcasm", "words": [], "closing": ""}]} +{"reference": "There is a dash missing from the second bullet, and a typo in the third one.", "disfluent": "There is a dash missing from the second bullet comma and a typo in the third one.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "bullet", "words": [], "closing": ""}]} +{"reference": "I said period, and it typed the word instead of the punctuation mark I wanted.", "disfluent": "I said period comma and it typed the word instead of the punctuation mark I wanted.", "operations": ["spoken:comma"], "commands": [{"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "period", "words": [], "closing": ""}]} +{"reference": "The Cretaceous PERIOD ENDED with an impact, which is roughly how the demo went.", "disfluent": "The Cretaceous caps on period ended caps off with an impact comma which is roughly how the demo went.", "operations": ["spoken:caps-on", "spoken:comma"], "commands": [{"spoken": "caps on", "mark": "", "kind": "caps", "anchor": "", "words": ["period", "ended"], "closing": "caps off"}, {"spoken": "comma", "mark": ",", "kind": "mark", "anchor": "impact", "words": [], "closing": ""}]} +{"reference": "Use a semicolon there; a comma is not strong enough to join those two clauses.", "disfluent": "Use a semicolon there semicolon a comma is not strong enough to join those two clauses.", "operations": ["spoken:semicolon"], "commands": [{"spoken": "semicolon", "mark": ";", "kind": "mark", "anchor": "there", "words": [], "closing": ""}]} diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index 622bb99..44ba381 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -138,11 +138,14 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: data.add_argument( "--source", default="nyra", - choices=(*corpus.SOURCES, "builtin"), + choices=(*corpus.SOURCES, "builtin", "punctuation"), help="which corpus to score against (default: nyra, the same hand-annotated " "Switchboard pairs with casing repaired, so the formatting axis is live and " "--metric blend scores two axes instead of degrading to content). builtin is a " - "bundled sample made disfluent by the injector, for the offline path", + "bundled sample made disfluent by the injector, for the offline path. punctuation is " + "a second bundled sample written for --punctuation-only: every row carries an " + "internal mark, because a dictated mark on the last word is one any instruction " + "would produce unprompted", ) data.add_argument( "--limit", @@ -184,11 +187,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: spoken.add_argument( "--spoken-punctuation", type=float, - default=0.0, + default=None, metavar="RATE", help="0..1; speak this share of the punctuation the reference licenses, turning " 'each mark into the words a dictation user says out loud ("comma", "question ' - 'mark", "all caps"). 0 (default) is off. Unlike --severity this applies to a ' + 'mark", "all caps"). Defaults to 0 — off — or to 1.0 under --punctuation-only. Unlike --severity this applies to a ' "PAIRED source too: the marks come from the corpus's own clean side, so the " "disfluencies stay the ones annotators marked by hand and only the punctuation " "task is synthetic. Below 1 on purpose — a corpus with no real marks left in the " @@ -204,6 +207,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "corpus. Only this operator edits the reference, because a target with no " "uppercase in it cannot pose the task. Ignored unless --spoken-punctuation is on", ) + spoken.add_argument( + "--punctuation-only", + action="store_true", + help="score ONLY the punctuation commands. The input then differs from the target " + "by nothing but the commands: no disfluencies are injected, and a paired source's " + "verbatim side is discarded in favour of its clean one, so nothing a cleanup does " + "about hesitation can move the number. Every row is guaranteed at least one " + "command and rows the injector cannot plant one in are dropped. Implies " + "--spoken-punctuation 1.0 unless you pass a rate; the utterance-final mark is " + "never spoken even then, so each row still contains real punctuation the " + "instruction has to leave alone", + ) evaluation = parser.add_argument_group("evaluation") evaluation.add_argument( @@ -406,7 +421,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "LDC-licensed Switchboard transcripts and should stay out of version control", ) parser.add_argument("--out", default=None, help="write the full results as JSON to this path") - return parser.parse_args(argv) + args = parser.parse_args(argv) + # `--punctuation-only` without a rate would load a corpus with no commands in it, + # which is the opposite of what it asks for. Resolved here rather than in `main` so + # anything reading `parse_args` sees the same coupling. + if args.spoken_punctuation is None: + args.spoken_punctuation = 1.0 if args.punctuation_only else 0.0 + return args #: Axis each corpus shape selects on when `--metric` is not given. @@ -632,6 +653,7 @@ def main(argv: list[str] | None = None) -> int: strip_formatting=args.strip_formatting, spoken_punctuation_rate=args.spoken_punctuation, spoken_caps_rate=args.spoken_caps_rate, + punctuation_only=args.punctuation_only, ) print(f"Loaded {len(loaded)} pairs from {loaded.source}: {loaded.detail}") print(f"{loaded.disfluent_fraction:.0%} of pairs differ from their target") diff --git a/evals/dictation-prompt/spoken_punctuation.py b/evals/dictation-prompt/spoken_punctuation.py index 1628326..48f0a7f 100644 --- a/evals/dictation-prompt/spoken_punctuation.py +++ b/evals/dictation-prompt/spoken_punctuation.py @@ -124,6 +124,14 @@ class Command: anchor: str = "" #: Normalized words a casing command uppercases; empty for a mark. words: tuple[str, ...] = () + #: The words that close a bracketed command (`caps off`), empty for every other kind. + #: + #: Held because the closing marker is part of the command's vocabulary and nothing + #: else records it: `spoken` is only the opener. Left out, the word "off" sat in the + #: input outside `command_words`, so a cleanup that left `caps off` standing was + #: charged one ordinary insertion for it instead of `metrics.COMMAND_WEIGHT` — the + #: same command, priced two ways depending on which half survived. + closing: str = "" def to_json(self) -> dict: """The command as plain JSON, for `corpus.dump_jsonl`. @@ -138,6 +146,7 @@ def to_json(self) -> dict: "kind": self.kind, "anchor": self.anchor, "words": list(self.words), + "closing": self.closing, } @classmethod @@ -150,6 +159,7 @@ def from_json(cls, row: dict) -> Command: kind=row["kind"], anchor=row.get("anchor", ""), words=tuple(row.get("words", ())), + closing=row.get("closing", ""), ) @property @@ -165,7 +175,7 @@ def spoken_words(self) -> tuple[str, ...]: not read "question mark" as an abandoned phrase and charge it five errors a word. See `metrics._is_abandoned`. """ - return tuple(metrics.normalize_text(self.spoken).split()) + return tuple(metrics.normalize_text(f"{self.spoken} {self.closing}").split()) @property def literal(self) -> tuple[str, ...]: @@ -175,8 +185,8 @@ def literal(self) -> tuple[str, ...]: (`("all", "caps", "urgent")`), so an instruction that happened to use the word elsewhere is not charged for it. """ - spoken = self.spoken_words - return (self.anchor, *spoken) if self.kind == "mark" else (*spoken, *self.words) + opener = tuple(metrics.normalize_text(self.spoken).split()) + return (self.anchor, *opener) if self.kind == "mark" else (*opener, *self.words) def _tokens(text: str) -> list[tuple[int, str]]: @@ -322,6 +332,7 @@ def inject( seed: int, rate: float = 0.8, caps_rate: float = 0.25, + require: bool = False, ) -> tuple[str, str, tuple[Command, ...]]: """Speak some of this pair's punctuation. Returns `(reference, input, commands)`. @@ -335,6 +346,23 @@ def inject( because a user shouting twice in one dictated sentence is not the common case and two commands in a 15-word utterance would make the operator most of the corpus. + The **last token is never spoken**, and that exclusion is most of what makes a command + a test of anything. A dictated "period" on the final word asks for a mark the model + would produce unprompted — any instruction that says "restore punctuation" ends a + sentence with a full stop — so obeying the command and ignoring it look identical, and + the row scores the same either way. Those marks were 55% of what the bundled corpus + licensed and 37% of nyra's. What survives is discriminative by construction: a + mid-utterance terminal mark forces a sentence split *and* the following capital, an + internal comma forces a placement inside the clause, and ALL CAPS forces uppercase, + which nothing produces by default. + + `require` guarantees at least one command whenever the pair licenses one, by speaking + a randomly chosen mark if the per-mark draws happened to select none. For a corpus + whose whole subject is punctuation commands, a row carrying none is not a hard example + — it is a row measuring nothing, diluting every mean it appears in. Off by default, + because on a mixed corpus a row where the speaker punctuated normally is a legitimate + negative: the instruction has to leave existing marks alone. + The reference comes back **unchanged** unless a casing command was planted, which is the invariant that lets this run over a real paired corpus at all. """ @@ -350,17 +378,35 @@ def inject( caps = _pick_caps_run(reference, disfluent, rng) if rng.random() < caps_rate else None caps_indices = frozenset(caps[1]) if caps else frozenset() + # Which marks get spoken is settled before anything is emitted, because `require` + # is a statement about the row as a whole: "none were selected" is only knowable + # once every draw has been made. + tokens = disfluent.split() + convertible = [ + index + for index, raw in enumerate(tokens) + if index not in caps_indices + and index != len(tokens) - 1 + and (split := _split_mark(raw)) + and (metrics.normalize_text(split[0]), split[1]) in licensed + ] + speaking = {index for index in convertible if rng.random() < rate} + if require and convertible and not speaking: + speaking = {rng.choice(convertible)} + out: list[str] = [] lowercase_next = False - for index, raw in enumerate(disfluent.split()): + for index, raw in enumerate(tokens): if caps and index == caps[1][0]: - out.append(CAPS_WORD if len(caps[2]) == 1 else CAPS_ON) + bracketed = len(caps[2]) > 1 + out.append(CAPS_ON if bracketed else CAPS_WORD) commands.append( Command( - spoken=CAPS_WORD if len(caps[2]) == 1 else CAPS_ON, + spoken=CAPS_ON if bracketed else CAPS_WORD, mark="", kind="caps", words=caps[2], + closing=CAPS_OFF if bracketed else "", ) ) @@ -377,12 +423,10 @@ def inject( # guaranteed cannot apply to these tokens. if index in caps_indices: out.append(token.lower()) - elif ( - (split := _split_mark(token)) - and (metrics.normalize_text(split[0]), split[1]) in licensed - and rng.random() < rate - ): - stem, mark = split + elif index in speaking: + # `convertible` already checked the shape and the licence; the only thing + # that can have changed `token` since is a leading capital. + stem, mark = token[:-1], token[-1] spoken = _speak(mark, rng) out.extend((stem, spoken)) commands.append( @@ -459,6 +503,53 @@ def outcome(command: Command, hypothesis: str) -> str: return "missing" +def undo(command: Command, reference: str) -> str: + """The reference with this one command's effect taken back out. + + The mark it produced is removed and the capital that followed it lowered; a casing + command's run is lowercased. What is left is the text a cleanup would emit if it had + dropped the command silently — the honest comparison for asking what obeying it buys. + """ + tokens = reference.split() + if command.kind == "mark": + for i, raw in enumerate(tokens): + if raw.endswith(command.mark) and metrics.normalize_text(raw[:-1]) == command.anchor: + tokens[i] = raw[:-1] + if command.mark in TERMINAL_MARKS and i + 1 < len(tokens): + following = tokens[i + 1] + if not _keeps_its_capital(following): + tokens[i + 1] = following[:1].lower() + following[1:] + break + return " ".join(tokens) + + words = [w for _, w in _tokens(reference)] + index = [i for i, _ in _tokens(reference)] + span = len(command.words) + for k in range(len(words) - span + 1): + if tuple(words[k : k + span]) == command.words: + for j in index[k : k + span]: + tokens[j] = tokens[j].lower() + break + return " ".join(tokens) + + +def effect(command: Command, reference: str) -> int: + """Format tokens that obeying `command` fixes — how much getting it right is worth. + + The marginal value of this one command: the reference against the reference with only + this command's effect undone. Zero would mean the command is decoration — the same + output scores the same whether the model understood it or not — and a corpus of those + measures nothing however many rows it has. + + Nothing here can be zero, which is what the exclusions in `inject` buy: a + mid-utterance terminal mark is worth 2 (the mark, and the capital behind it), an + internal mark 1, and a casing command one per word. It is checked rather than filtered + on for that reason — a zero means an invariant broke, not that the row is unusable. + """ + diff = metrics.align(metrics.surface(reference), metrics.surface(undo(command, reference))) + return diff.substitutions + diff.deletions + diff.insertions + + def tally(pairs) -> dict[str, float]: """Command outcomes over `(commands, hypothesis)` pairs, as shares of the total. diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index ff8f2e6..d33003c 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -1463,6 +1463,22 @@ def spoken_pair(**kwargs): return spoken_punctuation.inject(SPOKEN_REFERENCE, SPOKEN_DISFLUENT, **kwargs) +# A pair whose one convertible mark is internal, so exactly one command is planted and the +# spoken side doubles as the "left the command in" hypothesis. Single-sentence fixtures no +# longer work here: `inject` will not speak an utterance-final mark, because that is the +# one mark any instruction produces unprompted. +ONE_COMMAND_TEXT = "Is it ready? Let me know either way." + + +def one_command(text: str = ONE_COMMAND_TEXT): + """`(reference, spoken input, the single command)`.""" + reference, spoken, commands = spoken_punctuation.inject( + text, text, seed=1, rate=1.0, caps_rate=0.0 + ) + (command,) = commands + return reference, spoken, command + + def test_a_perfect_cleanup_is_still_the_reference_exactly(): """The invariant the whole module rests on: the planted task is achievable. @@ -1521,7 +1537,9 @@ def test_a_spoken_terminal_mark_takes_its_sentence_capital_with_it(): ) assert "today period nobody" in disfluent assert reference == "We shipped it today. Nobody noticed." - assert [c.mark for c in commands] == [".", "."] + # One command, not two: the second period ends the utterance and is left alone. + assert [c.mark for c in commands] == ["."] + assert disfluent.endswith("noticed.") def test_a_name_after_a_spoken_mark_is_lowercased_like_any_other_word(): @@ -1535,7 +1553,8 @@ def test_a_name_after_a_spoken_mark_is_lowercased_like_any_other_word(): following word appears capitalized mid-sentence somewhere in the corpus. """ _, disfluent, _ = spoken_pair(seed=5, rate=1.0, caps_rate=0.0) - assert "today period monday" in disfluent + assert "monday" in disfluent.split() + assert "Monday" not in disfluent # And the target still asks for the capital, so nothing about the task got easier. assert "today. Monday" in SPOKEN_REFERENCE @@ -1620,12 +1639,9 @@ def test_a_leftover_command_word_costs_its_own_weight(): """ # Long enough that the charge stays inside WER 1, where `1 - WER` is linear and the # error count can be read back off the score. Past that `from_error_rate` decays. - clean = "Can you send me the numbers before the review meeting tomorrow?" - reference, disfluent, (command,) = spoken_punctuation.inject( - clean, clean, seed=1, rate=1.0, caps_rate=0.0 - ) + reference, spoken, command = one_command() words = frozenset(command.spoken_words) - left_in = metrics.score(reference, clean[:-1] + " question mark", disfluent, words) + left_in = metrics.score(reference, spoken, spoken, words) charged = (1.0 - left_in.content) * left_in.content_alignment.reference_length assert charged == pytest.approx(metrics.COMMAND_WEIGHT * len(command.spoken_words)) assert left_in.uncorrected_commands == ("mark", "question") @@ -1637,12 +1653,10 @@ def test_leaving_a_command_in_is_clearly_worse_than_dropping_it(): `Is it ready` for `Is it ready?` is a missing mark. `Is it ready question mark` puts two words in the reader's document that the speaker never meant to write. """ - reference, disfluent, (command,) = spoken_punctuation.inject( - "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 - ) + reference, spoken, command = one_command() words = frozenset(command.spoken_words) - dropped = metrics.score(reference, "Is it ready", disfluent, words) - left_in = metrics.score(reference, "Is it ready question mark", disfluent, words) + dropped = metrics.score(reference, spoken_punctuation.undo(command, reference), spoken, words) + left_in = metrics.score(reference, spoken, spoken, words) assert left_in.format < dropped.format lost = dropped.format - left_in.format assert lost > (1.0 - dropped.format), "leaving it in must cost more than the mark alone" @@ -1680,12 +1694,10 @@ def test_the_command_weight_is_neutral_on_a_corpus_that_plants_nothing(): def test_feedback_names_a_leftover_command_apart_from_a_disfluency_and_says_what_it_cost(): """Calling "comma" a disfluency points the reflector at the wrong rule: it is not a hesitation the speaker made, it is an instruction they gave.""" - reference, disfluent, (command,) = spoken_punctuation.inject( - "Is it ready?", "um is it ready?", seed=1, rate=1.0, caps_rate=0.0 - ) + reference, spoken, command = one_command() words = frozenset(command.spoken_words) - scored = metrics.score(reference, "Um is it ready question mark", disfluent, words) - text = metrics.feedback(reference, disfluent, scored) + scored = metrics.score(reference, "Um " + spoken, "um " + spoken, words) + text = metrics.feedback(reference, "um " + spoken, scored) assert "dictated punctuation commands" in text assert f"{metrics.COMMAND_WEIGHT:g} errors" in text # The filler was in the input, so it is a leftover — reported, and separately. @@ -1699,15 +1711,12 @@ def test_the_note_and_the_feedback_do_not_both_report_a_leftover_command(): without its cost is half a fact. The note owns what metrics cannot see: a command whose words are gone and whose mark never appeared. """ - reference, disfluent, (command,) = spoken_punctuation.inject( - "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 - ) - hypothesis = "Is it ready question mark" - note = spoken_punctuation.feedback_note((command,), hypothesis) - assert note == "" - assert "left in" not in note + reference, spoken, command = one_command() + assert spoken_punctuation.feedback_note((command,), spoken) == "" # Dropped, on the other hand, is invisible to metrics on the content axis. - dropped = spoken_punctuation.feedback_note((command,), "Is it ready") + dropped = spoken_punctuation.feedback_note( + (command,), spoken_punctuation.undo(command, reference) + ) assert "did not carry out" in dropped @@ -1722,13 +1731,11 @@ def test_the_exemption_is_neutral_on_a_corpus_that_plants_nothing(): def test_the_utterance_hands_its_own_planted_commands_to_the_scorer(): """`Utterance.scored` is the one place that knows both sides and what was planted.""" - reference, disfluent, commands = spoken_punctuation.inject( - "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 - ) - utterance = corpus.Utterance(reference=reference, disfluent=disfluent, commands=commands) + reference, spoken, command = one_command() + utterance = corpus.Utterance(reference=reference, disfluent=spoken, commands=(command,)) assert utterance.command_words == frozenset({"question", "mark"}) - assert utterance.scored(disfluent) == metrics.score( - reference, disfluent, disfluent, utterance.command_words + assert utterance.scored(spoken) == metrics.score( + reference, spoken, spoken, utterance.command_words ) @@ -1742,31 +1749,27 @@ def test_the_reported_false_start_fraction_is_the_charged_one(): def test_a_command_left_in_is_told_apart_from_a_command_dropped(): - reference, disfluent, commands = spoken_punctuation.inject( - "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 + reference, spoken, command = one_command() + assert spoken_punctuation.outcome(command, reference) == "converted" + assert spoken_punctuation.outcome(command, spoken) == "literal" + assert ( + spoken_punctuation.outcome(command, spoken_punctuation.undo(command, reference)) + == "missing" ) - (command,) = commands - assert spoken_punctuation.outcome(command, "Is it ready?") == "converted" - assert spoken_punctuation.outcome(command, "Is it ready question mark") == "literal" - assert spoken_punctuation.outcome(command, "Is it ready") == "missing" def test_conversion_is_credited_on_the_anchor_not_on_any_mark_anywhere(): """Otherwise punctuating something else entirely reads as obeying the command.""" - _, _, (command,) = spoken_punctuation.inject( - "Is it ready?", "is it ready?", seed=1, rate=1.0, caps_rate=0.0 - ) - assert spoken_punctuation.outcome(command, "Is it? ready") == "missing" + _, _, command = one_command() + assert spoken_punctuation.outcome(command, "Is it? ready let me know either way") == "missing" def test_the_literal_match_is_anchored_so_an_ordinary_word_is_not_charged(): """ "the Cretaceous period was long" is content, not a command left in.""" - _, _, (command,) = spoken_punctuation.inject( - "It ended then.", "it ended then.", seed=1, rate=1.0, caps_rate=0.0 - ) + reference, _, command = one_command("It ended then. We moved on to the next thing.") assert command.anchor == "then" assert ( - spoken_punctuation.outcome(command, "The Cretaceous period was long. It ended then.") + spoken_punctuation.outcome(command, "The Cretaceous period was long. " + reference) == "converted" ) @@ -1850,13 +1853,18 @@ def test_a_corpus_without_the_flag_plants_nothing(): # fails this suite instead of quietly invalidating a dataset still sitting in the tree. FROZEN_DATASET = pathlib.Path(__file__).parent / "data" / "spoken-punctuation.jsonl" FROZEN_ARGV = ( - "--source builtin --limit 200 --spoken-punctuation 0.8 --severity 0.35 --seed 7" + "--source punctuation --punctuation-only --limit 200 --spoken-caps-rate 0.35 --seed 7" ).split() def frozen_corpus(): return corpus.load( - source="builtin", limit=200, spoken_punctuation_rate=0.8, severity=0.35, seed=7 + source="punctuation", + limit=200, + spoken_punctuation_rate=1.0, + spoken_caps_rate=0.35, + punctuation_only=True, + seed=7, ) @@ -1892,7 +1900,7 @@ def test_the_frozen_dataset_round_trips_through_the_loader(): def test_a_reloaded_dataset_is_not_injected_over_again(): """The flag is often still on the command line; injecting twice would speak marks that are no longer there and uppercase an ALL CAPS run a second time.""" - reloaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200, spoken_punctuation_rate=0.8) + reloaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200, spoken_punctuation_rate=1.0) generated = frozen_corpus() assert [u.disfluent for u in reloaded.utterances] == [u.disfluent for u in generated.utterances] @@ -1927,8 +1935,8 @@ def test_the_frozen_dataset_can_charge_over_conversion_where_nyra_cannot(): instruction that rewrites "the Jurassic period was long" into "the Jurassic. was long". These sentences were written to. """ - references = [u.reference for u in corpus.load(source="builtin", limit=200).utterances] - assert spoken_punctuation.literal_use_fraction(references) > 0.1 + assert spoken_punctuation.literal_use_fraction(corpus.PUNCTUATION_SAMPLE) > 0.1 + assert spoken_punctuation.literal_use_fraction(corpus.BUILTIN_SAMPLE) > 0.1 def test_the_dump_flag_works_without_a_key_or_a_network(tmp_path): @@ -1983,6 +1991,160 @@ def test_the_bundled_sample_exercises_every_mark_in_the_vocabulary(): assert set(spoken_punctuation.SPOKEN_FORMS) <= licensed +def test_an_utterance_final_mark_is_never_spoken(): + """It is the one command that tests nothing. + + A dictated "period" on the last word asks for a mark any instruction would produce + unprompted, so obeying it and ignoring it score the same. Those were 55% of what the + bundled corpus licensed. + """ + text = "We shipped it today. Nobody noticed." + _, spoken, commands = spoken_punctuation.inject(text, text, seed=1, rate=1.0, caps_rate=0.0) + assert spoken.endswith("noticed.") + assert len(commands) == 1 + + +def test_a_row_whose_only_mark_is_final_yields_no_command_at_all(): + """Which is why `--punctuation-only` drops such rows rather than keeping them.""" + text = "The build failed over the weekend and nobody noticed." + _, spoken, commands = spoken_punctuation.inject( + text, text, seed=1, rate=1.0, caps_rate=0.0, require=True + ) + assert commands == () + assert spoken == text + + +def test_require_guarantees_a_command_wherever_one_is_possible(): + """A row carrying none is not a hard example on this task; it measures nothing.""" + text = "We shipped it today. Monday was quiet, so nobody noticed." + without = sum( + not spoken_punctuation.inject(text, text, seed=n, rate=0.15, caps_rate=0.0)[2] + for n in range(200) + ) + with_require = sum( + not spoken_punctuation.inject(text, text, seed=n, rate=0.15, caps_rate=0.0, require=True)[2] + for n in range(200) + ) + assert without > 50 + assert with_require == 0 + + +def test_every_planted_command_changes_something(): + """The invariant that makes the corpus worth scoring at all. + + A command worth 0 is decoration — the same output scores the same whether the model + understood it or not. Mid-utterance terminal marks are worth 2 (the mark, and the + capital behind it), internal marks 1, casing commands one per word. + """ + loaded = corpus.load( + source="punctuation", + limit=200, + spoken_punctuation_rate=1.0, + spoken_caps_rate=0.35, + punctuation_only=True, + ) + worth = [ + spoken_punctuation.effect(command, utterance.reference) + for utterance in loaded.utterances + for command in utterance.commands + ] + assert worth + assert min(worth) >= 1, "a command that changes nothing measures nothing" + assert max(worth) >= 2, "no command forces a capital, so the split is untested" + + +def test_undoing_a_command_is_what_ignoring_it_would_produce(): + reference, _, command = one_command() + assert spoken_punctuation.undo(command, reference) == "Is it ready let me know either way." + + +def test_punctuation_only_leaves_nothing_but_commands_between_input_and_target(): + """The whole point of the mode: no score can be moved by disfluency removal. + + On a paired source that means discarding the verbatim side, so `nyra`'s hand-annotated + disfluencies are not in the input at all. + """ + loaded = corpus.load( + source="punctuation", limit=200, spoken_punctuation_rate=1.0, punctuation_only=True + ) + assert len(loaded) > 50 + for utterance in loaded.utterances: + assert utterance.commands, "every row carries at least one command" + assert all(op.startswith("spoken:") for op in utterance.operations) + # Stated on the alignment rather than by reconstructing the input: on the content + # axis the input must be the target plus command words and nothing else — no word + # substituted, none dropped, and every addition a command. Marks and casing are + # what the format axis sees, and they are the task. + diff = metrics.align( + metrics.normalize(utterance.reference), metrics.normalize(utterance.disfluent) + ) + assert diff.substitutions == 0, utterance.disfluent + assert diff.deletions == 0, utterance.disfluent + assert all(word in utterance.command_words for word in diff.inserted), utterance.disfluent + assert corpus.false_start_fraction(list(loaded.utterances)) == 0.0 + assert loaded.detail["punctuation_only"] is True + + +def test_punctuation_only_scores_a_perfect_cleanup_perfectly(): + loaded = corpus.load( + source="punctuation", limit=200, spoken_punctuation_rate=1.0, punctuation_only=True + ) + scored = metrics.mean([u.scored(u.reference) for u in loaded.utterances]) + assert scored["format"] == 1.0 + assert scored["content"] == 1.0 + + +def test_punctuation_only_discards_a_paired_source_s_verbatim_side(monkeypatch): + """`nyra`'s input side is its disfluent one; the mode has to reach past it.""" + rows = [ + { + "verbatim_transcript": "um we shipped it today. uh Monday was quiet.", + "intended_transcript": "We shipped it today. Monday was quiet, so nobody noticed.", + } + ] + monkeypatch.setattr(corpus, "_rows_via_api", lambda *a, **k: iter(rows)) + loaded = corpus.load( + source="nyra", + limit=1, + spoken_punctuation_rate=1.0, + spoken_caps_rate=0.0, + punctuation_only=True, + ) + (utterance,) = loaded.utterances + assert "um" not in metrics.normalize(utterance.disfluent) + assert "uh" not in metrics.normalize(utterance.disfluent) + assert utterance.commands + + +def test_punctuation_only_implies_a_rate_so_the_corpus_is_not_empty(): + """Without one the mode would load a corpus with no commands in it, which is the + opposite of what it asks for.""" + assert cli.parse_args(["--punctuation-only"]).spoken_punctuation == 1.0 + assert cli.parse_args([]).spoken_punctuation == 0.0 + # An explicit rate still wins. + assert ( + cli.parse_args(["--punctuation-only", "--spoken-punctuation", "0.5"]).spoken_punctuation + == 0.5 + ) + + +def test_the_punctuation_sample_is_all_internal_marks(): + """Its reason for existing apart from BUILTIN_SAMPLE, 55% of whose marks are final.""" + for text in corpus.PUNCTUATION_SAMPLE: + tokens = text.split() + internal = [ + i + for i, raw in enumerate(tokens[:-1]) + if spoken_punctuation._split_mark(raw) + and ( + metrics.normalize_text(raw[:-1]), + raw[-1], + ) + in spoken_punctuation.licensed_marks(text) + ] + assert internal, f"no internal mark to speak: {text!r}" + + def test_the_punctuation_candidates_join_the_table_only_when_the_corpus_asks(): """Keyed on the corpus, not the flag, so a frozen dataset read with --jsonl behaves the same as the corpus it was dumped from.""" From 9dcee7c3f78e909d037bd562a76ca7a26995c90e Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 18:47:25 -0700 Subject: [PATCH 4/8] feat: let a search resume a ranking instead of re-paying for it Both changes come from working out how to run --punctuation-only over the whole corpus, which is 3442 nyra rows -> 900 dev. `--candidates baseline` was silently ignored on a spoken run: resolve_candidates returned the full punctuation set whenever the corpus carried commands, so the flag was a lie exactly where it earns its keep. At 900 dev rows the default sweep is 7 x 900 = 6300 calls, and having already run it once, paying for it again to reach the search is the one thing anyone would reach for the flag to avoid. `--start` now takes any candidate name as well as prior-winner and best-candidate, validated against the run's own table before anything is spent. Together the two mean a search picks up where a ranking left off: --start punct-explicit --candidates baseline. The README's punctuation section now carries the arithmetic for four run shapes, from the 420-call committed fixture to the full 4000-row load, with a note that ~14% of rows drop because their only mark is utterance-final and that HF_TOKEN is needed for a load that size. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 84 +++++++++++++------ .../optimize_cleanup_prompt.py | 24 +++++- evals/dictation-prompt/test_eval.py | 25 ++++++ 3 files changed, 106 insertions(+), 27 deletions(-) diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 51afdda..30cef01 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -114,6 +114,42 @@ disfluency smoke corpus wants plain declaratives with room to inject hesitation punctuation corpus wants **internal** marks, and 37 of `BUILTIN_SAMPLE`'s 91 rows have none. Every row of `PUNCTUATION_SAMPLE` carries at least one, most carry several. +#### Running it on the whole corpus + +`nyra`'s train split is the large dataset, and `--punctuation-only` keeps ~86% of it — the +rest are rows whose only mark is their last one, which is not speakable. Set `HF_TOKEN` +before a load this size or the datasets-server will rate-limit it. + +```bash +# 3442 rows -> 1642 train / 450 optimizer valset / 900 dev / 450 test. +# Ranking first: 7 instructions x 900 dev + 2 x 450 test, about 7,200 calls. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ + --limit 4000 --optimizer none --num-threads 4 --out ranking.json + +# Then search from whichever won, without paying for that sweep twice. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ + --limit 4000 --start punct-explicit --candidates baseline --auto medium \ + --num-threads 4 --out search.json +``` + +The second command is the one worth understanding. A spoken run scores `BASELINE` plus +every punctuation candidate by default, which is right once and pure waste afterwards — at +900 dev rows that sweep is 6,300 calls to re-establish an ordering you already have. +`--candidates baseline` skips it and `--start NAME` seeds the search from the instruction +that actually won, so the paid work goes into the search instead of into re-ranking. + +Two cheaper shapes, when the full corpus is more than the question needs: + +```bash +# The committed fixture: 80 rows, ~420 calls, no download. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py \ + --jsonl evals/dictation-prompt/data/spoken-punctuation.jsonl --optimizer none + +# nyra at a resolution that still separates candidates, ~2,500 calls. +uv run evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ + --limit 900 --optimizer none --dev-fraction 300 --test-fraction 200 --num-threads 4 +``` + #### The two operators **Marks** are spoken on the **input side only**, and only where the reference **licenses** @@ -451,30 +487,30 @@ is `--model`. Its proposal prompts are multi-field, so they keep DSPy's marker p ## The knobs that matter -| Flag | Default | What it changes | -| ---------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | -| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | -| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | -| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | -| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | -| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | -| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | -| `--spoken-punctuation` | `0` (off), or `1.0` under `--punctuation-only` | Speak this share of the marks the reference licenses. Never the utterance-final one. | -| `--punctuation-only` | off | Score ONLY the commands: no disfluencies, and a paired source's verbatim side discarded. | -| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | -| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | -| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | -| `--start` | `prior-winner` | Which instruction GEPA evolves from — the compressed prior winner, or the best hand-written candidate. | -| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | -| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | -| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | -| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | -| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | -| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | -| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | -| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | -| `--seed` | `7` | Seeds injection and the train/dev/test split. | +| Flag | Default | What it changes | +| ---------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | +| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | +| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | +| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | +| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | +| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | +| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | +| `--spoken-punctuation` | `0` (off), or `1.0` under `--punctuation-only` | Speak this share of the marks the reference licenses. Never the utterance-final one. | +| `--punctuation-only` | off | Score ONLY the commands: no disfluencies, and a paired source's verbatim side discarded. | +| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | +| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | +| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | +| `--start` | `prior-winner` | Which instruction GEPA evolves from — `prior-winner`, `best-candidate`, or any candidate by name. | +| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | +| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | +| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | +| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | +| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | +| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | +| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | +| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | +| `--seed` | `7` | Seeds injection and the train/dev/test split. | Both optimizers run with few-shot demos disabled. `config.llm.instruction` is a single string the service applies in one pass, so an optimized program that depended on bundled examples diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index 44ba381..30f8e05 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -381,11 +381,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: search.add_argument( "--start", default="prior-winner", - choices=("prior-winner", "best-candidate"), + metavar="prior-winner|best-candidate|NAME", help="which instruction the optimizer starts from (default: prior-winner, the " "evolved instruction in candidates.py — it already scores well and only needs " "pruning under the character cap; best-candidate starts from whichever " - "hand-written candidate topped dev instead)", + "hand-written candidate topped dev instead). Any candidate name also works, which " + "is how a search picks up where a ranking run left off: --start punct-explicit " + "--candidates baseline skips a sweep you have already paid for", ) live_group = parser.add_argument_group("live verification (macOS, real endpoint)") @@ -526,6 +528,12 @@ def resolve_candidates( """ if args.candidates == "all": return list(CANDIDATES if table is None else table) + # Honoured on a spoken run too. It was not, which made the flag a lie exactly when it + # was worth using: on a large corpus the sweep is `len(scoring) x dev` calls, and + # having already run it once, paying for it again to reach the search is the one thing + # anyone would reach for this flag to avoid. + if args.candidates == "baseline": + return [BASELINE] if spoken: # BASELINE first so the table reads as "the bar, then the challengers". return [BASELINE, *SPOKEN_PUNCTUATION_CANDIDATES] @@ -664,6 +672,11 @@ def main(argv: list[str] | None = None) -> int: # for: a typo in candidates.py should not cost a paid sweep to discover. table = instruction_table(loaded) check_candidates(table) + if args.start not in ("prior-winner", "best-candidate") and args.start not in table: + raise SystemExit( + f"--start {args.start!r} is not a candidate this run scores. Available: " + + ", ".join(("prior-winner", "best-candidate", *table)) + ) train, dev, test = corpus.split( list(loaded.utterances), args.seed, args.dev_fraction, args.test_fraction @@ -757,7 +770,12 @@ def main(argv: list[str] | None = None) -> int: # sweep above and may legitimately win. Nothing to score separately, and no # need to keep it out of the selection: unlike the over-cap instruction this # replaced, shipping it is a real option. - seed_name = BASELINE if args.start == "prior-winner" else winner_name + if args.start == "prior-winner": + seed_name = BASELINE + elif args.start == "best-candidate": + seed_name = winner_name + else: + seed_name = args.start seed_instruction = table[seed_name] seed_scores = dict(dev_rows)[seed_name] diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index d33003c..c6a78ad 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -2207,6 +2207,31 @@ def test_a_spoken_punctuation_run_scores_the_bar_and_the_challengers(): assert set(scoring) < set(table) +def test_baseline_only_is_honoured_on_a_spoken_run(): + """It was silently ignored, which made the flag a lie exactly where it earns its keep: + on a large corpus the sweep is len(scoring) x dev calls.""" + loaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + scoring = cli.resolve_candidates( + cli.parse_args(["--candidates", "baseline"]), + cli.instruction_table(loaded), + spoken=True, + ) + assert scoring == [candidates.BASELINE] + + +def test_a_search_can_start_from_a_named_candidate(): + """So a search picks up where a ranking left off instead of re-paying for the sweep.""" + args = cli.parse_args(["--start", "punct-explicit", "--candidates", "baseline"]) + assert args.start == "punct-explicit" + assert cli.resolve_candidates(args, spoken=True) == [candidates.BASELINE] + + +def test_an_unknown_start_is_refused_before_anything_is_spent(): + with pytest.raises(SystemExit) as raised: + cli.main(["--source", "punctuation", "--punctuation-only", "--start", "nope", "--dry-run"]) + assert "not a candidate" in str(raised.value) + + def test_the_full_table_is_still_available_on_a_spoken_run(): scoring = cli.resolve_candidates( cli.parse_args(["--candidates", "all"]), From cff80bed94fb6965b35b140fe9d0a0abf2a57c1b Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 18:54:44 -0700 Subject: [PATCH 5/8] fix: wait out the datasets-server's intermittent 500s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RuntimeError: datasets-server returned HTTP 500 ... {"error":"Unexpected error."}` killed a 4000-row load. Probing six offsets of the same dataset minutes later, every page came back 200 and num_rows_total was 4458 — so the dataset was fine and the failure was a blip. `_fetch_page` retried 429 only, on the reasoning that every other status is a fact about the request that waiting will not change, and that a load which silently took four minutes to work around a broken upstream is harder to diagnose than one that said so. That was wrong twice over. It contradicted itself — the same docstring called the 500s intermittent — and the diagnosis argument is answered by printing each retry as it happens, not by abandoning the load. What it produced instead was a 45-page load discarding 30 fetched pages over one blip, *before the first model call*, which is precisely the failure the 429 backoff was written to prevent: a run set up to take hours dying at second zero. So the 5xx family is retried on the same bounded backoff, and a 4xx still raises at once — 403 and 404 really are facts about the request. The give-up message now says which kind it was and what to do about it, since "authenticate to raise the limit" is useless advice about a 500 and the opposite of what a caller needs to hear. Also fixes a bug in the new test's own helper, which is worth naming because it silently passed: `(sleeps or []).append(...)` appends to a throwaway list whenever `sleeps` is empty, which is exactly its state on the first call. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 9 +++ evals/dictation-prompt/corpus.py | 94 ++++++++++++++++++++--------- evals/dictation-prompt/test_eval.py | 73 ++++++++++++++++++++++ 3 files changed, 146 insertions(+), 30 deletions(-) diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 30cef01..2e1a1e2 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -442,6 +442,15 @@ short ones back to back — will start getting 429s. Authenticating lifts the li in the environment. Either works; the loader resolves a token the same way `huggingface_hub` does, so whichever one you reach for is the one it reads. +The server also returns `HTTP 500 {"error": "Unexpected error."}` now and again on a +dataset that is otherwise entirely healthy. Those are retried with the same bounded +backoff as a 429 — a 4000-row load is ~45 pages, and abandoning it over one bad page +throws away everything already fetched, before the first model call. Each retry prints, so +a slow load says why it was slow. A 4xx is raised at once instead: 403 and 404 are facts +about the request, and waiting does not change them. If a 500 survives all five attempts, +run it again before believing it — and use `--dump-corpus` once it loads, so a flaky server +cannot kill a later run. + [tok]: https://huggingface.co/settings/tokens The dry-run and the tests import nothing outside the standard library. That is structural, not diff --git a/evals/dictation-prompt/corpus.py b/evals/dictation-prompt/corpus.py index 00708bc..2543204 100644 --- a/evals/dictation-prompt/corpus.py +++ b/evals/dictation-prompt/corpus.py @@ -578,16 +578,27 @@ def hf_token() -> str | None: return None -#: Attempts per page against a 429, and the base of the backoff between them. +#: Attempts per page against a retryable failure, and the base of the backoff between +#: them. #: -#: The rows API rate-limits by volume, not just by anonymity: a 4000-row load is ~40 -#: pages, and a token raises the ceiling without removing it. Failing the whole load on -#: one 429 wastes the pages already fetched and, worse, fails *before* the first model -#: call — so a run set up to take an hour dies at second zero. Bounded, because a rate -#: limit that has not lifted in a minute of waiting is a quota rather than a burst, and -#: the error already says what to do about it. -_RATE_LIMIT_ATTEMPTS = 5 -_RATE_LIMIT_BACKOFF = 4.0 +#: Two kinds are retryable, for the same reason. The rows API rate-limits by volume, not +#: just by anonymity: a 4000-row load is ~45 pages, and a token raises the ceiling without +#: removing it. And the server returns a 500 `{"error": "Unexpected error."}` now and +#: again on a dataset that is otherwise entirely healthy — probed across six offsets of +#: `nyra` minutes after one, every page came back 200. +#: +#: Either way, failing the whole load on one bad page wastes every page already fetched +#: and, worse, fails *before the first model call* — so a run set up to take hours dies at +#: second zero. Bounded, because a rate limit that has not lifted in a minute of waiting +#: is a quota rather than a burst, and a 500 that persists that long is an outage; the +#: error says which it was and what to do about it. +_RETRY_ATTEMPTS = 5 +_RETRY_BACKOFF = 4.0 + +#: Statuses worth waiting out: the rate limit, and the 5xx family. Everything else — 401, +#: 403, 404, 422 — is a fact about the request or the dataset that no amount of waiting +#: changes, and surfacing it immediately is the whole point. +_RETRYABLE = frozenset({429, 500, 502, 503, 504}) def _rows_via_api(source: Source, limit: int): @@ -620,16 +631,44 @@ def _rows_via_api(source: Source, limit: int): offset += page -def _fetch_page(url: str, headers: dict, source: Source, token: str | None) -> dict: - """One page of rows, waiting out a rate limit rather than failing the load. +def _retry_advice(code: int, token: str | None) -> str: + """What to do about a failure that survived every retry.""" + if code != 429: + return ( + "The datasets-server does this intermittently on datasets that are otherwise " + "fine, so it is usually worth simply running again. If it persists, the " + "dataset's viewer on huggingface.co will say so. --dump-corpus freezes a " + "corpus once it has loaded, and --jsonl reads it back, so a flaky server " + "cannot kill a later run" + ) + if not token: + return ( + "Authenticating raises the limit: run `hf auth login`, or set HF_TOKEN to a " + "read token from https://huggingface.co/settings/tokens" + ) + return ( + "A token is already in use, so this is the authenticated ceiling — wait a few " + "minutes, or lower --limit" + ) + - Retries only 429. Every other HTTP status is a fact about the request or the - dataset that waiting will not change, and the 500s the server intermittently - returns are worth surfacing rather than papering over — a load that silently took - four minutes to work around a broken upstream is harder to diagnose than one that - said so. +def _fetch_page(url: str, headers: dict, source: Source, token: str | None) -> dict: + """One page of rows, waiting out a transient failure rather than failing the load. + + Retries `_RETRYABLE` — the rate limit and the 5xx family — and nothing else. A 404 or + a 403 is a fact about the request that waiting does not change, so it is raised at + once. + + The 500s were **not** retried at first, on the reasoning that a load which silently + took four minutes to work around a broken upstream is harder to diagnose than one that + said so. That was wrong twice over. It is self-contradicting — the same docstring + called them intermittent — and the diagnosis argument is answered by printing each + retry as it happens rather than by abandoning the load. What it actually produced was + a 45-page load discarding 30 fetched pages over one `{"error": "Unexpected error."}`, + before a single model call, on a dataset that answered 200 at every offset minutes + later. """ - for attempt in range(1, _RATE_LIMIT_ATTEMPTS + 1): + for attempt in range(1, _RETRY_ATTEMPTS + 1): try: with urllib.request.urlopen( # noqa: S310 — fixed https endpoint urllib.request.Request(url, headers=headers), timeout=60 @@ -637,24 +676,19 @@ def _fetch_page(url: str, headers: dict, source: Source, token: str | None) -> d return json.load(response) except urllib.error.HTTPError as error: body = error.read().decode("utf-8", "replace")[:400] - if error.code != 429: + if error.code not in _RETRYABLE: raise RuntimeError( f"datasets-server returned HTTP {error.code} for {source.dataset}: {body}" ) from error - if attempt == _RATE_LIMIT_ATTEMPTS: - advice = ( - "Authenticating raises the limit: run `hf auth login`, or set HF_TOKEN to a " - "read token from https://huggingface.co/settings/tokens" - if not token - else "A token is already in use, so this is the authenticated ceiling — wait " - "a few minutes, or lower --limit" - ) + if attempt == _RETRY_ATTEMPTS: raise RuntimeError( - f"rate limited by {ROWS_API} while reading {source.dataset}, still after " - f"{_RATE_LIMIT_ATTEMPTS} attempts. {advice}." + f"datasets-server returned HTTP {error.code} for {source.dataset}, still " + f"after {_RETRY_ATTEMPTS} attempts: {body}. " + f"{_retry_advice(error.code, token)}." ) from error - delay = _RATE_LIMIT_BACKOFF * 2 ** (attempt - 1) - print(f" rate limited; retrying in {delay:.0f}s ({attempt}/{_RATE_LIMIT_ATTEMPTS})") + delay = _RETRY_BACKOFF * 2 ** (attempt - 1) + reason = "rate limited" if error.code == 429 else f"HTTP {error.code}" + print(f" {reason}; retrying in {delay:.0f}s ({attempt}/{_RETRY_ATTEMPTS})") time.sleep(delay) except urllib.error.URLError as error: raise RuntimeError( diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index c6a78ad..d88246f 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -10,12 +10,14 @@ from __future__ import annotations +import contextlib import importlib.util import io import json import pathlib import re import sys +import urllib.error import pytest @@ -539,6 +541,77 @@ def test_the_paired_source_reads_both_repackaged_columns_and_undoes_its_markup() assert source.detag is corpus._detag_nyra +# -------------------------------------------------------------------------- +# datasets-server retries +# -------------------------------------------------------------------------- + + +def _http_error(code: int, body: str = '{"error":"Unexpected error."}'): + return urllib.error.HTTPError("https://x", code, "err", {}, io.BytesIO(body.encode())) + + +def _fetch_with(monkeypatch, statuses, sleeps=None): + """Drive `_fetch_page` through `statuses`, returning a page when one is 200.""" + codes = iter(statuses) + + def fake_urlopen(request, timeout=None): + code = next(codes) + if code != 200: + raise _http_error(code) + return contextlib.closing(io.BytesIO(b'{"rows": [{"row": {}}]}')) + + monkeypatch.setattr(corpus.urllib.request, "urlopen", fake_urlopen) + # `sleeps or []` would append to a throwaway list: an empty list is falsy, which is + # exactly the state it is in on the first call. + recorded = [] if sleeps is None else sleeps + monkeypatch.setattr(corpus.time, "sleep", recorded.append) + return corpus._fetch_page("https://x", {}, corpus.SOURCES["nyra"], "token") + + +def test_a_transient_server_error_is_waited_out_not_fatal(monkeypatch): + """A 45-page load used to discard 30 fetched pages over one "Unexpected error." — and + before the first model call, so a run set up to take hours died at second zero.""" + sleeps: list[float] = [] + assert _fetch_with(monkeypatch, [500, 200], sleeps) == {"rows": [{"row": {}}]} + assert sleeps == [corpus._RETRY_BACKOFF] + + +def test_every_retryable_status_is_retried(monkeypatch): + for code in sorted(corpus._RETRYABLE): + assert _fetch_with(monkeypatch, [code, 200]) == {"rows": [{"row": {}}]} + + +def test_a_request_level_failure_is_raised_at_once(monkeypatch): + """404 and 403 are facts about the request; waiting does not change them, and a retry + loop would only delay the message that says so.""" + for code in (401, 403, 404, 422): + with pytest.raises(RuntimeError) as raised: + _fetch_with(monkeypatch, [code, 200]) + assert str(code) in str(raised.value) + assert "attempts" not in str(raised.value) + + +def test_a_persistent_failure_gives_up_and_says_which_kind_it_was(monkeypatch): + with pytest.raises(RuntimeError) as raised: + _fetch_with(monkeypatch, [500] * corpus._RETRY_ATTEMPTS) + message = str(raised.value) + assert f"{corpus._RETRY_ATTEMPTS} attempts" in message + assert "intermittently" in message, "a 500 must not be described as a rate limit" + + with pytest.raises(RuntimeError) as raised: + _fetch_with(monkeypatch, [429] * corpus._RETRY_ATTEMPTS) + assert "ceiling" in str(raised.value) + + +def test_the_backoff_is_bounded_and_grows(monkeypatch): + sleeps: list[float] = [] + with pytest.raises(RuntimeError): + _fetch_with(monkeypatch, [503] * corpus._RETRY_ATTEMPTS, sleeps) + assert len(sleeps) == corpus._RETRY_ATTEMPTS - 1 + assert sleeps == sorted(sleeps) + assert sum(sleeps) < 120, "a wait this long is an outage, not a burst" + + # -------------------------------------------------------------------------- # Hugging Face auth # -------------------------------------------------------------------------- From 3a8a284530d8611f29234758107c46d7a1468396 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 19:31:48 -0700 Subject: [PATCH 6/8] fix: score the seed on dev when the sweep deliberately skipped it `--candidates baseline --start NAME` is the resume path added one commit ago, and it crashed: the seed's dev score was read with `dict(dev_rows)[seed_name]`, so skipping the sweep -- the entire point of `--candidates baseline` -- raised KeyError before the search began. It is the exact command the README recommends for a large-corpus run. The seed is scored instead of the lookup being made lenient, because the seed is what the search's result has to be measured against: "did evolving this beat starting from it" is the question the run answers, and a missing comparison would leave only the delta against BASELINE, which on this task is an instruction that has never heard of the task. One dev sweep, against the seven the full table costs. The dev winner is recomputed once the seed is in, so a search that fails falls back to the better of the seed and the bar rather than to whichever was measured first. Tested by driving `main` through the path with a fake `program` module injected into sys.modules, which keeps the suite free of DSPy -- the property two other tests here exist to protect -- while still exercising the CLI's control flow, where this mistake lived. Reverting the fix reproduces the KeyError under that test. Co-Authored-By: Claude Opus 5 (1M context) --- .../optimize_cleanup_prompt.py | 19 ++++- evals/dictation-prompt/test_eval.py | 71 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index 30f8e05..b1b93b9 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -777,7 +777,24 @@ def main(argv: list[str] | None = None) -> int: else: seed_name = args.start seed_instruction = table[seed_name] - seed_scores = dict(dev_rows)[seed_name] + scored_on_dev = dict(dev_rows) + if seed_name in scored_on_dev: + seed_scores = scored_on_dev[seed_name] + else: + # The resume path: `--candidates baseline --start NAME` deliberately skips the + # sweep, so the seed has no dev score yet — and the seed is precisely what the + # search's result has to be measured against, since "did evolving this beat + # starting from it" is the question. One dev sweep instead of the whole table's. + with Progress(len(dev), f"Scoring the seed {seed_name} on dev") as meter: + seed_scores = program.evaluate( + program.build(seed_instruction), dev, args.num_threads, on_example=meter.tick + ) + dev_rows.append((seed_name, seed_scores)) + # Recomputed, because the seed may well beat what the sweep scored: if the + # search then fails, the fallback should be the better of the two rather than + # whichever happened to be measured first. + winner_name, winner_scores = max(dev_rows, key=lambda row: row[1][axis]) + winner_instruction = table[winner_name] proposer = program.CappedInstructionProposer(INSTRUCTION_CHARACTER_CAP) diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index d88246f..e8badb3 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -17,6 +17,7 @@ import pathlib import re import sys +import types import urllib.error import pytest @@ -2299,6 +2300,76 @@ def test_a_search_can_start_from_a_named_candidate(): assert cli.resolve_candidates(args, spoken=True) == [candidates.BASELINE] +def _fake_program(scored: list[str], instruction: str = "an evolved instruction"): + """A stand-in for `program`, so `main` can be driven end to end with no model. + + Injected into `sys.modules` before `main`'s deferred import fires. That keeps the test + suite free of DSPy — the property two other tests here exist to protect — while still + exercising the CLI's own control flow, which is where the interesting mistakes are. + """ + module = types.ModuleType("program") + axes = dict.fromkeys(metrics.AXES, 0.5) + commands = { + "commands_converted": 0.5, + "commands_literal": 0.25, + "commands_missing": 0.25, + "commands_total": 4.0, + } + + def evaluate(built, rows, threads, on_example=None): + scored.append(built) + for _ in rows: + if on_example: + on_example() + return axes | commands + + class Proposer: + def __init__(self, cap, attempts=5, fields=()): + self.cap, self.fields = cap, fields or ("raw_transcript", "cleaned_transcript") + self.rejected = self.trimmed = self.abandoned = 0 + + class Optimized: + signature = types.SimpleNamespace(instructions=instruction) + + module.ModelSpec = lambda **kwargs: types.SimpleNamespace(**kwargs) + module.configure = lambda spec: None + module.build = lambda text: text + module.evaluate = evaluate + module.CappedInstructionProposer = Proposer + module.optimize = lambda *a, **k: Optimized() + return module + + +def test_the_seed_is_scored_on_dev_even_when_the_sweep_skipped_it(monkeypatch): + """`--candidates baseline --start NAME` is the resume path, and it used to crash. + + The seed's dev score was read out of the sweep with `dict(dev_rows)[seed_name]`, so + skipping the sweep — the entire point of `--candidates baseline` — raised KeyError + before the search ever began. The seed is precisely what the result has to be + compared against, so it is scored rather than the lookup being made lenient. + """ + scored: list[str] = [] + monkeypatch.setitem(sys.modules, "program", _fake_program(scored)) + assert ( + cli.main( + [ + "--jsonl", + str(FROZEN_DATASET), + "--candidates", + "baseline", + "--start", + "punct-explicit", + "--show-samples", + "0", + ] + ) + == 0 + ) + # The bar and the seed, both on dev — and the seed not skipped. + assert candidates.CANDIDATES[candidates.BASELINE] in scored + assert candidates.SPOKEN_PUNCTUATION_CANDIDATES["punct-explicit"] in scored + + def test_an_unknown_start_is_refused_before_anything_is_spent(): with pytest.raises(SystemExit) as raised: cli.main(["--source", "punctuation", "--punctuation-only", "--start", "nope", "--dry-run"]) From f25a3076502776030160de15eb3586bebf353040 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 20:33:08 -0700 Subject: [PATCH 7/8] feat: let a round's winner become the next round's bar `--baseline NAME` moves what a run compares against: the row scored on held-out test, what `--candidates baseline` scores, and the one instruction held to the safeguard requirement. Without it a second round spends a 900-call dev sweep re-confirming that prior-winner scores what the first round already said it scores -- measured on the first punctuation-only run, which learned 0.6124 for a second time. candidates.BASELINE does not move, deliberately. It names the string CleanupInstruction.text actually sends, and the whole "what does teaching this task buy over what ships" comparison depends on it continuing to mean that. A flag is the right place for "the bar for this run". Three details that follow. The promoted baseline is de-duplicated out of the punctuation sweep, since it is itself one of those candidates and would otherwise be scored twice. Whatever a run compares against is what it would fall back to shipping, so the safeguard check follows the flag rather than the constant. And `--start prior-winner` loses its special case: now that --start takes any candidate name, "prior-winner" is simply one of them, which removes a branch that only existed to spell BASELINE. Worth noting what was already true and is easy to misread: the seed is already the promotion bar. The dev winner is recomputed once the seed is scored, so an evolved instruction has to beat the instruction it evolved from. That is why a search returning 0.7706 against a 0.7781 seed was correctly refused, rather than being promoted for beating prior-winner's 0.6124. Co-Authored-By: Claude Opus 5 (1M context) --- evals/dictation-prompt/README.md | 71 +++++++++++------- .../optimize_cleanup_prompt.py | 51 ++++++++----- evals/dictation-prompt/test_eval.py | 74 +++++++++++++++++++ 3 files changed, 153 insertions(+), 43 deletions(-) diff --git a/evals/dictation-prompt/README.md b/evals/dictation-prompt/README.md index 2e1a1e2..37b7ca6 100644 --- a/evals/dictation-prompt/README.md +++ b/evals/dictation-prompt/README.md @@ -132,7 +132,27 @@ uv run evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ --num-threads 4 --out search.json ``` -The second command is the one worth understanding. A spoken run scores `BASELINE` plus +Once a round has produced a winner, make it the bar for the next one: + +```bash +uv run evals/dictation-prompt/optimize_cleanup_prompt.py --punctuation-only \ + --limit 4000 --baseline punct-explicit --start punct-explicit \ + --candidates baseline --auto medium --num-threads 4 --out round2.json +``` + +`--baseline` moves what the run compares against — the held-out row, what +`--candidates baseline` scores, and the one instruction held to the safeguard +requirement. Without it, a second round spends a 900-call dev sweep re-confirming that +the shipped instruction scores what the first round already said it scores. +`candidates.BASELINE` itself does not move: it names the string the Swift side actually +sends, and the "what does this buy over what ships" comparison depends on it meaning +that. + +The seed is already the promotion bar whichever way you set this — the dev winner is +recomputed once the seed is scored, so an evolved instruction has to beat the instruction +it evolved from, not merely beat `prior-winner`. + +The second command above is the one worth understanding. A spoken run scores `BASELINE` plus every punctuation candidate by default, which is right once and pure waste afterwards — at 900 dev rows that sweep is 6,300 calls to re-establish an ordering you already have. `--candidates baseline` skips it and `--start NAME` seeds the search from the instruction @@ -496,30 +516,31 @@ is `--model`. Its proposal prompts are multi-field, so they keep DSPy's marker p ## The knobs that matter -| Flag | Default | What it changes | -| ---------------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | -| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | -| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | -| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | -| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | -| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | -| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | -| `--spoken-punctuation` | `0` (off), or `1.0` under `--punctuation-only` | Speak this share of the marks the reference licenses. Never the utterance-final one. | -| `--punctuation-only` | off | Score ONLY the commands: no disfluencies, and a paired source's verbatim side discarded. | -| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | -| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | -| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | -| `--start` | `prior-winner` | Which instruction GEPA evolves from — `prior-winner`, `best-candidate`, or any candidate by name. | -| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | -| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | -| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | -| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | -| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | -| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | -| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | -| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | -| `--seed` | `7` | Seeds injection and the train/dev/test split. | +| Flag | Default | What it changes | +| ---------------------- | ------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| `--source` | `disfluency-speech` | Which corpus to score against — see the table above. | +| `--model` | `openai/qwen3.5-4b-32k-fast` | The LiteLLM model standing in for the service's rewrite model. | +| `--reflection-model` | `openai/claude-opus-4-8` | Writes the instructions during `--optimizer gepa`. Keep it stronger than `--model`. | +| `--api-base` | the AssemblyAI gateway | Endpoint for both models. `""` falls back to the provider's own. | +| `--metric` | `blend`, or `format` under `--spoken-punctuation` | `content` (words only), `format` (case and punctuation too), or 0.7/0.3 of both. | +| `--severity` | `0.35` | 0–1; how often a disfluency is injected. Reference-only sources only. | +| `--strip-formatting` | off | Also lowercase and unpunctuate, so restoring formatting is part of the task. | +| `--spoken-punctuation` | `0` (off), or `1.0` under `--punctuation-only` | Speak this share of the marks the reference licenses. Never the utterance-final one. | +| `--punctuation-only` | off | Score ONLY the commands: no disfluencies, and a paired source's verbatim side discarded. | +| `--spoken-caps-rate` | `0.25` | Chance a row also gets one ALL CAPS command. Only this operator edits a reference. | +| `--dump-corpus` | off | Write the loaded corpus to a JSONL file and carry on. Works under `--dry-run`. | +| `--optimizer` | `gepa` | `none` only ranks the candidates; both optimizers search instructions only. | +| `--baseline` | `prior-winner` | The bar: scored on held-out test, and what `--candidates baseline` means. Point it at a winner from a previous round. | +| `--start` | `prior-winner` | Which instruction GEPA evolves from — `prior-winner`, `best-candidate`, or any candidate by name. | +| `--auto` | `heavy` | Reflection trials: 10 / 18 / 27. The only knob that changes how many ideas get tried. | +| `--split` | `train` | The sources' own held-out splits are only ~250 rows — too few for the default `--limit`. | +| `--limit` | `2000` | Rows loaded, then sliced 1800 train / 50 dev / 150 test. Train rows cost nothing. | +| `--dev-fraction` | `150` (rows) | Fraction below 1, absolute count at 1 or above. Decides what ships; the search never sees it. | +| `--gepa-valset` | `50` (rows) | The optimizer's valset, taken off train. Multiplies search cost, adds no exploration. | +| `--test-fraction` | `150` (rows) | Same convention. Scored twice, and by nothing that makes a selection. | +| `--num-threads` | `1` | Serial by default — the gateway rate-limits. | +| `--max-tokens` | `8192` | Headroom for reasoning tokens. Too low silently corrupts a run rather than failing it. | +| `--seed` | `7` | Seeds injection and the train/dev/test split. | Both optimizers run with few-shot demos disabled. `config.llm.instruction` is a single string the service applies in one pass, so an optimized program that depended on bundled examples diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index b1b93b9..20a03cd 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -378,6 +378,18 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: "independent of corpus size. Shrinking --gepa-valset makes each trial cheaper; " "only this makes there be more of them", ) + search.add_argument( + "--baseline", + default=BASELINE, + metavar="NAME", + help=f"which instruction is the bar (default: {BASELINE}, what Blurt ships). It is " + "the row the winner is scored against on held-out test, what --candidates baseline " + "scores, and the one instruction held to the safeguard requirement. Point it at a " + "candidate that has already won a round and the run stops re-measuring something " + "you know: a search whose bar is still the shipped string spends a dev sweep " + "confirming the shipped string. candidates.BASELINE itself does not move, because " + "it is what the Swift side actually sends", + ) search.add_argument( "--start", default="prior-winner", @@ -464,7 +476,7 @@ def describe_length(instruction: str) -> str: return f"{len(instruction)} chars, {headroom} under the {INSTRUCTION_CHARACTER_CAP} cap" -def check_candidates(table: dict[str, str] | None = None) -> None: +def check_candidates(table: dict[str, str] | None = None, baseline: str = BASELINE) -> None: """Refuse to start if a hand-written candidate could never be shipped. Before any model call, because these are typo-class mistakes and paying for a full @@ -487,9 +499,9 @@ def check_candidates(table: dict[str, str] | None = None) -> None: for name, text in (CANDIDATES if table is None else table).items(): if overage(text): problems.append(f" {name}: {describe_length(text)}") - if name == BASELINE and (absent := missing_safeguards(text)): + if name == baseline and (absent := missing_safeguards(text)): stems = ", ".join(stem for stem, _ in absent) - problems.append(f" {name}: BASELINE is missing safeguard(s): {stems}") + problems.append(f" {name}: the baseline is missing safeguard(s): {stems}") if problems: raise SystemExit( "These candidates.py instructions could not be shipped as written:\n" @@ -532,11 +544,14 @@ def resolve_candidates( # was worth using: on a large corpus the sweep is `len(scoring) x dev` calls, and # having already run it once, paying for it again to reach the search is the one thing # anyone would reach for this flag to avoid. + baseline = getattr(args, "baseline", BASELINE) if args.candidates == "baseline": - return [BASELINE] + return [baseline] if spoken: - # BASELINE first so the table reads as "the bar, then the challengers". - return [BASELINE, *SPOKEN_PUNCTUATION_CANDIDATES] + # The bar first, so the table reads as "the bar, then the challengers" — and + # de-duplicated, since a baseline promoted from a previous round is itself one of + # them and would otherwise be scored twice. + return [baseline, *(n for n in SPOKEN_PUNCTUATION_CANDIDATES if n != baseline)] if ( args.candidates is None and args.optimizer == "none" @@ -544,7 +559,7 @@ def resolve_candidates( or args.start == "best-candidate" ): return list(CANDIDATES if table is None else table) - return [BASELINE] + return [baseline] def run_live_verification( @@ -671,11 +686,16 @@ def main(argv: list[str] | None = None) -> int: # contains, not by a flag. Still before any model call, which is what the check is # for: a typo in candidates.py should not cost a paid sweep to discover. table = instruction_table(loaded) - check_candidates(table) - if args.start not in ("prior-winner", "best-candidate") and args.start not in table: + check_candidates(table, args.baseline) + if args.baseline not in table: + raise SystemExit( + f"--baseline {args.baseline!r} is not a candidate this run scores. Available: " + + ", ".join(table) + ) + if args.start != "best-candidate" and args.start not in table: raise SystemExit( f"--start {args.start!r} is not a candidate this run scores. Available: " - + ", ".join(("prior-winner", "best-candidate", *table)) + + ", ".join(("best-candidate", *table)) ) train, dev, test = corpus.split( @@ -770,12 +790,7 @@ def main(argv: list[str] | None = None) -> int: # sweep above and may legitimately win. Nothing to score separately, and no # need to keep it out of the selection: unlike the over-cap instruction this # replaced, shipping it is a real option. - if args.start == "prior-winner": - seed_name = BASELINE - elif args.start == "best-candidate": - seed_name = winner_name - else: - seed_name = args.start + seed_name = winner_name if args.start == "best-candidate" else args.start seed_instruction = table[seed_name] scored_on_dev = dict(dev_rows) if seed_name in scored_on_dev: @@ -865,8 +880,8 @@ def main(argv: list[str] | None = None) -> int: # Held-out test scores for the winner and for the shipped-default proxy, so the # reported improvement is measured on data no selection decision saw. scored_on_test = [(winner_name, winner_instruction)] - if winner_name != BASELINE: - scored_on_test.append((BASELINE, table[BASELINE])) + if winner_name != args.baseline: + scored_on_test.append((args.baseline, table[args.baseline])) test_rows = [] with Progress(len(scored_on_test) * len(test), "Scoring on held-out test") as meter: for name, instruction in scored_on_test: diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index e8badb3..7612ace 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -2281,6 +2281,80 @@ def test_a_spoken_punctuation_run_scores_the_bar_and_the_challengers(): assert set(scoring) < set(table) +def test_the_bar_can_be_moved_to_a_candidate_that_already_won_a_round(): + """So a second round stops re-measuring what the first one established. + + `candidates.BASELINE` deliberately does not move: it is the string the Swift side + actually sends, and the "what does this buy over what ships" comparison depends on it + meaning that. `--baseline` moves the bar for one run. + """ + loaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + table = cli.instruction_table(loaded) + args = cli.parse_args(["--baseline", "punct-explicit", "--candidates", "baseline"]) + assert cli.resolve_candidates(args, table, spoken=True) == ["punct-explicit"] + assert candidates.BASELINE == "prior-winner" + + +def test_a_promoted_baseline_is_not_scored_twice(): + """It is itself one of the punctuation candidates, so the sweep would duplicate it.""" + loaded = corpus.load(jsonl=str(FROZEN_DATASET), limit=200) + scoring = cli.resolve_candidates( + cli.parse_args(["--baseline", "punct-explicit"]), + cli.instruction_table(loaded), + spoken=True, + ) + assert scoring[0] == "punct-explicit" + assert len(scoring) == len(set(scoring)) + assert set(scoring) == set(candidates.SPOKEN_PUNCTUATION_CANDIDATES) + + +def test_the_moved_bar_is_the_one_held_to_the_safeguards(monkeypatch): + """Whatever a run compares against is what it would fall back to shipping.""" + monkeypatch.setitem( + candidates.SPOKEN_PUNCTUATION_CANDIDATES, "punct-explicit", "Convert the commands." + ) + table = dict(candidates.CANDIDATES) | candidates.SPOKEN_PUNCTUATION_CANDIDATES + # Held to it as the bar... + with pytest.raises(SystemExit) as raised: + cli.check_candidates(table, baseline="punct-explicit") + assert "safeguard" in str(raised.value) + # ...and not held to it otherwise, like every other contrast candidate. + assert cli.check_candidates(table, baseline="prior-winner") is None + + +def test_an_unknown_baseline_is_refused_before_anything_is_spent(): + with pytest.raises(SystemExit) as raised: + cli.main( + ["--source", "punctuation", "--punctuation-only", "--baseline", "nope", "--dry-run"] + ) + assert "not a candidate" in str(raised.value) + + +def test_the_held_out_table_compares_against_the_moved_bar(monkeypatch): + scored: list[str] = [] + monkeypatch.setitem(sys.modules, "program", _fake_program(scored)) + assert ( + cli.main( + [ + "--jsonl", + str(FROZEN_DATASET), + "--baseline", + "punct-explicit", + "--candidates", + "baseline", + "--optimizer", + "none", + "--show-samples", + "0", + ] + ) + == 0 + ) + # The bar is the only thing scored, on dev and then on test — prior-winner never runs. + assert candidates.SPOKEN_PUNCTUATION_CANDIDATES["punct-explicit"] in scored + assert candidates.CANDIDATES["prior-winner"] not in scored + + def test_baseline_only_is_honoured_on_a_spoken_run(): """It was silently ignored, which made the flag a lie exactly where it earns its keep: on a large corpus the sweep is len(scoring) x dev calls.""" From 9234f6f4c061a8adee6873f718829174b9de7c58 Mon Sep 17 00:00:00 2001 From: Alex Kroman Date: Tue, 25 Aug 2026 20:44:51 -0700 Subject: [PATCH 8/8] feat: report per-utterance failure, not only per-command means "What percent of statements had an error?" could not be answered from a run's own output, which is the product question and a bad thing to be missing. `--out` recorded means only -- format 0.8081, commands_converted 0.682 -- and a mean of 0.682 over 2.26 commands a row is consistent with failures sitting in a third of rows or in nearly all of them. Those are different products and a per-command average cannot distinguish them. So `evaluate` now also reports, per candidate: rows_with_command_failure any command missed in this utterance rows_with_command_literal a command word pasted into this utterance rows_exact_content/format/blend came back exactly right The last of those is the one a user would recognise. The first two get their own columns in the command table, labelled ROWS to keep them from being read as more per-command rates. For scale on why this matters: at the measured 31.8% per-command failure rate, and the actual command-count distribution of the 450-row nyra test split (149 rows with one command, 136 with two, 95 with three, up to one row with nine), independent failures would put >=1 error in 53.7% of utterances. Clustering would lower that -- which is exactly the thing worth measuring rather than assuming. Also worth recording: the corpus load that produced that distribution hit four rate limits and an HTTP 502 and completed anyway, which is the retry added two commits ago earning its place on its first real outing. Co-Authored-By: Claude Opus 5 (1M context) --- .../optimize_cleanup_prompt.py | 14 ++++++++-- evals/dictation-prompt/program.py | 26 ++++++++++++------ evals/dictation-prompt/spoken_punctuation.py | 21 ++++++++++++--- evals/dictation-prompt/test_eval.py | 27 +++++++++++++++++++ 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/evals/dictation-prompt/optimize_cleanup_prompt.py b/evals/dictation-prompt/optimize_cleanup_prompt.py index 20a03cd..3f74c90 100644 --- a/evals/dictation-prompt/optimize_cleanup_prompt.py +++ b/evals/dictation-prompt/optimize_cleanup_prompt.py @@ -108,13 +108,23 @@ def print_command_table(rows: list[tuple[str, dict[str, float]]], axis: str) -> if not scored: return print("\nSpoken punctuation commands, per candidate") - print(f" {'candidate':<22} {'converted':>10} {'left as words':>14} {'dropped':>9}") - print(f" {'-' * 22} {'-' * 10} {'-' * 14} {'-' * 9}") + print( + f" {'candidate':<22} {'converted':>10} {'left as words':>14} {'dropped':>9}" + f" {'ROWS wrong':>11} {'ROWS w/ word':>13}" + ) + print(f" {'-' * 22} {'-' * 10} {'-' * 14} {'-' * 9} {'-' * 11} {'-' * 13}") for name, scores in sorted(scored, key=lambda row: row[1][axis], reverse=True): print( f" {name:<22} {scores['commands_converted']:>10.1%} " f"{scores['commands_literal']:>14.1%} {scores['commands_missing']:>9.1%}" + f" {scores.get('rows_with_command_failure', 0.0):>11.1%}" + f" {scores.get('rows_with_command_literal', 0.0):>13.1%}" ) + print( + " The last two columns are per utterance, not per command: how often a dictation\n" + " comes back with any command missed, and how often it comes back with a command\n" + " word pasted into it. A per-command mean cannot tell you either." + ) def print_samples(loaded: corpus.Corpus, count: int) -> None: diff --git a/evals/dictation-prompt/program.py b/evals/dictation-prompt/program.py index 6aeb554..b407b8f 100644 --- a/evals/dictation-prompt/program.py +++ b/evals/dictation-prompt/program.py @@ -252,20 +252,30 @@ def evaluate( counted = program if on_example is None else _Ticking(program, on_example) predictions = runner([(counted, {INPUT_FIELD: u.disfluent}) for u in utterances]) cleaned = [_cleaned(prediction) for prediction in predictions] - axes = metrics.mean( - [ - utterance.scored(hypothesis) - for utterance, hypothesis in zip(utterances, cleaned, strict=True) - ] - ) # Reported next to the axes rather than instead of them. WER answers "how close is # the text", which mixes the punctuation task into every other kind of error; the # tally answers "of the commands planted, how many were obeyed", which is the # question `--spoken-punctuation` is asking and which only a synthetic operator can # be asked. Zeros with `commands_total` at 0 mean "not asked", not "failed". - return axes | spoken_punctuation.tally( - (utterance.commands, hypothesis) + scored = [ + utterance.scored(hypothesis) for utterance, hypothesis in zip(utterances, cleaned, strict=True) + ] + # Row-level too. A mean of 0.81 is consistent with every row being slightly wrong and + # with a fifth being badly wrong, and those are different products. + exact = { + f"rows_exact_{axis}": sum(s.value(axis) >= 1.0 for s in scored) / len(scored) + if scored + else 0.0 + for axis in metrics.AXES + } + return ( + metrics.mean(scored) + | exact + | spoken_punctuation.tally( + (utterance.commands, hypothesis) + for utterance, hypothesis in zip(utterances, cleaned, strict=True) + ) ) diff --git a/evals/dictation-prompt/spoken_punctuation.py b/evals/dictation-prompt/spoken_punctuation.py index 48f0a7f..9b15280 100644 --- a/evals/dictation-prompt/spoken_punctuation.py +++ b/evals/dictation-prompt/spoken_punctuation.py @@ -563,12 +563,27 @@ def tally(pairs) -> dict[str, float]: they mean "not asked", not "failed". """ counts: dict[str, int] = dict.fromkeys(OUTCOMES, 0) + rows = failed_rows = literal_rows = 0 for commands, hypothesis in pairs: - for command in commands: - counts[outcome(command, hypothesis)] += 1 + rows += 1 + outcomes = [outcome(command, hypothesis) for command in commands] + for name in outcomes: + counts[name] += 1 + if commands: + failed_rows += any(name != "converted" for name in outcomes) + literal_rows += any(name == "literal" for name in outcomes) total = sum(counts.values()) stats = {f"commands_{name}": (counts[name] / total if total else 0.0) for name in OUTCOMES} - return stats | {"commands_total": float(total)} + # Per-**row** as well as per-command, because they answer different questions and only + # the second is the product's. "68% of commands converted" does not say how often a + # dictation comes back wrong: at 2.3 commands a row those failures could sit in a + # third of rows or in nearly all of them, and a mean cannot tell you which. + return stats | { + "commands_total": float(total), + "rows_total": float(rows), + "rows_with_command_failure": failed_rows / rows if rows else 0.0, + "rows_with_command_literal": literal_rows / rows if rows else 0.0, + } def feedback_note(commands: tuple[Command, ...], hypothesis: str) -> str: diff --git a/evals/dictation-prompt/test_eval.py b/evals/dictation-prompt/test_eval.py index 7612ace..fa6a512 100644 --- a/evals/dictation-prompt/test_eval.py +++ b/evals/dictation-prompt/test_eval.py @@ -1882,6 +1882,33 @@ def test_every_planted_command_is_answered_by_its_own_reference(): assert spoken_punctuation.outcome(command, target) == "converted", command +def test_the_tally_counts_rows_as_well_as_commands(): + """They answer different questions, and only the second is the product's. + + "68% of commands converted" does not say how often a dictation comes back wrong: at + 2.3 commands a row those failures could sit in a third of rows or in nearly all of + them, and a per-command mean cannot tell you which. + """ + text = "We shipped it today. Monday was quiet, so nobody noticed." + reference, spoken, commands = spoken_punctuation.inject( + text, text, seed=1, rate=1.0, caps_rate=0.0 + ) + assert len(commands) == 2 + # Two rows, four commands: one cleaned perfectly, one that left both commands in. + stats = spoken_punctuation.tally([(commands, reference), (commands, spoken)]) + assert stats["rows_total"] == 2.0 + assert stats["commands_total"] == 4.0 + assert stats["rows_with_command_failure"] == 0.5 + assert stats["rows_with_command_literal"] == 0.5 + assert stats["commands_literal"] == 0.5 + + +def test_a_row_with_no_commands_is_never_counted_as_a_failure(): + stats = spoken_punctuation.tally([((), "anything at all")]) + assert stats["rows_total"] == 1.0 + assert stats["rows_with_command_failure"] == 0.0 + + def test_the_tally_of_nothing_says_nothing_rather_than_zero_percent(): assert spoken_punctuation.tally([])["commands_total"] == 0.0 assert spoken_punctuation.tally([((), "anything")])["commands_converted"] == 0.0