From d80eb98fae65ea93d652cc3e800d6a75e0b0e50c Mon Sep 17 00:00:00 2001 From: ankitkumar Date: Wed, 17 Jun 2026 16:47:29 +0530 Subject: [PATCH] fix: corroborate ack-channel correction detection to kill self-ack false positives (v1.3.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Stop-hook auto-proposer treated the assistant's acknowledgment language (channel 2: "you're right", "my mistake", "i was wrong") as sufficient to arm a correction on its own. But that vocabulary also covers ordinary agreement and Claude self-correcting its OWN prior claim in conversation — the dominant false positive. A neutral user instruction answered apologetically would fire a rule nudge with no actual user correction. Channel 2 is now CONFIRMING, not self-sufficient: an assistant ack counts as a correction only when the user's message also shows doubt (a question mark, a hard CORRECTION_PATTERN, or a soft DOUBT_LEXICON term). Channel 1 (explicit user phrasing) still arms a correction alone at the unchanged low bar, so genuine low-tool corrections (e.g. the web-search-for-latest-versions rule) are preserved. Deliberately did NOT add a score/edit gate on rule capture: verified it would suppress real corrections that do no edits and score 0 (web-search confirmations). - add DOUBT_LEXICON + _user_shows_doubt() (opening-window scan) - score_turn: ack_is_correction = correction_ack and user_doubt - expose user_doubt in the signals dict for observability - 4 regression tests; 59/59 green Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude-plugin/plugin.json | 2 +- scripts/auto_propose.py | 78 +++++++++++++++++++++++++++++++++++--- tests/test_auto_propose.py | 78 +++++++++++++++++++++++++++++++++++++- 3 files changed, 149 insertions(+), 9 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 40cc288..94684d9 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin.json", "name": "compounded", - "version": "1.3.2", + "version": "1.3.3", "description": "Turns your agent into a trainable employee. Learns rules from your corrections (with your approval), and skills earn autonomy through demonstrated reliability \u2014 proposed \u2192 verified \u2192 trusted \u2192 autonomous. No daemon. No cloud.", "author": { "name": "Ankit Kumar", diff --git a/scripts/auto_propose.py b/scripts/auto_propose.py index 7fbadc4..5ea6d36 100644 --- a/scripts/auto_propose.py +++ b/scripts/auto_propose.py @@ -20,7 +20,8 @@ bash_count — Bash tool invocations plan_used — Plan/ExitPlanMode/EnterPlanMode tool usage recovery — at least one tool error followed by a successful retry - correction — user message before the assistant turn contains a correction signal + correction — a real correction: user phrasing (channel 1), OR an + assistant acknowledgment (channel 2) corroborated by user doubt Scoring (additive, threshold-based): +2 tool_uses >= 5 @@ -32,6 +33,17 @@ procedure capture: fire if score >= 3 AND correction == False rule capture: fire if correction == True AND tool_uses >= 1 +Correction has two channels, and they are NOT symmetric: + - Channel 1 (user phrasing) can arm a correction on its own — the user + explicitly flagged a problem. + - Channel 2 (assistant acknowledgment) is a CONFIRMING signal only. The + model's ack vocabulary ("you're right", "my mistake", "i was wrong") also + covers ordinary agreement and self-correction in meta-discussion, so an ack + counts as a correction only when the user's message also shows doubt + (a question, or a soft-doubt term). This kills the dominant false positive: + Claude saying "I was wrong" about its own prior claim while the user gave a + neutral instruction. See _user_shows_doubt. + Two capture kinds: procedure — a successful multi-step task worth saving as a replayable procedure (the original compounded behavior). @@ -144,10 +156,17 @@ # beats more keyword surgery. OPENING_WINDOW_CHARS = 250 -# Channel 2: the ASSISTANT's reaction. This is the channel that generalizes: -# however the user phrases a correction, the model's acknowledgment is highly -# standardized ("you're right", "my mistake", "good catch"). The LLM in the -# loop does the semantic understanding; the hook just reads its reaction. +# Channel 2: the ASSISTANT's reaction. However the user phrases a correction, +# the model's acknowledgment is highly standardized ("you're right", "my +# mistake", "good catch"). The LLM in the loop does the semantic understanding; +# the hook just reads its reaction. +# +# But this same vocabulary covers ordinary agreement and — the dominant false +# positive — Claude self-correcting its OWN prior claim in conversation ("I was +# wrong about that"). So channel 2 is CONFIRMING, not self-sufficient: it counts +# only when the user's message also shows doubt (see _user_shows_doubt / +# DOUBT_LEXICON). A genuine user correction Claude acknowledges still fires; a +# neutral instruction Claude happens to answer apologetically does not. ACK_PATTERNS = ( "you're right", "you are right", @@ -176,6 +195,25 @@ "correcting my", ) +# Corroboration lexicon for channel 2. An assistant ack is only treated as a +# correction when the user's message opening shows doubt — a question mark, any +# CORRECTION_PATTERN, or one of these softer doubt markers. Broader than +# CORRECTION_PATTERNS on purpose: these are too weak to arm a correction ALONE +# (channel 1), but strong enough to confirm that an ack reflects a real user +# correction rather than Claude's conversational politeness. +DOUBT_LEXICON = ( + "i think", + "i thought", + "maybe", + "not sure", + "unsure", + "wrong", + "actually", + "hmm", + "really?", + "sure?", +) + # ----------------------------------------------------------------------------- # Transcript parsing @@ -399,6 +437,28 @@ def _has_correction_signal(prior_user: dict | None) -> bool: return any(p in text for p in CORRECTION_PATTERNS) +def _user_shows_doubt(prior_user: dict | None) -> bool: + """Corroboration gate for channel 2: did the USER actually express doubt? + + Scans only the message opening (same OPENING_WINDOW_CHARS guard as the other + channels) for a question mark, a hard CORRECTION_PATTERN, or a softer + DOUBT_LEXICON term. Used to validate an assistant acknowledgment — never to + arm a correction on its own. Without this, Claude acknowledging its own + prior claim ("I was wrong about that") in a neutral exchange reads as a + user correction, which is the dominant false positive. + """ + if prior_user is None: + return False + text = _extract_text(prior_user).lower()[:OPENING_WINDOW_CHARS] + if not text: + return False + if "?" in text: + return True + if any(p in text for p in CORRECTION_PATTERNS): + return True + return any(p in text for p in DOUBT_LEXICON) + + def _assistant_acknowledged_correction(turn_events: list[dict]) -> bool: """Channel 2: the assistant's own text admits it was corrected. @@ -429,7 +489,12 @@ def score_turn(turn_events: list[dict], prior_user: dict | None) -> dict: recovery = _has_recovery(turn_events) correction_user = _has_correction_signal(prior_user) correction_ack = _assistant_acknowledged_correction(turn_events) - correction = correction_user or correction_ack + user_doubt = _user_shows_doubt(prior_user) + # Channel 1 (user phrasing) arms a correction alone. Channel 2 (assistant + # ack) only counts when corroborated by user doubt — otherwise it fires on + # Claude's conversational politeness and self-corrections. + ack_is_correction = correction_ack and user_doubt + correction = correction_user or ack_is_correction score = 0 if len(tool_uses) >= THRESHOLD_TOOL_USES: @@ -465,6 +530,7 @@ def score_turn(turn_events: list[dict], prior_user: dict | None) -> dict: "correction": correction, "correction_user": correction_user, "correction_ack": correction_ack, + "user_doubt": user_doubt, } diff --git a/tests/test_auto_propose.py b/tests/test_auto_propose.py index 56f6533..ed76f59 100644 --- a/tests/test_auto_propose.py +++ b/tests/test_auto_propose.py @@ -161,8 +161,9 @@ def test_soft_user_correction_captures_rule(self) -> None: self.assertEqual(signals["capture_kind"], "rule") def test_assistant_acknowledgment_captures_rule_for_any_phrasing(self) -> None: - # Channel 2: the user's phrasing matches NO pattern at all, but the - # assistant's reply acknowledges the correction — that generalizes. + # Channel 2: the user's phrasing matches NO hard correction pattern, but + # the message shows doubt ("?" / "maybe") and the assistant's reply + # acknowledges the correction. Doubt corroborates the ack → fires. events = [ _user("hmm gemini 2 maybe?"), _assistant( @@ -177,6 +178,79 @@ def test_assistant_acknowledgment_captures_rule_for_any_phrasing(self) -> None: self.assertTrue(signals["correction_ack"]) # Claude's reaction did self.assertEqual(signals["capture_kind"], "rule") + def test_ack_alone_without_user_doubt_does_not_fire(self) -> None: + # Regression (field): the dominant false positive. The user gives a + # neutral directive (no doubt, no question) and Claude's reply happens + # to open with ack vocabulary ("I was wrong …"). Channel 2 must NOT + # arm a correction without user-side corroboration. + events = [ + _user("yes, write up the plan and make sure we only improve it, not degrade it"), + _assistant( + tool_uses=[ + ("Read", {"file_path": "/a.py"}), + ("Bash", {"command": "ls"}), + ], + text="I was wrong about that earlier — fair point. Here is the corrected plan.", + ), + _tool_result(), + ] + turn, prior = self.auto_propose._split_into_turns(events) + signals = self.auto_propose.score_turn(turn, prior) + self.assertTrue(signals["correction_ack"]) # ack phrasing present + self.assertFalse(signals["user_doubt"]) # but user expressed none + self.assertFalse(signals["correction"]) # so no correction armed + self.assertIsNone(signals["capture_kind"]) + + def test_assistant_self_correction_in_meta_discussion_silent(self) -> None: + # Claude correcting its OWN prior analysis (no user pushback at all) + # must stay silent — there is no user correction to learn from. + events = [ + _user("find why it's misfiring"), + _assistant( + tool_uses=[("Read", {"file_path": "/auto_propose.py"})], + text="You're right to ask. I was wrong in my earlier read — here's the real cause.", + ), + _tool_result(), + ] + turn, prior = self.auto_propose._split_into_turns(events) + signals = self.auto_propose.score_turn(turn, prior) + self.assertFalse(signals["user_doubt"]) + self.assertNotEqual(signals["capture_kind"], "rule") + + def test_ack_with_user_question_mark_still_fires(self) -> None: + # Preserve channel 2 for genuine doubt: a bare question + ack still + # arms rule capture even with no hard correction keyword. + events = [ + _user("wait is that the current api?"), + _assistant( + tool_uses=[("WebSearch", {"query": "current anthropic api version"})], + text="Good catch — you're right, there's a newer one.", + ), + _tool_result(), + ] + turn, prior = self.auto_propose._split_into_turns(events) + signals = self.auto_propose.score_turn(turn, prior) + self.assertFalse(signals["correction_user"]) + self.assertTrue(signals["user_doubt"]) # via "?" + self.assertEqual(signals["capture_kind"], "rule") + + def test_ack_with_soft_doubt_no_question_mark_fires(self) -> None: + # Soft doubt without a question mark ("i think u are wrong …") still + # corroborates the ack. Mirrors the settle-race field case at unit level. + events = [ + _user("i think u are wrong, gemini already has a newer model, can u web search and tell"), + _assistant( + tool_uses=[("WebSearch", {"query": "latest gemini embedding model"})], + text="You were right — Google shipped a newer one.", + ), + _tool_result(), + ] + turn, prior = self.auto_propose._split_into_turns(events) + signals = self.auto_propose.score_turn(turn, prior) + self.assertFalse(signals["correction_user"]) # no hard pattern matches + self.assertTrue(signals["user_doubt"]) # via "i think" / "wrong" + self.assertEqual(signals["capture_kind"], "rule") + def test_pattern_buried_in_user_paste_is_not_a_correction(self) -> None: # Regression (field): the user pasted a transcript that contained # "Is there a newer [model / SDK / tool] than [X]?" ~1000 chars in —