From 76105fbaaea163c075b6c7bab09e74e781768474 Mon Sep 17 00:00:00 2001
From: Attila <29815676+arthapraha@users.noreply.github.com>
Date: Thu, 13 Aug 2026 21:47:38 +0100
Subject: [PATCH] fix(scorers): a digit inside a name is not a count

_extract_count_from_text accepted any answer containing exactly one number,
so "Chiller 6" parsed as 6 -- and so did "Boiler 6". Two differently-named
assets then compared equal on a digit that happened to be inside a name, and
a wrong asset scored a perfect strict match while a different wrong asset
scored zero.

The docstring already states the intent: count-only or nearly count-only.
This enforces it. Once the number and the usual count wording are removed,
any remaining word means the number was part of a phrase -- most importantly,
part of a name.

"The count is 6" and "There are 5 assets" still parse. "Chiller 6" no
longer does.

Note for reviewers: 6 tests in evaluation/tests/ already fail on main at
e11d1c1 (test_car_metadata_*, test_static_json_scorer_uses_car_metadata_score).
This change does not touch them; the suite goes from 6 failed / 90 passed to
6 failed / 92 passed.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Attila <29815676+arthapraha@users.noreply.github.com>
---
 src/evaluation/scorers/static_json.py         | 29 ++++++++++++++++++-
 .../tests/test_static_json_scorer.py          | 22 ++++++++++++++
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py
index 7afa9d56..5d377297 100644
--- a/src/evaluation/scorers/static_json.py
+++ b/src/evaluation/scorers/static_json.py
@@ -209,6 +209,16 @@ def _parse_json_or_python(content: str) -> Any:
     return _PARSE_MISSING
 
 
+# Wording that may surround a bare count without making the answer something
+# other than a count. Anything else left over means the number was part of a
+# phrase -- most importantly, part of a name.
+_COUNT_WORDS = re.compile(
+    r"\b(?:the|a|an|final|answer|count|result|total|is|are|there|of|"
+    r"assets?|items?|records?|rows?|entries)\b",
+    flags=re.IGNORECASE,
+)
+
+
 def _extract_count_from_text(content: str) -> int | float | None:
     """Extract a count when the answer is count-only or nearly count-only."""
     stripped = content.strip()
@@ -225,7 +235,24 @@ def _extract_count_from_text(content: str) -> int | float | None:
     )
     if len(numbers) == 1:
         number = numbers[0]
-        return float(number) if "." in number else int(number)
+        # A digit inside a name is not a count.
+        #
+        # This function is documented as accepting answers that are "count-only
+        # or nearly count-only", but a single number anywhere in the text
+        # satisfied that test. So "Chiller 6" parsed as 6 -- and so did
+        # "Boiler 6", making two differently-named assets compare equal on a
+        # digit that happened to be inside a name.
+        #
+        # Require the residue, once the number and the usual count wording are
+        # removed, to contain no remaining words: that is what "nearly
+        # count-only" means. "The count is 6" still parses; "Chiller 6" no
+        # longer does.
+        residue = re.sub(
+            r"(?<![A-Za-z0-9_])-?\d+(?:\.\d+)?(?![A-Za-z0-9_])", " ", stripped
+        )
+        residue = _COUNT_WORDS.sub(" ", residue)
+        if not re.search(r"[A-Za-z0-9]", residue):
+            return float(number) if "." in number else int(number)
 
     return None
 
diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py
index d28fa6de..74e62ffe 100644
--- a/src/evaluation/tests/test_static_json_scorer.py
+++ b/src/evaluation/tests/test_static_json_scorer.py
@@ -557,3 +557,25 @@ def test_static_json_scorer_uses_car_metadata_score():
     assert result.passed is True
     assert result.score == 1.0
     assert result.details["car_score"] == 1.0
+
+
+def test_digit_inside_an_asset_name_is_not_a_count():
+    """A number inside a name is not a count.
+
+    `_extract_count_from_text` accepted any text containing exactly one number,
+    so "Chiller 6" parsed as 6. Two differently-named assets then compared equal
+    on the digit, and a wrong asset scored as a perfect match while a different
+    wrong asset scored zero.
+    """
+    assert parse_structured_answer("Chiller 6") == "Chiller 6"
+    assert parse_structured_answer("Boiler 6") == "Boiler 6"
+    assert parse_structured_answer("Chiller 6") != parse_structured_answer("Boiler 6")
+
+
+def test_count_only_answers_still_parse_as_counts():
+    """The intended behaviour is preserved: count-only or nearly count-only."""
+    assert parse_structured_answer("6") == 6
+    assert parse_structured_answer("  6  ") == 6
+    assert parse_structured_answer("The count is 6") == 6
+    assert parse_structured_answer("Answer: 6.") == 6
+    assert parse_structured_answer("3.5") == 3.5