fix(scorers): a digit inside a name is not a count

src/evaluation/scorers/static_json.py

@@ -209,6 +209,16 @@ def _parse_json_or_python(content: str) -> Any:
     return _PARSE_MISSING
 
 
+# Wording that may surround a bare count without making the answer something
+# other than a count. Anything else left over means the number was part of a
+# phrase -- most importantly, part of a name.
+_COUNT_WORDS = re.compile(
+    r"\b(?:the|a|an|final|answer|count|result|total|is|are|there|of|"
+    r"assets?|items?|records?|rows?|entries)\b",
+    flags=re.IGNORECASE,
+)
+
+
 def _extract_count_from_text(content: str) -> int | float | None:
     """Extract a count when the answer is count-only or nearly count-only."""
     stripped = content.strip()
@@ -225,7 +235,24 @@ def _extract_count_from_text(content: str) -> int | float | None:
     )
     if len(numbers) == 1:
         number = numbers[0]
-        return float(number) if "." in number else int(number)
+        # A digit inside a name is not a count.
+        #
+        # This function is documented as accepting answers that are "count-only
+        # or nearly count-only", but a single number anywhere in the text
+        # satisfied that test. So "Chiller 6" parsed as 6 -- and so did
+        # "Boiler 6", making two differently-named assets compare equal on a
+        # digit that happened to be inside a name.
+        #
+        # Require the residue, once the number and the usual count wording are
+        # removed, to contain no remaining words: that is what "nearly
+        # count-only" means. "The count is 6" still parses; "Chiller 6" no
+        # longer does.
+        residue = re.sub(
+            r"(?<![A-Za-z0-9_])-?\d+(?:\.\d+)?(?![A-Za-z0-9_])", " ", stripped
+        )
+        residue = _COUNT_WORDS.sub(" ", residue)
+        if not re.search(r"[A-Za-z0-9]", residue):
+            return float(number) if "." in number else int(number)
 
     return None
 

src/evaluation/tests/test_static_json_scorer.py

@@ -557,3 +557,25 @@ def test_static_json_scorer_uses_car_metadata_score():
     assert result.passed is True
     assert result.score == 1.0
     assert result.details["car_score"] == 1.0
+
+
+def test_digit_inside_an_asset_name_is_not_a_count():
+    """A number inside a name is not a count.
+
+    `_extract_count_from_text` accepted any text containing exactly one number,
+    so "Chiller 6" parsed as 6. Two differently-named assets then compared equal
+    on the digit, and a wrong asset scored as a perfect match while a different
+    wrong asset scored zero.
+    """
+    assert parse_structured_answer("Chiller 6") == "Chiller 6"
+    assert parse_structured_answer("Boiler 6") == "Boiler 6"
+    assert parse_structured_answer("Chiller 6") != parse_structured_answer("Boiler 6")
+
+
+def test_count_only_answers_still_parse_as_counts():
+    """The intended behaviour is preserved: count-only or nearly count-only."""
+    assert parse_structured_answer("6") == 6
+    assert parse_structured_answer("  6  ") == 6
+    assert parse_structured_answer("The count is 6") == 6
+    assert parse_structured_answer("Answer: 6.") == 6
+    assert parse_structured_answer("3.5") == 3.5