From a4fedc9fd9e6cb71aa23b8d0d4b242aea252d046 Mon Sep 17 00:00:00 2001 From: anaghasimha Date: Wed, 29 Jul 2026 04:26:36 -0400 Subject: [PATCH] fix: cap model_max_length sentinel value to prevent OverflowError in Rust tokenizers backend Models like DeBERTa and DistilBERT don't define model_max_length, causing HuggingFace to assign VERY_LARGE_INTEGER (~1e30) as default. This overflows int32 when passed to the Rust tokenizers backend via enable_truncation(). Fixes #205 --- bert_score/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bert_score/utils.py b/bert_score/utils.py index 8af24be..f1387e6 100644 --- a/bert_score/utils.py +++ b/bert_score/utils.py @@ -188,6 +188,8 @@ def sent_encode(tokenizer, sent): "Encoding as sentence based on the tokenizer" sent = sent.strip() + if tokenizer.model_max_length > 1e30: + tokenizer.model_max_length = 512 if sent == "": return tokenizer.build_inputs_with_special_tokens([]) elif isinstance(tokenizer, GPT2Tokenizer) or isinstance(tokenizer, RobertaTokenizer):