Tiiiger#85 switch to model_max_length for future versions

kirzharov · Dec 4, 2020 · 9ffea67 · 9ffea67
1 parent ee78779
commit 9ffea67
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/bert_score/utils.py b/bert_score/utils.py
@@ -109,15 +109,17 @@ def sent_encode(tokenizer, sent):
 
         if LooseVersion(transformers.__version__) >= LooseVersion("3.0.0"):
             return tokenizer.encode(
-                sent, add_special_tokens=True, add_prefix_space=True, max_length=tokenizer.max_len, truncation=True
+                sent, add_special_tokens=True, add_prefix_space=True, max_length=tokenizer.model_max_length,
+                truncation=True
             )
         else:
             return tokenizer.encode(sent, add_special_tokens=True, add_prefix_space=True, max_length=tokenizer.max_len)
     else:
         import transformers
 
         if LooseVersion(transformers.__version__) >= LooseVersion("3.0.0"):
-            return tokenizer.encode(sent, add_special_tokens=True, max_length=tokenizer.max_len, truncation=True)
+            return tokenizer.encode(sent, add_special_tokens=True, max_length=tokenizer.model_max_length,
+                                    truncation=True)
         else:
             return tokenizer.encode(sent, add_special_tokens=True, max_length=tokenizer.max_len)