Add vocab size property to embeddingtokenizer

deepset-ai · Jun 18, 2020 · 06e45f9 · 06e45f9
1 parent b6e92a6
commit 06e45f9
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/farm/modeling/tokenization.py b/farm/modeling/tokenization.py
@@ -153,7 +153,10 @@ def __init__(
         self.unk_tok_idx = self.vocab[unk_token]
         self.ids_to_tokens = collections.OrderedDict([(ids, tok) for tok, ids in self.vocab.items()])
         self.do_lower_case = do_lower_case
-        self.vocab_size_farm = len(self.vocab)
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):