Support tokenizers that do not have a pad_token configured

- closes #37
krasserm · Feb 24, 2023 · f807b2e · f807b2e
1 parent 1847909
commit f807b2e
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/perceiver/data/text/collator.py b/perceiver/data/text/collator.py
@@ -59,7 +59,7 @@ def _prepare_sequence(self, sequence, max_length):
             sequence,
             add_special_tokens=False,
             return_token_type_ids=False,
-            padding=PaddingStrategy.MAX_LENGTH,
+            padding=False if self.tokenizer.pad_token is None else PaddingStrategy.MAX_LENGTH,
             max_length=max_length,
             truncation=True,
         )

diff --git a/perceiver/data/text/common.py b/perceiver/data/text/common.py
@@ -30,7 +30,7 @@ def preprocess(self, text):
     def preprocess_batch(self, text_batch):
         result = self.tokenizer(
             text_batch,
-            padding=True,
+            padding=self.tokenizer.pad_token is not None,
             truncation=True,
             add_special_tokens=self.add_special_tokens,
             return_token_type_ids=False,
@@ -261,6 +261,7 @@ def _tokenize_dataset(
         return_word_ids=True,
     ):
         def tokenize(examples):
+            examples["text"] = [text + self.tokenizer.eos_token for text in examples["text"]]
             encoding = self.tokenizer(
                 examples["text"],
                 padding=padding,

diff --git a/perceiver/model/text/clm.py b/perceiver/model/text/clm.py
@@ -183,7 +183,7 @@ def create(cls, config: CausalLanguageModelConfig, **kwargs: Any):
     def setup(self, stage: Optional[str] = None):
         dm = self.trainer.datamodule
 
-        if dm.tokenizer.padding_side != "left":
+        if dm.tokenizer.pad_token is not None and dm.tokenizer.padding_side != "left":
             raise ValueError(
                 "Causal language modeling with Perceiver AR requires a data module configured with padding_side=left"
             )