Merge pull request #20 from baai-open-internal/transform_tokenizer

Transform tokenizer

doc_zh/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md

@@ -118,7 +118,7 @@ class GLMTitleGenerationCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMTitleGenerationDataset(train_src,
                                    train_tgt)
 ```

doc_zh/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md

@@ -131,7 +131,7 @@ class GLMPoetryDynamicCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMPoetryDataset(train_src,
                                    train_tgt)
 ```

docs/TUTORIAL_12_GLM_EXAMPLE_TITLE_GENERATION.md

@@ -119,7 +119,7 @@ class GLMTitleGenerationCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMTitleGenerationCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMTitleGenerationDataset(train_src,
                                    train_tgt)
 ```

docs/TUTORIAL_13_GLM_EXAMPLE_PEOTRY_GENERATION.md

@@ -122,7 +122,7 @@ class GLMPoetryDynamicCollateFN():
 ```python
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
-my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command('pad').Id)
+my_collate_fn = GLMPoetryDynamicCollateFN(pad_id=tokenizer.get_command_id('pad'))
 train_dataset = GLMPoetryDataset(train_src,
                                    train_tgt)
 ```

examples/bert_title_generation_english/generate.py

@@ -14,7 +14,7 @@
 maxlen = 512
 auto_loader = AutoLoader(
     "seq2seq",
-    model_name="bert-base-uncased",
+    model_name="BERT-base-en",
     model_dir=model_dir,
 )
 model = auto_loader.get_model()

examples/clip/inference_clip.py

@@ -17,7 +17,7 @@
 def inference():
     image = Image.open("./CLIP.png")
     image = transform(image).unsqueeze(0).to(device)
-    text = tokenizer.tokenize(["a diagram", "a dog", "a cat"]).to(device)
+    text = tokenizer.tokenize_as_tensor(["a diagram", "a dog", "a cat"]).to(device)
 
     with torch.no_grad():
         image_features = model.encode_image(image)
@@ -27,4 +27,4 @@ def inference():
     print(text_probs.cpu().numpy()[0].tolist())
 
 if __name__=="__main__":
-    inference()
+    inference()

examples/clip/train_clip_deepspeed.py

@@ -26,7 +26,7 @@
     num_checkpoints=1,
     hostfile="./deepspeed/hostfile",
     training_script=__file__,
-    deepspeed_config="./deepspeed/deepspeed.json"
+    deepspeed_config="./deepspeed.json"
 )
 loader = AutoLoader(task_name="txt_img_matching",#contrastive learning
                     model_name="clip-base-p32-224",

examples/glm_blank_filling/glm_generate_samples.py

@@ -5,16 +5,17 @@
 import torch
 
 from flagai.model.glm_model import GLMModel
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.predictor.predictor import Predictor
 if __name__ == "__main__":
     """Main training program."""
     print('Generate Samples')
     # Random seeds for reproducability.
     # Model,
-    model = GLMModel.from_pretrain(model_name='GLM-large-ch',
+    model_name = 'GLM-large-ch'
+    model = GLMModel.from_pretrain(model_name=model_name,
                                    download_path="./state_dict/")
-    tokenizer = GLMLargeChTokenizer()
+    tokenizer = Tokenizer.from_pretrained(model_name)
 
     model.cuda(torch.cuda.current_device())
 

examples/glm_poetry_generation/train.py

@@ -130,7 +130,7 @@ def __call__(self, batch):
 train_src, train_tgt = read_file()
 print('-----------train data length:', len(train_src))
 my_collate_fn = GLMPoetryDynamicCollateFN(
-    pad_id=tokenizer.get_command('pad').Id)
+    pad_id=tokenizer.get_command_id('pad'))
 train_dataset = BertSeq2seqDataset(train_src, train_tgt)
 
 trainer.train(model, train_dataset=train_dataset, collate_fn=my_collate_fn)

examples/glm_pretrain/train.py

@@ -2,7 +2,7 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.model.glm_model import GLMForSeq2Seq
 from flagai.trainer import Trainer
 from flagai.data.dataset import ConstructBlockStrategy
@@ -24,14 +24,11 @@
                       load_dir=None,
                       lr=1e-4,
                       save_interval=10)
-
-    model = GLMForSeq2Seq.from_pretrain(model_name='GLM-large-ch')
-
+    model_name = 'GLM-large-ch'
+    tokenizer = Tokenizer.from_pretrained(model_name)
     ds_args = PretrainDatasetArguments()
-
-    tokenizer = GLMLargeChTokenizer()
-
     ds_args = add_args(ds_args, tokenizer)
+    model = GLMForSeq2Seq.from_pretrain(model_name=model_name)
 
     def create_dataset(tokenizer, should_split):
         dataset = get_dataset_lazy("./examples/glm_pretrain/data",
@@ -59,7 +56,7 @@ def create_dataset(tokenizer, should_split):
     collate_fn = None
     if ds_args.block_lm:
         collate_fn = ConstructBlockStrategy(
-            tokenizer, 512, eod_token=tokenizer.get_command('eos').Id)
+            tokenizer, 512, eod_token=tokenizer.get_command_id('eos'))
     metric_methods = DEFAULT_METRICS['pretrain']
     trainer.train(model,
                   collate_fn=collate_fn,

examples/glm_seq2seq/train.py

@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSeq2Seq
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.data.dataset import Seq2SeqDataset
 from flagai.test_utils import Seq2SeqCollateArguments
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, CH_TASKS
@@ -27,12 +27,12 @@
 print("downloading...")
 
 if task_name in CH_TASKS:
-    tokenizer = GLMLargeChTokenizer()
     model_name = 'GLM-large-ch'
 else:
-    tokenizer = GLMLargeEnWordPieceTokenizer()
     model_name = 'GLM-large-en'
 
+tokenizer = Tokenizer.from_pretrained(model_name)
+
 train_dataset = Seq2SeqDataset(task_name=task_name,
                                data_dir='./datasets/',
                                dataset_type='train',

examples/glm_superglue/train_10b_clue.py

@@ -4,7 +4,7 @@
 import os
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLMLargeChTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -21,11 +21,12 @@
                   save_dir="./glm_superglue_en",
                   save_interval=1)
 
+model_name = "GLM-large-ch"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
                                              model_name="GLM-large-ch")
 
 
-tokenizer =  GLMLargeChTokenizer()
+tokenizer = Tokenizer.from_pretrained("GLM-large-ch")
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',

examples/glm_superglue/train_10b_superglue.py

@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -28,11 +28,11 @@
                   # deepspeed_config='./deepspeed.json',
                   # training_script=__file__)
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
 
-tokenizer = GLMLargeEnWordPieceTokenizer()
-
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',

examples/glm_superglue/train_prefix.py

@@ -2,13 +2,12 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
-from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze, GLMForMultiTokenClozeFast, GLMForSequenceClassification
-from flagai.data.tokenizer import GLMLargeEnWordPieceTokenizer, GLMLargeChTokenizer
+from flagai.model.glm_model import GLMForSequenceClassification
+from flagai.data.tokenizer import Tokenizer
 
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
 from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
-import unittest
 from flagai.data.dataset import ConstructSuperglueStrategy
 
 
@@ -32,13 +31,10 @@
 
 if task_name in CH_TASKS:
     model_name = 'GLM-large-ch'
-    tokenizer = GLMLargeChTokenizer(add_block_symbols=True,
-                                    add_task_mask=False,
-                                    add_decoder_mask=False,
-                                    fix_command_token=True)
+    add_block_symbols=True,
 else:
     model_name = 'GLM-large-en'
-    tokenizer = GLMLargeEnWordPieceTokenizer()
+tokenizer = Tokenizer.from_pretrained(model_name)
 
 model = GLMForSequenceClassification.from_pretrain(model_name=model_name, spell_length=2,
                                                     class_num=3, tune_prefix_layers=1)

examples/glm_superglue/train_qqp_deepspeed.py

@@ -2,38 +2,46 @@
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
-from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.model.glm_model import GLMForSingleTokenCloze, GLMForMultiTokenCloze
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
+from flagai.data.dataset.superglue.control import DEFAULT_METRICS, MULTI_TOKEN_TASKS, CH_TASKS
 
-task_name = 'qqp'
+task_name = 'boolq'
 trainer = Trainer(env_type='deepspeed',
-                  epochs=10,
+                  epochs=1000,
                   batch_size=512,
                   eval_interval=100,
                   log_interval=10,
-                  save_interval = 1e5,
+                  save_interval=1e5,
                   gradient_accumulation_steps=5,
                   checkpoint_activations=True,
                   fp16=True,
                   warm_up=0.1,
                   weight_decay=0.1,
                   save_dir="./qqp",
                   master_ip='127.0.0.1',
-                  master_port=17887,
+                  master_port=17810,
                   num_nodes=1,
                   num_gpus=2,
                   hostfile='./hostfile',
                   deepspeed_config='./deepspeed.json',
                   training_script=__file__)
 
-model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+model_name = "GLM-large-en"
+tokenizer = Tokenizer.from_pretrained(model_name)
 
+if task_name in MULTI_TOKEN_TASKS:
+    model = GLMForMultiTokenCloze.from_pretrain(
+        download_path="/mnt/test_10b_models", model_name=model_name)
+else:
+    model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
+                                                 model_name=model_name)
 
-tokenizer = GLMLargeEnWordPieceTokenizer()
+# model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
+#                                              model_name="GLM-large-en")
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',

examples/glm_superglue/train_qqp_pytorch.py

@@ -4,8 +4,7 @@
 
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.model.bert_model import BertForClsClassifier
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -28,11 +27,12 @@
                   warm_up=0.1,
                   save_dir="./glm_large_qqp_pytorch")
 
+model_name = "GLM-large-en"
+tokenizer = Tokenizer.from_pretrained(model_name)
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
+
 
-#tokenizer = GLM10bENBPETokenizer()
-tokenizer = GLMLargeEnWordPieceTokenizer()
 
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',

examples/glm_superglue/train_qqp_pytorch_fp16.py

@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -24,9 +24,10 @@
                   warm_up=0.1,
                   save_dir="./glm_large_qqp_pytorch_fp16")
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
-tokenizer = GLMLargeEnWordPieceTokenizer()
+                                             model_name=model_name)
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',

examples/glm_superglue/train_qqp_pytorchddp.py

@@ -3,7 +3,7 @@
 # Licensed under the Apache License, Version 2.0 (the "License")
 from flagai.trainer import Trainer
 from flagai.model.glm_model import GLMForSingleTokenCloze
-from flagai.data.tokenizer import GLM10bENBPETokenizer, GLMLargeEnWordPieceTokenizer
+from flagai.data.tokenizer import Tokenizer
 from flagai.metrics import accuracy_metric
 from flagai.data.dataset import SuperGlueDataset
 from flagai.test_utils import CollateArguments
@@ -29,11 +29,11 @@
                   hostfile='./hostfile',
                   training_script=__file__)
 
+model_name = "GLM-large-en"
 model = GLMForSingleTokenCloze.from_pretrain(download_path="/mnt/test_10b_models",
-                                             model_name="GLM-large-en")
+                                             model_name=model_name)
 
-#tokenizer = GLM10bENBPETokenizer()
-tokenizer = GLMLargeEnWordPieceTokenizer()
+tokenizer = Tokenizer.from_pretrained(model_name)
 train_dataset = SuperGlueDataset(task_name=task_name,
                                  data_dir='./datasets/',
                                  dataset_type='train',

examples/glm_title_generation/train.py

@@ -134,7 +134,7 @@ def __call__(self, batch):
 
 sents_src, sents_tgt = read_file()
 my_collate_fn = GLMPoetryDynamicCollateFN(
-    pad_id=tokenizer.get_command('pad').Id)
+    pad_id=tokenizer.get_command_id('pad'))
 
 data_len = len(sents_tgt)
 train_size = int(data_len * 0.8)

examples/opt/generate_opt_1.3b.py

@@ -1,4 +1,3 @@
-
 from flagai.model.predictor.predictor import Predictor
 from flagai.auto_model.auto_loader import AutoLoader
 

examples/roberta_faq/1_construct_data.py

@@ -10,7 +10,6 @@
 import numpy as np
 from tqdm import tqdm
 import collections
-import faiss
 
 faq_data_path = "./data/financezhidao_filter.csv"
 answer_save_path = "./data/finance_fqa.json"