From 25b659ba89c7f58cdb5e58078dfd5faf143cf469 Mon Sep 17 00:00:00 2001 From: Zac Liu Date: Thu, 25 Aug 2022 09:39:28 +0800 Subject: [PATCH] Update glm_10b_en_tokenizer.py --- flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py index f005d5fc..a7c2a281 100644 --- a/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py +++ b/flagai/data/tokenizer/glm_10b_en/glm_10b_en_tokenizer.py @@ -148,7 +148,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs): else: special_tokens = kwargs.pop('special_tokens', []) - if not os.path.exists(resolved_merges_file): # 如果没有这个vocab文件， 那我们就要下载 + if not os.path.exists(resolved_merges_file): if pretrained_model_name_or_path in VOCAB_ARCHIVE_URLS_MAP: for key, url in VOCAB_ARCHIVE_URLS_MAP[ pretrained_model_name_or_path].items():