Removed uneeded vocab.tokenizer_clean_spaces assignment

ggerganov · Jul 19, 2024 · ea215c7 · ea215c7
1 parent 003fcaf
commit ea215c7
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -5520,7 +5520,6 @@ static void llm_load_vocab(
             } else if (
                 tokenizer_pre == "tekken") {
                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_TEKKEN;
-                vocab.tokenizer_clean_spaces = true;
                 vocab.tokenizer_ignore_merges = true;
                 vocab.tokenizer_add_bos = true;
             } else if (