From 66898d7d752deaed808d08b4365621968fc0e0a6 Mon Sep 17 00:00:00 2001 From: Kevin Gibbons Date: Sat, 7 Sep 2024 22:51:00 -0700 Subject: [PATCH] llama : set attrs of mislabelled EOT/EOM tokens (#9348) --- src/llama.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/llama.cpp b/src/llama.cpp index 190564fa4e69e..f590bcd3b9047 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -6399,6 +6399,11 @@ static void llm_load_vocab( ) ) { vocab.special_eot_id = t.second; + if ((vocab.id_to_token[t.second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { + LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n", + __func__, t.first.c_str()); + vocab.id_to_token[t.second].attr = LLAMA_TOKEN_ATTR_CONTROL; + } break; } } @@ -6412,6 +6417,11 @@ static void llm_load_vocab( const auto & t = vocab.token_to_id.find("<|eom_id|>"); if (t != vocab.token_to_id.end()) { vocab.special_eom_id = t->second; + if ((vocab.id_to_token[t->second].attr & LLAMA_TOKEN_ATTR_CONTROL) == 0) { + LLAMA_LOG_WARN("%s: control-looking token: '%s' was not control-type; this is probably a bug in the model. its type will be overridden\n", + __func__, t->first.c_str()); + vocab.id_to_token[t->second].attr = LLAMA_TOKEN_ATTR_CONTROL; + } } } }