Fix for ggerganov#3454 (ggerganov#3455)

Fix: `sentencepiece` tokenizers with added tokens failed with an incorrect assertion
yusiwen · Oct 7, 2023 · 240db82 · 240db82
1 parent 3ae5a56
commit 240db82
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/llama.cpp b/llama.cpp
@@ -8200,7 +8200,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
                 buf[0] = llama_token_to_byte(model->vocab, token);
                 return 1;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }
@@ -8216,7 +8218,9 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, ch
             } else if (llama_is_control_token(model->vocab, token)) {
                 ;
             } else {
-                GGML_ASSERT(false);
+                // TODO: for now we accept all unsupported token types,
+                // suppressing them like CONTROL tokens.
+                // GGML_ASSERT(false);
             }
             break;
         }