added support for eot token and unknown token (for an improperly toke…

…nised layla fine-tune)
l3utterfly · Aug 23, 2023 · f15f3e3 · f15f3e3
1 parent 3c6c65b
commit f15f3e3
Showing 1 changed file with 10 additions and 1 deletion.
diff --git a/examples/main/main.cpp b/examples/main/main.cpp
@@ -54,6 +54,15 @@ void sigint_handler(int signo) {
 }
 #endif
 
+
+llama_token llama_token_unk() {
+    return 0;
+}
+
+llama_token llama_token_eot() {
+    return 32000;
+}
+
 int main(int argc, char ** argv) {
     gpt_params params;
 
@@ -705,7 +714,7 @@ int main(int argc, char ** argv) {
             }
 
             // deal with end of text token in interactive mode
-            if (last_n_tokens.back() == llama_token_eos()) {
+            if (last_n_tokens.back() == llama_token_eos() || last_n_tokens.back() == llama_token_eot() || last_n_tokens.back() == llama_token_unk()) {
                 if (params.interactive) {
                     if (params.antiprompt.size() != 0) {
                         // tokenize and inject first reverse prompt