debug.

b4rtaz · Nov 17, 2024 · 89c1406 · 89c1406
1 parent 0fabc54
commit 89c1406
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
@@ -135,6 +135,7 @@ Tokenizer::Tokenizer(char* tokenizerPath, int modelVocabSize) {
         vocab[i][len] = '\0'; // add the string terminating token
     }
     fclose(file);
+    printf("Tokenizer loaded with %d tokens\n", vocabSize); fflush(stdout);
 }
 
 Tokenizer::~Tokenizer() {