fixed the inference issue for chatglm2

intel · Jan 18, 2024 · 1d29a9c · 1d29a9c
1 parent 89cc9eb
commit 1d29a9c
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/neural_speed/models/model_utils/model_files.h b/neural_speed/models/model_utils/model_files.h
@@ -962,8 +962,9 @@ struct gguf_loader {
       scores = (const float*)gguf_get_arr_data(ctx_gguf, score_idx);
     }
 
+    uint32_t default_n_vocab = 32000;
     const uint32_t n_vocab = gguf_get_arr_n(ctx_gguf, token_idx);
-    if (hparams.n_vocab != n_vocab) {
+    if ((hparams.n_vocab == default_n_vocab) && (hparams.n_vocab != n_vocab)) {
       hparams.n_vocab = n_vocab;
     }