fix(llama.cpp): Determine granite language 3b instruct by vocab size

Branch: GraniteLM Signed-off-by: Gabe Goodhart <[email protected]>
ggerganov · Sep 11, 2024 · e8a1073 · e8a1073
1 parent 2ec91ee
commit e8a1073
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -5401,7 +5401,7 @@ static void llm_load_hparams(
                         // granite uses a vocab with len 49152
                         case 32: model.type = hparams.n_vocab == 49152 ? e_model::MODEL_3B : (hparams.n_vocab < 40000 ? e_model::MODEL_7B : e_model::MODEL_8B); break;
                         case 36: model.type = e_model::MODEL_8B; break; // granite
-                        case 40: model.type = hparams.n_vocab == 49152 ? e_model::MODEL_3B : e_model::MODEL_13B; break;
+                        case 40: model.type = (hparams.n_vocab == 49152 || hparams.n_vocab == 49156) ? e_model::MODEL_3B : e_model::MODEL_13B; break;
                         case 48: model.type = e_model::MODEL_34B; break;
                         case 60: model.type = e_model::MODEL_30B; break;
                         case 80: model.type = hparams.n_head() == hparams.n_head_kv() ? e_model::MODEL_65B : e_model::MODEL_70B; break;