From 8841ce3f439de6e770f70319b7e08b6613197ea7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 27 Oct 2024 20:59:58 +0200 Subject: [PATCH 1/2] llama : switch KQ multiplication to F32 precision by default (#10015) ggml-ci --- src/llama.cpp | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/llama.cpp b/src/llama.cpp index 50eebc2c298f5..53979e83f8b87 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -9618,20 +9618,16 @@ static struct ggml_tensor * llm_build_kqv( cur = ggml_flash_attn_ext(ctx, q, k, v, kq_mask, kq_scale, hparams.f_max_alibi_bias, hparams.attn_soft_cap ? hparams.f_attn_logit_softcapping : 0.0f); - if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_GEMMA2) { - ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); - } + ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32); cur = ggml_reshape_2d(ctx, cur, n_embd_head_v*n_head, n_tokens); } else { struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); cb(kq, "kq", il); - if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_NEMOTRON || model.arch == LLM_ARCH_CHATGLM) { - // for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs - // ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847 - ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - } + // note: this op tends to require high floating point range + // while for some models F16 is enough, for others it is not, so we default to F32 here + ggml_mul_mat_set_prec(kq, GGML_PREC_F32); if (model.arch == LLM_ARCH_GROK) { // need to do the following: @@ -9640,9 +9636,6 @@ static struct ggml_tensor * llm_build_kqv( // kq = 30 * tanh(kq / 30) // before the softmax below - //try from phi2 - //ggml_mul_mat_set_prec(kq, GGML_PREC_F32); - kq = ggml_tanh(ctx, ggml_scale(ctx, kq, 0.08838834764831845f/30.0f)); kq = ggml_scale(ctx, kq, 30); } From 8125e6cbfcf2b3b9066e4d923aca9295526730f5 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Mon, 28 Oct 2024 08:49:32 +0200 Subject: [PATCH 2/2] server : don't overfill the batch during infill (#10018) ggml-ci --- examples/server/server.cpp | 1 + examples/server/utils.hpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/server/server.cpp b/examples/server/server.cpp index ff1d9b03cec5d..077c7ad1adffb 100644 --- a/examples/server/server.cpp +++ b/examples/server/server.cpp @@ -1880,6 +1880,7 @@ struct server_context { if (slot.state == SLOT_STATE_STARTED) { slot.t_start_process_prompt = ggml_time_us(); slot.t_start_generation = 0; + slot.n_past = 0; slot.n_prompt_tokens = prompt_tokens.size(); slot.state = SLOT_STATE_PROCESSING_PROMPT; diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp index 8112420624185..562635555e0ab 100644 --- a/examples/server/utils.hpp +++ b/examples/server/utils.hpp @@ -266,8 +266,10 @@ static llama_tokens format_infill( } // for now pick FIM context to fit in a batch (ratio prefix:suffix = 3:1, TODO: configurable?) - const int n_suffix_take = std::min(tokens_suffix.size(), (n_batch/4)); - const int n_prefix_take = std::min(tokens_prefix.size(), 3*(n_batch/4) - 3); + const int n_prefix_take = std::min(tokens_prefix.size(), 3*(n_batch/4)); + const int n_suffix_take = std::min(tokens_suffix.size(), std::max(0, (n_batch/4) - (2 + tokens_prompt.size()))); + + SRV_DBG("n_prefix_take = %d, n_suffix_take = %d, total = %d\n", n_prefix_take, n_suffix_take, (n_prefix_take + n_suffix_take)); // fill the rest of the context with extra chunks const int n_extra_take = std::min(std::max(0, n_ctx - (n_batch) - 2*n_predict), extra_tokens.size());