From f33f10e903d5b4c80a358f68a46b5573a23fcc90 Mon Sep 17 00:00:00 2001 From: Anastasiia Pnevskaia Date: Wed, 24 Jul 2024 10:26:47 +0200 Subject: [PATCH] Updated docstring. --- src/cpp/include/openvino/genai/scheduler_config.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cpp/include/openvino/genai/scheduler_config.hpp b/src/cpp/include/openvino/genai/scheduler_config.hpp index e813054c42..d9bf7a7b41 100644 --- a/src/cpp/include/openvino/genai/scheduler_config.hpp +++ b/src/cpp/include/openvino/genai/scheduler_config.hpp @@ -31,7 +31,12 @@ struct SchedulerConfig { // max number of scheduled sequences (you can think of it as "max batch size") std::size_t max_num_seqs = 256; - // enable caching of KV-blocks and reusage in next generations. + // Enable caching of KV-blocks. + // When turned on all previously calculated KV-caches are kept in memory for future usages. + // KV-caches can be rewritten if KV-cache limit is reached, but blocks are not released. + // This results in more RAM usage, maximum RAM usage is determined by cache_size or num_kv_blocks parameters. + // When turend off only KV-cache required for batch calculation is kept in memory and + // when a sequence has finished genegartion its cache is released. bool enable_prefix_caching = false; }; }