Skip to content

Commit

Permalink
Made sheduler config not needed for prompt lookup.
Browse files Browse the repository at this point in the history
  • Loading branch information
popovaan committed Dec 20, 2024
1 parent a1e4973 commit 38a42d6
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ int main(int argc, char* argv[]) try {
ov::genai::LLMPipeline pipe(
model_path,
device,
ov::genai::prompt_lookup(true),
ov::genai::scheduler_config(scheduler_config));
ov::genai::prompt_lookup(true));

auto streamer = [](std::string subword) {
std::cout << subword << std::flush;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,8 @@ def main():
args = parser.parse_args()

device = 'CPU'
scheduler_config = openvino_genai.SchedulerConfig()
# cache params
scheduler_config.cache_size = 2

pipe = openvino_genai.LLMPipeline(args.model_dir, device, scheduler_config=scheduler_config, prompt_lookup=True)
pipe = openvino_genai.LLMPipeline(args.model_dir, device, prompt_lookup=True)

config = openvino_genai.GenerationConfig()
config.max_new_tokens = 100
Expand Down
9 changes: 6 additions & 3 deletions src/cpp/src/llm_pipeline.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,8 @@ ov::genai::LLMPipeline::LLMPipeline(
){
auto start_time = std::chrono::steady_clock::now();
if (properties.find(ov::genai::scheduler_config.name()) != properties.end() ||
properties.find(utils::DRAFT_MODEL_ARG_NAME) != properties.end()) {
properties.find(utils::DRAFT_MODEL_ARG_NAME) != properties.end() ||
properties.find(ov::genai::prompt_lookup.name()) != properties.end()) {
auto [plugin_config, scheduler_config] = utils::split_scheduler_config(properties);
m_pimpl = std::make_unique<ContinuousBatchingAdapter>(models_path, tokenizer, scheduler_config, device, plugin_config);
} else if (device == "NPU") {
Expand All @@ -721,7 +722,8 @@ ov::genai::LLMPipeline::LLMPipeline(
auto start_time = std::chrono::steady_clock::now();

if (config.find(ov::genai::scheduler_config.name()) != config.end() ||
config.find(utils::DRAFT_MODEL_ARG_NAME) != config.end()) {
config.find(utils::DRAFT_MODEL_ARG_NAME) != config.end() ||
config.find(ov::genai::prompt_lookup.name()) != config.end()) {
auto [plugin_config, scheduler_config] = utils::split_scheduler_config(config);
m_pimpl = std::make_unique<ContinuousBatchingAdapter>(models_path, scheduler_config, device, plugin_config);
} else if (device == "NPU") {
Expand All @@ -745,7 +747,8 @@ ov::genai::LLMPipeline::LLMPipeline(

auto start_time = std::chrono::steady_clock::now();
if (plugin_config.find(ov::genai::scheduler_config.name()) != plugin_config.end() ||
plugin_config.find(utils::DRAFT_MODEL_ARG_NAME) != plugin_config.end()) {
plugin_config.find(utils::DRAFT_MODEL_ARG_NAME) != plugin_config.end() ||
plugin_config.find(ov::genai::prompt_lookup.name()) != plugin_config.end()){

auto [plugin_config_, scheduler_config] = utils::split_scheduler_config(plugin_config);
m_pimpl = std::make_unique<ContinuousBatchingAdapter>(model_str, weights_tensor,
Expand Down

0 comments on commit 38a42d6

Please sign in to comment.