Ref

openvinotoolkit · Jul 22, 2024 · 7998dd6 · 7998dd6
1 parent cc1c465
commit 7998dd6
Show file tree

Hide file tree

Showing 5 changed files with 4 additions and 33 deletions.
diff --git a/samples/cpp/speculative_decoding/speculative_decoding_sample.cpp b/samples/cpp/speculative_decoding/speculative_decoding_sample.cpp
@@ -58,7 +58,7 @@ int main(int argc, char* argv[]) try {
     std::vector<ov::genai::GenerationConfig> sampling_params_examples {
         // ov::genai::beam_search(),
         ov::genai::greedy(),
-        // ov::genai::multinomial(),
+        ov::genai::multinomial(),
     };
 
     std::vector<std::string> prompts(num_prompts);

diff --git a/src/cpp/continuous_batching/src/continuous_batching_pipeline.cpp b/src/cpp/continuous_batching/src/continuous_batching_pipeline.cpp
@@ -120,10 +120,6 @@ class ContinuousBatchingPipeline::Impl {
         return res;
     }
 
-    void set_to_free_sequences(bool is_free) {
-        is_free_seq = is_free;
-    }
-
     void set_speculative_decoding_mode() {
         m_sampler->set_validation_mode();
     }
@@ -197,15 +193,6 @@ class ContinuousBatchingPipeline::Impl {
         }
     }
 
-    void free_all_sequences() {
-        for (auto& r : m_requests) {
-            for (auto& s : r->get_running_sequences()) {
-                s->set_status(SequenceStatus::FINISHED);
-                m_scheduler->free_sequence(s->get_id());
-            }
-        }
-    }
-
     ov::genai::GenerationConfig get_config() const {
         return m_generation_config;
     }
@@ -312,10 +299,8 @@ class ContinuousBatchingPipeline::Impl {
                 for (auto & child_id : child_ids)
                     m_scheduler->fork_sequence(parent_id, child_id);
             }
-            if (is_free_seq) {
-                for (auto seq_id : sampler_output.m_dropped_sequences)
-                    m_scheduler->free_sequence(seq_id);
-            }
+            for (auto seq_id : sampler_output.m_dropped_sequences)
+                m_scheduler->free_sequence(seq_id);
             timer.end();
         }
 

diff --git a/src/cpp/continuous_batching/src/logit_processor.hpp b/src/cpp/continuous_batching/src/logit_processor.hpp
@@ -338,10 +338,6 @@ class LogitProcessor {
         ++m_generated_tokens;
     }
 
-    void set_gen_len(size_t new_len) {
-        m_generated_tokens = new_len;
-    }
-
     void register_new_generated_token(int64_t new_token_id) {
         auto it = m_unique_generated_token_ids->find(new_token_id);
         if (it == m_unique_generated_token_ids->end()) {

diff --git a/src/cpp/continuous_batching/src/sampler.hpp b/src/cpp/continuous_batching/src/sampler.hpp
@@ -203,16 +203,10 @@ class GroupBeamSearcher {
 class Sampler {
 
     std::vector<Token> _get_logit_vector(ov::Tensor logits, size_t batch_idx = 0, size_t token_offset = 0) {
-        if (token_offset) {
-            int a = 0;
-        }
         ov::Shape logits_shape = logits.get_shape();
         size_t batch_size = logits_shape[0], seq_len = logits_shape[1], vocab_size = logits_shape[2];
         OPENVINO_ASSERT(batch_idx <= batch_size);
-        // OPENVINO_ASSERT(token_offset <= seq_len);
-        if (token_offset > seq_len) {
-            auto a = 0;
-        }
+        OPENVINO_ASSERT(token_offset <= seq_len);
         size_t batch_offset = batch_idx * seq_len * vocab_size;
         size_t sequence_offset = (seq_len - token_offset - 1) * vocab_size;
         const float* logits_data = logits.data<const float>() + batch_offset + sequence_offset;

diff --git a/src/cpp/continuous_batching/src/scheduler.hpp b/src/cpp/continuous_batching/src/scheduler.hpp
@@ -331,10 +331,6 @@ class Scheduler {
                 size_t num_running_seqs = sequence_group->num_running_seqs();
                 // prompt phases can have a single running sequence
                 OPENVINO_ASSERT(num_running_seqs == 1);
-                // here we also assume that sequence must be scheduler in a single shot and has no already generated context
-                if (sequence_group->get_context_len() != 0) {
-                    auto a = 0;
-                }
                 OPENVINO_ASSERT(sequence_group->get_context_len() == 0);
 
                 size_t num_available_tokens_in_megabatch = m_config.max_num_batched_tokens - scheduler_output.m_total_num_scheduled_tokens;