streaming_decoder=False

.github/workflows/llm.yml

@@ -50,7 +50,7 @@ jobs:
           with open('pred.txt', 'r') as file:
               predictions = file.read()
           tokenizer = transformers.LlamaTokenizer.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/')
-          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('69', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20):
+          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('69', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
               ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len('69'):] + '\n'
               idx = predictions.find(ref)
               if -1 == idx:
@@ -65,14 +65,59 @@ jobs:
           with open('pred.txt', 'r') as file:
               predictions = file.read()
           tokenizer = transformers.LlamaTokenizer.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/')
-          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('Hi', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20):
+          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('Hi', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
               ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len('Hi'):] + '\n'
               idx = predictions.find(ref)
               if -1 == idx:
                   raise RuntimeError(f'Missing "{ref=}" from predictions')
               predictions = predictions[:idx] + predictions[idx + len(ref):]
           "
           echo Hi passed
+
+          timeout 25s ./build/llm/cpp/llm ./TinyLlama-1.1B-Chat-v0.6/openvino_model.xml ./tokenizer.xml ./detokenizer.xml "return 0" > ./pred.txt
+          python -c "
+          import transformers
+          with open('pred.txt', 'r') as file:
+              predictions = file.read()
+          tokenizer = transformers.LlamaTokenizer.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/')
+          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('return 0', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
+              ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len('return 0'):] + '\n'
+              idx = predictions.find(ref)
+              if -1 == idx:
+                  raise RuntimeError(f'Missing "{ref=}" from predictions')
+              predictions = predictions[:idx] + predictions[idx + len(ref):]
+          "
+          echo return 0 passed
+
+          ./build/llm/cpp/llm ./TinyLlama-1.1B-Chat-v0.6/openvino_model.xml ./tokenizer.xml ./detokenizer.xml "" > ./pred.txt
+          python -c "
+          import transformers
+          with open('pred.txt', 'r') as file:
+              predictions = file.read()
+          tokenizer = transformers.LlamaTokenizer.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/')
+          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
+              ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len(''):] + '\n'
+              idx = predictions.find(ref)
+              if -1 == idx:
+                  raise RuntimeError(f'Missing "{ref=}" from predictions')
+              predictions = predictions[:idx] + predictions[idx + len(ref):]
+          "
+          echo '""' passed
+
+          ./build/llm/cpp/llm ./TinyLlama-1.1B-Chat-v0.6/openvino_model.xml ./tokenizer.xml ./detokenizer.xml "你好！ 你好嗎？" > ./pred.txt
+          python -c "
+          import transformers
+          with open('pred.txt', 'r') as file:
+              predictions = file.read()
+          tokenizer = transformers.LlamaTokenizer.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/')
+          for beam in transformers.LlamaForCausalLM.from_pretrained('./TinyLlama-1.1B-Chat-v0.6/').generate(tokenizer('你好！ 你好嗎？', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
+              ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len('你好！ 你好嗎？'):] + '\n'
+              idx = predictions.find(ref)
+              if -1 == idx:
+                  raise RuntimeError(f'Missing "{ref=}" from predictions')
+              predictions = predictions[:idx] + predictions[idx + len(ref):]
+          "
+          echo 你好！ 你好嗎？ passed
   llm-cpp-windows:
     runs-on: windows-latest
     steps:
@@ -117,7 +162,7 @@ jobs:
           with open('pred.txt', 'r') as file:
               predictions = file.read()
           tokenizer = transformers.LlamaTokenizer.from_pretrained(r'.\TinyLlama-1.1B-Chat-v0.6\')
-          for beam in transformers.LlamaForCausalLM.from_pretrained(r'.\TinyLlama-1.1B-Chat-v0.6\').generate(tokenizer('69', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20):
+          for beam in transformers.LlamaForCausalLM.from_pretrained(r'.\TinyLlama-1.1B-Chat-v0.6\').generate(tokenizer('69', return_tensors='pt')['input_ids'], num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9):
               ref = ': ' + tokenizer.decode(beam, skip_special_tokens=True)[len('69'):] + '\n'
               idx = predictions.find(ref)
               if -1 == idx:

llm/cpp/convert_tokenizers.py

@@ -14,7 +14,7 @@ def main():
         raise RuntimeError("Usage: {sys.argv[0]} <user_ov_extensions_LIB> <SOURCE_MODEL_DIR>")
     ov_tokenizer.init_extension(sys.argv[1])
     tokenizer, detokenizer = ov_tokenizer.convert_tokenizer(
-        transformers.AutoTokenizer.from_pretrained(sys.argv[2]), with_decoder=True, streaming_decoder=True)
+        transformers.AutoTokenizer.from_pretrained(sys.argv[2]), with_decoder=True, streaming_decoder=False)
     openvino.save_model(tokenizer, "tokenizer.xml")
     openvino.save_model(detokenizer, "detokenizer.xml")
 

llm/cpp/group_beam_searcher.hpp

@@ -86,7 +86,7 @@ struct Parameters {
     float diversity_penalty = 1.0;
     size_t max_new_tokens = 20;
     StopCriteria stop_criteria = StopCriteria::heuristic;
-    float length_penalty = 0.0;
+    float length_penalty = 1.0;
     size_t no_repeat_ngram_size = std::numeric_limits<size_t>::max();
     // There's no way to extract special token values from the tokenizer for now
     int64_t eos_token = 2;
@@ -99,7 +99,7 @@ struct Group {
     std::vector<Beam> min_heap;  // The worst of the best completed beams is the first
     bool done = false;
     void finish(Beam&& beam, const Parameters& parameters) {
-        beam.score /= std::pow(float(parameters.prompt.size() + beam.tokens.size()), parameters.length_penalty);
+        beam.score /= std::pow(float(beam.tokens.size()), parameters.length_penalty);
         min_heap.push_back(std::move(beam));
         std::push_heap(min_heap.begin(), min_heap.end(), greater);
         if (min_heap.size() > parameters.group_size) {
@@ -111,7 +111,7 @@ struct Group {
         if (min_heap.size() < parameters.group_size) {
             return;
         }
-        size_t cur_len = parameters.prompt.size() + ongoing.front().tokens.size();
+        size_t cur_len = ongoing.front().tokens.size();
         float best_sum_logprobs = ongoing.front().score;
         float worst_score = min_heap.front().score;
         switch (parameters.stop_criteria) {
@@ -146,6 +146,7 @@ struct GroupBeamSearcher {
         for (Group& group : groups) {
             group.ongoing.resize(parameters.group_size);
             group.ongoing.front().score = 0.0;
+            group.ongoing.front().tokens = this->parameters.prompt;
         }
     }
     std::vector<TokenToBeam> process(const ov::Tensor& logits) {
@@ -156,9 +157,9 @@ struct GroupBeamSearcher {
             if (!group.done) {
                 for (Beam& beam : group.ongoing) {
                     beam.global_beam_idx = beam_count;
-                    // beam.tokens.empty() holds for the first process() call.
-                    // Every beam should be constructed from the single batch
-                    if (!beam.tokens.empty()) {
+                    // Every beam should be constructed from the single batch on first process() call
+                    if (group.ongoing.front().tokens.size() != parameters.prompt.size()) {
+                        // It's not the first process() call
                         ++beam_count;
                     }
                 }
@@ -182,11 +183,9 @@ struct GroupBeamSearcher {
                         tokens.at(size_t(prev_beam.tokens.back())).log_prob -= parameters.diversity_penalty;
                     }
                 }
-                std::vector<int64_t> full_text{parameters.prompt};
-                full_text.insert(full_text.end(), beam.tokens.begin(), beam.tokens.end());
-                if (full_text.size() > 1 && full_text.size() >= parameters.no_repeat_ngram_size) {
-                    auto tail_start = full_text.end() - ptrdiff_t(parameters.no_repeat_ngram_size) + 1;
-                    for (int64_t banned_token : kmp_search(full_text, {tail_start, full_text.end()})) {
+                if (beam.tokens.size() > 1 && beam.tokens.size() >= parameters.no_repeat_ngram_size) {
+                    auto tail_start = beam.tokens.end() - ptrdiff_t(parameters.no_repeat_ngram_size) + 1;
+                    for (int64_t banned_token : kmp_search(beam.tokens, {tail_start, beam.tokens.end()})) {
                         tokens.at(size_t(banned_token)).log_prob = -std::numeric_limits<float>::infinity();
                     }
                 }

llm/cpp/llm.cpp

@@ -32,7 +32,8 @@ int main(int argc, char* argv[]) try {
     }
     ov::Core core;
     core.add_extension(USER_OV_EXTENSIONS_PATH);  // USER_OV_EXTENSIONS_PATH is defined in root CMakeLists.txt
-    auto [input_ids, mask] = tokenize(core.compile_model(argv[2], "CPU").create_infer_request(), argv[4]);
+    std::string_view prompt = argv[4];
+    auto [input_ids, mask] = tokenize(core.compile_model(argv[2], "CPU").create_infer_request(), prompt);
     ov::InferRequest detokenizer = core.compile_model(argv[3], "CPU").create_infer_request();
     std::shared_ptr<ov::Model> model = core.read_model(argv[1]);
     std::map<size_t, ov::PartialShape> shapes = {
@@ -112,7 +113,12 @@ int main(int argc, char* argv[]) try {
         }
         std::cout << "Group:\n";
         for (const Beam& beam : group.min_heap) {
-            std::cout << beam.score << ": " << detokenize(detokenizer, beam.tokens) << '\n';
+            std::string detokenized = detokenize(detokenizer, beam.tokens);
+            if (detokenized.size() < prompt.size()) {
+                throw std::runtime_error("Detokenized sequence became smaller than the prompt which must be included");
+            }
+            std::string_view generated{detokenized.data() + prompt.size(), detokenized.size() - prompt.size()};
+            std::cout << beam.score << ": " << generated << '\n';
         }
     }
 } catch (const std::exception& error) {