fix TextCallbackStreamer when characters are eaten by regex

src/cpp/src/text_callback_streamer.cpp

@@ -15,6 +15,14 @@ bool TextCallbackStreamer::put(int64_t token) {
     std::stringstream res;
     m_tokens_cache.push_back(token);
     std::string text = m_tokenizer.decode(m_tokens_cache);
+
+    // In some cases adding the next token can shorten the text, 
+    // e.g. when apostrophe removing regex had worked after adding new tokens.
+    // Need to hold on before flushing if apostrophe is the last symbol.
+    if (text.size() > 1 && text.back() == '\'') {
+        return on_finalized_subword_callback(res.str());
+    }
+
     if (!text.empty() && '\n' == text.back() && text.size() > print_len) {
         // Flush the cache after the new line symbol
         res << std::string_view{text.data() + print_len, text.size() - print_len};