diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index f81f1a7..1d7c9f7 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -522,16 +522,17 @@ EosDetector::~EosDetector() { EosDetectorType EosDetector::append(int tokenId, const char* piece) { int pieceLength = strlen(piece); - int length = bufferPos + pieceLength + 1; - if (length > bufferSize) { - char* newBuffer = new char[length]; + int newSize = bufferPos + pieceLength + 1; + if (newSize > bufferSize) { + char* newBuffer = new char[newSize]; if (bufferPos > 0) memcpy(newBuffer, buffer, bufferPos); if (bufferSize > 0) delete[] buffer; buffer = newBuffer; + bufferSize = newSize; } - memcpy(buffer + bufferPos, piece, pieceLength + 1); + memcpy(&buffer[bufferPos], piece, pieceLength + 1); bufferPos += pieceLength; // detection