diff --git a/src/transformers/generation/stopping_criteria.py b/src/transformers/generation/stopping_criteria.py
index 7b32c018041443..7b06b64a94c297 100644
--- a/src/transformers/generation/stopping_criteria.py
+++ b/src/transformers/generation/stopping_criteria.py
@@ -254,6 +254,8 @@ def _stop_string_get_matching_positions(
     def _cleanup_token(token: str) -> str:
         if token[0] in ["▁", "Ġ"]:
             token = " " + token[1:]
+        elif token[0] == "##":
+            token = token[2:]
         return token
 
     reversed_filtered_tok_list = [_cleanup_token(token)[::-1] for token in tok_list]