diff --git a/tests/models/whisper/test_tokenization_whisper.py b/tests/models/whisper/test_tokenization_whisper.py
index f84c050cc039ab..7be9e983f9fee6 100644
--- a/tests/models/whisper/test_tokenization_whisper.py
+++ b/tests/models/whisper/test_tokenization_whisper.py
@@ -278,12 +278,10 @@ def test_timestamp_encoding(self):
         rust_tokenizer = self.get_rust_tokenizer()
 
         input_text = "<|0.00|> Whisper can do timestamps?<|2.60|>"
-        target_text = "<|startoftranscript|><|notimestamps|><|0.00|> Whisper can do timestamps?<|2.60|><|endoftext|>"
 
         # fmt: off
         EXPECTED_TOKENS = [
-            START_OF_TRANSCRIPT, NOTIMESTAMPS, NOTIMESTAMPS + 1, 2471, 271, 610,
-            393, 360, 220, 31208, 377, 23150, 30, 50494, END_OF_TRANSCRIPT,
+            NOTIMESTAMPS + 1, 2471, 271, 610, 393, 360, 220, 31208, 377, 23150, 30, 50494,
         ]
         # fmt: on
 
@@ -291,13 +289,13 @@ def test_timestamp_encoding(self):
         decoding = tokenizer.decode(encoding)
 
         self.assertEqual(EXPECTED_TOKENS, encoding)
-        self.assertEqual(target_text, decoding)
+        self.assertEqual(input_text, decoding)
 
         encoding = rust_tokenizer(input_text, split_special_tokens=False)
         decoding = rust_tokenizer.decode(encoding)
 
         self.assertEqual(EXPECTED_TOKENS, encoding)
-        self.assertEqual(target_text, decoding)
+        self.assertEqual(input_text, decoding)
 
 
 class SpeechToTextTokenizerMultilinguialTest(unittest.TestCase):