diff --git a/tests/python_tests/test_whisper_generate_api.py b/tests/python_tests/test_whisper_generate_api.py
index 5a68dd98b6..9a117bc939 100644
--- a/tests/python_tests/test_whisper_generate_api.py
+++ b/tests/python_tests/test_whisper_generate_api.py
@@ -25,7 +25,9 @@ def run_gc_after_test():
     yield
     gc.collect()
 
-@functools.lru_cache(1)
+# used whisper models are relatively small
+# cache them in memory to speedup tests
+@functools.lru_cache(3)
 def read_whisper_model(params, **tokenizer_kwargs):
     model_id, path = params