diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py
index bd8176be9ed..82407de358b 100644
--- a/server/text_generation_server/models/causal_lm.py
+++ b/server/text_generation_server/models/causal_lm.py
@@ -630,7 +630,7 @@ def fallback(
         if tokenizer.pad_token_id is None:
             if model.config.pad_token_id is not None:
                 tokenizer.pad_token_id = model.config.pad_token_id
-            elif model.config.eos_token_id is not None:
+            elif model.config.eos_token_id is not None and isinstance(model.config.eos_token_id, int):
                 tokenizer.pad_token_id = model.config.eos_token_id
             elif tokenizer.eos_token_id is not None:
                 tokenizer.pad_token_id = tokenizer.eos_token_id