diff --git a/tests/models/auto/test_modeling_auto.py b/tests/models/auto/test_modeling_auto.py
index a8e42d77f90e36..363028c7f22978 100644
--- a/tests/models/auto/test_modeling_auto.py
+++ b/tests/models/auto/test_modeling_auto.py
@@ -141,7 +141,7 @@ def test_lmhead_model_from_pretrained(self):
 
     @slow
     def test_model_for_causal_lm(self):
-        model_name = "google-bert/bert-base-uncased"
+        model_name = "openai-community/gpt2"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, GPT2Config)
@@ -165,7 +165,7 @@ def test_model_for_masked_lm(self):
 
     @slow
     def test_model_for_encoder_decoder_lm(self):
-        model_name = "google-bert/bert-base-uncased"
+        model_name = "google-t5/t5-base"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, T5Config)
diff --git a/tests/models/auto/test_modeling_tf_auto.py b/tests/models/auto/test_modeling_tf_auto.py
index 53a07b197057e7..a63d9fbe4cf14c 100644
--- a/tests/models/auto/test_modeling_tf_auto.py
+++ b/tests/models/auto/test_modeling_tf_auto.py
@@ -118,15 +118,15 @@ def test_lmhead_model_from_pretrained(self):
         model_name = "openai-community/gpt2"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
-        self.assertIsInstance(config, BertConfig)
+        self.assertIsInstance(config, GPT2Config)
 
         model = TFAutoModelWithLMHead.from_pretrained(model_name)
         self.assertIsNotNone(model)
-        self.assertIsInstance(model, TFBertForMaskedLM)
+        self.assertIsInstance(model, TFGPT2LMHeadModel)
 
     @slow
     def test_model_for_masked_lm(self):
-        model_name = "openai-community/gpt2"
+        model_name = "google-bert/bert-base-uncased"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, BertConfig)
@@ -138,7 +138,7 @@ def test_model_for_masked_lm(self):
 
     @slow
     def test_model_for_encoder_decoder_lm(self):
-        model_name = "openai-community/gpt2"
+        model_name = "google-t5/t5-base"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, T5Config)
diff --git a/tests/models/auto/test_modeling_tf_pytorch.py b/tests/models/auto/test_modeling_tf_pytorch.py
index 5b9036cbf1cf18..2c59c906db6b4f 100644
--- a/tests/models/auto/test_modeling_tf_pytorch.py
+++ b/tests/models/auto/test_modeling_tf_pytorch.py
@@ -103,7 +103,7 @@ def test_model_for_pretraining_from_pretrained(self):
 
     @slow
     def test_model_for_causal_lm(self):
-        model_name = "google-bert/bert-base-uncased"
+        model_name = "openai-community/gpt2"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, GPT2Config)
@@ -156,7 +156,7 @@ def test_model_for_masked_lm(self):
 
     @slow
     def test_model_for_encoder_decoder_lm(self):
-        model_name = "google-bert/bert-base-uncased"
+        model_name = "google-t5/t5-base"
         config = AutoConfig.from_pretrained(model_name)
         self.assertIsNotNone(config)
         self.assertIsInstance(config, T5Config)