diff --git a/optimum/exporters/tasks.py b/optimum/exporters/tasks.py
index 75c8a4da905..df768a8e59e 100644
--- a/optimum/exporters/tasks.py
+++ b/optimum/exporters/tasks.py
@@ -1754,9 +1754,9 @@ def infer_task_from_model(
                 cache_dir=cache_dir,
                 token=token,
             )
-        elif issubclass(model, (PreTrainedModel, TFPreTrainedModel, DiffusionPipeline)):
+        elif issubclass(model, object):
             task_name = cls._infer_task_from_model_or_model_class(model_class=model)
-        elif isinstance(model, (PreTrainedModel, TFPreTrainedModel, DiffusionPipeline)):
+        elif isinstance(model, object):
             task_name = cls._infer_task_from_model_or_model_class(model=model)
 
         if task_name is None: