diff --git a/sentence_transformers/cross_encoder/CrossEncoder.py b/sentence_transformers/cross_encoder/CrossEncoder.py
index c4345017a..3d7ee0abb 100644
--- a/sentence_transformers/cross_encoder/CrossEncoder.py
+++ b/sentence_transformers/cross_encoder/CrossEncoder.py
@@ -527,6 +527,11 @@ def rank(
                 'score': -5.082967,
                 'text': "The 'Harry Potter' series, which consists of seven fantasy novels written by British author J.K. Rowling, is among the most popular and critically acclaimed books of the modern era."}]
         """
+        if self.config.num_labels != 1:
+            raise ValueError(
+                "CrossEncoder.rank() only works for models with num_labels=1. "
+                "Consider using CrossEncoder.predict() with input pairs instead."
+            )
         query_doc_pairs = [[query, doc] for doc in documents]
         scores = self.predict(
             sentences=query_doc_pairs,