diff --git a/sphinx_needs/filter_common.py b/sphinx_needs/filter_common.py
index 250bc40aa..f39f478f7 100644
--- a/sphinx_needs/filter_common.py
+++ b/sphinx_needs/filter_common.py
@@ -309,6 +309,11 @@ def filter_single_need(
             result = bool(eval(filter_compiled, filter_context))
         else:
             result = bool(eval(filter_string, filter_context))
+        if result not in (True, False):
+            # Raises NeedsInvalidFilter if the result is not equal True/False
+            raise NeedsInvalidFilter(
+                f"Error when evaluating filter: expected output to have True/False but got a result <{result}>"
+            )
     except Exception as e:
         raise NeedsInvalidFilter(f"Filter {filter_string} not valid. Error: {e}.")
     return result