diff --git a/examples/text-extract/evaluate.py b/examples/text-extract/evaluate.py
index abb292b198e..357f101e387 100644
--- a/examples/text-extract/evaluate.py
+++ b/examples/text-extract/evaluate.py
@@ -6,7 +6,7 @@
 import openai
 
 import weave
-from weave.flow.scorer import MultiTaskBinaryClassificationF1
+from weave.scorers import MultiTaskBinaryClassificationF1
 
 
 class TextExtractModel(weave.Model):
diff --git a/examples/tutorial_scripts/05_eval_pipeline.py b/examples/tutorial_scripts/05_eval_pipeline.py
index ccb14126a03..0a6a5baf9ab 100644
--- a/examples/tutorial_scripts/05_eval_pipeline.py
+++ b/examples/tutorial_scripts/05_eval_pipeline.py
@@ -60,7 +60,7 @@ async def predict(self, sentence: str) -> dict:
 ]
 
 import weave
-from weave.flow.scorer import MultiTaskBinaryClassificationF1
+from weave.scorers import MultiTaskBinaryClassificationF1
 
 
 @weave.op()
diff --git a/examples/tutorial_scripts/06_eval_pipeline_all.py b/examples/tutorial_scripts/06_eval_pipeline_all.py
index 6be10f08a44..0d5fe8fd3b2 100644
--- a/examples/tutorial_scripts/06_eval_pipeline_all.py
+++ b/examples/tutorial_scripts/06_eval_pipeline_all.py
@@ -4,7 +4,7 @@
 import openai
 
 import weave
-from weave.flow.scorer import MultiTaskBinaryClassificationF1
+from weave.scorers import MultiTaskBinaryClassificationF1
 
 # We create a model class with one predict function.
 # All inputs, predictions and parameters are automatically captured for easy inspection.
diff --git a/tests/trace/test_evaluate.py b/tests/trace/test_evaluate.py
index f5ada25215f..ccb64b2da61 100644
--- a/tests/trace/test_evaluate.py
+++ b/tests/trace/test_evaluate.py
@@ -4,7 +4,7 @@
 
 import weave
 from weave import Dataset, Evaluation, Model
-from weave.flow.scorer import MultiTaskBinaryClassificationF1
+from weave.scorers import MultiTaskBinaryClassificationF1
 
 dataset_rows = [{"input": "1 + 2", "target": 3}, {"input": "2**4", "target": 15}]
 dataset = Dataset(rows=dataset_rows)
diff --git a/tests/trace/test_evaluations.py b/tests/trace/test_evaluations.py
index 92bd75f9700..49f8426cc65 100644
--- a/tests/trace/test_evaluations.py
+++ b/tests/trace/test_evaluations.py
@@ -9,7 +9,7 @@
 import weave
 from tests.trace.util import AnyIntMatcher
 from weave import Evaluation, Model
-from weave.flow.scorer import Scorer
+from weave.scorers import Scorer
 from weave.trace_server import trace_server_interface as tsi
 
 
diff --git a/weave/__init__.py b/weave/__init__.py
index 3b54ba97176..7cf5b49de48 100644
--- a/weave/__init__.py
+++ b/weave/__init__.py
@@ -15,6 +15,20 @@
 from weave.trace.util import Thread as Thread
 from weave.trace.util import ThreadPoolExecutor as ThreadPoolExecutor
 
+from typing import TYPE_CHECKING
+
+# Helper for IDEs
+if TYPE_CHECKING:
+    from weave.flow import scorers
+
+# Lazy import for the scorers module
+def __getattr__(name):
+    if name == "scorers":
+        from weave.flow import scorers
+        globals()["scorers"] = scorers
+        return scorers
+    raise AttributeError(f"module {__name__} has no attribute {name}")
+
 # Special object informing doc generation tooling which symbols
 # to document & to associate with this module.
 __docspec__ = [
diff --git a/weave/flow/eval.py b/weave/flow/eval.py
index 8ed5b24d1e3..9a807d47f98 100644
--- a/weave/flow/eval.py
+++ b/weave/flow/eval.py
@@ -13,7 +13,7 @@
 from weave.flow.dataset import Dataset
 from weave.flow.model import Model, get_infer_method
 from weave.flow.obj import Object
-from weave.flow.scorer import (
+from weave.scorers import (
     Scorer,
     auto_summarize,
     get_scorer_attributes,
diff --git a/weave/flow/scorer/__init__.py b/weave/flow/scorers/__init__.py
similarity index 92%
rename from weave/flow/scorer/__init__.py
rename to weave/flow/scorers/__init__.py
index d5702549ef7..68c423eea3c 100644
--- a/weave/flow/scorer/__init__.py
+++ b/weave/flow/scorers/__init__.py
@@ -1,4 +1,8 @@
-from weave.flow.scorer.base_scorer import Scorer, auto_summarize, get_scorer_attributes
+from weave.flow.scorer.base_scorer import (
+    Scorer,
+    auto_summarize,
+    get_scorer_attributes,
+)
 from weave.flow.scorer.classification_scorer import (
     MultiTaskBinaryClassificationF1,
     transpose,
diff --git a/weave/flow/scorer/base_scorer.py b/weave/flow/scorers/base_scorer.py
similarity index 100%
rename from weave/flow/scorer/base_scorer.py
rename to weave/flow/scorers/base_scorer.py
diff --git a/weave/flow/scorer/classification_scorer.py b/weave/flow/scorers/classification_scorer.py
similarity index 100%
rename from weave/flow/scorer/classification_scorer.py
rename to weave/flow/scorers/classification_scorer.py
diff --git a/weave/flow/scorer/hallucination_scorer.py b/weave/flow/scorers/hallucination_scorer.py
similarity index 100%
rename from weave/flow/scorer/hallucination_scorer.py
rename to weave/flow/scorers/hallucination_scorer.py
diff --git a/weave/flow/scorer/json_scorer.py b/weave/flow/scorers/json_scorer.py
similarity index 100%
rename from weave/flow/scorer/json_scorer.py
rename to weave/flow/scorers/json_scorer.py
diff --git a/weave/flow/scorer/llm_scorer.py b/weave/flow/scorers/llm_scorer.py
similarity index 100%
rename from weave/flow/scorer/llm_scorer.py
rename to weave/flow/scorers/llm_scorer.py
diff --git a/weave/flow/scorer/llm_utils.py b/weave/flow/scorers/llm_utils.py
similarity index 100%
rename from weave/flow/scorer/llm_utils.py
rename to weave/flow/scorers/llm_utils.py
diff --git a/weave/flow/scorer/moderation_scorer.py b/weave/flow/scorers/moderation_scorer.py
similarity index 100%
rename from weave/flow/scorer/moderation_scorer.py
rename to weave/flow/scorers/moderation_scorer.py
diff --git a/weave/flow/scorer/pydantic_scorer.py b/weave/flow/scorers/pydantic_scorer.py
similarity index 100%
rename from weave/flow/scorer/pydantic_scorer.py
rename to weave/flow/scorers/pydantic_scorer.py
diff --git a/weave/flow/scorer/ragas_scorer.py b/weave/flow/scorers/ragas_scorer.py
similarity index 100%
rename from weave/flow/scorer/ragas_scorer.py
rename to weave/flow/scorers/ragas_scorer.py
diff --git a/weave/flow/scorer/similarity_score.py b/weave/flow/scorers/similarity_score.py
similarity index 100%
rename from weave/flow/scorer/similarity_score.py
rename to weave/flow/scorers/similarity_score.py
diff --git a/weave/flow/scorer/string_scorer.py b/weave/flow/scorers/string_scorer.py
similarity index 100%
rename from weave/flow/scorer/string_scorer.py
rename to weave/flow/scorers/string_scorer.py
diff --git a/weave/flow/scorer/summarization_scorer.py b/weave/flow/scorers/summarization_scorer.py
similarity index 100%
rename from weave/flow/scorer/summarization_scorer.py
rename to weave/flow/scorers/summarization_scorer.py
diff --git a/weave/flow/scorer/utils.py b/weave/flow/scorers/utils.py
similarity index 100%
rename from weave/flow/scorer/utils.py
rename to weave/flow/scorers/utils.py
diff --git a/weave/flow/scorer/xml_scorer.py b/weave/flow/scorers/xml_scorer.py
similarity index 100%
rename from weave/flow/scorer/xml_scorer.py
rename to weave/flow/scorers/xml_scorer.py