feat: Add how to for elo evaluations refactor: Move elo evaluation lo…

…gic from "elo_evaluator.py" into "incremental_evaluator.py"

TASK: IL-502

src/documentation/how_tos/example_data.py

@@ -17,6 +17,12 @@
     SuccessfulExampleOutput,
 )
 from intelligence_layer.evaluation.aggregation.aggregator import AggregationLogic
+from intelligence_layer.evaluation.evaluation.evaluator.incremental_evaluator import (
+    ComparisonEvaluation,
+    EloEvaluationLogic,
+    Matches,
+    MatchOutcome,
+)
 
 
 class DummyExample(Example[str, str]):
@@ -42,6 +48,34 @@ def do_evaluate(
         )
 
 
+class DummyEloEvaluationLogic(EloEvaluationLogic[str, str, str]):
+    def grade(
+        self,
+        first: SuccessfulExampleOutput[str],
+        second: SuccessfulExampleOutput[str],
+        example: Example[str, str],
+    ) -> MatchOutcome:
+        return MatchOutcome.DRAW
+
+    def do_incremental_evaluate(
+        self,
+        example: Example[str, str],
+        outputs: list[SuccessfulExampleOutput[str]],
+        already_evaluated_outputs: list[list[SuccessfulExampleOutput[str]]],
+    ) -> Matches:
+        player_a = SuccessfulExampleOutput(run_id="1", example_id="1", output="1")
+        player_b = SuccessfulExampleOutput(run_id="2", example_id="2", output="2")
+        return Matches(
+            comparison_evaluations=[
+                ComparisonEvaluation(
+                    first_player="1",
+                    second_player="2",
+                    outcome=self.grade(player_a, player_b, example),
+                )
+            ]
+        )
+
+
 class DummyAggregation(BaseModel):
     num_evaluations: int
 

src/documentation/how_tos/how_to_implement_elo_evaluations.ipynb

@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from documentation.how_tos.example_data import DummyEloEvaluationLogic, example_data\n",
+    "from intelligence_layer.evaluation import (\n",
+    "    IncrementalEvaluator,\n",
+    "    InMemoryEvaluationRepository,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# How to implement elo evaluations\n",
+    "0. Run your tasks on the datasets you want to evaluate (see [here](./how_to_run_a_task_on_a_dataset.ipynb))\n",
+    "   - When evaluating multiple runs, all of them need the same data types \n",
+    "2. Initialize all necessary repositories for the `IncrementalEvaluator`, and an `EloEvaluationLogic` that is specific to your use case. \n",
+    "3. Run the evaluator to evaluate all examples and create a single `EvaluationOverview`\n",
+    "4. (Optional) Save the evaluation id for later use"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Example"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Step 0\n",
+    "\n",
+    "\n",
+    "my_example_data = example_data()\n",
+    "print()\n",
+    "run_ids = [my_example_data.run_overview_1.id, my_example_data.run_overview_2.id]\n",
+    "\n",
+    "# Step 1\n",
+    "dataset_repository = my_example_data.dataset_repository\n",
+    "run_repository = my_example_data.run_repository\n",
+    "evaluation_repository = InMemoryEvaluationRepository()\n",
+    "evaluation_logic = DummyEloEvaluationLogic()\n",
+    "\n",
+    "# Step 3\n",
+    "evaluator = IncrementalEvaluator(\n",
+    "    dataset_repository,\n",
+    "    run_repository,\n",
+    "    evaluation_repository,\n",
+    "    \"My dummy evaluation\",\n",
+    "    evaluation_logic,\n",
+    ")\n",
+    "\n",
+    "evaluation_overview = evaluator.evaluate_runs(*run_ids)\n",
+    "\n",
+    "# Step 4\n",
+    "print(evaluation_overview.id)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "intelligence-layer-aL2cXmJM-py3.11",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/intelligence_layer/evaluation/__init__.py

@@ -63,23 +63,28 @@
 from .evaluation.evaluator.async_evaluator import (
     AsyncEvaluationRepository as AsyncEvaluationRepository,
 )
-from .evaluation.evaluator.elo_evaluator import (
-    ComparisonEvaluation as ComparisonEvaluation,
-)
-from .evaluation.evaluator.elo_evaluator import EloEvaluationLogic as EloEvaluationLogic
-from .evaluation.evaluator.elo_evaluator import Matches as Matches
-from .evaluation.evaluator.elo_evaluator import MatchOutcome as MatchOutcome
 from .evaluation.evaluator.evaluator import EvaluationLogic as EvaluationLogic
 from .evaluation.evaluator.evaluator import Evaluator as Evaluator
 from .evaluation.evaluator.evaluator import (
     SingleOutputEvaluationLogic as SingleOutputEvaluationLogic,
 )
+from .evaluation.evaluator.incremental_evaluator import (
+    ComparisonEvaluation as ComparisonEvaluation,
+)
+from .evaluation.evaluator.incremental_evaluator import (
+    EloEvaluationLogic as EloEvaluationLogic,
+)
+from .evaluation.evaluator.incremental_evaluator import (
+    EloGradingInput as EloGradingInput,
+)
 from .evaluation.evaluator.incremental_evaluator import (
     IncrementalEvaluationLogic as IncrementalEvaluationLogic,
 )
 from .evaluation.evaluator.incremental_evaluator import (
     IncrementalEvaluator as IncrementalEvaluator,
 )
+from .evaluation.evaluator.incremental_evaluator import Matches as Matches
+from .evaluation.evaluator.incremental_evaluator import MatchOutcome as MatchOutcome
 from .evaluation.file_evaluation_repository import (
     AsyncFileEvaluationRepository as AsyncFileEvaluationRepository,
 )

src/intelligence_layer/evaluation/aggregation/elo_aggregation.py

@@ -7,7 +7,7 @@
 
 from intelligence_layer.evaluation.aggregation.accumulator import MeanAccumulator
 from intelligence_layer.evaluation.aggregation.aggregator import AggregationLogic
-from intelligence_layer.evaluation.evaluation.evaluator.elo_evaluator import (
+from intelligence_layer.evaluation.evaluation.evaluator.incremental_evaluator import (
     ComparisonEvaluation,
     Matches,
     MatchOutcome,

src/intelligence_layer/evaluation/evaluation/evaluator/argilla_evaluator.py

@@ -30,7 +30,7 @@
 from intelligence_layer.evaluation.evaluation.evaluator.base_evaluator import (
     EvaluationLogicBase,
 )
-from intelligence_layer.evaluation.evaluation.evaluator.elo_evaluator import (
+from intelligence_layer.evaluation.evaluation.evaluator.incremental_evaluator import (
     ComparisonEvaluation,
     MatchOutcome,
 )