deleted argilla evaluation repository

src/intelligence_layer/evaluation/__init__.py

@@ -4,17 +4,12 @@
 )
 from .aggregation.aggregator import AggregationLogic as AggregationLogic
 from .aggregation.aggregator import Aggregator as Aggregator
-from .aggregation.argilla_aggregator import (
-    AggregatedInstructComparison as AggregatedInstructComparison,
-)
-from .aggregation.argilla_aggregator import ArgillaAggregator as ArgillaAggregator
-from .aggregation.argilla_aggregator import (
-    InstructComparisonArgillaAggregationLogic as InstructComparisonArgillaAggregationLogic,
-)
-from .aggregation.argilla_aggregator import PlayerScore as PlayerScore
 from .aggregation.domain import AggregatedEvaluation as AggregatedEvaluation
 from .aggregation.domain import AggregationOverview as AggregationOverview
 from .aggregation.elo import EloCalculator as EloCalculator
+from .aggregation.elo import (
+    InstructComparisonEvaluation as InstructComparisonEvaluation,
+)
 from .aggregation.elo import MatchOutcome as MatchOutcome
 from .aggregation.elo import WinRateCalculator as WinRateCalculator
 from .aggregation.file_aggregation_repository import (
@@ -45,12 +40,6 @@
 from .dataset.single_huggingface_dataset_repository import (
     SingleHuggingfaceDatasetRepository as SingleHuggingfaceDatasetRepository,
 )
-from .evaluation.argilla_evaluation_repository import (
-    ArgillaEvaluationRepository as ArgillaEvaluationRepository,
-)
-from .evaluation.argilla_evaluation_repository import (
-    RecordDataSequence as RecordDataSequence,
-)
 from .evaluation.argilla_evaluator import (
     ArgillaEvaluationLogic as ArgillaEvaluationLogic,
 )

src/intelligence_layer/evaluation/aggregation/elo.py

@@ -1,8 +1,13 @@
-from collections import defaultdict
+import random
+from collections import Counter, defaultdict
 from enum import Enum
 from typing import Iterable, Mapping, Sequence
 
 import numpy as np
+from pydantic import BaseModel
+
+from intelligence_layer.evaluation.aggregation.accumulator import MeanAccumulator
+from intelligence_layer.evaluation.aggregation.aggregator import AggregationLogic
 
 
 class MatchOutcome(str, Enum):
@@ -96,3 +101,64 @@ def calculate(
             player: self.win_count[player] / match_count
             for player, match_count in self.match_count.items()
         }
+
+
+class PlayerScore(BaseModel):
+    elo: float
+    elo_standard_error: float
+    win_rate: float
+    num_matches: int
+
+
+class AggregatedInstructComparison(BaseModel):
+    scores: Mapping[str, PlayerScore]
+
+
+class InstructComparisonEvaluation(BaseModel):
+    first: str
+    second: str
+    winner: MatchOutcome
+
+
+class InstructComparisonAggregationLogic(
+    AggregationLogic[InstructComparisonEvaluation, AggregatedInstructComparison]
+):
+    def aggregate(
+        self, evaluations: Iterable[InstructComparisonEvaluation]
+    ) -> AggregatedInstructComparison:
+        flattened_evaluations = [
+            (
+                evaluation.first,
+                evaluation.second,
+                evaluation.winner,
+            )
+            for evaluation in evaluations
+        ]
+        player_counter = Counter(
+            player for match in flattened_evaluations for player in [match[0], match[1]]
+        )
+        player_counts = dict(player_counter)
+        players = player_counts.keys()
+
+        accumulators = {p: MeanAccumulator() for p in players}
+        for _ in range(100):
+            elo_calc = EloCalculator(players)
+            random.shuffle(flattened_evaluations)
+            elo_calc.calculate(flattened_evaluations)
+            for p in players:
+                accumulators[p].add(elo_calc.ratings[p])
+
+        win_rate_calc = WinRateCalculator(players)
+        win_rate = win_rate_calc.calculate(flattened_evaluations)
+
+        return AggregatedInstructComparison(
+            scores={
+                p: PlayerScore(
+                    elo=acc.extract(),
+                    elo_standard_error=acc.standard_error(),
+                    win_rate=win_rate[p],
+                    num_matches=player_counts[p],
+                )
+                for p, acc in accumulators.items()
+            },
+        )