Skip to content

Commit

Permalink
fix: exchange force_execution bool with allow_diff
Browse files Browse the repository at this point in the history
  • Loading branch information
JohannesWesch committed Nov 12, 2024
1 parent 5c98fae commit 4587d1c
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 6 deletions.
4 changes: 2 additions & 2 deletions src/intelligence_layer/evaluation/benchmark/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,15 @@ def get_benchmark(
benchmark_id: str,
eval_logic: EvaluationLogic[Input, Output, ExpectedOutput, Evaluation],
aggregation_logic: AggregationLogic[Evaluation, AggregatedEvaluation],
force_execution: bool = False,
allow_diff: bool = False,
) -> Benchmark:
"""Retrieves an existing benchmark from the repository.
Args:
benchmark_id: Unique identifier for the benchmark to retrieve.
eval_logic: Evaluation logic to apply.
aggregation_logic (AggregationLogic[Evaluation, AggregatedEvaluation]): Aggregation logic to apply.
force_execution: Execute the benchmark even though logics behaviour do not match.
allow_diff: Retrieve the benchmark even though logics behaviour do not match.
Returns:
The retrieved benchmark instance. Raises ValueError if no benchmark is found.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def __init__(
self.client = studio_client

def execute(self, task: Task[Input, Output], metadata: dict[str, Any]) -> str:
raise NotImplementedError # <- skip the impl here for now, not this is another ticket
raise NotImplementedError


class StudioBenchmarkRepository(BenchmarkRepository):
Expand Down Expand Up @@ -88,13 +88,11 @@ def get_benchmark(
benchmark_id: str,
eval_logic: EvaluationLogic[Input, Output, ExpectedOutput, Evaluation],
aggregation_logic: AggregationLogic[Evaluation, AggregatedEvaluation],
force_execution: bool = False,
allow_diff: bool = False,
) -> StudioBenchmark:
benchmark = self.client.get_benchmark(benchmark_id)
if benchmark is None:
raise ValueError("Benchmark not found")
# check if the logic is the same
# check force bool
return StudioBenchmark(
benchmark_id,
benchmark.dataset_id,
Expand Down

0 comments on commit 4587d1c

Please sign in to comment.