Fix template import in WildBenchAnnotator (#3231)

stanford-crfm · Dec 20, 2024 · c28a842 · c28a842
1 parent 1bf82b4
commit c28a842
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -5,3 +5,4 @@ recursive-include src/helm/benchmark/static/ *.css *.html *.js *.png *.yaml
 recursive-include src/helm/benchmark/static_build/ *.css *.html *.js *.png *.yaml
 recursive-include src/helm/config/ *.yaml
 recursive-include src/helm/benchmark/annotation/omni_math/ *.txt
+recursive-include src/helm/benchmark/annotation/wildbench/ *.md
diff --git a/src/helm/benchmark/annotation/wildbench_annotator.py b/src/helm/benchmark/annotation/wildbench_annotator.py
@@ -1,5 +1,6 @@
 import re
 from typing import Any
+from importlib.resources import files
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.annotation.annotator import Annotator
@@ -14,7 +15,8 @@ class WildBenchAnnotator(Annotator):
 
     def __init__(self, auto_client: AutoClient):
         self._auto_client = auto_client
-        with open("src/helm/benchmark/annotation/wildbench/eval_template.score.v2.md") as f:
+        template_path = files("src.helm.benchmark.annotation.wildbench").joinpath("eval_template.score.v2.md")
+        with template_path.open("r") as f:
             self._score_template = f.read()
         self._pattern = re.compile(
             r'"strengths"\s*:\s*"(.*?)"\s*,\s*"weaknesses"\s*:\s*"(.*?)"\s*,\s*"score"\s*:\s*(".*?"|\d+)', re.DOTALL