add multi level log

modelscope · Dec 27, 2024 · fc5574d · fc5574d
1 parent 08240b1
commit fc5574d
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 16 deletions.
diff --git a/evalscope/collections/evaluator.py b/evalscope/collections/evaluator.py
@@ -1,8 +1,8 @@
 import json
 import os
 import pandas as pd
-from collections import defaultdict
-from datetime import datetime
+from collections import OrderedDict, defaultdict
+from tabulate import tabulate
 from tqdm import tqdm
 
 from evalscope.benchmarks import Benchmark
@@ -93,25 +93,33 @@ def get_report(self, reviews):
 
         df = pd.DataFrame(data)
 
-        # Multi-level aggregation
-        subset_report_df = df.groupby(['task_type', 'dataset_name', 'subset_name']).agg(
-            average_score=('score', 'mean'), count=('score', 'size')).reset_index()
+        # Helper function for aggregation and sorting
+        def aggregate_and_sort(df, group_by_cols):
+            report_df = df.groupby(group_by_cols).agg(
+                average_score=('score', 'mean'), count=('score', 'size')).reset_index()
+            return report_df.sort_values(by='count', ascending=False)
 
-        dataset_report_df = df.groupby(['task_type', 'dataset_name']).agg(
-            average_score=('score', 'mean'), count=('score', 'size')).reset_index()
+        # Multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type'])
 
-        task_report_df = df.groupby(['task_type']).agg(
-            average_score=('score', 'mean'), count=('score', 'size')).reset_index()
+        # Explode tags and aggregate
+        df_exploded = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded, ['tags'])
 
-        # Combine all reports into a single dictionary
+        # Convert sorted DataFrames to Dict
         report = {
             'subset_level': subset_report_df.to_dict(orient='records'),
             'dataset_level': dataset_report_df.to_dict(orient='records'),
-            'task_level': task_report_df.to_dict(orient='records')
+            'task_level': task_report_df.to_dict(orient='records'),
+            'tag_level': tag_report_df.to_dict(orient='records')
         }
 
         # Log the report
-        logger.info(f"Report:\n{pd.DataFrame(report['subset_level']).to_markdown(index=False)}")
+        for level, data in report.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f"{level} Report:\n{table}")
 
         # Save the report to a JSON file
         report_file_path = os.path.join(self.outputs.reports_dir, 'data_collection.json')
@@ -140,7 +148,7 @@ def get_reviews(self, answers):
 
     @staticmethod
     def get_pred_score(review_d) -> float:
-        return review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT]
+        return float(review_d[AnswerKeys.CHOICES][0][ReviewKeys.REVIEW][ReviewKeys.RESULT])
 
     def eval(self, **kwargs):
         answers = self.get_answers()

diff --git a/evalscope/collections/schema.py b/evalscope/collections/schema.py
@@ -62,14 +62,22 @@ def from_dict(cls, data):
                 instance.datasets.append(DatasetInfo(**dataset))
         return instance
 
-    def flatten(self) -> List[DatasetInfo]:
+    def flatten(self, parent_names=None) -> List[DatasetInfo]:
+        if parent_names is None:
+            parent_names = []
+
         flat_datasets = []
+        current_names = parent_names + [self.name]
 
         for dataset in self.datasets:
             if isinstance(dataset, CollectionSchema):
-                nested_datasets = dataset.flatten()
+                nested_datasets = dataset.flatten(current_names)
                 flat_datasets.extend(nested_datasets)
             else:
+                # Add all parent CollectionSchema names to the tags of each DatasetInfo
+                for name in current_names:
+                    if name not in dataset.tags:
+                        dataset.tags.append(name)
                 flat_datasets.append(dataset)
         return flat_datasets
 

diff --git a/tests/cli/test_collection.py b/tests/cli/test_collection.py
@@ -48,7 +48,8 @@ def test_evaluate_collection(self):
             eval_type=EvalType.SERVICE,
             datasets=['data_collection'],
             dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
+                # 'local_path': 'outputs/mixed_data_test.jsonl'
+                'local_path': 'outputs/mixed_data.jsonl'
             }},
         )
         run_task(task_cfg=task_cfg)