From f06e051a6804714042e864eb60f91342d7d43d59 Mon Sep 17 00:00:00 2001 From: Carlo Date: Wed, 29 Nov 2023 17:53:00 +0100 Subject: [PATCH] added mean reciprocal rank calculation (MRR) --- codalab/apps/web/models.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/codalab/apps/web/models.py b/codalab/apps/web/models.py index 30ceb040f..607d8a38a 100644 --- a/codalab/apps/web/models.py +++ b/codalab/apps/web/models.py @@ -1238,9 +1238,13 @@ def sortkey(x): for result in results: for sdef in result['scoredefs']: if sdef.computed: - operation = getattr(models, sdef.computed_score.operation) + try: + operation = getattr(models, sdef.computed_score.operation) + operation_name = operation.name + except: + operation_name = sdef.computed_score.operation weights = sdef.computed_score.weights - if (operation.name == 'Avg'): + if (operation_name == 'Avg'): try: cnt = len(computed_deps[sdef.id]) if (cnt > 0): @@ -1263,6 +1267,18 @@ def sortkey(x): ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') except KeyError: pass + elif (operation_name == 'MRR'): + try: + cnt = len(computed_deps[sdef.id]) + if (cnt > 0): + computed_values = {} + for id in submission_ids: + # Mean reciprocal rank computation + computed_values[id] = sum([1.0/ranks[d.id][id] for d in computed_deps[sdef.id]]) / float(cnt) + values[sdef.id] = computed_values + ranks[sdef.id] = self.rank_values(submission_ids, computed_values, sort_ascending=sdef.sorting=='asc') + except KeyError: + pass # format values for result in results: @@ -2398,7 +2414,8 @@ def save(self, *args, **kwargs): class SubmissionComputedScore(models.Model): scoredef = models.OneToOneField(SubmissionScoreDef, related_name='computed_score', on_delete=models.CASCADE) operation = models.CharField(max_length=10, choices=(('Max', 'Max'), - ('Avg', 'Average'))) + ('Avg', 'Average'), + ('MRR', 'MRR'))) weights = models.CharField(max_length=200, null=True, blank=True)