Adding the ability to specify classes to filter user-skill calculation

panoptes_aggregation/reducers/reducer_wrapper.py

@@ -57,6 +57,11 @@ def wrapper(argument, **kwargs):
                 kwargs_details['mode'] = kwargs['mode'].strip("\'")
             if 'strategy' in kwargs:
                 kwargs_details['strategy'] = kwargs['strategy'].strip("\'")
+            if 'focus_classes' in kwargs:
+                focus_classes = kwargs['focus_classes']
+                if isinstance(focus_classes, str):
+                    focus_classes = ast.literal_eval(focus_classes)
+                kwargs_details['focus_classes'] = focus_classes
 
             no_version = kwargs.pop('no_version', False)
             if defaults_process is not None:

panoptes_aggregation/reducers/user_skill_reducer.py

@@ -12,7 +12,6 @@
 from ..feedback_strategies import FEEDBACK_STRATEGIES
 from sklearn.metrics import confusion_matrix
 
-
 # smallest possible value for difficulty so that
 # subjects have a non-negligible effect on user skill
 # in extreme cases
@@ -21,7 +20,7 @@
 
 @reducer_wrapper(relevant_reduction=True)
 def user_skill_reducer(extracts, relevant_reduction=[], mode='binary', null_class='NONE',
-                       skill_threshold=0.7, count_threshold=10, strategy='mean'):
+                       skill_threshold=0.7, count_threshold=10, strategy='mean', focus_classes=None):
     '''
         Parameters
         ----------
@@ -84,13 +83,19 @@ def user_skill_reducer(extracts, relevant_reduction=[], mode='binary', null_clas
 
     # remove the null class from the skill array to calculate the mean skill
     if mode == 'binary':
-        null_removed_classes = [classi for classi in classes if classi != 'False']
-        null_removed_counts = [ci for classi, ci in per_class_count.items() if classi != 'False']
-        mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16)
+        null_class = 'False'
     else:
+        null_class = null_class
+
+    # compute either on user-specified classes or perform mean skill calculation on all detected classes
+    if focus_classes is None:
         null_removed_classes = [classi for classi in classes if classi != null_class]
         null_removed_counts = [ci for classi, ci in per_class_count.items() if classi != null_class]
         mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16)
+    else:
+        null_removed_classes = [classi for classi in focus_classes if classi != null_class]
+        null_removed_counts = [ci for classi, ci in per_class_count.items() if classi in null_removed_classes]
+        mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16)
 
     # check the leveling up value
     if strategy == 'mean':

panoptes_aggregation/tests/reducer_tests/test_user_skill_reducer.py

@@ -505,3 +505,133 @@ def process(data):
     processed_type='list',
     test_name='TestOneToOneUserSkillReducer'
 )
+
+
+extracted_data = [
+    {
+        "1": 1,
+        "feedback": {
+            "strategy": "singleAnswerQuestion",
+            "true_answer": [
+                "1"
+            ],
+            "agreement_score": 1
+        },
+        "aggregation_version": "4.1.0"
+    },
+    {
+        "7": 1,
+        "aggregation_version": "4.1.0"
+    },
+    {
+        "1": 1,
+        "feedback": {
+            "strategy": "singleAnswerQuestion",
+            "true_answer": [
+                "2"
+            ],
+            "agreement_score": 0
+        },
+        "aggregation_version": "4.1.0"
+    },
+    {
+        "1": 1,
+        "aggregation_version": "4.1.0"
+    },
+    {
+        "1": 1,
+        "feedback": {
+            "strategy": "singleAnswerQuestion",
+            "true_answer": [
+                "1"
+            ],
+            "agreement_score": 1
+        },
+        "aggregation_version": "4.1.0"
+    }
+]
+
+kwargs_extra_data = {
+    "relevant_reduction": [
+        {
+            "data": {
+                "difficulty": [
+                    1
+                ],
+                "aggregation_version": "4.1.0"
+            }
+        },
+        {
+            "data": {
+                "aggregation_version": "4.1.0"
+            }
+        },
+        {
+            "data": {
+                "difficulty": [
+                    0
+                ],
+                "aggregation_version": "4.1.0"
+            }
+        },
+        {
+            "data": {
+                "aggregation_version": "4.1.0"
+            }
+        },
+        {
+            "data": {
+                "difficulty": [
+                    1
+                ],
+                "aggregation_version": "4.1.0"
+            }
+        }
+    ]
+}
+
+
+reduced_data = {
+    "classes": [
+        "1",
+        "2"
+    ],
+    "confusion_simple": [
+        [
+            2,
+            0
+        ],
+        [
+            1,
+            0
+        ]
+    ],
+    "weighted_skill": {
+        "1": 0.999999999999999,
+        "2": 0.0
+    },
+    "skill": {
+        "1": 1.0,
+        "2": 0.0
+    },
+    "count": {
+        "1": 2,
+        "2": 1
+    },
+    "mean_skill": 0.999999999999999,
+    "level_up": True
+}
+
+TestSubclassUserSkillReducer = ReducerTest(
+    user_skill_reducer,
+    process,
+    extracted_data,
+    extracted_data,
+    reduced_data,
+    'Test user skill reducer with class subsetting',
+    network_kwargs=kwargs_extra_data,
+    kwargs={'mode': 'one-to-one', 'strategy': 'all', 'skill_threshold': 0.2, 'count_threshold': 1, 'focus_classes': ["1"]},
+    add_version=False,
+    processed_type='list',
+    test_name='TestUserSkillReducer_SubsetClass'
+)