From f86c56bc7ecad0555880f9770f527dd4a1744be2 Mon Sep 17 00:00:00 2001 From: agentM-GEG Date: Tue, 29 Oct 2024 08:42:22 -0500 Subject: [PATCH 1/2] adding ability to specify classes for Uskill calc --- panoptes_aggregation/reducers/reducer_wrapper.py | 2 ++ .../reducers/user_skill_reducer.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/panoptes_aggregation/reducers/reducer_wrapper.py b/panoptes_aggregation/reducers/reducer_wrapper.py index 7d7c1b6f..fe775a6e 100644 --- a/panoptes_aggregation/reducers/reducer_wrapper.py +++ b/panoptes_aggregation/reducers/reducer_wrapper.py @@ -57,6 +57,8 @@ def wrapper(argument, **kwargs): kwargs_details['mode'] = kwargs['mode'].strip("\'") if 'strategy' in kwargs: kwargs_details['strategy'] = kwargs['strategy'].strip("\'") + if 'focus_classes' in kwargs: + kwargs_details['focus_classes'] = ast.literal_eval(kwargs['focus_classes']) no_version = kwargs.pop('no_version', False) if defaults_process is not None: diff --git a/panoptes_aggregation/reducers/user_skill_reducer.py b/panoptes_aggregation/reducers/user_skill_reducer.py index b780ccf3..13bb7c24 100644 --- a/panoptes_aggregation/reducers/user_skill_reducer.py +++ b/panoptes_aggregation/reducers/user_skill_reducer.py @@ -12,7 +12,6 @@ from ..feedback_strategies import FEEDBACK_STRATEGIES from sklearn.metrics import confusion_matrix - # smallest possible value for difficulty so that # subjects have a non-negligible effect on user skill # in extreme cases @@ -21,7 +20,7 @@ @reducer_wrapper(relevant_reduction=True) def user_skill_reducer(extracts, relevant_reduction=[], mode='binary', null_class='NONE', - skill_threshold=0.7, count_threshold=10, strategy='mean'): + skill_threshold=0.7, count_threshold=10, strategy='mean', focus_classes=None): ''' Parameters ---------- @@ -84,13 +83,19 @@ def user_skill_reducer(extracts, relevant_reduction=[], mode='binary', null_clas # remove the null class from the skill array to calculate the mean skill if mode == 'binary': - null_removed_classes = [classi for classi in classes if classi != 'False'] - null_removed_counts = [ci for classi, ci in per_class_count.items() if classi != 'False'] - mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16) + null_class = 'False' else: + null_class = null_class + + # compute either on user-specified classes or perform mean skill calculation on all detected classes + if focus_classes is None: null_removed_classes = [classi for classi in classes if classi != null_class] null_removed_counts = [ci for classi, ci in per_class_count.items() if classi != null_class] mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16) + else: + null_removed_classes = [classi for classi in focus_classes if classi != null_class] + null_removed_counts = [ci for classi, ci in per_class_count.items() if classi in null_removed_classes] + mean_skill = np.sum([weighted_per_class_skill_dict[key] for key in null_removed_classes]) / (len(null_removed_classes) + 1.e-16) # check the leveling up value if strategy == 'mean': From 217d90307a2c0eef0762836c9c658f0e54824390 Mon Sep 17 00:00:00 2001 From: agentM-GEG Date: Thu, 7 Nov 2024 14:09:35 -0600 Subject: [PATCH 2/2] adding tests for user skill reducer focus classes --- .../reducers/reducer_wrapper.py | 5 +- .../reducer_tests/test_user_skill_reducer.py | 130 ++++++++++++++++++ 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/panoptes_aggregation/reducers/reducer_wrapper.py b/panoptes_aggregation/reducers/reducer_wrapper.py index fe775a6e..cff1f511 100644 --- a/panoptes_aggregation/reducers/reducer_wrapper.py +++ b/panoptes_aggregation/reducers/reducer_wrapper.py @@ -58,7 +58,10 @@ def wrapper(argument, **kwargs): if 'strategy' in kwargs: kwargs_details['strategy'] = kwargs['strategy'].strip("\'") if 'focus_classes' in kwargs: - kwargs_details['focus_classes'] = ast.literal_eval(kwargs['focus_classes']) + focus_classes = kwargs['focus_classes'] + if isinstance(focus_classes, str): + focus_classes = ast.literal_eval(focus_classes) + kwargs_details['focus_classes'] = focus_classes no_version = kwargs.pop('no_version', False) if defaults_process is not None: diff --git a/panoptes_aggregation/tests/reducer_tests/test_user_skill_reducer.py b/panoptes_aggregation/tests/reducer_tests/test_user_skill_reducer.py index cc0e7ed6..cb9cb771 100644 --- a/panoptes_aggregation/tests/reducer_tests/test_user_skill_reducer.py +++ b/panoptes_aggregation/tests/reducer_tests/test_user_skill_reducer.py @@ -505,3 +505,133 @@ def process(data): processed_type='list', test_name='TestOneToOneUserSkillReducer' ) + + +extracted_data = [ + { + "1": 1, + "feedback": { + "strategy": "singleAnswerQuestion", + "true_answer": [ + "1" + ], + "agreement_score": 1 + }, + "aggregation_version": "4.1.0" + }, + { + "7": 1, + "aggregation_version": "4.1.0" + }, + { + "1": 1, + "feedback": { + "strategy": "singleAnswerQuestion", + "true_answer": [ + "2" + ], + "agreement_score": 0 + }, + "aggregation_version": "4.1.0" + }, + { + "1": 1, + "aggregation_version": "4.1.0" + }, + { + "1": 1, + "feedback": { + "strategy": "singleAnswerQuestion", + "true_answer": [ + "1" + ], + "agreement_score": 1 + }, + "aggregation_version": "4.1.0" + } +] + +kwargs_extra_data = { + "relevant_reduction": [ + { + "data": { + "difficulty": [ + 1 + ], + "aggregation_version": "4.1.0" + } + }, + { + "data": { + "aggregation_version": "4.1.0" + } + }, + { + "data": { + "difficulty": [ + 0 + ], + "aggregation_version": "4.1.0" + } + }, + { + "data": { + "aggregation_version": "4.1.0" + } + }, + { + "data": { + "difficulty": [ + 1 + ], + "aggregation_version": "4.1.0" + } + } + ] +} + + +reduced_data = { + "classes": [ + "1", + "2" + ], + "confusion_simple": [ + [ + 2, + 0 + ], + [ + 1, + 0 + ] + ], + "weighted_skill": { + "1": 0.999999999999999, + "2": 0.0 + }, + "skill": { + "1": 1.0, + "2": 0.0 + }, + "count": { + "1": 2, + "2": 1 + }, + "mean_skill": 0.999999999999999, + "level_up": True +} + +TestSubclassUserSkillReducer = ReducerTest( + user_skill_reducer, + process, + extracted_data, + extracted_data, + reduced_data, + 'Test user skill reducer with class subsetting', + network_kwargs=kwargs_extra_data, + kwargs={'mode': 'one-to-one', 'strategy': 'all', 'skill_threshold': 0.2, 'count_threshold': 1, 'focus_classes': ["1"]}, + add_version=False, + processed_type='list', + test_name='TestUserSkillReducer_SubsetClass' +)