diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py index 0baf003e1d..d2bb0c87de 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py @@ -1,20 +1,19 @@ -''' +""" Created on April 24, 2015 @author: alexandru-m-g -''' +""" import json import logging import ckan.lib.dictization as d import ckan.lib.helpers as helpers import ckan.lib.navl.dictization_functions -import ckan.logic as logic import ckan.model as model import ckan.plugins.toolkit as tk - import ckanext.hdx_org_group.dao.indicator_access as indicator_access import ckanext.hdx_org_group.dao.widget_data_service as widget_data_service +import ckanext.hdx_org_group.helpers.country_helper as country_helper import ckanext.hdx_org_group.helpers.organization_helper as org_helper from ckan.common import c from ckanext.hdx_theme.helpers.caching import cached_make_rest_api_request as cached_make_rest_api_request @@ -36,7 +35,7 @@ @side_effect_free def hdx_datasets_for_group(context, data_dict): - ''' + """ Returns a paginated list of datasets for a group with 25 items per page. Options for sorting are: metadata_modified desc, title_case_insensitive desc, title_case_insensitive asc, views_recent desc, score desc ( only useful if query string is specified, should be combined @@ -52,11 +51,11 @@ def hdx_datasets_for_group(context, data_dict): :param type: 'all', 'indicators', 'datasets'. Defaults to 'all' :type q: string :return: - ''' + """ skipped_keys = ['q', 'id', 'sort', 'type', 'page'] - id = _get_or_bust(data_dict, "id") + id = _get_or_bust(data_dict, 'id') limit = 25 @@ -74,31 +73,31 @@ def hdx_datasets_for_group(context, data_dict): new_data_dict['ext_indicator'] = u'0' search_param_list = [ - key + ":" + value for key, value in data_dict.iteritems() if key not in skipped_keys] + key + ':' + value for key, value in data_dict.iteritems() if key not in skipped_keys] search_param_list.append(u'groups:{}'.format(id)) if search_param_list != None: - new_data_dict['fq'] = " ".join( + new_data_dict['fq'] = ' '.join( search_param_list) + ' +dataset_type:dataset' if data_dict.get('q', None): new_data_dict['q'] = data_dict['q'] - query = get_action("package_search")(context, new_data_dict) + query = get_action('package_search')(context, new_data_dict) return query @side_effect_free def hdx_topline_num_for_group(context, data_dict): - ''' + """ :param id: the id of the group for which top line numbers are requested :type id: string :return: a dict of top line numbers. Please note that depending on the selected group the source of the data ( either the datastore or CPS/indicators ) might be different. The data will have some fields that are specific to the source. - ''' - id = _get_or_bust(data_dict, "id") + """ + id = _get_or_bust(data_dict, 'id') grp_result = get_group(id) group_info = grp_result.get('group_info') # custom_dict = grp_result.get('custom_dict') @@ -124,14 +123,14 @@ def hdx_topline_num_for_group(context, data_dict): def __get_toplines_for_active_country(group_info, common_format): - ''' + """ :param group_info: :type group_info: dict :param common_format: :type common_format: bool :return: :rtype: list - ''' + """ # source is rw top_line_data_list = widget_data_service.build_widget_data_access(group_info).get_dataset_results() @@ -162,14 +161,14 @@ def _parse_integer_value(item): def __get_toplines_for_standard_country(group_info, common_format): - ''' + """ :param group_info: :type group_info: dict :param common_format: :type common_format: bool :return: :rtype: list - ''' + """ # source is configured in 'hdx.locations.toplines_url' # ckan_site_url = config.get('ckan.site_url') raw_top_line_items = widget_data_service.build_widget_data_access(group_info).get_dataset_results() @@ -197,13 +196,13 @@ def __get_toplines_for_standard_country(group_info, common_format): @side_effect_free def hdx_light_group_show(context, data_dict): - ''' + """ Return a lightweight ( less resource intensive,faster but without datasets ) version of the group details :param id: the id of the group for which top line numbers are requested :type id: string - ''' + """ - id = _get_or_bust(data_dict, "id") + id = _get_or_bust(data_dict, 'id') group_dict = {} group = model.Group.get(id) if not group: @@ -226,13 +225,13 @@ def hdx_light_group_show(context, data_dict): dictized = d.table_dictize(extra, context) if not extra.state == 'active': continue - value = dictized["value"] + value = dictized['value'] result_list.append(dictized) # Keeping the above for backwards compatibility - group_dict[name] = dictized["value"] + group_dict[name] = dictized['value'] - group_dict['extras'] = sorted(result_list, key=lambda x: x["key"]) + group_dict['extras'] = sorted(result_list, key=lambda x: x['key']) return group_dict @@ -283,20 +282,20 @@ def hdx_get_locations_info_from_rw(context, data_dict): return cached_make_rest_api_request(url) return None except: - log.error("RW file was not found or can not be accessed") + log.error('RW file was not found or can not be accessed') return None @side_effect_free def hdx_organization_follower_list(context, data_dict): - '''Return the list of users that are following the given organization. + """Return the list of users that are following the given organization. :param id: the id or name of the organization :type id: string :rtype: list of dictionaries - ''' + """ _check_access('hdx_organization_follower_list', context, data_dict) context['keep_email'] = True return _follower_list( @@ -336,3 +335,14 @@ def _user_list_dictize(obj_list, context, # user_dict.pop('email', None) result_list.append(user_dict) return sorted(result_list, key=sort_key, reverse=reverse) + +@side_effect_free +def hdx_datagrid_show(context, data_dict): + id = _get_or_bust(data_dict, 'id') + if id: + grp_dict = get_action('hdx_light_group_show')(context, {'id':id}) + data_completeness = country_helper._get_data_completeness(grp_dict.get('name')) + replaced_data_completeness = country_helper.hdx_replace_datagrid_labels(data_completeness, grp_dict) + return replaced_data_completeness + else: + raise NotFound('Group was not found.') diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py index e132cce2be..a7c7041e27 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py @@ -2,7 +2,7 @@ from dogpile.cache import make_region import ckan.plugins.toolkit as tk -import ckanext.hdx_org_group.helpers.data_completness as data_completness +import ckanext.hdx_org_group.helpers.data_completeness as data_completeness from ckanext.hdx_theme.helpers.caching import dogpile_standard_config, dogpile_config_filter, \ HDXRedisInvalidationStrategy @@ -23,13 +23,13 @@ @dogpile_country_region.cache_on_arguments() def cached_data_completeness(location_code): - log.info('Fetching data completness for ' + location_code) + log.info('Fetching data completeness for ' + location_code) url_pattern = config.get('hdx.datagrid.config_url_pattern') for_prod = config.get('hdx.datagrid.prod') branch = 'master' if for_prod else location_code url = url_pattern.format(branch=branch, iso=location_code) - return data_completness.DataCompletness(location_code, url).get_config() + return data_completeness.DataCompleteness(location_code, url).get_config() @dogpile_country_region.cache_on_arguments() diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py index 80ede8bdad..6bcd61ee3c 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py @@ -5,7 +5,7 @@ import ckan.plugins.toolkit as tk import ckanext.hdx_org_group.helpers.caching as caching # import ckanext.hdx_package.helpers.screenshot as screenshot - +from datetime import datetime from ckanext.hdx_org_group.controller_logic.group_search_logic import GroupSearchLogic import ckan.lib.helpers as h from ckanext.hdx_search.helpers.constants import DEFAULT_SORTING @@ -39,11 +39,11 @@ def _sort_datasets_by_is_good(data_completeness): - categories = data_completeness.get("categories") + categories = data_completeness.get('categories') for cat in categories: - if cat.get("data_series"): - for ds in cat.get("data_series"): - datasets_list = ds.get("datasets") + if cat.get('data_series'): + for ds in cat.get('data_series'): + datasets_list = ds.get('datasets') if datasets_list: datasets_sorted_list = sorted(datasets_list, key=lambda item: item['is_good'] == False) ds['datasets'] = datasets_sorted_list @@ -52,7 +52,7 @@ def _sort_datasets_by_is_good(data_completeness): def country_topline(id): - log.info("The id of the page is: " + id) + log.info('The id of the page is: ' + id) country_dict = get_country(id) top_line_data_list = caching.cached_topline_numbers(id) @@ -88,11 +88,11 @@ def get_template_data(country_dict, not_filtered_facet_info): # f_organization_list = _get_org_list_for_featured_from_facets(not_filtered_facet_info) # f_tag_list = _get_tag_list_for_featured_from_facets(not_filtered_facet_info) - data_completness = _get_data_completeness(country_dict.get('name')) \ + data_completeness = _get_data_completeness(country_dict.get('name')) \ if country_dict.get('data_completeness') == 'active' else None - if data_completness: - data_completness = _sort_datasets_by_is_good(data_completness) + if data_completeness: + data_completeness = _sort_datasets_by_is_good(data_completeness) template_data = { 'data': { @@ -117,7 +117,7 @@ def get_template_data(country_dict, not_filtered_facet_info): # 'tag_list': f_tag_list[:10], # 'show': len(f_organization_list) > 0 or len(f_tag_list) > 0 # }, - 'data_completness': data_completness, + 'data_completeness': data_completeness, }, 'errors': None, @@ -248,8 +248,8 @@ def get_country(id): def _db_to_form_schema(group_type=None): - '''This is an interface to manipulate data from the database - into a format suitable for the form (optional)''' + """This is an interface to manipulate data from the database + into a format suitable for the form (optional)""" return lookup_group_plugin(group_type).db_to_form_schema() @@ -290,3 +290,52 @@ def hdx_datagrid_org_get_display_text(dataseries_dict): if flag.get('key', '') == 'not_applicable' and flag.get('display_text'): return flag.get('display_text') return 'Not applicable' + +DATA_COMPLETENESS_LABELS_DICT = { + 'good_datasets_num': 'available_datasets_num', + 'dataset_goodness_percentage':'dataset_availability_percentage', + 'good_dataseries_num':'available_dataseries_num', + 'not_good_dataseries_num': 'outdated_dataseries_num', + 'good_dataseries_text': 'available_dataseries_text', + 'not_good_dataseries_text': 'outdated_dataseries_text', + 'dataseries_good_percentage':'dataseries_available_percentage', + 'dataseries_not_good_percentage':'dataseries_outdated_percentage', +} + +DATA_COMPLETENESS_STATE_DICT = { + 'good':'available', + 'not_good':'outdated', + 'empty':'empty', +} + + +def transform_data_completeness(data): + if isinstance(data, dict): + new_dict = {} + for key, value in data.items(): + # Replace keys if present in DATA_COMPLETENESS_LABELS_DICT + new_key = DATA_COMPLETENESS_LABELS_DICT.get(key, key) + + # Replace state values if applicable + if key == 'state' and isinstance(value, str): + value = DATA_COMPLETENESS_STATE_DICT.get(value, value) + + # Recursively transform values + new_dict[new_key] = transform_data_completeness(value) + return new_dict + elif isinstance(data, list): + return [transform_data_completeness(item) for item in data] + return data + + +def hdx_replace_datagrid_labels(data_completeness, grp_dict): + # Replace keys in the dictionary + updated_data = transform_data_completeness(data_completeness) + updated_data.pop("inherits_from", None) + updated_data.pop("name", None) + updated_data.pop("title", None) + updated_data.pop("description", None) + updated_data['date'] = datetime.now() + updated_data['iso3'] = grp_dict.get('name') + updated_data['title'] = grp_dict.get('title') or grp_dict.get('display_name') + return updated_data diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completness.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completeness.py similarity index 99% rename from ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completness.py rename to ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completeness.py index ae72439ab9..8332a2f7dc 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completness.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completeness.py @@ -14,7 +14,7 @@ FLAG_NOT_APPLICABLE = 'not_applicable' -class DataCompletness(object): +class DataCompleteness(object): basic_query_params = { 'start': 0, diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py index 97d4a224b6..fe0007b6cd 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py @@ -69,7 +69,8 @@ def get_actions(self): 'invalidate_data_completeness_for_location': update_actions.invalidate_data_completeness_for_location, 'hdx_organization_follower_list': get_actions.hdx_organization_follower_list, 'hdx_user_invite': create_actions.hdx_user_invite, - 'member_create': create_actions.hdx_member_create + 'member_create': create_actions.hdx_member_create, + 'hdx_datagrid_show': get_actions.hdx_datagrid_show, } diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_data_completeness.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_data_completeness.py index 5291b32833..c4ec8fe033 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_data_completeness.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_data_completeness.py @@ -9,7 +9,7 @@ import ckanext.hdx_org_group.helpers.country_helper as grp_h -from ckanext.hdx_org_group.helpers.data_completness import DataCompletness, FLAG_NOT_APPLICABLE +from ckanext.hdx_org_group.helpers.data_completeness import DataCompleteness, FLAG_NOT_APPLICABLE from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST _get_action = tk.get_action @@ -112,7 +112,7 @@ def keep_db_tables_on_clean(): model.repo.tables_created_and_initialised = True -class MockedDataCompleteness(DataCompletness): +class MockedDataCompleteness(DataCompleteness): def __init__(self, yaml_dict): self.yaml_dict = yaml_dict @@ -125,7 +125,7 @@ def _fetch_yaml(self): @pytest.mark.usefixtures("keep_db_tables_on_clean", "clean_db", "clean_index", "setup_data") class TestDataCompleteness(object): - @mock.patch('ckanext.hdx_org_group.helpers.data_completness.DataCompletness') + @mock.patch('ckanext.hdx_org_group.helpers.data_completeness.DataCompleteness') def test_data_completeness(self, patched_DataCompleteness): data = self.__compute_data_completeness(_generate_test_yaml_dict(), patched_DataCompleteness) @@ -148,7 +148,7 @@ def test_data_completeness(self, patched_DataCompleteness): assert subcategory2_stats['good_datasets_num'] == 0 assert subcategory2_stats['total_datasets_num'] == 1 - @mock.patch('ckanext.hdx_org_group.helpers.data_completness.DataCompletness') + @mock.patch('ckanext.hdx_org_group.helpers.data_completeness.DataCompleteness') def test_data_completeness_force_incomplete(self, patched_DataCompleteness): yaml_dict = _generate_test_yaml_dict() incomplete_dataset = 'dataset2-category1' @@ -169,7 +169,7 @@ def test_data_completeness_force_incomplete(self, patched_DataCompleteness): dataset = next(d for d in subcategory1['datasets'] if d['name'] == incomplete_dataset) assert dataset['general_comment'] == incomplete_comment - @mock.patch('ckanext.hdx_org_group.helpers.data_completness.DataCompletness') + @mock.patch('ckanext.hdx_org_group.helpers.data_completeness.DataCompleteness') def test_data_completeness_force_complete(self, patched_DataCompleteness): yaml_dict = _generate_test_yaml_dict() complete_dataset = 'dataset1-category1' @@ -190,7 +190,7 @@ def test_data_completeness_force_complete(self, patched_DataCompleteness): dataset = next(d for d in subcategory1['datasets'] if d['name'] == complete_dataset) assert dataset['general_comment'] == complete_comment - @mock.patch('ckanext.hdx_org_group.helpers.data_completness.DataCompletness') + @mock.patch('ckanext.hdx_org_group.helpers.data_completeness.DataCompleteness') def test_data_completeness_not_available(self, patched_DataCompleteness): yaml_dict = _generate_test_yaml_dict() not_applicable_comment = 'not applicable comment' @@ -217,7 +217,7 @@ def test_data_completeness_not_available(self, patched_DataCompleteness): assert subcategory1_stats['good_datasets_num'] == 0 assert subcategory1_stats['total_datasets_num'] == 0 - @mock.patch('ckanext.hdx_org_group.helpers.data_completness.DataCompletness') + @mock.patch('ckanext.hdx_org_group.helpers.data_completeness.DataCompleteness') def test_data_completeness_dataset_up_to_date(self, patched_DataCompleteness): review_date = datetime.datetime.utcnow() - datetime.timedelta(days=31) _generate_dataset_dict('dataset3-category1', ORG, LOCATION, review_date) diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/completeness_list.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/completeness_list.html index fbe2bea817..b25b65d403 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/completeness_list.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/completeness_list.html @@ -1,6 +1,6 @@