HDX-10398 export data completeness and rename "completness"

ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py

@@ -1,20 +1,19 @@
-'''
+"""
 Created on April 24, 2015
 
 @author: alexandru-m-g
-'''
+"""
 import json
 import logging
 
 import ckan.lib.dictization as d
 import ckan.lib.helpers as helpers
 import ckan.lib.navl.dictization_functions
-import ckan.logic as logic
 import ckan.model as model
 import ckan.plugins.toolkit as tk
-
 import ckanext.hdx_org_group.dao.indicator_access as indicator_access
 import ckanext.hdx_org_group.dao.widget_data_service as widget_data_service
+import ckanext.hdx_org_group.helpers.country_helper as country_helper
 import ckanext.hdx_org_group.helpers.organization_helper as org_helper
 from ckan.common import c
 from ckanext.hdx_theme.helpers.caching import cached_make_rest_api_request as cached_make_rest_api_request
@@ -36,7 +35,7 @@
 
 @side_effect_free
 def hdx_datasets_for_group(context, data_dict):
-    '''
+    """
     Returns a paginated list of datasets for a group with 25 items per page.
     Options for sorting are: metadata_modified desc, title_case_insensitive desc, title_case_insensitive asc,
     views_recent desc, score desc ( only useful if query string is specified, should be combined
@@ -52,11 +51,11 @@ def hdx_datasets_for_group(context, data_dict):
     :param type: 'all', 'indicators', 'datasets'. Defaults to 'all'
     :type q: string
     :return:
-    '''
+    """
 
     skipped_keys = ['q', 'id', 'sort', 'type', 'page']
 
-    id = _get_or_bust(data_dict, "id")
+    id = _get_or_bust(data_dict, 'id')
 
     limit = 25
 
@@ -74,31 +73,31 @@ def hdx_datasets_for_group(context, data_dict):
         new_data_dict['ext_indicator'] = u'0'
 
     search_param_list = [
-        key + ":" + value for key, value in data_dict.iteritems() if key not in skipped_keys]
+        key + ':' + value for key, value in data_dict.iteritems() if key not in skipped_keys]
     search_param_list.append(u'groups:{}'.format(id))
 
     if search_param_list != None:
-        new_data_dict['fq'] = " ".join(
+        new_data_dict['fq'] = ' '.join(
             search_param_list) + ' +dataset_type:dataset'
 
     if data_dict.get('q', None):
         new_data_dict['q'] = data_dict['q']
 
-    query = get_action("package_search")(context, new_data_dict)
+    query = get_action('package_search')(context, new_data_dict)
 
     return query
 
 
 @side_effect_free
 def hdx_topline_num_for_group(context, data_dict):
-    '''
+    """
     :param id: the id of the group for which top line numbers are requested
     :type id: string
     :return: a dict of top line numbers. Please note that depending on the selected group the source
     of the data ( either the datastore or CPS/indicators ) might be different. The data will have some fields
      that are specific to the source.
-    '''
-    id = _get_or_bust(data_dict, "id")
+    """
+    id = _get_or_bust(data_dict, 'id')
     grp_result = get_group(id)
     group_info = grp_result.get('group_info')
     # custom_dict = grp_result.get('custom_dict')
@@ -124,14 +123,14 @@ def hdx_topline_num_for_group(context, data_dict):
 
 
 def __get_toplines_for_active_country(group_info, common_format):
-    '''
+    """
     :param group_info:
     :type group_info: dict
     :param common_format:
     :type common_format: bool
     :return:
     :rtype: list
-    '''
+    """
 
     # source is rw
     top_line_data_list = widget_data_service.build_widget_data_access(group_info).get_dataset_results()
@@ -162,14 +161,14 @@ def _parse_integer_value(item):
 
 
 def __get_toplines_for_standard_country(group_info, common_format):
-    '''
+    """
     :param group_info:
     :type group_info: dict
     :param common_format:
     :type common_format: bool
     :return:
     :rtype: list
-    '''
+    """
     # source is configured in 'hdx.locations.toplines_url'
     # ckan_site_url = config.get('ckan.site_url')
     raw_top_line_items = widget_data_service.build_widget_data_access(group_info).get_dataset_results()
@@ -197,13 +196,13 @@ def __get_toplines_for_standard_country(group_info, common_format):
 
 @side_effect_free
 def hdx_light_group_show(context, data_dict):
-    '''
+    """
     Return a lightweight ( less resource intensive,faster but without datasets ) version of the group details
     :param id: the id of the group for which top line numbers are requested
     :type id: string
-    '''
+    """
 
-    id = _get_or_bust(data_dict, "id")
+    id = _get_or_bust(data_dict, 'id')
     group_dict = {}
     group = model.Group.get(id)
     if not group:
@@ -226,13 +225,13 @@ def hdx_light_group_show(context, data_dict):
         dictized = d.table_dictize(extra, context)
         if not extra.state == 'active':
             continue
-        value = dictized["value"]
+        value = dictized['value']
         result_list.append(dictized)
 
         # Keeping the above for backwards compatibility
-        group_dict[name] = dictized["value"]
+        group_dict[name] = dictized['value']
 
-    group_dict['extras'] = sorted(result_list, key=lambda x: x["key"])
+    group_dict['extras'] = sorted(result_list, key=lambda x: x['key'])
     return group_dict
 
 
@@ -283,20 +282,20 @@ def hdx_get_locations_info_from_rw(context, data_dict):
             return cached_make_rest_api_request(url)
         return None
     except:
-        log.error("RW file was not found or can not be accessed")
+        log.error('RW file was not found or can not be accessed')
         return None
 
 
 @side_effect_free
 def hdx_organization_follower_list(context, data_dict):
-    '''Return the list of users that are following the given organization.
+    """Return the list of users that are following the given organization.
 
     :param id: the id or name of the organization
     :type id: string
 
     :rtype: list of dictionaries
 
-    '''
+    """
     _check_access('hdx_organization_follower_list', context, data_dict)
     context['keep_email'] = True
     return _follower_list(
@@ -336,3 +335,14 @@ def _user_list_dictize(obj_list, context,
         # user_dict.pop('email', None)
         result_list.append(user_dict)
     return sorted(result_list, key=sort_key, reverse=reverse)
+
+@side_effect_free
+def hdx_datagrid_show(context, data_dict):
+    id = _get_or_bust(data_dict, 'id')
+    if id:
+        grp_dict = get_action('hdx_light_group_show')(context, {'id':id})
+        data_completeness = country_helper._get_data_completeness(grp_dict.get('name'))
+        replaced_data_completeness = country_helper.hdx_replace_datagrid_labels(data_completeness, grp_dict)
+        return replaced_data_completeness
+    else:
+        raise NotFound('Group was not found.')

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/caching.py

@@ -2,7 +2,7 @@
 from dogpile.cache import make_region
 
 import ckan.plugins.toolkit as tk
-import ckanext.hdx_org_group.helpers.data_completness as data_completness
+import ckanext.hdx_org_group.helpers.data_completeness as data_completeness
 from ckanext.hdx_theme.helpers.caching import dogpile_standard_config, dogpile_config_filter, \
     HDXRedisInvalidationStrategy
 
@@ -23,13 +23,13 @@
 
 @dogpile_country_region.cache_on_arguments()
 def cached_data_completeness(location_code):
-    log.info('Fetching data completness for ' + location_code)
+    log.info('Fetching data completeness for ' + location_code)
     url_pattern = config.get('hdx.datagrid.config_url_pattern')
     for_prod = config.get('hdx.datagrid.prod')
     branch = 'master' if for_prod else location_code
     url = url_pattern.format(branch=branch, iso=location_code)
 
-    return data_completness.DataCompletness(location_code, url).get_config()
+    return data_completeness.DataCompleteness(location_code, url).get_config()
 
 
 @dogpile_country_region.cache_on_arguments()

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/country_helper.py

@@ -5,7 +5,7 @@
 import ckan.plugins.toolkit as tk
 import ckanext.hdx_org_group.helpers.caching as caching
 # import ckanext.hdx_package.helpers.screenshot as screenshot
-
+from datetime import datetime
 from ckanext.hdx_org_group.controller_logic.group_search_logic import GroupSearchLogic
 import ckan.lib.helpers as h
 from ckanext.hdx_search.helpers.constants import DEFAULT_SORTING
@@ -39,11 +39,11 @@
 
 
 def _sort_datasets_by_is_good(data_completeness):
-    categories = data_completeness.get("categories")
+    categories = data_completeness.get('categories')
     for cat in categories:
-        if cat.get("data_series"):
-            for ds in cat.get("data_series"):
-                datasets_list = ds.get("datasets")
+        if cat.get('data_series'):
+            for ds in cat.get('data_series'):
+                datasets_list = ds.get('datasets')
                 if datasets_list:
                     datasets_sorted_list = sorted(datasets_list, key=lambda item: item['is_good'] == False)
                     ds['datasets'] = datasets_sorted_list
@@ -52,7 +52,7 @@ def _sort_datasets_by_is_good(data_completeness):
 
 
 def country_topline(id):
-    log.info("The id of the page is: " + id)
+    log.info('The id of the page is: ' + id)
 
     country_dict = get_country(id)
     top_line_data_list = caching.cached_topline_numbers(id)
@@ -88,11 +88,11 @@ def get_template_data(country_dict, not_filtered_facet_info):
     # f_organization_list = _get_org_list_for_featured_from_facets(not_filtered_facet_info)
     # f_tag_list = _get_tag_list_for_featured_from_facets(not_filtered_facet_info)
 
-    data_completness = _get_data_completeness(country_dict.get('name')) \
+    data_completeness = _get_data_completeness(country_dict.get('name')) \
         if country_dict.get('data_completeness') == 'active' else None
 
-    if data_completness:
-        data_completness = _sort_datasets_by_is_good(data_completness)
+    if data_completeness:
+        data_completeness = _sort_datasets_by_is_good(data_completeness)
 
     template_data = {
         'data': {
@@ -117,7 +117,7 @@ def get_template_data(country_dict, not_filtered_facet_info):
             #     'tag_list': f_tag_list[:10],
             #     'show': len(f_organization_list) > 0 or len(f_tag_list) > 0
             # },
-            'data_completness': data_completness,
+            'data_completeness': data_completeness,
 
         },
         'errors': None,
@@ -248,8 +248,8 @@ def get_country(id):
 
 
 def _db_to_form_schema(group_type=None):
-    '''This is an interface to manipulate data from the database
-    into a format suitable for the form (optional)'''
+    """This is an interface to manipulate data from the database
+    into a format suitable for the form (optional)"""
     return lookup_group_plugin(group_type).db_to_form_schema()
 
 
@@ -290,3 +290,52 @@ def hdx_datagrid_org_get_display_text(dataseries_dict):
                     if flag.get('key', '') == 'not_applicable' and flag.get('display_text'):
                         return flag.get('display_text')
     return 'Not applicable'
+
+DATA_COMPLETENESS_LABELS_DICT = {
+    'good_datasets_num': 'available_datasets_num',
+    'dataset_goodness_percentage':'dataset_availability_percentage',
+    'good_dataseries_num':'available_dataseries_num',
+    'not_good_dataseries_num': 'outdated_dataseries_num',
+    'good_dataseries_text': 'available_dataseries_text',
+    'not_good_dataseries_text': 'outdated_dataseries_text',
+    'dataseries_good_percentage':'dataseries_available_percentage',
+    'dataseries_not_good_percentage':'dataseries_outdated_percentage',
+}
+
+DATA_COMPLETENESS_STATE_DICT = {
+    'good':'available',
+    'not_good':'outdated',
+    'empty':'empty',
+}
+
+
+def transform_data_completeness(data):
+    if isinstance(data, dict):
+        new_dict = {}
+        for key, value in data.items():
+            # Replace keys if present in DATA_COMPLETENESS_LABELS_DICT
+            new_key = DATA_COMPLETENESS_LABELS_DICT.get(key, key)
+
+            # Replace state values if applicable
+            if key == 'state' and isinstance(value, str):
+                value = DATA_COMPLETENESS_STATE_DICT.get(value, value)
+
+            # Recursively transform values
+            new_dict[new_key] = transform_data_completeness(value)
+        return new_dict
+    elif isinstance(data, list):
+        return [transform_data_completeness(item) for item in data]
+    return data
+
+
+def hdx_replace_datagrid_labels(data_completeness, grp_dict):
+    # Replace keys in the dictionary
+    updated_data = transform_data_completeness(data_completeness)
+    updated_data.pop("inherits_from", None)
+    updated_data.pop("name", None)
+    updated_data.pop("title", None)
+    updated_data.pop("description", None)
+    updated_data['date'] = datetime.now()
+    updated_data['iso3'] = grp_dict.get('name')
+    updated_data['title'] = grp_dict.get('title') or grp_dict.get('display_name')
+    return updated_data

ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completness.py → ckanext-hdx_org_group/ckanext/hdx_org_group/helpers/data_completeness.py

@@ -14,7 +14,7 @@
 FLAG_NOT_APPLICABLE = 'not_applicable'
 
 
-class DataCompletness(object):
+class DataCompleteness(object):
 
     basic_query_params = {
         'start': 0,

ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py

@@ -69,7 +69,8 @@ def get_actions(self):
             'invalidate_data_completeness_for_location': update_actions.invalidate_data_completeness_for_location,
             'hdx_organization_follower_list': get_actions.hdx_organization_follower_list,
             'hdx_user_invite': create_actions.hdx_user_invite,
-            'member_create': create_actions.hdx_member_create
+            'member_create': create_actions.hdx_member_create,
+            'hdx_datagrid_show': get_actions.hdx_datagrid_show,
 
         }