Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDX-10398 export data completeness and rename "completness" #6496

Merged
merged 7 commits into from
Dec 11, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 36 additions & 26 deletions ckanext-hdx_org_group/ckanext/hdx_org_group/actions/get.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
'''
"""
Created on April 24, 2015

@author: alexandru-m-g
'''
"""
import json
import logging

import ckan.lib.dictization as d
import ckan.lib.helpers as helpers
import ckan.lib.navl.dictization_functions
import ckan.logic as logic
import ckan.model as model
import ckan.plugins.toolkit as tk

import ckanext.hdx_org_group.dao.indicator_access as indicator_access
import ckanext.hdx_org_group.dao.widget_data_service as widget_data_service
import ckanext.hdx_org_group.helpers.country_helper as country_helper
import ckanext.hdx_org_group.helpers.organization_helper as org_helper
from ckan.common import c
from ckanext.hdx_theme.helpers.caching import cached_make_rest_api_request as cached_make_rest_api_request
Expand All @@ -36,7 +35,7 @@

@side_effect_free
def hdx_datasets_for_group(context, data_dict):
'''
"""
Returns a paginated list of datasets for a group with 25 items per page.
Options for sorting are: metadata_modified desc, title_case_insensitive desc, title_case_insensitive asc,
views_recent desc, score desc ( only useful if query string is specified, should be combined
Expand All @@ -52,11 +51,11 @@ def hdx_datasets_for_group(context, data_dict):
:param type: 'all', 'indicators', 'datasets'. Defaults to 'all'
:type q: string
:return:
'''
"""

skipped_keys = ['q', 'id', 'sort', 'type', 'page']

id = _get_or_bust(data_dict, "id")
id = _get_or_bust(data_dict, 'id')

limit = 25

Expand All @@ -74,31 +73,31 @@ def hdx_datasets_for_group(context, data_dict):
new_data_dict['ext_indicator'] = u'0'

search_param_list = [
key + ":" + value for key, value in data_dict.iteritems() if key not in skipped_keys]
key + ':' + value for key, value in data_dict.iteritems() if key not in skipped_keys]
search_param_list.append(u'groups:{}'.format(id))

if search_param_list != None:
new_data_dict['fq'] = " ".join(
new_data_dict['fq'] = ' '.join(
search_param_list) + ' +dataset_type:dataset'

if data_dict.get('q', None):
new_data_dict['q'] = data_dict['q']

query = get_action("package_search")(context, new_data_dict)
query = get_action('package_search')(context, new_data_dict)

return query


@side_effect_free
def hdx_topline_num_for_group(context, data_dict):
'''
"""
:param id: the id of the group for which top line numbers are requested
:type id: string
:return: a dict of top line numbers. Please note that depending on the selected group the source
of the data ( either the datastore or CPS/indicators ) might be different. The data will have some fields
that are specific to the source.
'''
id = _get_or_bust(data_dict, "id")
"""
id = _get_or_bust(data_dict, 'id')
grp_result = get_group(id)
group_info = grp_result.get('group_info')
# custom_dict = grp_result.get('custom_dict')
Expand All @@ -124,14 +123,14 @@ def hdx_topline_num_for_group(context, data_dict):


def __get_toplines_for_active_country(group_info, common_format):
'''
"""
:param group_info:
:type group_info: dict
:param common_format:
:type common_format: bool
:return:
:rtype: list
'''
"""

# source is rw
top_line_data_list = widget_data_service.build_widget_data_access(group_info).get_dataset_results()
Expand Down Expand Up @@ -162,14 +161,14 @@ def _parse_integer_value(item):


def __get_toplines_for_standard_country(group_info, common_format):
'''
"""
:param group_info:
:type group_info: dict
:param common_format:
:type common_format: bool
:return:
:rtype: list
'''
"""
# source is configured in 'hdx.locations.toplines_url'
# ckan_site_url = config.get('ckan.site_url')
raw_top_line_items = widget_data_service.build_widget_data_access(group_info).get_dataset_results()
Expand Down Expand Up @@ -197,13 +196,13 @@ def __get_toplines_for_standard_country(group_info, common_format):

@side_effect_free
def hdx_light_group_show(context, data_dict):
'''
"""
Return a lightweight ( less resource intensive,faster but without datasets ) version of the group details
:param id: the id of the group for which top line numbers are requested
:type id: string
'''
"""

id = _get_or_bust(data_dict, "id")
id = _get_or_bust(data_dict, 'id')
group_dict = {}
group = model.Group.get(id)
if not group:
Expand All @@ -226,13 +225,13 @@ def hdx_light_group_show(context, data_dict):
dictized = d.table_dictize(extra, context)
if not extra.state == 'active':
continue
value = dictized["value"]
value = dictized['value']
result_list.append(dictized)

# Keeping the above for backwards compatibility
group_dict[name] = dictized["value"]
group_dict[name] = dictized['value']

group_dict['extras'] = sorted(result_list, key=lambda x: x["key"])
group_dict['extras'] = sorted(result_list, key=lambda x: x['key'])
return group_dict


Expand Down Expand Up @@ -283,20 +282,20 @@ def hdx_get_locations_info_from_rw(context, data_dict):
return cached_make_rest_api_request(url)
return None
except:
log.error("RW file was not found or can not be accessed")
log.error('RW file was not found or can not be accessed')
return None


@side_effect_free
def hdx_organization_follower_list(context, data_dict):
'''Return the list of users that are following the given organization.
"""Return the list of users that are following the given organization.

:param id: the id or name of the organization
:type id: string

:rtype: list of dictionaries

'''
"""
_check_access('hdx_organization_follower_list', context, data_dict)
context['keep_email'] = True
return _follower_list(
Expand Down Expand Up @@ -336,3 +335,14 @@ def _user_list_dictize(obj_list, context,
# user_dict.pop('email', None)
result_list.append(user_dict)
return sorted(result_list, key=sort_key, reverse=reverse)

@side_effect_free
def hdx_datagrid_show(context, data_dict):
id = _get_or_bust(data_dict, 'id')
if id:
grp_dict = get_action('hdx_light_group_show')(context, {'id':id})
data_completeness = country_helper._get_data_completeness(grp_dict.get('name'))
replaced_data_completeness = country_helper.hdx_replace_datagrid_labels(data_completeness, grp_dict)
return replaced_data_completeness
else:
raise NotFound('Group was not found.')
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from dogpile.cache import make_region

import ckan.plugins.toolkit as tk
import ckanext.hdx_org_group.helpers.data_completness as data_completness
import ckanext.hdx_org_group.helpers.data_completeness as data_completeness
from ckanext.hdx_theme.helpers.caching import dogpile_standard_config, dogpile_config_filter, \
HDXRedisInvalidationStrategy

Expand All @@ -23,13 +23,13 @@

@dogpile_country_region.cache_on_arguments()
def cached_data_completeness(location_code):
log.info('Fetching data completness for ' + location_code)
log.info('Fetching data completeness for ' + location_code)
url_pattern = config.get('hdx.datagrid.config_url_pattern')
for_prod = config.get('hdx.datagrid.prod')
branch = 'master' if for_prod else location_code
url = url_pattern.format(branch=branch, iso=location_code)

return data_completness.DataCompletness(location_code, url).get_config()
return data_completeness.DataCompleteness(location_code, url).get_config()


@dogpile_country_region.cache_on_arguments()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import ckan.plugins.toolkit as tk
import ckanext.hdx_org_group.helpers.caching as caching
# import ckanext.hdx_package.helpers.screenshot as screenshot

from datetime import datetime
from ckanext.hdx_org_group.controller_logic.group_search_logic import GroupSearchLogic
import ckan.lib.helpers as h
from ckanext.hdx_search.helpers.constants import DEFAULT_SORTING
Expand Down Expand Up @@ -39,11 +39,11 @@


def _sort_datasets_by_is_good(data_completeness):
categories = data_completeness.get("categories")
categories = data_completeness.get('categories')
for cat in categories:
if cat.get("data_series"):
for ds in cat.get("data_series"):
datasets_list = ds.get("datasets")
if cat.get('data_series'):
for ds in cat.get('data_series'):
datasets_list = ds.get('datasets')
if datasets_list:
datasets_sorted_list = sorted(datasets_list, key=lambda item: item['is_good'] == False)
ds['datasets'] = datasets_sorted_list
Expand All @@ -52,7 +52,7 @@ def _sort_datasets_by_is_good(data_completeness):


def country_topline(id):
log.info("The id of the page is: " + id)
log.info('The id of the page is: ' + id)

country_dict = get_country(id)
top_line_data_list = caching.cached_topline_numbers(id)
Expand Down Expand Up @@ -88,11 +88,11 @@ def get_template_data(country_dict, not_filtered_facet_info):
# f_organization_list = _get_org_list_for_featured_from_facets(not_filtered_facet_info)
# f_tag_list = _get_tag_list_for_featured_from_facets(not_filtered_facet_info)

data_completness = _get_data_completeness(country_dict.get('name')) \
data_completeness = _get_data_completeness(country_dict.get('name')) \
if country_dict.get('data_completeness') == 'active' else None

if data_completness:
data_completness = _sort_datasets_by_is_good(data_completness)
if data_completeness:
data_completeness = _sort_datasets_by_is_good(data_completeness)

template_data = {
'data': {
Expand All @@ -117,7 +117,7 @@ def get_template_data(country_dict, not_filtered_facet_info):
# 'tag_list': f_tag_list[:10],
# 'show': len(f_organization_list) > 0 or len(f_tag_list) > 0
# },
'data_completness': data_completness,
'data_completeness': data_completeness,

},
'errors': None,
Expand Down Expand Up @@ -248,8 +248,8 @@ def get_country(id):


def _db_to_form_schema(group_type=None):
'''This is an interface to manipulate data from the database
into a format suitable for the form (optional)'''
"""This is an interface to manipulate data from the database
into a format suitable for the form (optional)"""
return lookup_group_plugin(group_type).db_to_form_schema()


Expand Down Expand Up @@ -290,3 +290,53 @@ def hdx_datagrid_org_get_display_text(dataseries_dict):
if flag.get('key', '') == 'not_applicable' and flag.get('display_text'):
return flag.get('display_text')
return 'Not applicable'

DATA_COMPLETENESS_LABELS_DICT = {
'data_series': 'subcategories',
'good_datasets_num': 'available_datasets_num',
'dataset_goodness_percentage':'dataset_availability_percentage',
'good_dataseries_num':'available_dataseries_num',
'not_good_dataseries_num': 'outdated_dataseries_num',
'good_dataseries_text': 'available_dataseries_text',
'not_good_dataseries_text': 'outdated_dataseries_text',
'dataseries_good_percentage':'dataseries_available_percentage',
'dataseries_not_good_percentage':'dataseries_outdated_percentage',
}

DATA_COMPLETENESS_STATE_DICT = {
'good':'available',
'not_good':'outdated',
'empty':'empty',
}


def transform_data_completeness(data):
if isinstance(data, dict):
new_dict = {}
for key, value in data.items():
# Replace keys if present in DATA_COMPLETENESS_LABELS_DICT
new_key = DATA_COMPLETENESS_LABELS_DICT.get(key, key)

# Replace state values if applicable
if key == 'state' and isinstance(value, str):
value = DATA_COMPLETENESS_STATE_DICT.get(value, value)

# Recursively transform values
new_dict[new_key] = transform_data_completeness(value)
return new_dict
elif isinstance(data, list):
return [transform_data_completeness(item) for item in data]
return data


def hdx_replace_datagrid_labels(data_completeness, grp_dict):
# Replace keys in the dictionary
updated_data = transform_data_completeness(data_completeness)
updated_data.pop("inherits_from", None)
updated_data.pop("name", None)
updated_data.pop("title", None)
updated_data.pop("description", None)
updated_data['date'] = datetime.now()
updated_data['iso3'] = grp_dict.get('name')
updated_data['title'] = grp_dict.get('title') or grp_dict.get('display_name')
return updated_data
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
FLAG_NOT_APPLICABLE = 'not_applicable'


class DataCompletness(object):
class DataCompleteness(object):

basic_query_params = {
'start': 0,
Expand Down
3 changes: 2 additions & 1 deletion ckanext-hdx_org_group/ckanext/hdx_org_group/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ def get_actions(self):
'invalidate_data_completeness_for_location': update_actions.invalidate_data_completeness_for_location,
'hdx_organization_follower_list': get_actions.hdx_organization_follower_list,
'hdx_user_invite': create_actions.hdx_user_invite,
'member_create': create_actions.hdx_member_create
'member_create': create_actions.hdx_member_create,
'hdx_datagrid_show': get_actions.hdx_datagrid_show,

}

Expand Down
Loading
Loading