Merge branch 'dev' into feature/HDX-9990-implement-new-contact-contri…

…butor-form

.github/dependabot.yml

@@ -0,0 +1,8 @@
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10

ckanext-hdx_package/ckanext/hdx_package/actions/authorize.py

@@ -124,6 +124,11 @@ def hdx_fs_check_resource_revise(context, data_dict=None):
     '''
     return {'success': False, 'msg': _('Only sysadmins can change the file structure check info')}
 
+def hdx_qa_hapi_report_view(context, data_dict=None):
+    '''
+    Only sysadmins are allowed to call this action
+    '''
+    return {'success': False, 'msg': _('Only sysadmins can change the file structure check info')}
 
 def hdx_cod_update(context, data_dict):
     return _check_hdx_user_permission(context, Permissions.PERMISSION_MANAGE_COD)

ckanext-hdx_package/ckanext/hdx_package/actions/get.py

@@ -526,7 +526,7 @@ def resource_show(context, data_dict):
     return resource_dict
 
 
-def _additional_hdx_resource_show_processing(context, resource_dict):
+def _additional_hdx_resource_show_processing(context, resource_dict, just_for_reindexing=False):
     # if _should_manually_load_property_value(context, resource_dict, 'size'):
     #     resource_dict['size'] = _get_resource_filesize(resource_dict)
     # if _should_manually_load_property_value(context, resource_dict, 'revision_last_updated'):
@@ -549,6 +549,12 @@ def _additional_hdx_resource_show_processing(context, resource_dict):
             del resource_dict['apihighways_url']
     if resource_dict.get('url'):
         _process_url(context, resource_dict)
+    if not just_for_reindexing:
+        try:
+            _check_access('hdx_qa_hapi_report_view', context, {})
+        except NotAuthorized:
+            if 'qa_hapi_report' in resource_dict:
+                del resource_dict['qa_hapi_report']
 
 
 # process urls for resource in case of in quarantine

ckanext-hdx_package/ckanext/hdx_package/actions/update.py

@@ -5,7 +5,6 @@
 '''
 
 import datetime
-import json
 import logging
 
 from six import text_type
@@ -31,6 +30,7 @@
 from ckanext.hdx_package.helpers.analytics import QACompletedAnalyticsSender
 from ckanext.hdx_package.helpers.constants import FILE_WAS_UPLOADED, \
     BATCH_MODE, BATCH_MODE_DONT_GROUP, BATCH_MODE_KEEP_OLD
+from ckanext.hdx_package.helpers.resource_processors.csrf_field_remover import remove_unwanted_csrf_field
 from ckanext.hdx_package.helpers.resource_triggers import \
     BEFORE_PACKAGE_UPDATE_LISTENERS, AFTER_PACKAGE_UPDATE_LISTENERS, VERSION_CHANGE_ACTIONS
 from ckanext.hdx_package.helpers.file_removal import file_remove, find_filename_in_url
@@ -225,6 +225,7 @@ def package_update(
 
     process_batch_mode(context, data_dict)
     process_skip_validation(context, data_dict)
+    remove_unwanted_csrf_field(data_dict)
 
     model = context['model']
     session = context['session']

ckanext-hdx_package/ckanext/hdx_package/controller_logic/dataset_view_logic.py

@@ -15,8 +15,10 @@ def process_shapes(resources, id=None):
 
             res_pbf_url = res_pbf_template_url.replace('{resource_id}', shp_info['layer_id'])
             name = resource['name']
+            res_format = resource['format']
             shp_dict = {
                 'resource_name': name,
+                'resource_format': res_format,
                 'url': res_pbf_url,
                 'bounding_box': shp_info['bounding_box'],
                 'layer_fields': shp_info.get('layer_fields', []),

ckanext-hdx_package/ckanext/hdx_package/helpers/custom_validator.py

@@ -397,6 +397,31 @@ def hdx_resource_keep_prev_value_unless_sysadmin(key, data, errors, context):
     if key not in data:
         raise StopOnError
 
+def hdx_resource_keep_prev_value_if_exist_unless_sysadmin(key, data, errors, context):
+    '''
+    By default, this should inject the value from the previous version.
+    The exception is if the user is a sysadmin, then the new value is used.
+    '''
+
+    user = context.get('user')
+    ignore_auth = context.get('ignore_auth')
+    allowed_to_change = ignore_auth or (user and authz.is_sysadmin(user))
+
+    if not allowed_to_change or data[key] is missing:
+        data.pop(key, None)
+        resource_id = data.get(key[:-1] + ('id',))
+        package_id = data.get(('id',))
+        if resource_id:
+            specific_key = key[2]
+            context_key = 'resource_' + resource_id
+            resource_dict = context.get(context_key)
+            if not resource_dict:
+                resource_dict = __get_previous_resource_dict(context, package_id, resource_id)
+                context[context_key] = resource_dict
+            if resource_dict:
+                old_value = resource_dict.get(specific_key)
+                if old_value is not None:
+                    data[key] = old_value
 
 # def hdx_update_field_if_value_wrapper(context_field, value):
 #     def hdx_update_field_if_value(key, data, errors, context):

ckanext-hdx_package/ckanext/hdx_package/helpers/resource_processors/csrf_field_remover.py

@@ -0,0 +1,17 @@
+from typing import Dict
+
+from ckan.types import Context
+
+
+def remove_unwanted_csrf_field(dataset_dict: Dict):
+    resources = dataset_dict.get('resources')
+    if resources:
+        for resource_dict in resources:
+            key = None
+            for k in resource_dict.keys():
+                if 'csrf' in k:
+                    key = k
+                    break
+            if key:
+                resource_dict.pop(key, None)
+

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -282,6 +282,14 @@ def _modify_package_schema(self, schema):
                     tk.get_validator('hdx_in_hapi_flag_values'),
                     tk.get_validator('hdx_delete_if_marked_with_no_data'),
                 ],
+                'qa_hapi_report': [
+                    # tk.get_validator('ignore_missing'),  # if None, don't save 'None' string
+                    tk.get_validator('hdx_resource_keep_prev_value_if_exist_unless_sysadmin'),
+                    tk.get_validator('hdx_reset_on_file_upload'),
+                    tk.get_validator('ignore_missing'),  # if None, don't save 'None' string
+
+
+                ],
             }
         )
 
@@ -341,6 +349,9 @@ def show_package_schema(self):
                     tk.get_validator('ignore_missing'),
                     tk.get_validator('boolean_validator'),
                 ],
+                'qa_hapi_report': [
+                    tk.get_validator('ignore_missing'),
+                ],
             }
         )
 
@@ -489,6 +500,7 @@ def get_validators(self):
             'hdx_assume_missing_is_true': vd.hdx_assume_missing_is_true,
             'hdx_isodate_to_string_converter': vd.hdx_isodate_to_string_converter,
             'hdx_resource_keep_prev_value_unless_sysadmin': vd.hdx_resource_keep_prev_value_unless_sysadmin,
+            'hdx_resource_keep_prev_value_if_exist_unless_sysadmin': vd.hdx_resource_keep_prev_value_if_exist_unless_sysadmin,
             'hdx_reset_on_file_upload': vd.reset_on_file_upload,
             'hdx_keep_prev_value_if_empty': vd.hdx_keep_prev_value_if_empty,
             'hdx_delete_unless_allow_broken_link': vd.hdx_delete_unless_field_in_context('allow_broken_link_field'),
@@ -552,6 +564,7 @@ def get_auth_functions(self):
             'hdx_p_coded_resource_update': authorize.hdx_p_coded_resource_update,
             'hdx_mark_resource_in_hapi': authorize.hdx_mark_resource_in_hapi,
             'hdx_request_access': authorize.hdx_request_access,
+            'hdx_qa_hapi_report_view': authorize.hdx_qa_hapi_report_view,
         }
 
     def make_middleware(self, app, config):

ckanext-hdx_package/ckanext/hdx_package/tests/conftest.py

@@ -1 +1 @@
-from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean
+from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean, dataset_with_uploaded_resource

ckanext-hdx_package/ckanext/hdx_package/tests/test_actions/test_resource_csrf_not_stored.py

@@ -0,0 +1,24 @@
+import pytest
+
+import ckan.plugins.toolkit as tk
+import ckan.model as model
+
+from typing import Dict, cast
+from ckan.types import Context
+
+_get_action = tk.get_action
+
+
+@pytest.mark.usefixtures('keep_db_tables_on_clean', 'clean_db', 'clean_index')
+def test_csrf_not_stored_in_resource(dataset_with_uploaded_resource: Dict):
+    resource_dict: Dict = dataset_with_uploaded_resource['resources'][0]
+    for key in resource_dict.keys():
+        assert 'csrf_token' not in key
+
+    context = cast(Context, {'model': model, 'session': model.Session, 'user': 'test_hdx_sysadmin_user'})
+    modified_resource_dict = _get_action('resource_patch')(context, {
+        'id': resource_dict['id'],
+        '_csrf_token': 'abcdef'
+    })
+    for key in modified_resource_dict.keys():
+        assert 'csrf_token' not in key, 'csrf_token should not be saved in resource'

ckanext-hdx_package/ckanext/hdx_package/tests/test_dataset_special_fields/test_resource_qa_hapi_report.py

@@ -0,0 +1,163 @@
+import pytest
+import os
+from werkzeug.datastructures import FileStorage
+import dateutil.parser as dateutil_parser
+
+import ckan.tests.factories as factories
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+from ckan.types import Context
+
+from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST
+
+_get_action = tk.get_action
+ValidationError = tk.ValidationError
+
+STANDARD_USER = 'some_standard_user'
+SYSADMIN_USER = 'sysadmin_user'
+DATASET_NAME_UPLOADED_RESOURCE = 'dataset_name_for_resource_url_change_with_uploaded_resource'
+
+LOCATION_NAME = 'some_location_for_resource_url_change'
+ORG_NAME = 'org_name_for_resource_url_change'
+RESOURCE_LINKED_URL = 'http://test.ckan.test/test.csv'
+
+UPLOADED_RESOURCE = {
+    'url': 'data1.csv',  # tk.config.get('ckan.site_url', '') + '/storage/f/test_folder/data1.csv',
+    'resource_type': 'file.upload',
+    'url_type': 'upload',
+    'format': 'CSV',
+    'name': 'data1.csv',
+    'qa_hapi_report': 'initial string',
+}
+
+
+def _get_dataset_dict(external_resource: bool):
+    dataset_name = DATASET_NAME_UPLOADED_RESOURCE
+    resource = UPLOADED_RESOURCE
+    return {
+        "package_creator": "test function",
+        "private": False,
+        "dataset_date": "[1960-01-01 TO 2012-12-31]",
+        "caveats": "These are the caveats",
+        "license_other": "TEST OTHER LICENSE",
+        "methodology": "This is a test methodology",
+        "dataset_source": "Test data",
+        "license_id": "hdx-other",
+        "name": dataset_name,
+        "notes": "This is a test dataset",
+        "title": "Test Dataset " + dataset_name,
+        "owner_org": ORG_NAME,
+        "groups": [{"name": LOCATION_NAME}],
+        "data_update_frequency": "30",
+        "maintainer": STANDARD_USER,
+        "resources": [
+            resource
+        ],
+    }
+
+
+#         user = model.User.by_name('sysadmin_user')
+#         user.email = '[email protected]'
+#         model.Session.commit()
+#         auth = {'Authorization': self.testsysadmin_token}
+# testsysadmin_token = factories.APIToken(user='testsysadmin', expires_in=2, unit=60 * 60)['token']
+
+@pytest.fixture()
+def setup_data():
+    factories.User(name=STANDARD_USER, email='[email protected]')
+    factories.User(name=SYSADMIN_USER, email='[email protected]', sysadmin=True)
+
+    syadmin_obj = model.User.get('[email protected]')
+    syadmin_obj.apikey = 'SYSADMIN_API_KEY'
+    model.Session.commit()
+
+    user_obj = model.User.get('[email protected]')
+    user_obj.apikey = 'USER_API_KEY'
+    model.Session.commit()
+    group = factories.Group(name=LOCATION_NAME)
+    factories.Organization(
+        name=ORG_NAME,
+        title='ORG NAME FOR HDX_REL_URL',
+        users=[
+            {'name': STANDARD_USER, 'capacity': 'editor'},
+        ],
+        hdx_org_type=ORGANIZATION_TYPE_LIST[0][1],
+        org_url='https://hdx.hdxtest.org/'
+    )
+
+    context: Context = {'model': model, 'session': model.Session, 'user': STANDARD_USER}
+    # external_dataset_dict = _get_action('package_create')(context, _get_dataset_dict(external_resource=True))
+    uploaded_dataset_dict = _get_action('package_create')(context, _get_dataset_dict(external_resource=False))
+    assert 'qa_hapi_report' not in uploaded_dataset_dict.get('resources')[0]
+
+    context_sysadmin: Context = {'model': model, 'session': model.Session, 'user': STANDARD_USER}
+    package_dict = _get_action('package_show')(context, {'id': DATASET_NAME_UPLOADED_RESOURCE})
+
+
+def _get_user_context():
+    context: Context = {'model': model, 'session': model.Session, 'user': STANDARD_USER}
+    return context
+
+
+def _get_sysadmin_context():
+    context: Context = {'model': model, 'session': model.Session, 'user': SYSADMIN_USER}
+    return context
+
+
+@pytest.mark.usefixtures("keep_db_tables_on_clean", "clean_db", "clean_index", "setup_data")
+def test_last_modified_change_for_uploaded_resource():
+    package_dict = _get_action('package_show')(_get_user_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+
+    # try to update resource with qa_hapi_report as regular editor
+    resource_dict_modified = _get_action('resource_patch')(_get_user_context(), {
+        'id': package_dict['resources'][0]['id'],
+        'description': 'new description',  # changing description
+        'url': 'data1.csv',
+        'name': 'data2.csv',  # changing the name
+        'qa_hapi_report': 'regular user qa hapi report'
+    })
+    # qa_hapi_report not exist
+    package_dict = _get_action('package_show')(_get_user_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+    assert 'qa_hapi_report' not in package_dict.get('resources')[0]
+    # qa_hapi_report
+    package_dict = _get_action('package_show')(_get_sysadmin_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+    assert 'qa_hapi_report' not in package_dict.get('resources')[0]
+
+    # try to update resource with qa_hapi_report as sysadmin editor
+    resource_dict_modified_hapi_report = _get_action('resource_patch')(_get_sysadmin_context(), {
+        'id': package_dict['resources'][0]['id'],
+        'description': 'new description',  # changing description
+        'url': 'data1.csv',
+        'name': 'data2.csv',  # changing the name
+        'qa_hapi_report': 'sysadmin user qa hapi report'
+    })
+    # qa_hapi_report not displayed for regular user
+    package_dict = _get_action('package_show')(_get_user_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+    assert 'qa_hapi_report' not in package_dict.get('resources')[0]
+
+    # qa_hapi_report displays value for sysadmin only
+    package_dict = _get_action('package_show')(_get_sysadmin_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+    assert 'qa_hapi_report' in package_dict.get('resources')[0]
+    assert package_dict.get('resources')[0].get('qa_hapi_report') == 'sysadmin user qa hapi report'
+
+    # qa_hapi_report will be removed/reset when upload
+    res_id = package_dict['resources'][0]['id']
+    file_path = os.path.join(os.path.dirname(__file__), 'data1.csv')
+    with open(file_path, 'rb') as f:
+        file_upload = FileStorage(f)
+        resource_dict = _get_action('resource_update')(_get_user_context(),
+                                                       {
+                                                           'id': res_id,
+                                                           'name': 'data1.csv',
+                                                           'url_type': 'upload',
+                                                           'resource_type': 'file.upload',
+                                                           'upload': file_upload,
+                                                           'qa_hapi_report': 'updated user hapi report'
+
+                                                       }
+                                                       )
+        package_dict = _get_action('package_show')(_get_user_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+        assert 'qa_hapi_report' not in package_dict.get('resources')[0]
+
+        package_dict = _get_action('package_show')(_get_sysadmin_context(), {'id': DATASET_NAME_UPLOADED_RESOURCE})
+        assert 'qa_hapi_report' not in package_dict.get('resources')[0]