Merge branch 'dev' into fix/hdx-9070-google-search-optimize

ckanext-hdx_dataviz/ckanext/hdx_dataviz/tests/test_dataviz_page_load.py

@@ -1,5 +1,4 @@
 import pytest
-import six
 
 import ckan.plugins.toolkit as tk
 

ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_controller/test_custom_org_controller.py

@@ -140,7 +140,7 @@ def mock_get_top_line_items():
         org_dict = _get_action('organization_update')(context, org_dict)
 
         data_access_cls.return_value.get_top_line_items.side_effect = mock_get_top_line_items
-        req_mock.args = {}
+        req_mock.args = mock.MagicMock()
         tk.g.user = sysadmin_user
         tk.g.userobj = None
         org_helper_c_mock.user = sysadmin_user

ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_controller/test_member_controller.py

@@ -138,7 +138,6 @@ def test_members(self, render, app):
         context = {'model': model, 'session': model.Session, 'user': orgadmin}
         orgadmin_token = factories.APIToken(user='orgadmin', expires_in=2, unit=60 * 60)['token']
         auth = {'Authorization': orgadmin_token}
-        # test_client = self.get_backwards_compatible_test_client()
 
         member_with_name_list = _get_action('member_list')(context, {
             'id': 'hdx-test-org',
@@ -236,16 +235,17 @@ def _populate_member_names(self, members, member_with_name_list):
         ret = [next(u[4] for u in member_with_name_list if u[0] == member[0]) for member in members]
         return ret
 
+    @pytest.mark.usefixtures('with_request_context')
     @mock.patch('ckanext.hdx_users.helpers.mailer._mail_recipient_html')
-    def test_request_membership(self, _mail_recipient_html):
+    def test_request_membership(self, _mail_recipient_html, app):
         test_sysadmin = 'testsysadmin'
         test_username = 'johndoe1'
-        test_client = self.get_backwards_compatible_test_client()
+        test_username_token = factories.APIToken(user=test_username, expires_in=2, unit=60 * 60)['token']
         context = {'model': model, 'session': model.Session, 'user': test_sysadmin}
 
         # removing one member from organization
         url = h.url_for('hdx_members.member_delete', id='hdx-test-org')
-        test_client.post(url, params={'user': 'johndoe1'}, extra_environ={"REMOTE_USER": test_sysadmin})
+        app.post(url, params={'user': 'johndoe1'}, extra_environ={"REMOTE_USER": test_sysadmin})
 
         member_list = self._get_action('member_list')(context, {
             'id': 'hdx-test-org',
@@ -262,9 +262,9 @@ def test_request_membership(self, _mail_recipient_html):
 
         # send a membership request
         url = h.url_for('ytp_request.new')
-        ret_page = test_client.post(url, params={'organization': 'hdx-test-org', 'role': 'member', 'save': 'save',
-                                                 'message': 'add me to your organization'},
-                                    extra_environ={"REMOTE_USER": test_username})
+        ret_page = app.post(url, params={'organization': 'hdx-test-org', 'role': 'member', 'save': 'save',
+                                         'message': 'add me to your organization'},
+                            headers={'Authorization': test_username_token})
         member_requests = self._get_action('member_request_list')(context, {'group': 'hdx-test-org'})
         assert len(member_requests) == 1, 'Exactly one member request should exist for this org'
         assert member_requests[0].get('user_name') == test_username
@@ -280,16 +280,17 @@ def _populate_member_names(self, members, member_with_name_list):
         ret = [next(u[4] for u in member_with_name_list if u[0] == member[0]) for member in members]
         return ret
 
+    @pytest.mark.usefixtures('with_request_context')
     @mock.patch('ckanext.hdx_users.helpers.mailer._mail_recipient_html')
-    def test_request_membership(self, _mail_recipient_html):
+    def test_request_membership(self, _mail_recipient_html, app):
         test_sysadmin = 'testsysadmin'
         test_username = 'johndoe1'
-        test_client = self.get_backwards_compatible_test_client()
+        test_username_token = factories.APIToken(user=test_username, expires_in=2, unit=60 * 60)['token']
         context = {'model': model, 'session': model.Session, 'user': test_sysadmin}
 
         # removing one member from organization
         url = h.url_for('hdx_members.member_delete', id='hdx-test-org')
-        test_client.post(url, params={'user': 'johndoe1'}, extra_environ={"REMOTE_USER": test_sysadmin})
+        app.post(url, params={'user': 'johndoe1'}, extra_environ={"REMOTE_USER": test_sysadmin})
 
         member_list = self._get_action('member_list')(context, {
             'id': 'hdx-test-org',
@@ -306,9 +307,9 @@ def test_request_membership(self, _mail_recipient_html):
 
         # send a membership request
         url = h.url_for('ytp_request.new')
-        ret_page = test_client.post(url, params={'organization': 'hdx-test-org', 'role': 'editor', 'save': 'save',
-                                                 'message': 'add me to your organization'},
-                                    extra_environ={"REMOTE_USER": test_username})
+        ret_page = app.post(url, params={'organization': 'hdx-test-org', 'role': 'editor', 'save': 'save',
+                                         'message': 'add me to your organization'},
+                            headers={'Authorization': test_username_token})
         member_requests = self._get_action('member_request_list')(context, {'group': 'hdx-test-org'})
         assert len(member_requests) == 1, 'Exactly one member request should exist for this org'
         assert member_requests[0].get('user_name') == test_username

ckanext-hdx_org_group/ckanext/hdx_org_group/tests/test_pages/test_dataset_search_params.py

@@ -46,7 +46,7 @@ def test_search_params(self):
         end_str = '</section>'
         search_item = 'name="q"'
 
-        count = hdx_test_util.count_string_occurences(page, search_item,
+        count = hdx_test_util.count_string_occurrences(page, search_item,
                                                       begin_str, end_str)
 
         assert count == 1, 'There should be exactly one input with name q in the form'

ckanext-hdx_package/ckanext/hdx_package/helpers/custom_validator.py

@@ -685,6 +685,26 @@ def hdx_resources_not_allowed_if_requested_data(key, data, errors, context):
         raise df.Invalid(_('By request - HDX Connect datasets can not store resources'))
 
 
+def hdx_disable_live_frequency_filestore_resources_only(key, data, errors, context):
+    """
+    Validates that a dataset marked as 'Live' has at least one external resource
+    """
+    if data[key] == '0':  # '0' means 'Live'
+        has_resources = False
+        has_external_resource = False
+
+        for resource_key, value in data.items():
+            if resource_key[0] == 'resources':
+                has_resources = True
+                if resource_key[-1] == 'url_type':
+                    if value == 'api':
+                        has_external_resource = True
+                        break
+
+        if has_resources and not has_external_resource:
+            raise Invalid(_('Live datasets should have at least one external resource'))
+
+
 DATASERIES_TITLE_PATTERN = re.compile('^[\w ,-]+$', re.UNICODE)
 def hdx_dataseries_title_validator(value, context):
     if value:

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -155,7 +155,8 @@ def _modify_package_schema(self, schema):
                 tk.get_validator('not_empty'),
                 tk.get_validator('unicode_safe'),
                 tk.get_validator('hdx_in_update_frequency_values'),
-                tk.get_converter('convert_to_extras')
+                tk.get_validator('hdx_disable_live_frequency_filestore_resources_only'),
+                tk.get_converter('convert_to_extras'),
             ],
             'batch': [tk.get_validator('ignore_missing'), tk.get_converter('convert_to_extras')],
             'maintainer': [tk.get_validator('hdx_find_package_maintainer'), tk.get_validator('not_empty')],
@@ -518,6 +519,7 @@ def get_validators(self):
             'hdx_add_update_fs_check_info': vd.hdx_add_update_fs_check_info,
             'hdx_tag_name_approved_validator': vd.hdx_tag_name_approved_validator,
             'hdx_update_last_modified_if_url_changed': vd.hdx_update_last_modified_if_url_changed,
+            'hdx_disable_live_frequency_filestore_resources_only': vd.hdx_disable_live_frequency_filestore_resources_only,
         }
 
     def get_auth_functions(self):

ckanext-hdx_package/ckanext/hdx_package/tests/test_aws/hdx_s3_test_base.py

@@ -135,7 +135,7 @@ def _create_package_by_user(cls, pkg_name, user, create_org_and_group=True):
             "owner_org": "test_owner_org",
             "groups": [{"name": "test_group1"}],
             "maintainer": "testsysadmin",
-            "data_update_frequency": "0",
+            "data_update_frequency": "-1",
             "resources": [
                 {
                     'package_id': pkg_name,

ckanext-hdx_package/ckanext/hdx_package/tests/test_controller/test_contribute_flow.py

@@ -124,7 +124,7 @@ def _get_dataset_post_param(dataset_name):
             'maintainer': 'testsysadmin',
             'license_id': 'cc-by',
             'methodology': 'Census',
-            'data_update_frequency': 0,
+            'data_update_frequency': -1,
             'resources__0__position': 0,
             'resources__0__url': 'http://yahoo.com',
             'resources__0__format': 'link',

ckanext-hdx_package/ckanext/hdx_package/tests/test_dataset_special_fields/test_data_update_frequency.py

@@ -0,0 +1,90 @@
+import pytest
+
+import ckan.tests.factories as factories
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+
+import ckanext.hdx_theme.tests.hdx_test_base as hdx_test_base
+
+from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST
+
+config = tk.config
+NotAuthorized = tk.NotAuthorized
+ValidationError = tk.ValidationError
+
+
+class TestDataUpdateFrequency(hdx_test_base.HdxBaseTest):
+    NORMAL_USER = 'normal_user'
+    LIVE_UPDATE_FREQUENCY = '0'
+
+    INTERNAL_RESOURCE = {
+        'package_id': 'test_private_dataset_1',
+        'url': config.get('ckan.site_url', '') + '/storage/f/test_folder/hdx_test.csv',
+        'resource_type': 'file.upload',
+        'format': 'CSV',
+        'name': 'hdx_test.csv'
+    }
+    EXTERNAL_RESOURCE = {
+        'package_id': 'test_private_dataset_1',
+        'url': 'http://test.ckan.test/test.csv',
+        'resource_type': 'api',
+        'url_type': 'api',
+        'format': 'CSV',
+        'name': 'data1.csv',
+    }
+    PACKAGE = {
+        'package_creator': 'test function',
+        'private': False,
+        'dataset_date': '01/01/1960-12/31/2012',
+        'caveats': 'These are the caveats',
+        'license_other': 'TEST OTHER LICENSE',
+        'methodology': 'This is a test methodology',
+        'dataset_source': 'Test data',
+        'license_id': 'hdx-other',
+        'notes': 'This is a test dataset',
+        'data_update_frequency': LIVE_UPDATE_FREQUENCY,
+        'title': 'Test Dataset for Update Frequency',
+        'owner_org': 'org_name_4_update_frequency',
+        'groups': [{'name': 'roger'}],
+    }
+
+    @classmethod
+    def _get_action(cls, action_name):
+        return tk.get_action(action_name)
+
+    @classmethod
+    def setup_class(cls):
+        super(TestDataUpdateFrequency, cls).setup_class()
+        factories.User(name=cls.NORMAL_USER, email='[email protected]')
+        factories.Organization(
+            name='org_name_4_update_frequency',
+            title='ORG NAME FOR UPDATE FREQUENCY',
+            users=[
+                {'name': cls.NORMAL_USER, 'capacity': 'admin'},
+            ],
+            hdx_org_type=ORGANIZATION_TYPE_LIST[0][1],
+            org_url='https://hdx.hdxtest.org/'
+        )
+
+    def test_valid_live_dataset_with_external_resource(self):
+        context = {'model': model, 'session': model.Session, 'user': self.NORMAL_USER}
+
+        self.PACKAGE['name'] = 'test_dataset_1'
+        self.PACKAGE['resources'] = [self.INTERNAL_RESOURCE, self.EXTERNAL_RESOURCE]
+
+        dataset_dict = self._get_action('package_create')(context, self.PACKAGE)
+        assert dataset_dict.get(
+            'data_update_frequency') == self.LIVE_UPDATE_FREQUENCY, 'Live dataset should be created successfully with ' \
+                                                                    'an external resource'
+
+    def test_invalid_live_dataset_with_only_internal_resources(self):
+        context = {'model': model, 'session': model.Session, 'user': self.NORMAL_USER}
+
+        self.PACKAGE['name'] = 'test_dataset_2'
+        self.PACKAGE['resources'] = [self.INTERNAL_RESOURCE]
+
+        try:
+            self._get_action('package_create')(context, self.PACKAGE)
+            assert False, 'Validation error should be raised for live datasets without any external resource'
+        except ValidationError as e:
+            assert True, 'Live datasets should have at least one external resource'

ckanext-hdx_pages/ckanext/hdx_pages/tests/test_controller.py

@@ -244,7 +244,7 @@ def test_page_delete(self, app):
         eldeleted_page = _get_action('page_show')(context_sysadmin, {'id': page_eldeleted.get('name')})
         try:
             url = h.url_for(u'hdx_custom_page.delete_page', id=eldeleted_page.get('id'))
-            page_delete = app.post(url, extra_environ={"REMOTE_USER": USER})
+            page_delete = app.post(url, headers={'Authorization': self._get_token_for_user(USER)})
             assert 'Page not found' in page_delete.body, 'page doesn\'t exist'
             assert '404 Not Found'.lower() in page_delete.status.lower()
         except logic.NotAuthorized:

ckanext-hdx_search/ckanext/hdx_search/controller_logic/search_logic.py

@@ -407,7 +407,7 @@ def _page_number(self):
     def _params_nopage(self):
         params_to_skip = ['_show_filters']
         # most search operations should reset the page counter:
-        return [(k, v) for k, v in request.args.items()
+        return [(k, v) for k, v in request.args.items(multi=True)
                 if k != 'page' and k not in params_to_skip]
 
     def _set_search_url_params(self):

ckanext-hdx_search/ckanext/hdx_search/tests/conftest.py

@@ -0,0 +1 @@
+from ckanext.hdx_theme.tests.conftest import keep_db_tables_on_clean

ckanext-hdx_search/ckanext/hdx_search/tests/test_pages/test_pagination.py

@@ -0,0 +1,88 @@
+import pytest
+import ckan.tests.factories as factories
+
+import ckan.plugins.toolkit as tk
+import ckan.model as model
+
+from ckan.types import Context
+from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST
+from ckanext.hdx_theme.tests import hdx_test_util
+
+_url_for = tk.url_for
+_get_action = tk.get_action
+
+STANDARD_USER = 'some_standard_user'
+LOCATION_NAME = 'some_location'
+ORG_NAME1 = 'some_org1'
+ORG_NAME2 = 'some_org2'
+
+
+def _get_dataset_dict(dataset_name, org_name):
+    return {
+        "package_creator": "test function",
+        "private": False,
+        "dataset_date": "[1960-01-01 TO 2012-12-31]",
+        "caveats": "These are the caveats",
+        "license_other": "TEST OTHER LICENSE",
+        "methodology": "This is a test methodology",
+        "dataset_source": "Test data",
+        "license_id": "hdx-other",
+        "name": dataset_name,
+        "notes": "This is a test dataset",
+        "title": "Test Dataset " + dataset_name,
+        "owner_org": org_name,
+        "groups": [{"name": LOCATION_NAME}],
+        "data_update_frequency": "30",
+        "maintainer": STANDARD_USER,
+        "resources": [
+            {
+                'url': 'http://test.ckan.test/test.csv',
+                'resource_type': 'api',
+                'url_type': 'api',
+                'format': 'CSV',
+                'name': 'data1.csv',
+            }
+        ],
+    }
+
+
+@pytest.fixture()
+def setup_data():
+    factories.User(name=STANDARD_USER, email='[email protected]')
+    group = factories.Group(name=LOCATION_NAME)
+    for org_name in [ORG_NAME1, ORG_NAME2]:
+        factories.Organization(
+            name=org_name,
+            title='ORG NAME FOR HDX_REL_URL',
+            users=[
+                {'name': STANDARD_USER, 'capacity': 'editor'},
+            ],
+            hdx_org_type=ORGANIZATION_TYPE_LIST[0][1],
+            org_url='https://hdx.hdxtest.org/'
+        )
+
+    context: Context = {'model': model, 'session': model.Session, 'user': STANDARD_USER}
+    _get_action('package_create')(context, _get_dataset_dict(dataset_name='dataset1',
+                                                                                     org_name=ORG_NAME1))
+    _get_action('package_create')(context, _get_dataset_dict(dataset_name='dataset2',
+                                                                                     org_name=ORG_NAME2))
+
+
+@pytest.mark.usefixtures('keep_db_tables_on_clean', 'clean_db', 'clean_index', 'setup_data')
+def test_pagination_2_valued_filter(app):
+    url = _url_for('hdx_dataset.search', organization=[ORG_NAME1, ORG_NAME2], ext_page_size=1)
+    response = app.get(url)
+    assert response.status_code == 200
+
+    page = response.body
+
+    begin_str = '<a class="page-link" href='
+    end_str = '</a>'
+    search_item1 = f'organization={ORG_NAME1}'
+    search_item2 = f'organization={ORG_NAME2}'
+
+    count_org1 = hdx_test_util.count_string_occurrences(page, search_item1, begin_str, end_str)
+    assert count_org1 == 1
+
+    count_org2 = hdx_test_util.count_string_occurrences(page, search_item2, begin_str, end_str)
+    assert count_org2 == 1

ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/admin/carousel.js

@@ -49,6 +49,7 @@
             options.emulateJSON = true; // Important because your sending formdata
             options.processData = false;
             options.contentType = false;
+            options.headers = hdxUtil.net.getCsrfTokenAsObject();
 
             return Backbone.Model.prototype.sync.call(this, method, model, options);
             // return Backbone.sync.apply(this, arguments);

ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/admin/package_links.js

@@ -51,6 +51,7 @@
             options.emulateJSON = true; // Important because your sending formdata
             options.processData = false;
             options.contentType = false;
+            options.headers = hdxUtil.net.getCsrfTokenAsObject();
 
             return Backbone.Model.prototype.sync.call(this, method, model, options);
             // return Backbone.sync.apply(this, arguments);

ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/admin/quick_links.js

@@ -50,6 +50,7 @@
             options.emulateJSON = true; // Important because your sending formdata
             options.processData = false;
             options.contentType = false;
+            options.headers = hdxUtil.net.getCsrfTokenAsObject();
 
             return Backbone.Model.prototype.sync.call(this, method, model, options);
             // return Backbone.sync.apply(this, arguments);