Merge pull request #6373 from OCHA-DAP/dev

dev into prod for 1.82.5

ckanext-hdx_package/ckanext/hdx_package/helpers/custom_validator.py

@@ -685,6 +685,26 @@ def hdx_resources_not_allowed_if_requested_data(key, data, errors, context):
         raise df.Invalid(_('By request - HDX Connect datasets can not store resources'))
 
 
+def hdx_disable_live_frequency_filestore_resources_only(key, data, errors, context):
+    """
+    Validates that a dataset marked as 'Live' has at least one external resource
+    """
+    if data[key] == '0':  # '0' means 'Live'
+        has_resources = False
+        has_external_resource = False
+
+        for resource_key, value in data.items():
+            if resource_key[0] == 'resources':
+                has_resources = True
+                if resource_key[-1] == 'url_type':
+                    if value == 'api':
+                        has_external_resource = True
+                        break
+
+        if has_resources and not has_external_resource:
+            raise Invalid(_('Live datasets should have at least one external resource'))
+
+
 DATASERIES_TITLE_PATTERN = re.compile('^[\w ,-]+$', re.UNICODE)
 def hdx_dataseries_title_validator(value, context):
     if value:

ckanext-hdx_package/ckanext/hdx_package/plugin.py

@@ -155,7 +155,8 @@ def _modify_package_schema(self, schema):
                 tk.get_validator('not_empty'),
                 tk.get_validator('unicode_safe'),
                 tk.get_validator('hdx_in_update_frequency_values'),
-                tk.get_converter('convert_to_extras')
+                tk.get_validator('hdx_disable_live_frequency_filestore_resources_only'),
+                tk.get_converter('convert_to_extras'),
             ],
             'batch': [tk.get_validator('ignore_missing'), tk.get_converter('convert_to_extras')],
             'maintainer': [tk.get_validator('hdx_find_package_maintainer'), tk.get_validator('not_empty')],
@@ -518,6 +519,7 @@ def get_validators(self):
             'hdx_add_update_fs_check_info': vd.hdx_add_update_fs_check_info,
             'hdx_tag_name_approved_validator': vd.hdx_tag_name_approved_validator,
             'hdx_update_last_modified_if_url_changed': vd.hdx_update_last_modified_if_url_changed,
+            'hdx_disable_live_frequency_filestore_resources_only': vd.hdx_disable_live_frequency_filestore_resources_only,
         }
 
     def get_auth_functions(self):

ckanext-hdx_package/ckanext/hdx_package/tests/test_aws/hdx_s3_test_base.py

@@ -135,7 +135,7 @@ def _create_package_by_user(cls, pkg_name, user, create_org_and_group=True):
             "owner_org": "test_owner_org",
             "groups": [{"name": "test_group1"}],
             "maintainer": "testsysadmin",
-            "data_update_frequency": "0",
+            "data_update_frequency": "-1",
             "resources": [
                 {
                     'package_id': pkg_name,

ckanext-hdx_package/ckanext/hdx_package/tests/test_controller/test_contribute_flow.py

@@ -124,7 +124,7 @@ def _get_dataset_post_param(dataset_name):
             'maintainer': 'testsysadmin',
             'license_id': 'cc-by',
             'methodology': 'Census',
-            'data_update_frequency': 0,
+            'data_update_frequency': -1,
             'resources__0__position': 0,
             'resources__0__url': 'http://yahoo.com',
             'resources__0__format': 'link',

ckanext-hdx_package/ckanext/hdx_package/tests/test_dataset_special_fields/test_data_update_frequency.py

@@ -0,0 +1,90 @@
+import pytest
+
+import ckan.tests.factories as factories
+import ckan.model as model
+import ckan.plugins.toolkit as tk
+
+import ckanext.hdx_theme.tests.hdx_test_base as hdx_test_base
+
+from ckanext.hdx_org_group.helpers.static_lists import ORGANIZATION_TYPE_LIST
+
+config = tk.config
+NotAuthorized = tk.NotAuthorized
+ValidationError = tk.ValidationError
+
+
+class TestDataUpdateFrequency(hdx_test_base.HdxBaseTest):
+    NORMAL_USER = 'normal_user'
+    LIVE_UPDATE_FREQUENCY = '0'
+
+    INTERNAL_RESOURCE = {
+        'package_id': 'test_private_dataset_1',
+        'url': config.get('ckan.site_url', '') + '/storage/f/test_folder/hdx_test.csv',
+        'resource_type': 'file.upload',
+        'format': 'CSV',
+        'name': 'hdx_test.csv'
+    }
+    EXTERNAL_RESOURCE = {
+        'package_id': 'test_private_dataset_1',
+        'url': 'http://test.ckan.test/test.csv',
+        'resource_type': 'api',
+        'url_type': 'api',
+        'format': 'CSV',
+        'name': 'data1.csv',
+    }
+    PACKAGE = {
+        'package_creator': 'test function',
+        'private': False,
+        'dataset_date': '01/01/1960-12/31/2012',
+        'caveats': 'These are the caveats',
+        'license_other': 'TEST OTHER LICENSE',
+        'methodology': 'This is a test methodology',
+        'dataset_source': 'Test data',
+        'license_id': 'hdx-other',
+        'notes': 'This is a test dataset',
+        'data_update_frequency': LIVE_UPDATE_FREQUENCY,
+        'title': 'Test Dataset for Update Frequency',
+        'owner_org': 'org_name_4_update_frequency',
+        'groups': [{'name': 'roger'}],
+    }
+
+    @classmethod
+    def _get_action(cls, action_name):
+        return tk.get_action(action_name)
+
+    @classmethod
+    def setup_class(cls):
+        super(TestDataUpdateFrequency, cls).setup_class()
+        factories.User(name=cls.NORMAL_USER, email='[email protected]')
+        factories.Organization(
+            name='org_name_4_update_frequency',
+            title='ORG NAME FOR UPDATE FREQUENCY',
+            users=[
+                {'name': cls.NORMAL_USER, 'capacity': 'admin'},
+            ],
+            hdx_org_type=ORGANIZATION_TYPE_LIST[0][1],
+            org_url='https://hdx.hdxtest.org/'
+        )
+
+    def test_valid_live_dataset_with_external_resource(self):
+        context = {'model': model, 'session': model.Session, 'user': self.NORMAL_USER}
+
+        self.PACKAGE['name'] = 'test_dataset_1'
+        self.PACKAGE['resources'] = [self.INTERNAL_RESOURCE, self.EXTERNAL_RESOURCE]
+
+        dataset_dict = self._get_action('package_create')(context, self.PACKAGE)
+        assert dataset_dict.get(
+            'data_update_frequency') == self.LIVE_UPDATE_FREQUENCY, 'Live dataset should be created successfully with ' \
+                                                                    'an external resource'
+
+    def test_invalid_live_dataset_with_only_internal_resources(self):
+        context = {'model': model, 'session': model.Session, 'user': self.NORMAL_USER}
+
+        self.PACKAGE['name'] = 'test_dataset_2'
+        self.PACKAGE['resources'] = [self.INTERNAL_RESOURCE]
+
+        try:
+            self._get_action('package_create')(context, self.PACKAGE)
+            assert False, 'Validation error should be raised for live datasets without any external resource'
+        except ValidationError as e:
+            assert True, 'Live datasets should have at least one external resource'