Merge pull request #59 from vrk-kpa/LIKA-592_fix_harvester

LIKA-592: fix harvester

.github/workflows/test.yml

@@ -88,7 +88,7 @@ jobs:
           sed -i -e 's/#ckan.plugins/ckan.plugins/' test.ini
           
           ckan -c test.ini db init
-          ckan -c test.ini harvester initdb
+          ckan -c test.ini db upgrade -p harvest
           ckan -c test.ini xroad init-db
           
           # Disable plugins in test.ini

ckanext/xroad_integration/harvesters/xroad_harvester.py

@@ -360,7 +360,7 @@ def import_stage(self, harvest_object):
                 'name': name,
                 'xroad_servicecode': service.service_code,
                 'xroad_serviceversion': service.service_version,
-                'xroad_service_type': service.service_type,
+                # TODO: use data from xroad catalog 'xroad_service_type': service.service_type,
                 'harvested_from_xroad': True,
                 'access_restriction_level': 'public'
             }

ckanext/xroad_integration/harvesters/xroad_types.py

@@ -48,7 +48,7 @@ class Service(Base):
     changed: datetime
     fetched: datetime
     service_version: Optional[str] = field(default=None)
-    service_type: Optional[str] = field(default=None)
+    serviceType: Optional[str] = field(default=None)
     wsdl: Optional[ServiceDescription] = field(default=None)
     openapi: Optional[ServiceDescription] = field(default=None)
     removed: Optional[datetime] = field(default=None)

ckanext/xroad_integration/harvesters/xroad_types_utils.py

@@ -47,7 +47,7 @@ def from_dict(cls, d):
     @classmethod
     def deserialize(cls, data):
         obj = pickle.loads(base64.decodebytes(data.encode('utf-8')))
-        if type(obj) != cls:
+        if not isinstance(obj, cls):
             raise ValueError(f'Deserialized data describes a {type(obj)}, expected {cls}')
         return obj
 

ckanext/xroad_integration/tests/test_plugin.py

@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
 @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester')
 def test_base(xroad_rest_adapter_mocks):
 
@@ -47,7 +47,7 @@ def test_base(xroad_rest_adapter_mocks):
     assert len(results['TEST.ORG.000003-3.LargeSubsystem']['dataset']['resources']) == 4
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
 @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester')
 def test_base_twice(xroad_rest_adapter_mocks):
     harvester = XRoadHarvesterPlugin()
@@ -57,7 +57,7 @@ def test_base_twice(xroad_rest_adapter_mocks):
     run_harvest(url=url, harvester=harvester, config=config)
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 def test_delete(xroad_rest_adapter_mocks):
@@ -94,7 +94,7 @@ def test_delete(xroad_rest_adapter_mocks):
     assert should_not_be_removed_org.get('xroad_removed') is False
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
 @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester')
 def test_listmembers_on_error(xroad_rest_adapter_mocks):
     harvester = XRoadHarvesterPlugin()
@@ -107,7 +107,7 @@ def test_listmembers_on_error(xroad_rest_adapter_mocks):
     assert "There was an error on xroad catalog" in harvest_report['gather_errors'][0]['message']
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
@@ -135,8 +135,8 @@ def test_xroad_errors(xroad_rest_adapter_mocks, xroad_rest_mocks, migrate_db_for
 
 
 @pytest.mark.freeze_time('2023-01-17')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 def test_list_xroad_errors_for_organization(migrate_db_for):
     migrate_db_for('xroad_integration')
 
@@ -168,8 +168,8 @@ def test_list_xroad_errors_for_organization(migrate_db_for):
 
 
 @pytest.mark.freeze_time('2023-01-17')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 def test_view_xroad_errors_for_organization(migrate_db_for, app):
     migrate_db_for('xroad_integration')
 
@@ -200,7 +200,7 @@ def test_view_xroad_errors_for_organization(migrate_db_for, app):
 
 @pytest.mark.freeze_time('2022-01-02')
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getListOfServices'))
 def test_fetch_xroad_service_list(xroad_rest_mocks, xroad_database_setup):
     result = call_action('fetch_xroad_service_list')
@@ -254,7 +254,7 @@ def test_fetch_xroad_service_list(xroad_rest_mocks, xroad_database_setup):
     ]
 )
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
                          xroad_rest_service_url('getListOfServices'))
 def test_fetch_xroad_service_list_with_date_ranges(xroad_rest_mocks,
@@ -265,7 +265,7 @@ def test_fetch_xroad_service_list_with_date_ranges(xroad_rest_mocks,
 
 @pytest.mark.freeze_time('2022-01-02')
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getServiceStatistics'))
 def test_fetch_xroad_service_statistics(xroad_rest_mocks, xroad_database_setup):
     result = call_action('fetch_xroad_stats')
@@ -314,7 +314,7 @@ def test_fetch_xroad_service_statistics(xroad_rest_mocks, xroad_database_setup):
     ]
 )
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
                          xroad_rest_service_url('getServiceStatistics'))
 def test_fetch_xroad_service_statistics_with_date_ranges(xroad_rest_mocks,
@@ -325,7 +325,7 @@ def test_fetch_xroad_service_statistics_with_date_ranges(xroad_rest_mocks,
 
 @pytest.mark.freeze_time('2022-01-02')
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
                          xroad_rest_service_url('getDistinctServiceStatistics'))
 def test_fetch_xroad_distinct_service_statistics(xroad_rest_mocks, xroad_database_setup):
@@ -373,7 +373,7 @@ def test_fetch_xroad_distinct_service_statistics(xroad_rest_mocks, xroad_databas
     ]
 )
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
                          xroad_rest_service_url('getDistinctServiceStatistics'))
 def test_fetch_xroad_distinct_service_statistics_with_date_ranges(xroad_rest_mocks,
@@ -383,15 +383,15 @@ def test_fetch_xroad_distinct_service_statistics_with_date_ranges(xroad_rest_moc
 
 
 @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup')
-@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration')
+@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('heartbeat'))
 def test_xroad_heartbeat(xroad_rest_mocks):
     result = call_action('fetch_xroad_heartbeat')
     assert result['heartbeat'] is True
     assert result['success'] is True
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
@@ -419,7 +419,7 @@ def test_xroad_get_organizations_organization_data(xroad_rest_mocks):
     assert updated_organization['webpage_address']['en'] == ""
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
@@ -444,7 +444,7 @@ def test_xroad_get_organizations_company_data(xroad_rest_adapter_mocks, xroad_re
     assert 'old_business_ids' not in updated_organization
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address',
@@ -462,7 +462,7 @@ def test_xroad_get_organizations_company_data_with_business_id_changes(xroad_res
     assert updated_organization['old_business_ids'] == ['124567-8', '7654321-8']
 
 
-@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup')
+@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup')
 @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest '
                                          'xroad_harvester xroad_integration')
 @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getOrganizationEmptyData'))

conftest.py

@@ -1,6 +1,14 @@
 # -*- coding: utf-8 -*-
+import pytest
+
 
 pytest_plugins = [
     u'ckanext.harvest.tests.fixtures',
     u'ckanext.xroad_integration.tests.fixtures'
 ]
+
+
+@pytest.fixture
+def clean_db(reset_db, migrate_db_for):
+    reset_db()
+    migrate_db_for("harvest")

dev-requirements.txt

@@ -1,3 +1,3 @@
 pytest-ckan
 pytest-cov
-types-requests
+types-requests==2.25.1

test.ini

@@ -14,7 +14,7 @@ use = config:/usr/lib/ckan/default/src/ckan/test-core.ini
 # Insert any custom config settings to be used when running your extension's
 # tests here.
 ckan.locale_default = fi
-#ckan.plugins = apicatalog scheming_datasets scheming_organizations fluent harvest xroad_harvester xroad_integration
+ckan.plugins = harvest apicatalog scheming_datasets scheming_organizations fluent xroad_harvester xroad_integration
 ckanext.xroad_integration.xroad_environment = 'FI-TEST'
 ckanext.xroad_integration.xroad_catalog_address = http://localhost
 ckanext.xroad_integration.xroad_client_id = someid