diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 934ddff..64ad0b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: sed -i -e 's/#ckan.plugins/ckan.plugins/' test.ini ckan -c test.ini db init - ckan -c test.ini harvester initdb + ckan -c test.ini db upgrade -p harvest ckan -c test.ini xroad init-db # Disable plugins in test.ini diff --git a/ckanext/xroad_integration/harvesters/xroad_harvester.py b/ckanext/xroad_integration/harvesters/xroad_harvester.py index 0bcc06a..dc7b25b 100644 --- a/ckanext/xroad_integration/harvesters/xroad_harvester.py +++ b/ckanext/xroad_integration/harvesters/xroad_harvester.py @@ -360,7 +360,7 @@ def import_stage(self, harvest_object): 'name': name, 'xroad_servicecode': service.service_code, 'xroad_serviceversion': service.service_version, - 'xroad_service_type': service.service_type, + # TODO: use data from xroad catalog 'xroad_service_type': service.service_type, 'harvested_from_xroad': True, 'access_restriction_level': 'public' } diff --git a/ckanext/xroad_integration/harvesters/xroad_types.py b/ckanext/xroad_integration/harvesters/xroad_types.py index 0bb5eb9..aad69c2 100644 --- a/ckanext/xroad_integration/harvesters/xroad_types.py +++ b/ckanext/xroad_integration/harvesters/xroad_types.py @@ -48,7 +48,7 @@ class Service(Base): changed: datetime fetched: datetime service_version: Optional[str] = field(default=None) - service_type: Optional[str] = field(default=None) + serviceType: Optional[str] = field(default=None) wsdl: Optional[ServiceDescription] = field(default=None) openapi: Optional[ServiceDescription] = field(default=None) removed: Optional[datetime] = field(default=None) diff --git a/ckanext/xroad_integration/harvesters/xroad_types_utils.py b/ckanext/xroad_integration/harvesters/xroad_types_utils.py index 9bb5e78..03d70c0 100644 --- a/ckanext/xroad_integration/harvesters/xroad_types_utils.py +++ b/ckanext/xroad_integration/harvesters/xroad_types_utils.py @@ -47,7 +47,7 @@ def from_dict(cls, d): @classmethod def deserialize(cls, data): obj = pickle.loads(base64.decodebytes(data.encode('utf-8'))) - if type(obj) != cls: + if not isinstance(obj, cls): raise ValueError(f'Deserialized data describes a {type(obj)}, expected {cls}') return obj diff --git a/ckanext/xroad_integration/tests/test_plugin.py b/ckanext/xroad_integration/tests/test_plugin.py index 1b83999..62abdab 100644 --- a/ckanext/xroad_integration/tests/test_plugin.py +++ b/ckanext/xroad_integration/tests/test_plugin.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester') def test_base(xroad_rest_adapter_mocks): @@ -47,7 +47,7 @@ def test_base(xroad_rest_adapter_mocks): assert len(results['TEST.ORG.000003-3.LargeSubsystem']['dataset']['resources']) == 4 -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester') def test_base_twice(xroad_rest_adapter_mocks): harvester = XRoadHarvesterPlugin() @@ -57,7 +57,7 @@ def test_base_twice(xroad_rest_adapter_mocks): run_harvest(url=url, harvester=harvester, config=config) -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') def test_delete(xroad_rest_adapter_mocks): @@ -94,7 +94,7 @@ def test_delete(xroad_rest_adapter_mocks): assert should_not_be_removed_org.get('xroad_removed') is False -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') @pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_harvester') def test_listmembers_on_error(xroad_rest_adapter_mocks): harvester = XRoadHarvesterPlugin() @@ -107,7 +107,7 @@ def test_listmembers_on_error(xroad_rest_adapter_mocks): assert "There was an error on xroad catalog" in harvest_report['gather_errors'][0]['message'] -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', @@ -135,8 +135,8 @@ def test_xroad_errors(xroad_rest_adapter_mocks, xroad_rest_mocks, migrate_db_for @pytest.mark.freeze_time('2023-01-17') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') def test_list_xroad_errors_for_organization(migrate_db_for): migrate_db_for('xroad_integration') @@ -168,8 +168,8 @@ def test_list_xroad_errors_for_organization(migrate_db_for): @pytest.mark.freeze_time('2023-01-17') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') def test_view_xroad_errors_for_organization(migrate_db_for, app): migrate_db_for('xroad_integration') @@ -200,7 +200,7 @@ def test_view_xroad_errors_for_organization(migrate_db_for, app): @pytest.mark.freeze_time('2022-01-02') @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getListOfServices')) def test_fetch_xroad_service_list(xroad_rest_mocks, xroad_database_setup): result = call_action('fetch_xroad_service_list') @@ -254,7 +254,7 @@ def test_fetch_xroad_service_list(xroad_rest_mocks, xroad_database_setup): ] ) @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getListOfServices')) def test_fetch_xroad_service_list_with_date_ranges(xroad_rest_mocks, @@ -265,7 +265,7 @@ def test_fetch_xroad_service_list_with_date_ranges(xroad_rest_mocks, @pytest.mark.freeze_time('2022-01-02') @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getServiceStatistics')) def test_fetch_xroad_service_statistics(xroad_rest_mocks, xroad_database_setup): result = call_action('fetch_xroad_stats') @@ -314,7 +314,7 @@ def test_fetch_xroad_service_statistics(xroad_rest_mocks, xroad_database_setup): ] ) @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getServiceStatistics')) def test_fetch_xroad_service_statistics_with_date_ranges(xroad_rest_mocks, @@ -325,7 +325,7 @@ def test_fetch_xroad_service_statistics_with_date_ranges(xroad_rest_mocks, @pytest.mark.freeze_time('2022-01-02') @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getDistinctServiceStatistics')) def test_fetch_xroad_distinct_service_statistics(xroad_rest_mocks, xroad_database_setup): @@ -373,7 +373,7 @@ def test_fetch_xroad_distinct_service_statistics(xroad_rest_mocks, xroad_databas ] ) @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getDistinctServiceStatistics')) def test_fetch_xroad_distinct_service_statistics_with_date_ranges(xroad_rest_mocks, @@ -383,7 +383,7 @@ def test_fetch_xroad_distinct_service_statistics_with_date_ranges(xroad_rest_moc @pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup') -@pytest.mark.ckan_config('ckan.plugins', 'xroad_integration') +@pytest.mark.ckan_config('ckan.plugins', 'harvest xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('heartbeat')) def test_xroad_heartbeat(xroad_rest_mocks): result = call_action('fetch_xroad_heartbeat') @@ -391,7 +391,7 @@ def test_xroad_heartbeat(xroad_rest_mocks): assert result['success'] is True -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', @@ -419,7 +419,7 @@ def test_xroad_get_organizations_organization_data(xroad_rest_mocks): assert updated_organization['webpage_address']['en'] == "" -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', @@ -444,7 +444,7 @@ def test_xroad_get_organizations_company_data(xroad_rest_adapter_mocks, xroad_re assert 'old_business_ids' not in updated_organization -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', @@ -462,7 +462,7 @@ def test_xroad_get_organizations_company_data_with_business_id_changes(xroad_res assert updated_organization['old_business_ids'] == ['124567-8', '7654321-8'] -@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'xroad_database_setup') +@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'xroad_database_setup') @pytest.mark.ckan_config('ckan.plugins', 'apicatalog scheming_datasets scheming_organizations fluent harvest ' 'xroad_harvester xroad_integration') @pytest.mark.ckan_config('ckanext.xroad_integration.xroad_catalog_address', xroad_rest_service_url('getOrganizationEmptyData')) diff --git a/ckanext/xroad_integration/tests/xroad-catalog-mock-responses b/ckanext/xroad_integration/tests/xroad-catalog-mock-responses index 73c9161..813a656 160000 --- a/ckanext/xroad_integration/tests/xroad-catalog-mock-responses +++ b/ckanext/xroad_integration/tests/xroad-catalog-mock-responses @@ -1 +1 @@ -Subproject commit 73c916150f375dcd44d8130495b475a5dbf0099e +Subproject commit 813a656971ff6817042973b9e0d4eeca8a9396d5 diff --git a/conftest.py b/conftest.py index bb77e86..144d184 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,14 @@ # -*- coding: utf-8 -*- +import pytest + pytest_plugins = [ u'ckanext.harvest.tests.fixtures', u'ckanext.xroad_integration.tests.fixtures' ] + + +@pytest.fixture +def clean_db(reset_db, migrate_db_for): + reset_db() + migrate_db_for("harvest") diff --git a/dev-requirements.txt b/dev-requirements.txt index 76512f6..409cf32 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pytest-ckan pytest-cov -types-requests \ No newline at end of file +types-requests==2.25.1 \ No newline at end of file diff --git a/test.ini b/test.ini index a64a61f..dcc6725 100644 --- a/test.ini +++ b/test.ini @@ -14,7 +14,7 @@ use = config:/usr/lib/ckan/default/src/ckan/test-core.ini # Insert any custom config settings to be used when running your extension's # tests here. ckan.locale_default = fi -#ckan.plugins = apicatalog scheming_datasets scheming_organizations fluent harvest xroad_harvester xroad_integration +ckan.plugins = harvest apicatalog scheming_datasets scheming_organizations fluent xroad_harvester xroad_integration ckanext.xroad_integration.xroad_environment = 'FI-TEST' ckanext.xroad_integration.xroad_catalog_address = http://localhost ckanext.xroad_integration.xroad_client_id = someid