diff --git a/bin/travis-run.sh b/bin/travis-run.sh index b27a3b3a..a8c6a860 100644 --- a/bin/travis-run.sh +++ b/bin/travis-run.sh @@ -1,7 +1,6 @@ #!/bin/sh -e nosetests --ckan \ - --nologcapture \ --with-pylons=subdir/test.ini \ --with-coverage \ --cover-package=ckanext.xloader \ diff --git a/ckanext/xloader/cli.py b/ckanext/xloader/cli.py index 3df91124..53dee8a9 100644 --- a/ckanext/xloader/cli.py +++ b/ckanext/xloader/cli.py @@ -184,10 +184,13 @@ def _submit_resource(self, resource, user, indent=0): r=resource)) return dataset_ref = resource.get('package_name', resource['package_id']) - print('{indent}Submitting /dataset/{dataset}/resource/{r[id]}\n' - '{indent} url={r[url]}\n' - '{indent} format={r[format]}' - .format(dataset=dataset_ref, r=resource, indent=' ' * indent)) + print(u'{indent}Submitting /dataset/{dataset}/resource/{r[id]}\n' + u'{indent} url={url}\n' + u'{indent} format={format}' + .format(dataset=dataset_ref, + format=r['format'].decode('utf-8'), + url=r['url'].decode('utf-8'), + indent=' ' * indent)) data_dict = { 'resource_id': resource['id'], 'ignore_hash': True, diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index e554a328..68f3bc30 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -158,7 +158,7 @@ def xloader_data_into_datastore_(input, job_dict): ) # fetch the resource data - logger.info('Fetching from: {0}'.format(url)) + logger.info(u'Fetching from: {0}'.format(url)) try: headers = {} if resource.get('url_type') == 'upload': diff --git "a/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" "b/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" new file mode 100644 index 00000000..4879119e --- /dev/null +++ "b/ckanext/xloader/tests/samples/umlaut_name_\303\244\303\266\303\274.csv" @@ -0,0 +1,11 @@ +SET_ID,SET_NAME,INDIKATOR_ID,INDIKATOR_NAME,INDIKATOR_BESCHREIB,KOERPERSCHAFT_ID,KOERPERSCHAFT_NAME,JAHR,VALUE +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",433,"Berufswahlschule Bezirk Horgen",1995,192230 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",440,"Schulpsychologischer Dienst des Bezirks Horgen",1995,97576 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",456,"Gemeindezentrum Brüelmatt",1995,128003 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",474,"Schulzweckverband Bezirk Affoltern",1995,1074294 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",510,"Berufswahlschule Limmattal (BWL)",1995,338128 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",841,"Fürsorgeverband Weiningen",1996,335003 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",6,"Friedhofverband Weiningen",1996,287025 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",7,"Seniorenzentrum Im Morgen Weiningen",1996,899498 +6,"Erfolgsrechnung",101,"31 Sachaufwand [Fr.]","Sachaufwand",10,"Regionale Tierkörpersammelstelle Regensdorf",1996,32213 + diff --git a/ckanext/xloader/tests/test_jobs.py b/ckanext/xloader/tests/test_jobs.py index edca32d4..450558f8 100644 --- a/ckanext/xloader/tests/test_jobs.py +++ b/ckanext/xloader/tests/test_jobs.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + import os import json import random @@ -21,47 +23,49 @@ SOURCE_URL = 'http://www.example.com/static/file' -def mock_actions(func): - ''' - Decorator that mocks actions used by these tests - Based on ckan.test.helpers.mock_action - ''' - def wrapper(*args, **kwargs): - # Mock CKAN's resource_show API - from ckan.logic import get_action as original_get_action - - def side_effect(called_action_name): - if called_action_name == 'resource_show': - def mock_resource_show(context, data_dict): - return { - 'id': data_dict['id'], - 'name': 'short name', - 'url': SOURCE_URL, - 'format': 'CSV', - 'package_id': 'test-pkg', - } - return mock_resource_show - elif called_action_name == 'package_show': - def mock_package_show(context, data_dict): - return { - 'id': data_dict['id'], - 'name': 'pkg-name', - } - return mock_package_show - else: - return original_get_action(called_action_name) - try: - with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action: - mock_get_action.side_effect = side_effect - - return_value = func(*args, **kwargs) - finally: - pass - # Make sure to stop the mock, even with an exception - # mock_action.stop() - return return_value - - return make_decorator(func)(wrapper) +def mock_actions(resource_url=None): + def decorator(func): + ''' + Decorator that mocks actions used by these tests + Based on ckan.test.helpers.mock_action + ''' + def wrapper(*args, **kwargs): + # Mock CKAN's resource_show API + from ckan.logic import get_action as original_get_action + + def side_effect(called_action_name): + if called_action_name == 'resource_show': + def mock_resource_show(context, data_dict): + return { + 'id': data_dict['id'], + 'name': 'short name', + 'url': resource_url, + 'format': 'CSV', + 'package_id': 'test-pkg', + } + return mock_resource_show + elif called_action_name == 'package_show': + def mock_package_show(context, data_dict): + return { + 'id': data_dict['id'], + 'name': 'pkg-name', + } + return mock_package_show + else: + return original_get_action(called_action_name) + try: + with mock.patch('ckanext.xloader.jobs.get_action') as mock_get_action: + mock_get_action.side_effect = side_effect + + return_value = func(*args, **kwargs) + finally: + pass + # Make sure to stop the mock, even with an exception + # mock_action.stop() + return return_value + + return make_decorator(func)(wrapper) + return decorator class TestxloaderDataIntoDatastore(util.PluginsMixin): @@ -87,7 +91,8 @@ def teardown_class(cls): connection.close() def register_urls(self, filename='simple.csv', - content_type='application/csv'): + content_type='application/csv', + resource_url=None): """Mock some test URLs with responses. Mocks some URLs related to a data file and a CKAN resource that @@ -98,10 +103,13 @@ def register_urls(self, filename='simple.csv', resource_show URL for the resource that contains the data file """ + if not resource_url: + resource_url = SOURCE_URL + responses.add_passthru(config['solr_url']) # A URL that just returns a static file - responses.add(responses.GET, SOURCE_URL, + responses.add(responses.GET, resource_url, body=get_sample_file(filename), content_type=content_type) @@ -157,7 +165,7 @@ def get_load_logs(self, task_id): .where(logs.c.job_id == task_id)) return Logs(result.fetchall()) - @mock_actions + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_simple_csv(self): # Test not only the load and xloader_hook is called at the end @@ -216,7 +224,7 @@ def test_simple_csv(self): eq_(job['status'], u'complete') eq_(job['error'], None) - @mock_actions + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_umlaut_and_extra_comma(self): self.register_urls(filename='umlaut_and_extra_comma.csv') @@ -260,7 +268,52 @@ def test_umlaut_and_extra_comma(self): eq_(job['status'], u'complete') eq_(job['error'], None) - @mock_actions + @mock_actions(resource_url=u'http://example.com/umlaut_name_äöü.csv') + @responses.activate + def test_resource_url_with_umlaut(self): + # test that xloader can handle URLs with umlauts + # e.g. http://www.web.statistik.zh.ch/ogd/data/KANTON_ZUERICH_gpfi_Jahresrechung_Zweckverbände.csv + self.register_urls( + filename=u'umlaut_name_äöü.csv', + resource_url=u'http://example.com/umlaut_name_%C3%A4%C3%B6%C3%BC.csv' + ) + data = { + 'api_key': self.api_key, + 'job_type': 'xloader_to_datastore', + 'result_url': self.callback_url, + 'metadata': { + 'ckan_url': 'http://%s/' % self.host, + 'resource_id': self.resource_id + } + } + job_id = 'test{}'.format(random.randint(0, 1e5)) + + with mock.patch('ckanext.xloader.jobs.set_datastore_active_flag') \ + as mocked_set_datastore_active_flag: + # in tests we call jobs directly, rather than use rq, so mock + # get_current_job() + with mock.patch('ckanext.xloader.jobs.get_current_job', + return_value=mock.Mock(id=job_id)): + result = jobs.xloader_data_into_datastore(data) + assert result is None, jobs_db.get_job(job_id)['error']['message'].decode('utf-8') + + # Check it said it was successful + eq_(responses.calls[-1].request.url, 'http://www.ckan.org/api/3/action/xloader_hook') + job_dict = json.loads(responses.calls[-1].request.body) + assert job_dict['status'] == u'complete', job_dict + eq_(job_dict, + {u'metadata': {u'ckan_url': u'http://www.ckan.org/', + u'resource_id': u'foo-bar-42'}, + u'status': u'complete'}) + + logs = self.get_load_logs(job_id) + logs.assert_no_errors() + + job = jobs_db.get_job(job_id) + eq_(job['status'], u'complete') + eq_(job['error'], None) + + @mock_actions(resource_url=SOURCE_URL) @responses.activate def test_first_request_is_202_pending_response(self): # when you first get the CSV it returns this 202 response, which is