From a162871e788a2dc141e20bb76ed617ac92ac9cff Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" Date: Wed, 2 Nov 2022 15:21:49 +0000 Subject: [PATCH 01/13] Output localised reporting organisation name --- iati_datastore/iatilib/frontend/serialize/csv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/serialize/csv.py b/iati_datastore/iatilib/frontend/serialize/csv.py index 787cff30..b6aac91c 100644 --- a/iati_datastore/iatilib/frontend/serialize/csv.py +++ b/iati_datastore/iatilib/frontend/serialize/csv.py @@ -234,7 +234,13 @@ def reporting_org_ref(activity): def reporting_org_name(activity): try: - return activity.reporting_org.name + locale = request.args.get("locale", "en") + if activity.reporting_org.name_all_values and locale in activity.reporting_org.name_all_values: + return activity.reporting_org.name_all_values[locale] + elif activity.reporting_org.name_all_values and 'default' in activity.reporting_org.name_all_values: + return activity.reporting_org.name_all_values['default'] + else: + return activity.reporting_org.name except AttributeError: return "" From b5ece377223caaee41fe94943306b4dced373830 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" Date: Thu, 3 Nov 2022 09:47:17 +0000 Subject: [PATCH 02/13] Test localised reporting organisation name --- .../test/fixtures/localised-org-names.xml | 1 + iati_datastore/iatilib/test/test_api.py | 31 +++++++++++++++++++ iati_datastore/iatilib/test/test_parser.py | 6 ++-- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml index acbe19f2..91b5c5da 100644 --- a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml +++ b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml @@ -4,6 +4,7 @@ Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ) Federal Ministry for Economic Cooperation and Development (BMZ) + Ministère fédéral de la Coopération économique et du Développement (BMZ) <narrative xml:lang="pt">Esgotamento Sanitário Pernambuco</narrative> diff --git a/iati_datastore/iatilib/test/test_api.py b/iati_datastore/iatilib/test/test_api.py index b828bdef..0df969d9 100644 --- a/iati_datastore/iatilib/test/test_api.py +++ b/iati_datastore/iatilib/test/test_api.py @@ -972,6 +972,7 @@ def test_french_description_target_groups(self): output[1][i] ) + class TestActivityCurrencyConversionOutput(ClientTestCase): """Test new functionality to output USD and EUR in activities""" @@ -1097,3 +1098,33 @@ def test_eur_currency_fields(self): self.assertEquals(u'3924408.74', output[4][csv_headers.index('budget-value-EUR')]) # 2019-12-10: 3343511 GBP self.assertEquals(u'3999715.84', output[5][csv_headers.index('budget-value-EUR')]) # 2020-12-10: 3587780 GBP +class TestActivityLocalesOrganisationNames(ClientTestCase): + """Test new functionality to output locale appropriate organisation names""" + + base_url = '/api/1/access/activity.csv' + + def test_csv_activity_count(self): + load_fix("localised-org-names.xml") + resp = self.client.get(self.base_url) + self.assertEquals(2, resp.get_data(as_text=True).count("\n")) + + def test_english_reporting_org(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url).get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('reporting-org') + self.assertEquals( + u'Federal Ministry for Economic Cooperation and Development (BMZ)', + output[1][i] + ) + + def test_french_reporting_org(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url + '?locale=fr').get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('reporting-org') + self.assertEquals( + u"Ministère fédéral de la Coopération économique et du Développement (BMZ)", + output[1][i] + ) + diff --git a/iati_datastore/iatilib/test/test_parser.py b/iati_datastore/iatilib/test/test_parser.py index 5c731ecb..2bbda11b 100644 --- a/iati_datastore/iatilib/test/test_parser.py +++ b/iati_datastore/iatilib/test/test_parser.py @@ -1118,14 +1118,16 @@ def test_reporting_org_name(self): self.assertEquals( self.act.reporting_org.name_all_values, {'de': 'Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)', - 'en': 'Federal Ministry for Economic Cooperation and Development (BMZ)'} + 'en': 'Federal Ministry for Economic Cooperation and Development (BMZ)', + 'fr': 'Ministère fédéral de la Coopération économique et du Développement (BMZ)'} ) def test_participating_orgs(self): self.assertEquals( self.act.participating_orgs[0].organisation.name_all_values, {"de": 'Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)', - "en": 'Federal Ministry for Economic Cooperation and Development (BMZ)'} + "en": 'Federal Ministry for Economic Cooperation and Development (BMZ)', + "fr": 'Ministère fédéral de la Coopération économique et du Développement (BMZ)'} ) self.assertEquals( self.act.participating_orgs[1].organisation.name_all_values, From 11cadb19080f1c2ffd2923b788000c574ce8b4a2 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Thu, 3 Nov 2022 10:29:34 +0000 Subject: [PATCH 03/13] Output localised participating organisation names --- .../iatilib/frontend/serialize/csv.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/iati_datastore/iatilib/frontend/serialize/csv.py b/iati_datastore/iatilib/frontend/serialize/csv.py index b6aac91c..e73f0529 100644 --- a/iati_datastore/iatilib/frontend/serialize/csv.py +++ b/iati_datastore/iatilib/frontend/serialize/csv.py @@ -259,13 +259,29 @@ def reporting_org_type_code(activity): return "" +def participating_org_localised_name(org): + try: + locale = request.args.get("locale", "en") + if org.name_all_values and locale in org.name_all_values: + return org.name_all_values[locale] + elif org.name_all_values and 'default' in org.name_all_values: + return org.name_all_values['default'] + else: + return org.name + except AttributeError: + return "" + + def participating_org(attr, role, activity): activity_by_role = dict( [(a.role.value, a) for a in activity.participating_orgs]) participant = activity_by_role.get(role.value, "") if participant: - return getattr(participant.organisation, attr) + if attr == "name": + return participating_org_localised_name(participant.organisation) + else: + return getattr(participant.organisation, attr) else: return '' From 5fd5838b42adbc9734f0a2580aa2dbc66f7bf666 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Thu, 3 Nov 2022 11:33:35 +0000 Subject: [PATCH 04/13] Test localised participating organisation names --- .../test/fixtures/localised-org-names.xml | 1 + iati_datastore/iatilib/test/test_api.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml index 91b5c5da..534e4ce0 100644 --- a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml +++ b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml @@ -13,6 +13,7 @@ <participating-org ref="DE-1" role="1" type="10"> <narrative xml:lang="de">Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)</narrative> <narrative xml:lang="en">Federal Ministry for Economic Cooperation and Development (BMZ)</narrative> + <narrative xml:lang="fr">Ministère fédéral de la Coopération économique et du Développement (BMZ)</narrative> </participating-org> <participating-org ref="XM-DAC-5-2" role="2"> <narrative xml:lang="de">KfW Bankengruppe (KfW)</narrative> diff --git a/iati_datastore/iatilib/test/test_api.py b/iati_datastore/iatilib/test/test_api.py index 0df969d9..233e9043 100644 --- a/iati_datastore/iatilib/test/test_api.py +++ b/iati_datastore/iatilib/test/test_api.py @@ -1128,3 +1128,22 @@ def test_french_reporting_org(self): output[1][i] ) + def test_english_participating_org(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url).get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('participating-org (Funding)') + self.assertEquals( + u'Federal Ministry for Economic Cooperation and Development (BMZ)', + output[1][i] + ) + + def test_french_participating_org(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url + '?locale=fr').get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('participating-org (Funding)') + self.assertEquals( + u'Ministère fédéral de la Coopération économique et du Développement (BMZ)', + output[1][i] + ) From a21c481250502e25f7077393c1d7aacf616ce168 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Thu, 3 Nov 2022 12:30:11 +0000 Subject: [PATCH 05/13] Output localised provider/receiver organisation names --- .../iatilib/frontend/serialize/csv.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/iati_datastore/iatilib/frontend/serialize/csv.py b/iati_datastore/iatilib/frontend/serialize/csv.py index e73f0529..235c78d9 100644 --- a/iati_datastore/iatilib/frontend/serialize/csv.py +++ b/iati_datastore/iatilib/frontend/serialize/csv.py @@ -141,6 +141,24 @@ def transaction_org(field, transaction): receiver_org = partial(transaction_org, 'receiver_org') +def transaction_org_name(field, transaction): + org = getattr(transaction, field) + if org: + locale = request.args.get("locale", "en") + if org.name_all_values and locale in org.name_all_values: + return org.name_all_values[locale] + elif org.name_all_values and 'default' in org.name_all_values: + return org.name_all_values['default'] + else: + return org.name + else: + return "" + + +provider_org_name = partial(transaction_org_name, 'provider_org') +receiver_org_name = partial(transaction_org_name, 'receiver_org') + + def title(activity): """Select most appropriate title for request locale""" locale = request.args.get("locale", "en") @@ -829,11 +847,13 @@ def wrapper(args): (u'transaction_ref', lambda t: t.ref), (u'transaction_value_currency', value_currency), (u'transaction_value_value-date', lambda t: t.value_date), - (u'transaction_provider-org', lambda t: t.provider_org_text), + #(u'transaction_provider-org', lambda t: t.provider_org_text), + (u'transaction_provider-org', provider_org_name), (u'transaction_provider-org_ref', provider_org), (u'transaction_provider-org_provider-activity-id', lambda t: t.provider_org_activity_id), - (u'transaction_receiver-org', lambda t: t.receiver_org_text), + #(u'transaction_receiver-org', lambda t: t.receiver_org_text), + (u'transaction_receiver-org', receiver_org_name), (u'transaction_receiver-org_ref', receiver_org), (u'transaction_receiver-org_receiver-activity-id', lambda t: t.receiver_org_activity_id), From b01d8890a4567b165ebec36088214552a4f5c5f1 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Tue, 29 Nov 2022 15:03:20 +0000 Subject: [PATCH 06/13] Test localised provider/receiver organisation names --- .../test/fixtures/localised-org-names.xml | 4 +++ iati_datastore/iatilib/test/test_api.py | 34 +++++++++++++++++++ iati_datastore/iatilib/test/test_parser.py | 12 ++++--- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml index 534e4ce0..93c04c7a 100644 --- a/iati_datastore/iatilib/test/fixtures/localised-org-names.xml +++ b/iati_datastore/iatilib/test/fixtures/localised-org-names.xml @@ -30,9 +30,11 @@ </description> <provider-org ref="NO-BRC-971277882" type="10"> <narrative xml:lang="en">Norwegian Agency for Development Cooperation (NORAD)</narrative> + <narrative xml:lang="fr">Agence norvégienne de coopération au développement (NORAD)</narrative> </provider-org> <receiver-org ref="DK-CVR-12921047" receiver-activity-id="DK-CVR-12921047-2018-21LotCivSPA" type="22"> <narrative xml:lang="en">Care Danmark</narrative> + <narrative xml:lang="fr">Soins Danemark</narrative> </receiver-org> </transaction> <transaction> @@ -44,9 +46,11 @@ </description> <provider-org ref="BD-NAB-0210" provider-activity-id="BD-NAB-0210-POWER" type="21"> <narrative xml:lang="en">ActionAid Bangladesh</narrative> + <narrative xml:lang="es">Ayuda en Acción Bangladesh</narrative> </provider-org> <receiver-org type="22"> <narrative xml:lang="en">SKS Foundation</narrative> + <narrative xml:lang="fr">Fondation SKS</narrative> </receiver-org> </transaction> </iati-activity> diff --git a/iati_datastore/iatilib/test/test_api.py b/iati_datastore/iatilib/test/test_api.py index 233e9043..373b4291 100644 --- a/iati_datastore/iatilib/test/test_api.py +++ b/iati_datastore/iatilib/test/test_api.py @@ -1147,3 +1147,37 @@ def test_french_participating_org(self): u'Ministère fédéral de la Coopération économique et du Développement (BMZ)', output[1][i] ) + + +class TestTransactionLocalesOrganisationNames(ClientTestCase): + """Test new functionality to output locale appropriate organisation names""" + + base_url = '/api/1/access/transaction.csv' + + def test_provider_org_output(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url).get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('transaction_provider-org') + self.assertEquals(u'Norwegian Agency for Development Cooperation (NORAD)', output[1][i]) + + def test_provider_org_french_output(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url + '?locale=fr').get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('transaction_provider-org') + self.assertEquals(u'Agence norvégienne de coopération au développement (NORAD)', output[1][i]) + + def test_receiver_org_output(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url).get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('transaction_receiver-org') + self.assertEquals(u'SKS Foundation', output[2][i]) + + def test_receiver_org_french_output(self): + load_fix("localised-org-names.xml") + output = list(csv.reader(StringIO(self.client.get(self.base_url + '?locale=fr').get_data(as_text=True)))) + csv_headers = output[0] + i = csv_headers.index('transaction_receiver-org') + self.assertEquals(u'Fondation SKS', output[2][i]) diff --git a/iati_datastore/iatilib/test/test_parser.py b/iati_datastore/iatilib/test/test_parser.py index 2bbda11b..b38b8f41 100644 --- a/iati_datastore/iatilib/test/test_parser.py +++ b/iati_datastore/iatilib/test/test_parser.py @@ -1140,15 +1140,19 @@ def test_participating_orgs(self): def test_transaction_reciever_org_name(self): self.assertEquals(self.act.transactions[0].provider_org.name_all_values, - {"en": 'Norwegian Agency for Development Cooperation (NORAD)'} + {"en": 'Norwegian Agency for Development Cooperation (NORAD)', + "fr": 'Agence norvégienne de coopération au développement (NORAD)'} ) self.assertEquals(self.act.transactions[0].receiver_org.name_all_values, - {"en": 'Care Danmark'} + {"en": 'Care Danmark', + "fr": 'Soins Danemark'} ) self.assertEquals(self.act.transactions[1].provider_org.name_all_values, - {"en": 'ActionAid Bangladesh'} + {"en": 'ActionAid Bangladesh', + "es": 'Ayuda en Acción Bangladesh'} ) self.assertEquals(self.act.transactions[1].receiver_org.name_all_values, - {"en": 'SKS Foundation'} + {"en": 'SKS Foundation', + "fr": 'Fondation SKS'} ) From 69d828904956da879153e18d01593bdd71028843 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Thu, 1 Dec 2022 13:16:51 +0000 Subject: [PATCH 07/13] Force update of Organisations for ignore-hashes update --- iati_datastore/iatilib/crawler.py | 8 ++++++-- iati_datastore/iatilib/model.py | 8 +++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/iati_datastore/iatilib/crawler.py b/iati_datastore/iatilib/crawler.py index 2f78a2d1..e60a8378 100644 --- a/iati_datastore/iatilib/crawler.py +++ b/iati_datastore/iatilib/crawler.py @@ -224,7 +224,7 @@ def parse_resource(resource): return resource # , new_identifiers -def update_activities(dataset_name): +def update_activities(dataset_name, ignore_hashes=False): ''' Parses and stores the raw XML associated with a resource [see parse_resource()], or logs the invalid resource :param resource_url: @@ -239,6 +239,9 @@ def update_activities(dataset_name): dataset = Dataset.query.get(dataset_name) resource = dataset.resources[0] + + if ignore_hashes: db.session._update_all_unique = True + try: db.session.query(Log).filter(sa.and_( Log.logger.in_( @@ -262,6 +265,7 @@ def update_activities(dataset_name): )) db.session.commit() + if ignore_hashes: db.session._update_all_unique = False def update_dataset(dataset_name, ignore_hashes): ''' @@ -307,7 +311,7 @@ def update_dataset(dataset_name, ignore_hashes): if resource.last_status_code == 200 and not resource.last_parsed: queue.enqueue( - update_activities, args=(dataset_name,), + update_activities, args=(dataset_name, ignore_hashes), result_ttl=0, job_timeout=100000) diff --git a/iati_datastore/iatilib/model.py b/iati_datastore/iatilib/model.py index 1dde6592..4987fee4 100644 --- a/iati_datastore/iatilib/model.py +++ b/iati_datastore/iatilib/model.py @@ -37,7 +37,13 @@ def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): key = (cls, hashfunc(*arg, **kw)) if key in cache: - return cache[key] + if getattr(session, '_update_all_unique', False): + obj = cache[key] + for property in kw: + obj[property] = kw[property] + return obj + else: + return cache[key] else: with session.no_autoflush: q = session.query(cls) From e8745cb8d2f5b413b0c91bacedcb9e65abc2f219 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Thu, 1 Dec 2022 19:08:28 +0000 Subject: [PATCH 08/13] Tweaks to updating of Organisations objects --- iati_datastore/iatilib/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iati_datastore/iatilib/model.py b/iati_datastore/iatilib/model.py index 4987fee4..ba4e5428 100644 --- a/iati_datastore/iatilib/model.py +++ b/iati_datastore/iatilib/model.py @@ -39,8 +39,8 @@ def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): if key in cache: if getattr(session, '_update_all_unique', False): obj = cache[key] - for property in kw: - obj[property] = kw[property] + for name, value in kw.items(): + setattr(obj, name, value) return obj else: return cache[key] From 2bbb6fdd7ff2fb11526bd02be4cd6b884b2dea24 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Wed, 21 Dec 2022 16:34:13 +0000 Subject: [PATCH 09/13] More tweaks to updating of Organisations objects --- iati_datastore/iatilib/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/iati_datastore/iatilib/model.py b/iati_datastore/iatilib/model.py index ba4e5428..777c049e 100644 --- a/iati_datastore/iatilib/model.py +++ b/iati_datastore/iatilib/model.py @@ -52,6 +52,9 @@ def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw): if not obj: obj = constructor(*arg, **kw) session.add(obj) + elif getattr(session, '_update_all_unique', False): + for name, value in kw.items(): + setattr(obj, name, value) cache[key] = obj return obj From 215f7df1ff558b730fd306a0389546e981f42597 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Tue, 10 Jan 2023 14:06:15 +0000 Subject: [PATCH 10/13] Use activity default language for organistaion names --- iati_datastore/iatilib/parse.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/iati_datastore/iatilib/parse.py b/iati_datastore/iatilib/parse.py index 3f10f4d3..df462c48 100644 --- a/iati_datastore/iatilib/parse.py +++ b/iati_datastore/iatilib/parse.py @@ -103,26 +103,26 @@ def xpath_decimal(xpath, xml, resource=None, major_version='1'): return None -def xvals_lang(xml, major_version): +def xvals_lang(xml, major_version, default_lang="default"): ret = {} if major_version == '1': for ele in xml.xpath("."): - lang = xval(ele, "@xml:lang", "default") + lang = xval(ele, "@xml:lang", default_lang) value = xval(ele, "text()") ret[lang] = value else: for ele in xml.xpath("./narrative"): - lang = xval(ele, "@xml:lang", "default") + lang = xval(ele, "@xml:lang", default_lang) value = xval(ele, "text()") ret[lang] = value return ret -def parse_org(xml, resource=no_resource, major_version='1'): +def parse_org(xml, resource=no_resource, major_version='1', default_lang="default"): data = { "ref": xval(xml, "@ref", u""), "name": xval(xml, TEXT_ELEMENT[major_version], u""), - "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version]) + "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version], default_lang=default_lang) } try: data['type'] = codelists.by_major_version[major_version].OrganisationType.from_string(xval(xml, "@type")) @@ -131,7 +131,7 @@ def parse_org(xml, resource=no_resource, major_version='1'): return Organisation.as_unique(db.session, **data) -def reporting_org(element, resource=no_resource, major_version='1'): +def reporting_org(element, resource=no_resource, major_version='1', default_lang="default"): try: xml = element.xpath("./reporting-org")[0] except IndexError: @@ -141,7 +141,7 @@ def reporting_org(element, resource=no_resource, major_version='1'): data = { "ref": xval(xml, "@ref"), "name": xval(xml, TEXT_ELEMENT[major_version], u""), - "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version]) + "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version], default_lang=default_lang) } try: data.update({ @@ -160,7 +160,7 @@ def reporting_org(element, resource=no_resource, major_version='1'): return Organisation.as_unique(db.session, **data) -def participating_orgs(xml, resource=None, major_version='1'): +def participating_orgs(xml, resource=None, major_version='1', default_lang="default"): ret = [] seen = set() for ele in xml.xpath("./participating-org"): @@ -181,7 +181,7 @@ def participating_orgs(xml, resource=None, major_version='1'): role = codelists.by_major_version['1'].OrganisationRole.from_string(value) else: role = codelists.by_major_version[major_version].OrganisationRole.from_string(xval(ele, "@role").title()) - organisation = parse_org(ele, major_version=major_version) + organisation = parse_org(ele, major_version=major_version, default_lang=default_lang) if not (role, organisation.ref) in seen: seen.add((role, organisation.ref)) ret.append(Participation(role=role, organisation=organisation)) @@ -327,14 +327,14 @@ def description_all_values(xml, resource=None, major_version='1'): return ret -def transactions(xml, resource=no_resource, major_version='1'): +def transactions(xml, resource=no_resource, major_version='1', default_lang="default"): def from_cl(code, codelist): return codelist.from_string(code) if code is not None else None def from_org(path, ele, resource=None, major_version='1'): organisation = ele.xpath(path) if organisation: - return parse_org(organisation[0], major_version=major_version) + return parse_org(organisation[0], major_version=major_version, default_lang=default_lang) # return Organisation.as_unique(db.session, ref=org) if org else Nonejk def process(ele): @@ -554,6 +554,8 @@ def activity(xml, resource=no_resource, major_version='1', version=None): Expects xml argument of type lxml.etree._Element """ + default_lang = xval(xml, "@xml:lang", "default") + if major_version == '2': start_planned = partial(xval_date, "./activity-date[@type='1']") start_actual = partial(xval_date, "./activity-date[@type='2']") @@ -585,12 +587,12 @@ def activity(xml, resource=no_resource, major_version='1', version=None): "hierarchy": hierarchy, "last_updated_datetime": last_updated_datetime, "default_language": default_language, - "reporting_org": reporting_org, + "reporting_org": partial(reporting_org, default_lang=default_lang), "websites": websites, - "participating_orgs": participating_orgs, + "participating_orgs": partial(participating_orgs, default_lang=default_lang), "recipient_country_percentages": recipient_country_percentages, "recipient_region_percentages": recipient_region_percentages, - "transactions": transactions, + "transactions": partial(transactions, default_lang=default_lang), "start_planned": start_planned, "end_planned": end_planned, "start_actual": start_actual, From f0fc1512a2b1be9d918070f0e1d4529870633f26 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" <ed.lloyd-davies@opendataservices.coop> Date: Tue, 10 Jan 2023 14:43:44 +0000 Subject: [PATCH 11/13] Test activity default language for organistaion names --- .../fixtures/localised-org-names-default.xml | 51 +++++++++++++++++++ iati_datastore/iatilib/test/test_parser.py | 43 +++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 iati_datastore/iatilib/test/fixtures/localised-org-names-default.xml diff --git a/iati_datastore/iatilib/test/fixtures/localised-org-names-default.xml b/iati_datastore/iatilib/test/fixtures/localised-org-names-default.xml new file mode 100644 index 00000000..7a1e5254 --- /dev/null +++ b/iati_datastore/iatilib/test/fixtures/localised-org-names-default.xml @@ -0,0 +1,51 @@ +<iati-activities version="2.03"> + <iati-activity default-currency="EUR" hierarchy="1" xml:lang="de"> + <iati-identifier>DE-1-998966376</iati-identifier> + <reporting-org ref="DE-1" secondary-reporter="0" type="10"> + <narrative>Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)</narrative> + <narrative xml:lang="en">Federal Ministry for Economic Cooperation and Development (BMZ)</narrative> + </reporting-org> + <title> + <narrative xml:lang="pt">Esgotamento Sanitário Pernambuco</narrative> + <narrative>Abwasserentsorgung Pernambuco (Invest.)</narrative> + + + Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ) + Federal Ministry for Economic Cooperation and Development (BMZ) + + + KfW Bankengruppe (KfW) + + + KfW Bankengruppe (KfW) + + + + + 1749796.82 + + Aid from other bilateral donors + + + Norwegische Agentur für Entwicklungszusammenarbeit (NORAD) + + + Pflege Dänemark + + + + + + 4555468.00 + + Aid from other bilateral donors + + + ActionAid Bangladesch + + + SKS-Stiftung + + + + diff --git a/iati_datastore/iatilib/test/test_parser.py b/iati_datastore/iatilib/test/test_parser.py index b38b8f41..b447eb8d 100644 --- a/iati_datastore/iatilib/test/test_parser.py +++ b/iati_datastore/iatilib/test/test_parser.py @@ -1099,7 +1099,7 @@ def test_budget_conversion_usd(self): def test_budget_conversion_eur(self): self.assertEquals(548485.69, self.act.budgets[0].value_eur) # 2011-08-01: GBP 480637 -class TestLocalsedOrganisationNames(AppTestCase): +class TestLocalisedOrganisationNames(AppTestCase): def setUp(self): super().setUp() self.activities = list(parse.document_from_file(fixture_filename("localised-org-names.xml"))) @@ -1156,3 +1156,44 @@ def test_transaction_reciever_org_name(self): "fr": 'Fondation SKS'} ) +class TestLocalisedOrganisationNamesActivityDefault(AppTestCase): + def setUp(self): + super().setUp() + self.activities = list(parse.document_from_file(fixture_filename("localised-org-names-default.xml"))) + self.act = self.activities[0] + + def test_reporting_org_name(self): + self.assertEquals( + self.act.reporting_org.name_all_values, + {'de': 'Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)', + 'en': 'Federal Ministry for Economic Cooperation and Development (BMZ)'} + ) + + def test_participating_orgs(self): + self.assertEquals( + self.act.participating_orgs[0].organisation.name_all_values, + {"de": 'Bundesministerium für wirtschaftliche Zusammenarbeit und Entwicklung (BMZ)', + "en": 'Federal Ministry for Economic Cooperation and Development (BMZ)'} + ) + self.assertEquals( + self.act.participating_orgs[1].organisation.name_all_values, + {"de": 'KfW Bankengruppe (KfW)'} + ) + self.assertEquals( + self.act.participating_orgs[2].organisation.name_all_values, + {"de": 'KfW Bankengruppe (KfW)'} + ) + + def test_transaction_reciever_org_name(self): + self.assertEquals(self.act.transactions[0].provider_org.name_all_values, + {"de": 'Norwegische Agentur für Entwicklungszusammenarbeit (NORAD)'} + ) + self.assertEquals(self.act.transactions[0].receiver_org.name_all_values, + {"de": 'Pflege Dänemark'} + ) + self.assertEquals(self.act.transactions[1].provider_org.name_all_values, + {"de": 'ActionAid Bangladesch'} + ) + self.assertEquals(self.act.transactions[1].receiver_org.name_all_values, + {"de": 'SKS-Stiftung'} + ) From be4e206a2f6b4515393f317d93ce81189a666298 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" Date: Tue, 10 Jan 2023 15:34:46 +0000 Subject: [PATCH 12/13] Remove commented out lines --- iati_datastore/iatilib/frontend/serialize/csv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/iati_datastore/iatilib/frontend/serialize/csv.py b/iati_datastore/iatilib/frontend/serialize/csv.py index 235c78d9..faa90ec2 100644 --- a/iati_datastore/iatilib/frontend/serialize/csv.py +++ b/iati_datastore/iatilib/frontend/serialize/csv.py @@ -847,12 +847,10 @@ def wrapper(args): (u'transaction_ref', lambda t: t.ref), (u'transaction_value_currency', value_currency), (u'transaction_value_value-date', lambda t: t.value_date), - #(u'transaction_provider-org', lambda t: t.provider_org_text), (u'transaction_provider-org', provider_org_name), (u'transaction_provider-org_ref', provider_org), (u'transaction_provider-org_provider-activity-id', lambda t: t.provider_org_activity_id), - #(u'transaction_receiver-org', lambda t: t.receiver_org_text), (u'transaction_receiver-org', receiver_org_name), (u'transaction_receiver-org_ref', receiver_org), (u'transaction_receiver-org_receiver-activity-id', @@ -872,7 +870,6 @@ def wrapper(args): (u"transaction_sector", sector), (u"transaction_sector-vocabulary", sector_vocabulary), (u"transaction_sector-vocabulary-code", sector_vocabulary_code), - # (u'reporting-org', lambda t: t.activity.reporting_org_ref), ) _transaction_fields = ( From 769f0c7587704c6596d920c999148b7362e3bf96 Mon Sep 17 00:00:00 2001 From: "Ed (ODSC)" Date: Tue, 17 Jan 2023 15:25:55 +0000 Subject: [PATCH 13/13] Fix alembic two heads issue --- .../864375812164_merging_two_heads.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 migrations/versions/864375812164_merging_two_heads.py diff --git a/migrations/versions/864375812164_merging_two_heads.py b/migrations/versions/864375812164_merging_two_heads.py new file mode 100644 index 00000000..071dcd65 --- /dev/null +++ b/migrations/versions/864375812164_merging_two_heads.py @@ -0,0 +1,24 @@ +"""merging two heads + +Revision ID: 864375812164 +Revises: c6fe2cf16adb, 3daa1d4ab046 +Create Date: 2022-12-13 17:03:56.861063 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '864375812164' +down_revision = ('c6fe2cf16adb', '3daa1d4ab046') +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass