i18n: Add locale key to API and start translating CSV/XLSX headers

#341

Alter JS so locale key added to URL's

Ignore locale key when checking for valid filters

Start using flask-babel

Set up our 2 current translations, with no translated strings.

Set up custom extractor to get CSV column headings.

Translate headers in CSV & XLSX

Update README with instructions

Add compile strings stage to deploy

Fix tests

.gitignore

@@ -2,6 +2,8 @@
 /iati_datastore/iatilib/frontend/docs
 /iati_datastore/iatilib/frontend/querybuilder
 /dump.rdb
+/iati_datastore/iatilib/messages.pot
+/iati_datastore/iatilib/translations/*/LC_MESSAGES/messages.mo
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -347,3 +347,22 @@ Then in the 2 requirements*.txt files, look for the line:
 And edit them to:
 
     -e iati_datastore
+
+I18N - Flask Application
+------------------------
+
+    cd iati_datastore/iatilib
+
+To add a new locale:
+
+    pybabel extract -F babel.cfg -o messages.pot .
+    pybabel init -i messages.pot -d translations -l fr
+
+If strings change in app and you want to reparse the app for new strings, run:
+
+    pybabel extract -F babel.cfg  -k lazy_gettext -o messages.pot .
+    pybabel update -i messages.pot -d translations
+
+When .po files change with new content, or when deploying the app:
+
+    pybabel compile -d translations

fabfile.py

@@ -22,6 +22,8 @@ def deploy(conn):
         conn.run('iati db upgrade')
         # build the docs
         conn.run('iati build-docs')
+        # create translations
+        conn.run('(cd iati_datastore/iatilib && pybabel compile -d translations)')
         # build the query builder
         conn.run('iati build-query-builder --deploy-url https://datastore.codeforiati.org')
         # webserver

iati_datastore/iatilib/babel.cfg

@@ -0,0 +1,7 @@
+[extractors]
+csv_column_headings = iatilib.frontend.serialize.babel:extract_csv_column_headings
+
+[csv_column_headings: frontend/serialize/csv.py]
+
+[python: **.py]
+

iati_datastore/iatilib/frontend/api1.py

@@ -287,6 +287,7 @@ def validate_args(self):
         if not hasattr(self, "_valid_args"):
             args = MultiDict(request.args)
             args.pop("ref", None)
+            args.pop("locale", None)
             self._valid_args = validators.activity_api_args(args)
         return self._valid_args
 

iati_datastore/iatilib/frontend/app.py

@@ -1,4 +1,5 @@
-from flask import Flask, render_template
+from flask import Flask, render_template, request
+from flask_babel import Babel
 from flask_cors import CORS
 
 from iatilib import db, rq, migrate
@@ -13,6 +14,8 @@
 def create_app(config_object='iatilib.config.Config'):
     app = Flask(__name__.split('.')[0])
     app.config.from_object(config_object)
+    babel = Babel(app, configure_jinja=False)
+    babel.localeselector(get_locale)
     register_extensions(app)
     register_blueprints(app)
     register_error_handlers(app)
@@ -40,3 +43,7 @@ def register_error_handlers(app):
     for code in (500, 501, 502, 503, 504):
         app.register_error_handler(
             code, lambda x: (render_template('error/5xx.html'), code))
+
+
+def get_locale():
+    return request.args.get("locale", "en")

iati_datastore/iatilib/frontend/serialize/babel.py

@@ -0,0 +1,19 @@
+from .csv import _activity_fields, _activity_by_country_fields, _activity_by_sector_fields, \
+    _transaction_fields, _transaction_by_country_fields, _transaction_by_sector_fields, \
+    _budget_fields, _budget_by_country_fields, _budget_by_sector_fields
+
+
+def extract_csv_column_headings(fileobj, keywords, comment_tags, options):
+    out = []
+    list_names = [
+        '_activity_fields','_activity_by_country_fields','_activity_by_sector_fields',
+        '_transaction_fields','_transaction_by_country_fields','_transaction_by_sector_fields',
+        '_budget_fields','_budget_by_country_fields','_budget_by_sector_fields',
+   ]
+    for list_name in list_names:
+        for x in globals()[list_name]:
+            if isinstance(x, tuple):
+                out.append((1, '', x[0], ['A CSV/Excel column header; in '+list_name]))
+            else:
+                out.append((1, '', x, ['A CSV/Excel column header; in '+list_name]))
+    return iter(out)

iati_datastore/iatilib/frontend/serialize/csv.py

@@ -7,6 +7,7 @@
 from iatilib import codelists
 from pyexcelerate import Workbook
 from openpyxl_copy.utils import get_column_letter
+from flask_babel import gettext
 
 
 def total(column):
@@ -414,7 +415,7 @@ def line(row):
             writer = unicodecsv.writer(out)
             writer.writerow(row)
             return out.getvalue()
-        yield line(self.fields_by_major_version['1'].keys())
+        yield line([gettext(label) for label in self.fields_by_major_version['1'].keys()])
         get_major_version = self.get_major_version
         for obj in data.items:
             row = [accessor(obj) for accessor in self.fields_by_major_version[get_major_version(obj)].values()]
@@ -436,7 +437,7 @@ def __call__(self, data, wrapped=True):
         wb = Workbook()
         ws = wb.new_sheet("data")
         # Headers
-        headers = self.fields_by_major_version['1'].keys()
+        headers = [gettext(label) for label in self.fields_by_major_version['1'].keys()]
         final_column = get_column_letter(len(headers))
         ws.range("A1", final_column+"1").value = [headers]
         # Data

iati_datastore/iatilib/test/__init__.py

@@ -28,6 +28,30 @@ class TestConfig(Config):
     RQ_CONNECTION_CLASS = "fakeredis.FakeStrictRedis"
 
 
+class AppTestCaseNoDb(unittest.TestCase):
+    def __init__(self, methodName='runTest'):
+        super().__init__(methodName)
+        self.addTypeEqualityFunc(lxml_etree.Element, self.assertXMLEqual)
+        self.addTypeEqualityFunc(xml_etree.Element, self.assertXMLEqual)
+
+    def setUp(self):
+        global _app
+        if _app is None:
+            _app = create_app(TestConfig)
+            _app.app_context().push()
+
+        self.app = _app
+        if os.environ.get("SA_ECHO", "False") == "True":
+            db.engine.echo = True
+
+    def assertXMLEqual(self, x1, x2, msg=None):
+        sio = StringIO()
+        if not xml_compare(x1, x2, sio.write):
+            if msg is None:
+                msg = sio.getvalue()
+            raise self.failureException(msg)
+
+
 class AppTestCase(unittest.TestCase):
     def __init__(self, methodName='runTest'):
         super().__init__(methodName)

iati_datastore/iatilib/test/test_serializers/__init__.py

@@ -13,7 +13,8 @@ def load_csv(data):
 
 class CSVTstMixin(object):
     def process(self, data):
-        csv_str = u"".join(self.serialize(TestWrapper(data, 0, 0, 0))).encode('utf-8')
+        with self.app.test_request_context('/'):
+            csv_str = u"".join(self.serialize(TestWrapper(data, 0, 0, 0))).encode('utf-8')
         return load_csv(csv_str)
 
     def serialize(self, data):

iati_datastore/iatilib/test/test_serializers/test_csv_activities.py

@@ -1,18 +1,17 @@
 import datetime
 import inspect
-from unittest import TestCase
 from collections import namedtuple
 
 from . import CSVTstMixin as _CSVTstMixin
 
-from iatilib.test import factories as fac
+from iatilib.test import factories as fac, AppTestCaseNoDb
 
 
 from iatilib.frontend import serialize
 from iatilib import codelists as cl
 
 
-class TestCSVStream(TestCase):
+class TestCSVStream(AppTestCaseNoDb):
     def test_stream(self):
         self.assertTrue(inspect.isgenerator(serialize.csv([])))
 
@@ -22,7 +21,7 @@ def serialize(self, data):
         return serialize.csv(data)
 
 
-class TestCSVSerializer(CSVTstMixin, TestCase):
+class TestCSVSerializer(CSVTstMixin, AppTestCaseNoDb):
     def test_empty(self):
         data = self.process([])
         self.assertEquals(0, len(data))
@@ -72,7 +71,7 @@ def test_no_description(self):
         self.assertField({"description": ""}, data[0])
 
 
-class TestCSVExample(CSVTstMixin, TestCase):
+class TestCSVExample(CSVTstMixin, AppTestCaseNoDb):
     # these tests are based around an example from IATI
     # https://docs.google.com/a/okfn.org/spreadsheet/ccc?key=0AqR8dXc6Ji4JdHJIWDJtaXhBV0IwOG56N0p1TE04V2c#gid=4
 
@@ -492,7 +491,7 @@ def test_column_list(self):
 cl2 = cl.by_major_version['2']
 
 
-class TestCSVExample2(CSVTstMixin, TestCase):
+class TestCSVExample2(CSVTstMixin, AppTestCaseNoDb):
     def test_sector_vocabulary(self):
         data = self.process([fac.ActivityFactory.build(
             sector_percentages=[
@@ -621,7 +620,7 @@ def example(self):
         return activity
 
 
-class TestActivityByCountry(CSVTstMixin, ActivityExample, TestCase):
+class TestActivityByCountry(CSVTstMixin, ActivityExample, AppTestCaseNoDb):
     def serialize(self, data):
         return serialize.csv_activity_by_country(data)
 
@@ -787,7 +786,7 @@ def test_total_commitment(self):
         self.assertField({"total-Commitment": u"130000"}, data[0])
 
 
-class TestActivityBySector(CSVTstMixin, ActivityExample, TestCase):
+class TestActivityBySector(CSVTstMixin, ActivityExample, AppTestCaseNoDb):
     def serialize(self, data):
         return serialize.csv_activity_by_sector(data)
 
@@ -963,68 +962,68 @@ def test_many_currencies(self):
         self.assertField({self.csv_field: "!Mixed currency"}, data[0])
 
 
-class TestTotalDisbursement(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalDisbursement(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.disbursement
     transaction_code = "D"
     csv_field = "total-Disbursement"
 
 
-class TestTotalExpenditure(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalExpenditure(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.expenditure
     csv_field = "total-Expenditure"
 
 
-class TestTotalIncomingFunds(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalIncomingFunds(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.incoming_funds
     csv_field = "total-Incoming Funds"
 
 
-class TestTotalInterestRepayment(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalInterestRepayment(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.interest_repayment
     csv_field = "total-Interest Repayment"
 
 
-class TestTotalLoanRepayment(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalLoanRepayment(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.loan_repayment
     csv_field = "total-Loan Repayment"
 
 
-class TestTotalReimbursement(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalReimbursement(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     transaction_type = cl.TransactionType.reimbursement
     csv_field = "total-Reimbursement"
 
 
-class TestTotalDisbursement2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalDisbursement2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.disbursement
     csv_field = "total-Disbursement"
 
 
-class TestTotalExpenditure2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalExpenditure2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.expenditure
     csv_field = "total-Expenditure"
 
 
-class TestTotalIncomingFunds2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalIncomingFunds2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.incoming_funds
     csv_field = "total-Incoming Funds"
 
 
-class TestTotalInterestRepayment2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalInterestRepayment2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.interest_payment
     csv_field = "total-Interest Repayment"
 
 
-class TestTotalLoanRepayment2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalLoanRepayment2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.loan_repayment
     csv_field = "total-Loan Repayment"
 
 
-class TestTotalReimbursement2(CSVTstMixin, TotalFieldMixin, TestCase):
+class TestTotalReimbursement2(CSVTstMixin, TotalFieldMixin, AppTestCaseNoDb):
     cl = cl2
     transaction_type = cl2.TransactionType.reimbursement
     csv_field = "total-Reimbursement"

iati_datastore/iatilib/test/test_serializers/test_csv_budgets.py

@@ -1,10 +1,9 @@
 import datetime
-from unittest import TestCase
 from collections import namedtuple
 
 from . import CSVTstMixin as _CSVTstMixin
 
-from iatilib.test import factories as fac
+from iatilib.test import factories as fac, AppTestCaseNoDb
 from iatilib.frontend import serialize
 from iatilib import codelists as cl
 
@@ -70,7 +69,7 @@ def example():
     return activity
 
 
-class TestCSVBudgetExample(TestCase, CSVTstMixin):
+class TestCSVBudgetExample(AppTestCaseNoDb, CSVTstMixin):
     # See example here: https://docs.google.com/a/okfn.org/spreadsheet/ccc?key=0AqR8dXc6Ji4JdHJIWDJtaXhBV0IwOG56N0p1TE04V2c&usp=sharing#gid=5
     def test_start(self):
         data = self.process([
@@ -208,7 +207,7 @@ def test_sector_percentage(self):
         self.assertField({"sector-percentage": "20"}, data[0])
 
 
-class TestBudgetByCountry(TestCase, CSVTstMixin):
+class TestBudgetByCountry(AppTestCaseNoDb, CSVTstMixin):
     def serialize(self, data):
         return serialize.csv_budget_by_country(data)
 
@@ -252,7 +251,7 @@ def test_identifier(self):
         self.assertField({"iati-identifier": "GB-1-123"}, data[2])
 
 
-class TestBudgetBySector(TestCase, CSVTstMixin):
+class TestBudgetBySector(AppTestCaseNoDb, CSVTstMixin):
     def serialize(self, data):
         return serialize.csv_budget_by_sector(data)