i18n: Add locale key to API and start translating CSV/XLSX headers

#341

Alter JS so locale key added to URL's

Ignore locale key when checking for valid filters

Start using flask-babel

Set up our 2 current translations, with only one translated string (from Google Translate)

Translate headers in CSV & XLSX

So headers are picked up, duplicate at the top of iati_datastore/iatilib/frontend/serialize/csv.py
(Added TODO to fix this later)

Update README with instructions

Add compile strings stage to deploy

.gitignore

@@ -2,6 +2,8 @@
 /iati_datastore/iatilib/frontend/docs
 /iati_datastore/iatilib/frontend/querybuilder
 /dump.rdb
+/iati_datastore/iatilib/messages.pot
+/iati_datastore/iatilib/translations/*/LC_MESSAGES/messages.mo
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

README.md

@@ -347,3 +347,22 @@ Then in the 2 requirements*.txt files, look for the line:
 And edit them to:
 
     -e iati_datastore
+
+I18N - Flask Application
+------------------------
+
+    cd iati_datastore/iatilib
+
+To add a new locale:
+
+    pybabel extract -F babel.cfg -o messages.pot .
+    pybabel init -i messages.pot -d translations -l fr
+
+If strings change in app and you want to reparse the app for new strings, run:
+
+    pybabel extract -F babel.cfg  -k lazy_gettext -o messages.pot .
+    pybabel update -i messages.pot -d translations
+
+When .po files change with new content, or when deploying the app:
+
+    pybabel compile -d translations

fabfile.py

@@ -22,6 +22,8 @@ def deploy(conn):
         conn.run('iati db upgrade')
         # build the docs
         conn.run('iati build-docs')
+        # create translations
+        conn.run('(cd iati_datastore/iatilib && pybabel compile -d translations)')
         # build the query builder
         conn.run('iati build-query-builder --deploy-url https://datastore.codeforiati.org')
         # webserver

iati_datastore/iatilib/babel.cfg

@@ -0,0 +1 @@
+[python: **.py]

iati_datastore/iatilib/frontend/api1.py

@@ -287,6 +287,7 @@ def validate_args(self):
         if not hasattr(self, "_valid_args"):
             args = MultiDict(request.args)
             args.pop("ref", None)
+            args.pop("locale", None)
             self._valid_args = validators.activity_api_args(args)
         return self._valid_args
 

iati_datastore/iatilib/frontend/app.py

@@ -1,4 +1,5 @@
-from flask import Flask, render_template
+from flask import Flask, render_template, request
+from flask_babel import Babel
 from flask_cors import CORS
 
 from iatilib import db, rq, migrate
@@ -13,6 +14,8 @@
 def create_app(config_object='iatilib.config.Config'):
     app = Flask(__name__.split('.')[0])
     app.config.from_object(config_object)
+    babel = Babel(app, configure_jinja=False)
+    babel.localeselector(get_locale)
     register_extensions(app)
     register_blueprints(app)
     register_error_handlers(app)
@@ -40,3 +43,7 @@ def register_error_handlers(app):
     for code in (500, 501, 502, 503, 504):
         app.register_error_handler(
             code, lambda x: (render_template('error/5xx.html'), code))
+
+
+def get_locale():
+    return request.args.get("locale", "en")

iati_datastore/iatilib/frontend/serialize/csv.py

@@ -7,6 +7,71 @@
 from iatilib import codelists
 from pyexcelerate import Workbook
 from openpyxl_copy.utils import get_column_letter
+from flask_babel import gettext, lazy_gettext
+
+
+# I18N
+# Call lazy_gettext on everything we are using as column header. This way Babel will pick it up as a translate-able string.
+# TODO this is a pain as we have to duplicate all headers. Instead, write a custom extraction method later: https://babel.pocoo.org/en/latest/messages.html#writing-extraction-methods
+lazy_gettext("iati-identifier")
+lazy_gettext("hierarchy")
+lazy_gettext("last-updated-datetime")
+lazy_gettext("default-language")
+lazy_gettext("reporting-org")
+lazy_gettext("reporting-org-ref")
+lazy_gettext("reporting-org-type")
+lazy_gettext("reporting-org-type-code")
+lazy_gettext("title")
+lazy_gettext("description")
+lazy_gettext("activity-status-code")
+lazy_gettext("start-planned")
+lazy_gettext("end-planned")
+lazy_gettext("start-actual")
+lazy_gettext("end-actual")
+lazy_gettext("participating-org (Accountable)")
+lazy_gettext("participating-org-ref (Accountable)")
+lazy_gettext("participating-org-type (Accountable)")
+lazy_gettext("participating-org-type-code (Accountable)")
+lazy_gettext("participating-org (Funding)")
+lazy_gettext("participating-org-ref (Funding)")
+lazy_gettext("participating-org-type (Funding)")
+lazy_gettext("participating-org-type-code (Funding)")
+lazy_gettext("participating-org (Extending)")
+lazy_gettext("participating-org-ref (Extending)")
+lazy_gettext("participating-org-type (Extending)")
+lazy_gettext("participating-org-type-code (Extending)")
+lazy_gettext("participating-org (Implementing)")
+lazy_gettext("participating-org-ref (Implementing)")
+lazy_gettext("participating-org-type (Implementing)")
+lazy_gettext("participating-org-type-code (Implementing)")
+lazy_gettext("recipient-country-code")
+lazy_gettext("recipient-country")
+lazy_gettext("recipient-country-percentage")
+lazy_gettext("sector-code")
+lazy_gettext("sector")
+lazy_gettext("sector-percentage")
+lazy_gettext("sector-vocabulary")
+lazy_gettext("sector-vocabulary-code")
+lazy_gettext("collaboration-type-code")
+lazy_gettext("default-finance-type-code")
+lazy_gettext("default-flow-type-code")
+lazy_gettext("default-aid-type-code")
+lazy_gettext("default-tied-status-code")
+lazy_gettext("recipient-country-code")
+lazy_gettext("recipient-country")
+lazy_gettext("recipient-country-percentage")
+lazy_gettext("recipient-region-code")
+lazy_gettext("recipient-region")
+lazy_gettext("recipient-region-percentage")
+lazy_gettext("default-currency")
+lazy_gettext("currency")
+lazy_gettext('total-Commitment')
+lazy_gettext("total-Disbursement")
+lazy_gettext("total-Expenditure")
+lazy_gettext("total-Incoming Funds")
+lazy_gettext("total-Interest Repayment")
+lazy_gettext("total-Loan Repayment")
+lazy_gettext("total-Reimbursement")
 
 
 def total(column):
@@ -414,7 +479,7 @@ def line(row):
             writer = unicodecsv.writer(out)
             writer.writerow(row)
             return out.getvalue()
-        yield line(self.fields_by_major_version['1'].keys())
+        yield line([gettext(label) for label in self.fields_by_major_version['1'].keys()])
         get_major_version = self.get_major_version
         for obj in data.items:
             row = [accessor(obj) for accessor in self.fields_by_major_version[get_major_version(obj)].values()]
@@ -436,7 +501,7 @@ def __call__(self, data, wrapped=True):
         wb = Workbook()
         ws = wb.new_sheet("data")
         # Headers
-        headers = self.fields_by_major_version['1'].keys()
+        headers = [gettext(label) for label in self.fields_by_major_version['1'].keys()]
         final_column = get_column_letter(len(headers))
         ws.range("A1", final_column+"1").value = [headers]
         # Data