diff --git a/.github/workflows/hypha-ci.yml b/.github/workflows/hypha-ci.yml index 8d182996df..2d6f33ea33 100644 --- a/.github/workflows/hypha-ci.yml +++ b/.github/workflows/hypha-ci.yml @@ -37,6 +37,7 @@ jobs: DJANGO_SETTINGS_MODULE: hypha.settings.test SEND_MESSAGES: false PYTHONDONTWRITEBYTECODE: 1 + APPLICATION_TRANSLATIONS_ENABLED: 1 # Run tests for machine translation logic services: postgres: @@ -75,6 +76,7 @@ jobs: run: | uv venv uv pip install -r requirements-dev.txt + uv pip install -r requirements-translate.txt - name: Check Django migrations if: matrix.group == 1 diff --git a/docs/setup/administrators/configuration.md b/docs/setup/administrators/configuration.md index f4603a9d31..59aa689c68 100644 --- a/docs/setup/administrators/configuration.md +++ b/docs/setup/administrators/configuration.md @@ -245,6 +245,16 @@ Set this to enable Djangos settings for secure cookies. COOKIE_SECURE = env.bool('COOKIE_SECURE', False) +---- + +Machine translation settings for applications + +See [here](machine-translations.md) for more information on setting up machine translations + + APPLICATION_TRANSLATIONS_ENABLED = env.bool("APPLICATION_TRANSLATIONS_ENABLED", False) + +---- + ## Slack settings SLACK_TOKEN = env.str('SLACK_TOKEN', None) diff --git a/docs/setup/administrators/machine-translations.md b/docs/setup/administrators/machine-translations.md new file mode 100644 index 0000000000..f2cad1297f --- /dev/null +++ b/docs/setup/administrators/machine-translations.md @@ -0,0 +1,29 @@ +# Machine translations + +Hypha has the ability to utilize [argostranslate](https://github.com/argosopentech/argos-translate) for machine translations of submitted application content. This is disabled by default and the dependencies are not installed to prevent unneeded bloat due to [PyTorch](https://pytorch.org/)'s large language models. + + +## Installing dependencies + +As referenced in the [production deployment guide](../deployment/production/stand-alone.md), it is required to install the dependencies needed for machine translation dependencies via + +```bash +python3 -m pip install -r requirements-translate.txt +``` + +This requirements file will specifically attempt to install the CPU version of [PyTorch](https://pytorch.org/) if available on the detected platform to play better with heroku (doesn't support GPU processing) and to minimize package bloat (CPU package is ~300MB less than the normal GPU). Depending on your use case, you may want to adjust this. + + +## Installing languages + +Argostranslate handles translations via it's own packages - ie. Arabic -> English translation would be one package, while English -> Arabic would be another. + +Installing/uninstalling these packages can be done with the management commands `install_languages`/`uninstall_languages` respectively, utilizing the format of _. For example, installing the Arabic -> English & French -> English packages would look like: + +```bash +python3 manage.py install_languages ar_en fr_en +``` + +## Enabling on the system + +To enable machine translations on an instance, the proper configuration variables need to be set. These can be found in the [configuration options](configuration.md#hypha-custom-settings) \ No newline at end of file diff --git a/docs/setup/deployment/development/stand-alone.md b/docs/setup/deployment/development/stand-alone.md index 438affb2c9..048c8f29e5 100644 --- a/docs/setup/deployment/development/stand-alone.md +++ b/docs/setup/deployment/development/stand-alone.md @@ -278,6 +278,12 @@ source venv/bin/activate python3 -m pip install -r requirements-dev.txt ``` +If utilizing application machine translations, install the required dependencies: + +```shell +python3 -m pip install -r requirements-translate.txt +``` + Run: ```shell make serve-docs diff --git a/docs/setup/deployment/production/stand-alone.md b/docs/setup/deployment/production/stand-alone.md index 2f4fdcb165..b457830ebf 100644 --- a/docs/setup/deployment/production/stand-alone.md +++ b/docs/setup/deployment/production/stand-alone.md @@ -105,6 +105,12 @@ Next, install the required packages using: python3 -m pip install -r requirements.txt ``` +If utilizing application machine translations, install the required dependencies: + +```shell +python3 -m pip install -r requirements-translate.txt +``` + ### Install Node packages All the needed Node packages are listed in `package.json`. Install them with this command. diff --git a/hypha/apply/funds/templates/funds/applicationsubmission_admin_detail.html b/hypha/apply/funds/templates/funds/applicationsubmission_admin_detail.html index cadddf3ffe..62463ee0f4 100644 --- a/hypha/apply/funds/templates/funds/applicationsubmission_admin_detail.html +++ b/hypha/apply/funds/templates/funds/applicationsubmission_admin_detail.html @@ -1,5 +1,5 @@ {% extends "funds/applicationsubmission_detail.html" %} -{% load i18n static workflow_tags review_tags determination_tags heroicons %} +{% load i18n static workflow_tags review_tags determination_tags translate_tags heroicons %} {% block extra_css %} @@ -98,4 +98,8 @@
{% trans "Reminders" %}
+ + {% if request.user|can_translate_submission %} + + {% endif %} {% endblock %} diff --git a/hypha/apply/funds/templates/funds/applicationsubmission_detail.html b/hypha/apply/funds/templates/funds/applicationsubmission_detail.html index d025aa7db6..3533fc6c49 100644 --- a/hypha/apply/funds/templates/funds/applicationsubmission_detail.html +++ b/hypha/apply/funds/templates/funds/applicationsubmission_detail.html @@ -1,5 +1,5 @@ {% extends "base-apply.html" %} -{% load i18n static workflow_tags wagtailcore_tags statusbar_tags archive_tags submission_tags %} +{% load i18n static workflow_tags wagtailcore_tags statusbar_tags archive_tags submission_tags translate_tags %} {% load heroicons %} {% load can from permission_tags %} @@ -148,8 +148,15 @@
{% blocktrans with stage=object.previous.stage %}Your {{ stage }} applicatio {% endif %} - - {% include "funds/includes/rendered_answers.html" %} + {% if request.user|can_translate_submission %} +
+ {% include "funds/includes/rendered_answers.html" %} +
+ {% else %} +
+ {% include "funds/includes/rendered_answers.html" %} +
+ {% endif %} {% endif %} diff --git a/hypha/apply/funds/templates/funds/includes/admin_primary_actions.html b/hypha/apply/funds/templates/funds/includes/admin_primary_actions.html index 89892ff9b3..6e7ccc47ab 100644 --- a/hypha/apply/funds/templates/funds/includes/admin_primary_actions.html +++ b/hypha/apply/funds/templates/funds/includes/admin_primary_actions.html @@ -1,5 +1,5 @@ {% load i18n %} -{% load heroicons primaryactions_tags %} +{% load heroicons primaryactions_tags translate_tags %}
{% trans "Actions to take" %}
@@ -84,6 +84,13 @@
{% trans "Actions to take" %}
{% trans "More actions" %} {% trans "Revisions" %} + {% if request.user|can_translate_submission %} + + {% endif %} + + + + {% heroicon_outline "information-circle" aria_hidden="true" size=15 stroke_width=2 class="inline align-baseline me-1" %} + + + + \ No newline at end of file diff --git a/hypha/apply/funds/templates/submissions/partials/submission-title.html b/hypha/apply/funds/templates/submissions/partials/submission-title.html new file mode 100644 index 0000000000..e3038ceb93 --- /dev/null +++ b/hypha/apply/funds/templates/submissions/partials/submission-title.html @@ -0,0 +1 @@ +

{{ object.title }} #{{ object.public_id|default:object.id }}

\ No newline at end of file diff --git a/hypha/apply/funds/templatetags/translate_tags.py b/hypha/apply/funds/templatetags/translate_tags.py new file mode 100644 index 0000000000..652c9e8da0 --- /dev/null +++ b/hypha/apply/funds/templatetags/translate_tags.py @@ -0,0 +1,18 @@ +from django import template +from django.conf import settings + +register = template.Library() + + +@register.filter +def can_translate_submission(user) -> bool: + """Verify that system settings & user role allows for submission translations. + + Args: + user: the user to check the role of. + + Returns: + bool: true if submission can be translated, false if not. + + """ + return bool(settings.APPLICATION_TRANSLATIONS_ENABLED and user.is_org_faculty) diff --git a/hypha/apply/funds/tests/test_tags.py b/hypha/apply/funds/tests/test_tags.py index c85a5408a4..905cc1bd96 100644 --- a/hypha/apply/funds/tests/test_tags.py +++ b/hypha/apply/funds/tests/test_tags.py @@ -1,7 +1,8 @@ from django.template import Context, Template -from django.test import TestCase +from django.test import RequestFactory, TestCase, override_settings from hypha.apply.funds.tests.factories import ApplicationSubmissionFactory +from hypha.apply.users.tests.factories import ApplicantFactory, StaffFactory class TestTemplateTags(TestCase): @@ -26,3 +27,42 @@ def test_submission_tags(self): output == f'Lorem ipsum dolor {submission.title} #{submission.public_id or submission.id} sit amet.' ) + + @override_settings(APPLICATION_TRANSLATIONS_ENABLED=True) + def test_translate_tags_as_applicant(self): + submission = ApplicationSubmissionFactory() + request = RequestFactory().get(submission.get_absolute_url()) + request.user = ApplicantFactory() + template = Template( + "{% load translate_tags %}{% if request.user|can_translate_submission %}

some translation stuff

{% endif %}" + ) + context = Context({"request": request}) + output = template.render(context) + + self.assertEqual(output, "") + + @override_settings(APPLICATION_TRANSLATIONS_ENABLED=True) + def test_translate_tags_as_staff(self): + submission = ApplicationSubmissionFactory() + request = RequestFactory().get(submission.get_absolute_url()) + request.user = StaffFactory() + template = Template( + "{% load translate_tags %}{% if request.user|can_translate_submission %}

some translation stuff

{% endif %}" + ) + context = Context({"request": request}) + output = template.render(context) + + self.assertEqual(output, "

some translation stuff

") + + @override_settings(APPLICATION_TRANSLATIONS_ENABLED=False) + def test_translate_tags_disabled(self): + submission = ApplicationSubmissionFactory() + request = RequestFactory().get(submission.get_absolute_url()) + request.user = StaffFactory() + template = Template( + "{% load translate_tags %}{% if request.user|can_translate_submission %}

some translation stuff

{% endif %}" + ) + context = Context({"request": request}) + output = template.render(context) + + self.assertEqual(output, "") diff --git a/hypha/apply/funds/tests/test_views.py b/hypha/apply/funds/tests/test_views.py index 26d6a190de..642f6b83a2 100644 --- a/hypha/apply/funds/tests/test_views.py +++ b/hypha/apply/funds/tests/test_views.py @@ -671,6 +671,40 @@ def test_applicant_can_see_application_draft_status(self): response = SubmissionDetailView.as_view()(request, pk=submission.pk) self.assertEqual(response.status_code, 200) + @override_settings(APPLICATION_TRANSLATIONS_ENABLED=True) + def test_staff_can_see_translate_primary_action(self): + def assert_view_translate_displayed(submission): + response = self.get_page(submission) + buttons = ( + BeautifulSoup(response.content, "html5lib") + .find("div", attrs={"data-testid": "sidebar-primary-actions"}) + .find_all("button") + ) + + self.assertEqual( + len([button.text for button in buttons if "Translate" in button.text]), + 1, + ) + + assert_view_translate_displayed(self.submission) + + @override_settings(APPLICATION_TRANSLATIONS_ENABLED=False) + def test_staff_cant_see_translate_primary_action(self): + def assert_view_translate_displayed(submission): + response = self.get_page(submission) + buttons = ( + BeautifulSoup(response.content, "html5lib") + .find("div", attrs={"data-testid": "sidebar-primary-actions"}) + .find_all("button") + ) + + self.assertEqual( + len([button.text for button in buttons if "Translate" in button.text]), + 0, + ) + + assert_view_translate_displayed(self.submission) + class TestReviewersUpdateView(BaseSubmissionViewTestCase): user_factory = StaffFactory diff --git a/hypha/apply/funds/urls.py b/hypha/apply/funds/urls.py index 55f7d89210..a6cf4f4d05 100644 --- a/hypha/apply/funds/urls.py +++ b/hypha/apply/funds/urls.py @@ -27,6 +27,7 @@ SubmissionResultView, SubmissionsByStatus, SubmissionSealedView, + TranslateSubmissionView, UpdateLeadView, UpdateMetaTermsView, UpdatePartnersView, @@ -50,6 +51,7 @@ partial_submission_activities, partial_submission_answers, partial_submission_lead, + partial_translate_answers, sub_menu_bulk_update_lead, sub_menu_bulk_update_reviewers, sub_menu_category_options, @@ -198,6 +200,11 @@ partial_meta_terms_card, name="partial-meta-terms-card", ), + path( + "partial/translate/answers", + partial_translate_answers, + name="partial-translate-answers", + ), path( "project/create/", CreateProjectView.as_view(), @@ -213,6 +220,11 @@ ReminderCreateView.as_view(), name="create_reminder", ), + path( + "translate/", + TranslateSubmissionView.as_view(), + name="translate", + ), path( "progress/", ProgressSubmissionView.as_view(), name="progress" ), diff --git a/hypha/apply/funds/views.py b/hypha/apply/funds/views.py index cecfbe4d61..aa743ec5c0 100644 --- a/hypha/apply/funds/views.py +++ b/hypha/apply/funds/views.py @@ -139,6 +139,15 @@ review_statuses, ) +if settings.APPLICATION_TRANSLATIONS_ENABLED: + from hypha.apply.translate.forms import TranslateSubmissionForm + from hypha.apply.translate.utils import ( + get_lang_name, + get_language_choices_json, + get_translation_params, + translate_application_form_data, + ) + User = get_user_model() @@ -985,6 +994,83 @@ def post(self, *args, **kwargs): ) +@method_decorator(staff_required, name="dispatch") +class TranslateSubmissionView(View): + template = "funds/includes/translate_application_form.html" + + if settings.APPLICATION_TRANSLATIONS_ENABLED: + + def dispatch(self, request, *args, **kwargs): + self.submission = get_object_or_404( + ApplicationSubmission, id=kwargs.get("pk") + ) + if not request.user.is_org_faculty: + messages.warning( + self.request, + "User attempted to translate submission but is not org faculty", + ) + return HttpResponseRedirect(self.submission.get_absolute_url()) + return super(TranslateSubmissionView, self).dispatch( + request, *args, **kwargs + ) + + def get(self, *args, **kwargs): + translate_form = TranslateSubmissionForm() + return render( + self.request, + self.template, + context={ + "form": translate_form, + "value": _("Update"), + "object": self.submission, + "json_choices": get_language_choices_json(self.request), + }, + ) + + def post(self, request, *args, **kwargs): + form = TranslateSubmissionForm(self.request.POST) + + if form.is_valid(): + FROM_LANG_KEY = "from_lang" + TO_LANG_KEY = "to_lang" + + from_lang = form.cleaned_data[FROM_LANG_KEY] + to_lang = form.cleaned_data[TO_LANG_KEY] + + return HttpResponse( + status=204, + headers={ + "HX-Trigger": json.dumps( + { + "translateSubmission": { + FROM_LANG_KEY: from_lang, + TO_LANG_KEY: to_lang, + } + } + ), + }, + ) + + return render( + self.request, + self.template, + context={ + "form": form, + "value": _("Update"), + "object": self.submission, + "json_choices": get_language_choices_json(self.request), + }, + status=400, + ) + else: + + def get(self, *args, **kwargs): + raise Http404 + + def post(self, *args, **kwargs): + raise Http404 + + @login_required @user_passes_test(is_apply_staff) @require_http_methods(["GET"]) @@ -1093,6 +1179,33 @@ def dispatch(self, request, *args, **kwargs): redirect = SubmissionSealedView.should_redirect(request, submission) return redirect or super().dispatch(request, *args, **kwargs) + if settings.APPLICATION_TRANSLATIONS_ENABLED: + + def get(self, request: HttpRequest, *args, **kwargs) -> HttpResponse: + self.object = self.get_object() + + extra_context = {} + + # Check for language params - if they exist and are valid then update the context + if lang_params := get_translation_params(request=request): + from_lang, to_lang = lang_params + try: + self.object.form_data = translate_application_form_data( + self.object, from_lang, to_lang + ) + extra_context.update( + { + "from_lang_name": get_lang_name(from_lang), + "to_lang_name": get_lang_name(to_lang), + } + ) + except ValueError: + # Language package isn't valid or installed, redirect to the submission w/o params + return redirect(self.object.get_absolute_url()) + + context = self.get_context_data(object=self.object, **extra_context) + return self.render_to_response(context) + def get_context_data(self, **kwargs): other_submissions = ( self.model.objects.filter(user=self.object.user) diff --git a/hypha/apply/funds/views_partials.py b/hypha/apply/funds/views_partials.py index b49c204590..39442df6d6 100644 --- a/hypha/apply/funds/views_partials.py +++ b/hypha/apply/funds/views_partials.py @@ -1,13 +1,16 @@ import functools +import json from urllib.parse import parse_qs, urlparse +from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required from django.db.models import Count, Q -from django.http import HttpRequest, HttpResponse +from django.http import Http404, HttpRequest, HttpResponse, QueryDict from django.shortcuts import get_object_or_404, render from django.urls import reverse_lazy from django.utils.text import slugify +from django.utils.translation import gettext as _ from django.views.decorators.http import require_GET, require_http_methods from django_htmx.http import ( HttpResponseClientRefresh, @@ -28,6 +31,13 @@ from hypha.apply.review.options import REVIEWER from hypha.apply.users.roles import REVIEWER_GROUP_NAME +if settings.APPLICATION_TRANSLATIONS_ENABLED: + from hypha.apply.translate.utils import ( + get_lang_name, + get_translation_params, + translate_application_form_data, + ) + from . import services from .models import ApplicationSubmission, Round from .permissions import can_change_external_reviewers @@ -514,3 +524,93 @@ def partial_screening_card(request, pk): "no_screening_options": no_screening_statuses, } return render(request, "funds/includes/screening_status_block.html", ctx) + + +@login_required +def partial_translate_answers(request: HttpRequest, pk: int) -> HttpResponse: + """Partial to translate submissions's answers + + Args: + request: HttpRequest object + pk: pk of the submission to translate + + """ + if not settings.APPLICATION_TRANSLATIONS_ENABLED: + raise Http404 + + submission = get_object_or_404(ApplicationSubmission, pk=pk) + + if not request.user.is_org_faculty or request.method != "GET": + return HttpResponse(status=204) + + ctx = {"object": submission} + + # The existing params that were in the URL when the request was made + prev_params = get_translation_params(request.headers.get("Hx-Current-Url", "")) + # The requested params provided in the GET request + params = get_translation_params(request=request) + + updated_url = submission.get_absolute_url() + + message = None + + if params and not params[0] == params[1] and not params == prev_params: + from_lang, to_lang = params + try: + submission.form_data = translate_application_form_data( + submission, from_lang, to_lang + ) + + if current_url := request.headers.get("Hx-Current-Url"): + updated_params = QueryDict(urlparse(current_url).query, mutable=True) + updated_params["fl"] = from_lang + updated_params["tl"] = to_lang + updated_url = f"{updated_url}?{updated_params.urlencode()}" + + to_lang_name = get_lang_name(to_lang) + from_lang_name = get_lang_name(from_lang) + + message = _("Submission translated from {fl} to {tl}.").format( + fl=from_lang_name, tl=to_lang_name + ) + + ctx.update( + { + "object": submission, + "from_lang_name": from_lang_name, + "to_lang_name": to_lang_name, + } + ) + except ValueError: + # TODO: WA Error/failed message type rather than success + message = _("Submission translation failed. Contact your Administrator.") + return HttpResponse( + status=400, + headers={"HX-Trigger": json.dumps({"showMessage": {message}})}, + ) + + elif params == prev_params: + message = _("Translation cleared.") + + response = render(request, "funds/includes/rendered_answers.html", ctx) + + trigger_dict = {} + if title := submission.form_data.get("title"): + trigger_dict.update( + { + "translatedSubmission": { + "appTitle": title, + "docTitle": submission.title_text_display, + } + } + ) + + if message: + trigger_dict.update({"showMessage": message}) + + if trigger_dict: + response["HX-Trigger"] = json.dumps(trigger_dict) + + response["HX-Replace-Url"] = updated_url + + return response diff --git a/hypha/apply/translate/__init__.py b/hypha/apply/translate/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hypha/apply/translate/fields.py b/hypha/apply/translate/fields.py new file mode 100644 index 0000000000..629c99deff --- /dev/null +++ b/hypha/apply/translate/fields.py @@ -0,0 +1,39 @@ +from typing import Any, Iterable, Literal, Optional + +from argostranslate.package import Package +from django import forms + + +class LanguageChoiceField(forms.ChoiceField): + def __init__( + self, + role: Literal["to", "from"], + available_packages: Iterable[Package], + choices: Optional[Iterable[str]] = set(), + **kwargs, + ) -> None: + self.available_packages = available_packages + + # Ensure the given language is either "to" or "from" + if role not in ["to", "from"]: + raise ValueError(f'Invalid role "{role}", must be "to" or "from"') + + self.role = role + + super().__init__(choices=choices, **kwargs) + self.widget.attrs.update({"data-placeholder": f"{role.capitalize()}..."}) + + def validate(self, value: Any) -> None: + """Basic validation to ensure the language (depending on role) is available in the installed packages + + Only checks the language exists as from/to code, doesn't validate based on from -> to. + """ + if self.role == "from": + valid_list = [package.from_code for package in self.available_packages] + else: + valid_list = [package.to_code for package in self.available_packages] + + if value not in valid_list: + raise forms.ValidationError( + "The specified language is either invalid or not installed" + ) diff --git a/hypha/apply/translate/forms.py b/hypha/apply/translate/forms.py new file mode 100644 index 0000000000..d5bd4dd7ca --- /dev/null +++ b/hypha/apply/translate/forms.py @@ -0,0 +1,30 @@ +from django import forms + +from hypha.apply.translate.fields import LanguageChoiceField +from hypha.apply.translate.utils import get_available_translations + + +class TranslateSubmissionForm(forms.Form): + available_packages = get_available_translations() + + from_lang = LanguageChoiceField("from", available_packages) + to_lang = LanguageChoiceField("to", available_packages) + + def clean(self): + form_data = self.cleaned_data + try: + from_code = form_data["from_lang"] + to_code = form_data["to_lang"] + + to_packages = get_available_translations([from_code]) + + if to_code not in [package.to_code for package in to_packages]: + self.add_error( + "to_lang", + "The specified language is either invalid or not installed", + ) + + return form_data + except KeyError as err: + # If one of the fields could not be parsed, there is likely bad input being given + raise forms.ValidationError("Invalid input selected") from err diff --git a/hypha/apply/translate/management/__init__.py b/hypha/apply/translate/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hypha/apply/translate/management/commands/__init__.py b/hypha/apply/translate/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hypha/apply/translate/management/commands/install_languages.py b/hypha/apply/translate/management/commands/install_languages.py new file mode 100644 index 0000000000..b927f15fc2 --- /dev/null +++ b/hypha/apply/translate/management/commands/install_languages.py @@ -0,0 +1,142 @@ +import argparse +from typing import List + +import argostranslate.package +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = ( + "Delete all drafts that haven't been modified in the specified time (in days)" + ) + + available_packages = argostranslate.package.get_available_packages() + installed_packages = argostranslate.package.get_installed_packages() + + def __validate_language(self, value): + """Used to validate `from_to_language` argument""" + try: + from_code, to_code = value.split("_") + except ValueError: + raise argparse.ArgumentTypeError( + f'Invalid language package "{value}", expected "_" in ISO 639 format' + ) from None + + package = next( + filter( + lambda x: x.from_code == from_code and x.to_code == to_code, + self.available_packages, + ), + None, + ) + + if not package: + raise argparse.ArgumentTypeError( + f'Package "{value}" is not available for install' + ) + + return package + + def add_arguments(self, parser): + parser.add_argument( + "languages", + action="store", + nargs="*", + type=self.__validate_language, + help='Language packages to install in the format of "_" in ISO 639 format', + ) + + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Do not prompt the user for confirmation", + required=False, + ) + + parser.add_argument( + "--all", + action="store_true", + help="Install all available language packages", + required=False, + ) + + def __print_package_list(self, packages: List[argostranslate.package.Package]): + for package in packages: + self.stdout.write(f"{package.from_name} ➜ {package.to_name}") + + def handle(self, *args, **options): + interactive = options["interactive"] + packages = options["languages"] + verbosity = options["verbosity"] + all = options["all"] + + # Require either languages or "--all" to be specified + if not bool(packages) ^ bool(all): + raise argparse.ArgumentTypeError("A language selection must be specified") + + if all: + packages = self.available_packages + + existing_packages = [ + lang for lang in packages if lang in self.installed_packages + ] + pending_packages = [ + lang for lang in packages if lang not in self.installed_packages + ] + + if existing_packages: + if verbosity > 1: + self.stdout.write( + f"The following package{'s are' if len(existing_packages) > 1 else ' is'} already installed:" + ) + self.__print_package_list(existing_packages) + elif ( + not pending_packages + ): # Only notify the user if no packages will be installed + self.stdout.write( + f"The specified package{'s are' if len(existing_packages) > 1 else ' is'} already installed." + ) + + if not pending_packages: + return + + if pending_packages: + if verbosity > 1: + self.stdout.write( + f"The following package{'s' if len(pending_packages) > 1 else ''} will be installed:" + ) + self.__print_package_list(pending_packages) + elif ( + interactive + ): # Only log what will be installed if prompting the user to confirm + self.stdout.write( + f"{len(pending_packages)} package{'s' if len(pending_packages) > 1 else ''} will be installed." + ) + + if interactive: + confirm = input( + "Are you sure you want to do this?\n\nType 'yes' to continue, or 'no' to cancel: " + ) + else: + confirm = "yes" + + if confirm == "yes": + for package in pending_packages: + argostranslate.package.install_from_path(package.download()) + + successful_installs = len( + argostranslate.package.get_installed_packages() + ) - len(self.installed_packages) + + success_msg = f"{successful_installs} new package{'s' if successful_installs > 1 else ''} installed" + + if existing_packages: + success_msg = f"{success_msg}, while {len(existing_packages)} package{'s were' if len(existing_packages) > 1 else ' was'} already installed." + else: + success_msg = f"{success_msg}." + + self.stdout.write(success_msg) + else: + self.stdout.write("Installation cancelled.") diff --git a/hypha/apply/translate/management/commands/uninstall_languages.py b/hypha/apply/translate/management/commands/uninstall_languages.py new file mode 100644 index 0000000000..585e882ca9 --- /dev/null +++ b/hypha/apply/translate/management/commands/uninstall_languages.py @@ -0,0 +1,109 @@ +import argparse +from typing import List + +import argostranslate.package +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + help = ( + "Delete all drafts that haven't been modified in the specified time (in days)" + ) + + installed_packages = argostranslate.package.get_installed_packages() + + def __validate_language(self, value): + """Used to validate `from_to_language` argument""" + try: + from_code, to_code = value.split("_") + except ValueError: + raise argparse.ArgumentTypeError( + f'Invalid language package "{value}", expected "_" in ISO 639 format' + ) from None + + package = next( + filter( + lambda x: x.from_code == from_code and x.to_code == to_code, + self.installed_packages, + ), + None, + ) + + if not package: + raise argparse.ArgumentTypeError(f'Package "{value}" is not installed') + + return package + + def add_arguments(self, parser): + parser.add_argument( + "languages", + action="store", + nargs="*", + type=self.__validate_language, + help='Language packages to uninstall in the format of "_" in ISO 639 format', + ) + + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Do not prompt the user for confirmation", + required=False, + ) + + parser.add_argument( + "--all", + action="store_true", + help="Uninstall all installed language packages", + required=False, + ) + + def __print_package_list(self, packages: List[argostranslate.package.Package]): + for package in packages: + self.stdout.write(f"{package.from_name} ➜ {package.to_name}") + + def handle(self, *args, **options): + interactive = options["interactive"] + packages = options["languages"] + verbosity = options["verbosity"] + + # Require either languages or "--all" to be specified + if not bool(packages) ^ bool(all): + raise argparse.ArgumentTypeError("A language selection must be specified") + + if all: + packages = self.installed_packages + + if verbosity > 1: + self.stdout.write( + f"The following package{'s' if len(packages) > 1 else ''} will be uninstalled:" + ) + self.__print_package_list(packages) + elif ( + interactive + ): # Only log what will be uninstalled if prompting the user to confirm + self.stdout.write( + f"{len(packages)} package{'s' if len(packages) > 1 else ''} will be uninstalled." + ) + + if interactive: + confirm = input( + "Are you sure you want to do this?\n\nType 'yes' to continue, or 'no' to cancel: " + ) + else: + confirm = "yes" + + if confirm == "yes": + for package in packages: + argostranslate.package.uninstall(package) + + successful_uninstalls = len(self.installed_packages) - len( + argostranslate.package.get_installed_packages() + ) + + self.stdout.write( + f"{successful_uninstalls} package{'s' if successful_uninstalls > 1 else ''} uninstalled." + ) + else: + self.stdout.write("Removal cancelled.") diff --git a/hypha/apply/translate/tests/__init__.py b/hypha/apply/translate/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hypha/apply/translate/tests/test_translate.py b/hypha/apply/translate/tests/test_translate.py new file mode 100644 index 0000000000..86bcb4987c --- /dev/null +++ b/hypha/apply/translate/tests/test_translate.py @@ -0,0 +1,79 @@ +from typing import List +from unittest import skipUnless +from unittest.mock import Mock, patch + +from django.conf import settings +from django.test import SimpleTestCase + +if settings.APPLICATION_TRANSLATIONS_ENABLED: + from hypha.apply.translate.translate import translate + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TestTranslate(SimpleTestCase): + @staticmethod + def mocked_translate(string: str, from_code, to_code): + """Use pig latin for all test translations - ie. 'hypha is cool' -> 'yphahay isway oolcay' + https://en.wikipedia.org/wiki/Pig_Latin + """ + vowels = {"a", "e", "i", "o", "u"} + string = string.lower() + pl = [ + f"{word}way" if word[0] in vowels else f"{word[1:]}{word[0]}ay" + for word in string.split() + ] + return " ".join(pl) + + @staticmethod + def mocked_get_available_translations(codes: List[str]): + mocked_packages = [ + Mock(from_code="ar", to_code="en"), + Mock(from_code="fr", to_code="en"), + Mock(from_code="en", to_code="ar"), + Mock(from_code="zh", to_code="en"), + Mock(from_code="en", to_code="fr"), + ] + + return list(filter(lambda x: x.from_code in codes, mocked_packages)) + + @classmethod + def setUpClass(cls): + """Used to patch & mock all the methods called from argostranslate & hypha.apply.translate.utils""" + + cls.patcher = [ + patch( + "hypha.apply.translate.utils.get_available_translations", + side_effect=cls.mocked_get_available_translations, + ), + patch( + "argostranslate.translate.translate", side_effect=cls.mocked_translate + ), + ] + + for patched in cls.patcher: + patched.start() + + @classmethod + def tearDownClass(cls): + for patched in cls.patcher: + patched.stop() + + def test_valid_translate(self): + self.assertEqual(translate("hey there", "fr", "en"), "eyhay heretay") + + def test_duplicate_code_translate(self): + with self.assertRaises(ValueError) as context: + translate("hey there", "fr", "fr") + + self.assertEqual( + "Translation from_code cannot match to_code", str(context.exception) + ) + + def test_invalid_code_translate(self): + with self.assertRaises(ValueError) as context: + translate("hey there", "test", "test2") + + self.assertIn("is not installed", str(context.exception)) diff --git a/hypha/apply/translate/tests/test_utils.py b/hypha/apply/translate/tests/test_utils.py new file mode 100644 index 0000000000..db5e599793 --- /dev/null +++ b/hypha/apply/translate/tests/test_utils.py @@ -0,0 +1,448 @@ +import json +from typing import Optional +from unittest import skipUnless +from unittest.mock import Mock, patch +from urllib.parse import parse_qs, urlparse + +from django.conf import settings +from django.http import QueryDict +from django.test import RequestFactory, SimpleTestCase, TestCase, override_settings + +from hypha.apply.funds.tests.factories import ApplicationSubmissionFactory +from hypha.apply.users.tests.factories import ApplicantFactory + +if settings.APPLICATION_TRANSLATIONS_ENABLED: + from hypha.apply.translate.utils import ( + get_available_translations, + get_lang_name, + get_language_choices_json, + get_translation_params, + translate_application_form_data, + ) + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TesGetAvailableTranslations(SimpleTestCase): + @classmethod + def setUpClass(cls): + mock_packages = [ + Mock(from_code="ar", to_code="en"), + Mock(from_code="fr", to_code="en"), + Mock(from_code="en", to_code="ar"), + Mock(from_code="zh", to_code="en"), + Mock(from_code="fr", to_code="zh"), + ] + + cls.patcher = patch( + "argostranslate.package.get_installed_packages", return_value=mock_packages + ) + cls.patcher.start() + + @classmethod + def tearDownClass(cls): + cls.patcher.stop() + + def test_get_available_translations(self): + codes = {(p.from_code, p.to_code) for p in get_available_translations()} + self.assertEqual( + codes, + {("ar", "en"), ("fr", "en"), ("en", "ar"), ("zh", "en"), ("fr", "zh")}, + ) + + def test_get_available_translations_with_codes(self): + codes = {(p.from_code, p.to_code) for p in get_available_translations(["fr"])} + self.assertEqual(codes, {("fr", "en"), ("fr", "zh")}) + + codes = { + (p.from_code, p.to_code) for p in get_available_translations(["en", "zh"]) + } + self.assertEqual(codes, {("en", "ar"), ("zh", "en")}) + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TestGetTranslationParams(SimpleTestCase): + def get_test_get_request(self, extra_params: Optional[str] = None) -> Mock: + extra_params = f"{extra_params}&" if extra_params else "" + return RequestFactory().get( + "/test/", data=QueryDict(f"{extra_params}fl=ar&tl=en") + ) + + def get_test_url(self, extra_params: Optional[str] = None) -> str: + extra_params = f"{extra_params}&" if extra_params else "" + return f"https://hyphaiscool.org/apply/submissions/6/?{extra_params}fl=ar&tl=en" + + def test_get_translation_params_with_request(self): + self.assertEqual( + get_translation_params(request=self.get_test_get_request()), ("ar", "en") + ) + + # Ensure param extraction works even when unrelated params are present + mock_request = self.get_test_get_request(extra_params="ref=table-view") + self.assertEqual(get_translation_params(request=mock_request), ("ar", "en")) + + def test_get_translation_params_with_url(self): + self.assertEqual(get_translation_params(url=self.get_test_url()), ("ar", "en")) + + # Ensure param extraction works even when unrelated params are present + url = self.get_test_url("ref=table-view") + self.assertEqual(get_translation_params(url=url), ("ar", "en")) + + def test_get_translation_params_with_invalid_args(self): + # Should fail with no args given... + with self.assertRaises(ValueError): + get_translation_params() + + # ...and with both args given + with self.assertRaises(ValueError): + get_translation_params(self.get_test_url(), self.get_test_get_request()) + + def test_get_translation_params_with_invalid_params(self): + # Testing using params that hypha can give but are unrelated to translations + mock_request = RequestFactory().get("/test/", data=QueryDict("ref=table-view")) + self.assertIsNone(get_translation_params(request=mock_request)) + + url = "https://hyphaiscool.org/apply/submissions/6/?ref=table-view" + self.assertIsNone(get_translation_params(url=url)) + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TestGetLangName(SimpleTestCase): + def test_get_lang_name(self): + # "!" added to ensure mock is working rather than actually calling argos + language_mock = Mock() + language_mock.name = ( + "Arabic!" # Done this way as `name` is an attribute of Mock() objects + ) + with patch( + "argostranslate.translate.get_language_from_code", + return_value=language_mock, + ) as from_code_mock: + self.assertEqual(get_lang_name("ar"), "Arabic!") + from_code_mock.assert_called_once_with("ar") + + def test_get_lang_name_invalid_code(self): + with patch( + "argostranslate.translate.get_language_from_code", + side_effect=AttributeError(), + ) as from_code_mock: + self.assertIsNone(get_lang_name("nope")) + from_code_mock.assert_called_once_with("nope") + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TestTranslateSubmissionFormData(TestCase): + @staticmethod + def mocked_translate(string: str, from_code, to_code): + """Use pig latin for all test translations - ie. 'hypha is cool' -> 'yphahay isway oolcay' + https://en.wikipedia.org/wiki/Pig_Latin + """ + valid_codes = ["en", "fr", "zh", "es"] + if from_code == to_code or not ( + from_code in valid_codes and to_code in valid_codes + ): + raise ValueError() + + vowels = {"a", "e", "i", "o", "u"} + string = string.lower() + pl = [ + f"{word}way" if word[0] in vowels else f"{word[1:]}{word[0]}ay" + for word in string.split() + ] + return " ".join(pl) + + @classmethod + def setUpClass(cls): + """Used to patch & mock all the methods called from hypha.apply.translate""" + cls.patcher = patch( + "hypha.apply.translate.translate.translate", + side_effect=cls.mocked_translate, + ) + cls.patcher.start() + + @classmethod + def tearDownClass(cls): + cls.patcher.stop() + + def setUp(self): + self.applicant = ApplicantFactory( + email="test@hyphaiscool.com", full_name="Johnny Doe" + ) + self.application = ApplicationSubmissionFactory(user=self.applicant) + + @property + def form_data(self): + return self.application.live_revision.form_data + + def test_translate_application_form_data_plaintext_fields(self): + uuid = "97c51cea-ab47-4a64-a64a-15d893788ef2" # random uuid + self.application.form_data[uuid] = "Just a plain text field" + + translated_form_data = translate_application_form_data( + self.application, "en", "fr" + ) + + self.assertEqual( + translated_form_data[uuid], "ustjay away lainpay exttay ieldfay" + ) + + def test_translate_application_form_data_html_fields(self): + uuid_mixed_format = "ed89378g-3b54-4444-abcd-37821f58ed89" # random uuid + self.application.form_data[uuid_mixed_format] = ( + "

Hello from a Hyper Text Markup Language field

" + ) + + uuid_same_format = "9191fc65-02c6-46e0-9fc8-3b778113d19f" # random uuid + self.application.form_data[uuid_same_format] = ( + "

Hypha rocks

yeah

" + ) + + translated_form_data = translate_application_form_data( + self.application, "en", "fr" + ) + + self.assertEqual( + translated_form_data[uuid_mixed_format], + "

ellohay romfay away yperhay exttay arkupmay anguagelay ieldfay

", + ) + self.assertEqual( + translated_form_data[uuid_same_format], + "

yphahay ocksray

eahyay

", + ) + + def test_translate_application_form_data_skip_info_fields(self): + self.application.form_data["random"] = "don't translate me pls" + + name = self.form_data["full_name"] + email = self.form_data["email"] + random = self.form_data["random"] + + translated_form_data = translate_application_form_data( + self.application, "en", "fr" + ) + self.assertEqual(translated_form_data["full_name"], name) + self.assertEqual(translated_form_data["email"], email) + self.assertEqual(translated_form_data["random"], random) + + def test_translate_application_form_data_skip_non_str_fields(self): + uuid = "4716ddd4-ce87-4964-b82d-bf2db75bdbc3" # random uuid + self.application.form_data[uuid] = {"test": "dict field"} + + translated_form_data = translate_application_form_data( + self.application, "en", "fr" + ) + self.assertEqual(translated_form_data[uuid], {"test": "dict field"}) + + def test_translate_application_form_data_error_bubble_up(self): + """Ensure errors bubble up from underlying translate func""" + application = ApplicationSubmissionFactory() + with self.assertRaises(ValueError): + # duplicate language code + translate_application_form_data(application, "en", "en") + + with self.assertRaises(ValueError): + # language code not in `mocked_translate` + translate_application_form_data(application, "de", "en") + + +# Setup for testing `get_language_choices_json` + + +def equal_ignore_order(a: list, b: list) -> bool: + """Used to compare two lists that are unsortable & unhashable + + Primarily used when comparing the result of json.loads + + Args: + a: the first unhashable & unsortable list + b: the second unhashable & unsortable list + + Returns: + bool: true when lists are equal in length & content + """ + if len(a) != len(b): + return False + unmatched = list(b) + for element in a: + try: + unmatched.remove(element) + except ValueError: + return False + return not unmatched + + +@skipUnless( + settings.APPLICATION_TRANSLATIONS_ENABLED, + "Attempts to import translate dependencies", +) +class TestGetLanguageChoices(SimpleTestCase): + @staticmethod + def mocked_get_lang_name(code): + # Added "!" to ensure the mock is being called rather than the actual get_lang + codes_to_lang = { + "en": "English!", + "ar": "Arabic!", + "zh": "Chinese!", + "fr": "French!", + } + return codes_to_lang[code] + + @staticmethod + def mocked_get_translation_params(url): + query_dict = {k: v[0] for (k, v) in parse_qs(urlparse(url).query).items()} + if (to_lang := query_dict.get("tl")) and (from_lang := query_dict.get("fl")): + return (from_lang, to_lang) + + @classmethod + def setUpClass(cls): + """Used to patch & mock all the methods called from hypha.apply.translate.utils""" + available_packages = [ + Mock(from_code="ar", to_code="en"), + Mock(from_code="fr", to_code="en"), + Mock(from_code="en", to_code="ar"), + Mock(from_code="zh", to_code="en"), + Mock(from_code="en", to_code="fr"), + ] + + cls.patcher = [ + patch( + "hypha.apply.translate.utils.get_lang_name", + side_effect=cls.mocked_get_lang_name, + ), + patch( + "hypha.apply.translate.utils.get_available_translations", + return_value=available_packages, + ), + patch( + "hypha.apply.translate.utils.get_translation_params", + side_effect=cls.mocked_get_translation_params, + ), + ] + + for patched in cls.patcher: + patched.start() + + @classmethod + def tearDownClass(cls): + for patched in cls.patcher: + patched.stop() + + def test_get_language_choices_json(self): + expected_json = [ + { + "label": "English!", + "selected": False, + "to": [ + {"label": "Arabic!", "selected": True, "value": "ar"}, + {"label": "French!", "selected": False, "value": "fr"}, + ], + "value": "en", + }, + { + "label": "Arabic!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "ar", + }, + { + "label": "Chinese!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "zh", + }, + { + "label": "French!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "fr", + }, + ] + request = RequestFactory().get("/test/") + + json_out = get_language_choices_json(request) + self.assertTrue(equal_ignore_order(json.loads(json_out), expected_json)) + + def test_get_language_choices_json_with_current_url(self): + expected_json = [ + { + "label": "English!", + "selected": False, + "to": [ + {"label": "Arabic!", "selected": True, "value": "ar"}, + {"label": "French!", "selected": False, "value": "fr"}, + ], + "value": "en", + }, + { + "label": "Arabic!", + "selected": True, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "ar", + }, + { + "label": "Chinese!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "zh", + }, + { + "label": "French!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "fr", + }, + ] + current_url = "https://hyphaiscool.org/apply/submissions/6/?fl=ar&tl=en" + request = RequestFactory().get( + "/test/", headers={"Hx-Current-Url": current_url} + ) + + json_out = get_language_choices_json(request) + self.assertTrue(equal_ignore_order(json.loads(json_out), expected_json)) + + @override_settings(LANGUAGE_CODE="fr") + def test_get_language_choices_json_with_language_code(self): + expected_json = [ + { + "label": "English!", + "selected": False, + "to": [ + {"label": "Arabic!", "selected": False, "value": "ar"}, + {"label": "French!", "selected": True, "value": "fr"}, + ], + "value": "en", + }, + { + "label": "Arabic!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "ar", + }, + { + "label": "Chinese!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "zh", + }, + { + "label": "French!", + "selected": False, + "to": [{"label": "English!", "selected": True, "value": "en"}], + "value": "fr", + }, + ] + request = RequestFactory().get("/test/") + + json_out = get_language_choices_json(request) + self.assertTrue(equal_ignore_order(json.loads(json_out), expected_json)) diff --git a/hypha/apply/translate/translate.py b/hypha/apply/translate/translate.py new file mode 100644 index 0000000000..74b448348f --- /dev/null +++ b/hypha/apply/translate/translate.py @@ -0,0 +1,35 @@ +import argostranslate.translate + +from . import utils + + +def translate(string: str, from_code: str, to_code: str) -> str: + """Translate a string from one language to another + + Requires the request language's argostranslate package to be installed first + + Args: + string: the string to translate + from_code: the ISO 639 code of the original language + to_code: the ISO 639 code of the language to translate to + + Returns: + str: the translated string + + Raises: + ValueError: if the requested language translation package is not installed or request is invalid + """ + + if from_code == to_code: + raise ValueError("Translation from_code cannot match to_code") + + available_translations = utils.get_available_translations([from_code]) + + if not available_translations or to_code not in [ + package.to_code for package in available_translations + ]: + raise ValueError(f"Package {from_code} -> {to_code} is not installed") + + translated_text = argostranslate.translate.translate(string, from_code, to_code) + + return translated_text diff --git a/hypha/apply/translate/utils.py b/hypha/apply/translate/utils.py new file mode 100644 index 0000000000..fda338d5a4 --- /dev/null +++ b/hypha/apply/translate/utils.py @@ -0,0 +1,234 @@ +import json +import re +from typing import List, Optional, Tuple +from urllib.parse import parse_qs, urlparse + +import argostranslate +from bs4 import BeautifulSoup, element +from django.conf import settings +from django.http import HttpRequest + +from . import translate + + +def get_available_translations( + from_codes: Optional[List[str]] = None, +) -> List[argostranslate.package.Package]: + """Get languages available for translation + + Args: + from_codes: optionally specify a list of languages to view available translations to + + Returns: + A list of argostranslate package objects that are installed and available. + """ + + available_packages = argostranslate.package.get_installed_packages() + + if not from_codes: + return available_packages + + return list(filter(lambda x: x.from_code in from_codes, available_packages)) + + +def get_translation_params( + url: str = None, request: HttpRequest = None +) -> Tuple[str, str] | None: + r"""Attempts to extract the `fl` (from language) & `tl` (to language) params from the provided URL or request object + + Return values are *not* validated to ensure languages are valid & packages exist. + + Args: + url: the URL to extract the params from + + Returns: + tuple: in the format of (\, \) + + Raises: + ValueError: If `url`/`request` are not provided OR if both are provided + """ + + # Ensure either url or request is provided but not both. + if not (bool(url) ^ bool(request)): + raise ValueError("Either a URL or HttpRequest must be provided.") + + if url: + query_dict = {k: v[0] for (k, v) in parse_qs(urlparse(url).query).items()} + else: + query_dict = request.GET + + if (to_lang := query_dict.get("tl")) and (from_lang := query_dict.get("fl")): + return (from_lang, to_lang) + + return None + + +def get_lang_name(code: str) -> str | None: + try: + return argostranslate.translate.get_language_from_code(code).name + except AttributeError: + return None + + +def get_language_choices_json(request: HttpRequest) -> str: + """Generate a JSON output of available translation options + + Utilized for populating the reactive form fields on the client side + + Args: + request: an `HttpRequest` containing an "Hx-Current-Url" header to extract current translation params from + + Returns: + A JSON string in the format of: + + ``` + [ + { + "value": "", + "label": "", + "to": [ + { + "value": "", + "label": "" + "selected": + } + ], + "selected": + }, + ... + ] + ``` + """ + available_translations = get_available_translations() + from_langs = {package.from_code for package in available_translations} + default_to_lang = settings.LANGUAGE_CODE if settings.LANGUAGE_CODE else None + default_from_lang = None + + # If there's existing lang params, use those as the default in the form + # ie. the user has an active translation for ar -> en, those should be selected in the form + if (current_url := request.headers.get("Hx-Current-Url")) and ( + params := get_translation_params(current_url) + ): + default_from_lang, default_to_lang = params + + choices = [] + for lang in from_langs: + to_langs = [ + package.to_code + for package in available_translations + if package.from_code == lang + ] + + # Set the default selection to be the default_to_lang if it exists in the to_langs list, + # otherwise use the first value in the list. + selected_to = ( + default_to_lang + if default_to_lang and default_to_lang in to_langs + else to_langs[0] + ) + + to_choices = [ + { + "value": to_lang, + "label": get_lang_name(to_lang), + "selected": to_lang == selected_to, + } + for to_lang in to_langs + ] + + choices.append( + { + "value": lang, + "label": get_lang_name(lang), + "to": to_choices, + "selected": lang == default_from_lang, + } + ) + + return json.dumps(choices) + + +def translate_application_form_data(application, from_code: str, to_code: str) -> dict: + """Translate the content of an application's live revision `form_data`. + Will parse fields that contain both plaintext & HTML, extracting & replacing strings. + + NOTE: Mixed formatting like `

Hey from Hypha

` will result in a + string that is stripped of text formatting (untranslated: `

Hey from Hypha

`). On + the other hand, unmixed strings like `

Hey from Hypha

` will be + replaced within formatting tags. + + Args: + application: the application to translate + from_code: the ISO 639 code of the original language + to_code: the ISO 639 code of the language to translate to + + Returns: + The `form_data` with values translated (including nested HTML strings) + + Raises: + ValueError if an invalid `from_code` or `to_code` is requested + """ + form_data: dict = application.live_revision.form_data + + translated_form_data = form_data.copy() + + # Only translate content fields or the title - don't with name, email, etc. + translated_form_data["title"] = translate.translate( + form_data["title"], from_code, to_code + ) + + # RegEx to match wagtail's generated field UIDs - ie. "97c51cea-ab47-4a64-a64a-15d893788ef2" + uid_regex = re.compile(r"([a-z]|\d){8}(-([a-z]|\d){4}){3}-([a-z]|\d){12}") + fields_to_translate = [ + key + for key in form_data + if uid_regex.match(key) and isinstance(form_data[key], str) + ] + + for key in fields_to_translate: + field_html = BeautifulSoup(form_data[key], "html.parser") + if field_html.find(): # Check if BS detected any HTML + for field in field_html.find_all(has_valid_str): + # Removes formatting if mixed into the tag to prioritize context in translation + # ie. `

Hey y'all

` -> `

Hey y'all

` (but translated) + to_translate = field.string if field.string else field.text + field.clear() + field.string = translate.translate(to_translate, from_code, to_code) + + translated_form_data[key] = str(field_html) + # Ensure the field value isn't empty & translate as is + elif form_data[key].strip(): + translated_form_data[key] = translate.translate( + form_data[key], from_code, to_code + ) + + return translated_form_data + + +def has_valid_str(tag: element.Tag) -> bool: + """Checks that an Tag contains a valid text element and/or string. + + Args: + tag: a `bs4.element.Tag` + Returns: + bool: True if has a valid string that isn't whitespace or `-` + """ + text_elem = tag.name in ["span", "p", "strong", "em", "td", "a"] + + try: + # try block logic handles elements that have text directly in them + # ie. `

test

` or `yeet!` would return true as string values would be contained in tag.string + ret = bool( + text_elem + and tag.find(string=True, recursive=False) + and tag.string.strip(" -\n") + ) + return ret + except AttributeError: + # except block logic handles embedded tag strings where tag.string == None but the specified tag DOES contain a string + # ie. `

Hypha is cool

` contains the string "Hypha is" but due to the strong tag being mixed in will + # have None for the tag.string value. + # tags like `

Hypha rocks

` will return false as the

tag contains no valid strings, it's child does. + tag_contents = "".join(tag.find_all(string=True, recursive=False)) + ret = bool(tag.text and tag.text.strip() and tag_contents.strip()) + return ret diff --git a/hypha/settings/base.py b/hypha/settings/base.py index f62b2ccdff..5e438b945b 100644 --- a/hypha/settings/base.py +++ b/hypha/settings/base.py @@ -185,6 +185,10 @@ # The corrosponding locale dir is named: en, en_GB, en_US LANGUAGE_CODE = env.str("LANGUAGE_CODE", "en") +# Machine translation settings +# NOTE: Ensure the packages in `requirements-translate.txt` have been installed! +APPLICATION_TRANSLATIONS_ENABLED = env.bool("APPLICATION_TRANSLATIONS_ENABLED", False) + # Number of seconds that password reset and account activation links are valid (default 259200, 3 days). PASSWORD_RESET_TIMEOUT = env.int("PASSWORD_RESET_TIMEOUT", 259200) diff --git a/hypha/settings/django.py b/hypha/settings/django.py index 81fa428b5f..e1d35ed418 100644 --- a/hypha/settings/django.py +++ b/hypha/settings/django.py @@ -21,6 +21,7 @@ "hypha.apply.determinations", "hypha.apply.stream_forms", "hypha.apply.todo", + "hypha.apply.translate", "hypha.apply.utils.apps.UtilsConfig", "hypha.apply.projects.apps.ProjectsConfig", "hypha.public.funds", @@ -88,6 +89,7 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + "django.middleware.common.CommonMiddleware", "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "elevate.middleware.ElevateMiddleware", diff --git a/hypha/static_src/javascript/translate-application.js b/hypha/static_src/javascript/translate-application.js new file mode 100644 index 0000000000..27e3daaf47 --- /dev/null +++ b/hypha/static_src/javascript/translate-application.js @@ -0,0 +1,13 @@ +(function () { + // eslint-disable-next-line no-undef + htmx.on("translatedSubmission", (event) => { + if (event.detail?.appTitle) { + document.getElementById("app-title").textContent = + event.detail.appTitle; + } + + if (event.detail?.docTitle) { + document.title = event.detail.docTitle; + } + }); +})(); diff --git a/requirements-translate.txt b/requirements-translate.txt new file mode 100644 index 0000000000..7591180bfc --- /dev/null +++ b/requirements-translate.txt @@ -0,0 +1,6 @@ +# Requirements for machine translations + +# Only install the CPU version of torch when available (linux) +--find-links https://download.pytorch.org/whl/cpu/torch_stable.html +torch==2.3.1+cpu; sys_platform == 'linux' +argostranslate==1.9.6 \ No newline at end of file