diff --git a/.github/workflows/release-project-sync.yml b/.github/workflows/release-project-sync.yml new file mode 100644 index 00000000000..8ec13199d9b --- /dev/null +++ b/.github/workflows/release-project-sync.yml @@ -0,0 +1,44 @@ +# This workflow runs release_project_sync.py on demand + +name: Sync reviews and tags into release project + +on: + workflow_dispatch: # by request + inputs: + release: + description: 'Release name: e.g. zebrawood' + required: true + type: string + language: + description: 'Language code e.g. "ar" or "zh_CN". Leave empty to sync all languages.' + required: false + default: '' + type: string + resource: + description: 'Resource slug e.g. "AudioXBlock" or "frontend-app-learning". Leave empty to sync all languages' + required: false + default: '' + type: string + +jobs: + release-project-sync: + runs-on: ubuntu-latest + + steps: + # Clones the openedx-translations repo + - name: clone openedx/openedx-translations + uses: actions/checkout@v3 + + # Sets up Python + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: '3.8' + + # Run the script + - name: Sync reviews for release project + env: + TRANSIFEX_API_TOKEN: ${{ secrets.TRANSIFEX_API_TOKEN }} + run: | + make translations_scripts_requirements + python ./scripts/release_project_sync.py --resource="${{ inputs.resource }}" --language="${{ inputs.language }}" "${{ inputs.release }}" diff --git a/scripts/release_project_sync.py b/scripts/release_project_sync.py new file mode 100644 index 00000000000..b8e6ab2ee7f --- /dev/null +++ b/scripts/release_project_sync.py @@ -0,0 +1,292 @@ +""" +Release cut script to sync the main openedx-translations Transifex project into the\ +openedx-translations- release projects. + + - Project links: + * Main: https://app.transifex.com/open-edx/openedx-translations/ + * Release projects: https://app.transifex.com/open-edx/openedx-translations-/ + +Variable names meaning: + + - main_translation: translation in the main "open-edx/openedx-translations" project + - release_translation: translation in the release project "open-edx/openedx-translations-" + +""" + +import argparse +import configparser +import os +from os.path import expanduser + +from transifex.api import transifex_api +from transifex.api.jsonapi import exceptions + +ORGANIZATION_SLUG = 'open-edx' +MAIN_PROJECT_SLUG = 'openedx-translations' +RELEASE_PROJECT_SLUG_TEMPLATE = 'openedx-translations-{release_name}' + + +class Command: + + def __init__(self, tx_api, dry_run, resource, language, release_name, environ): + self.dry_run = dry_run + self.release_name = release_name + self.resource = resource + self.language = language + self.tx_api = tx_api + self.environ = environ + + def is_dry_run(self): + """ + Check if the script is running in dry-run mode. + """ + return self.dry_run + + def get_resource_url(self, resource): + resource.fetch('project') + project = resource.project + return f'https://www.transifex.com/{ORGANIZATION_SLUG}/{project.slug}/{resource.slug}' + + def get_release_project_slug(self): + return RELEASE_PROJECT_SLUG_TEMPLATE.format(release_name=self.release_name) + + def get_transifex_project(self, project_slug): + """ + Get openedx-translations projects from Transifex. + """ + tx_api_token = self.environ.get('TX_API_TOKEN') + if not tx_api_token: + config = configparser.ConfigParser() + config.read(expanduser('~/.transifexrc')) + tx_api_token = config['https://www.transifex.com']['password'] + + if not tx_api_token: + raise Exception( + 'Error: No auth token found. ' + 'Set transifex API token via TX_API_TOKEN environment variable or via the ~/.transifexrc file.' + ) + + try: + self.tx_api.setup(auth=tx_api_token) + project = self.tx_api.Project.get(id=f'o:{ORGANIZATION_SLUG}:p:{project_slug}') + return project + except exceptions.DoesNotExist as error: + print(f'Error: Project not found: {project_slug}. Error: {error}') + raise + + def get_resource(self, project, resource_slug): + resource_id = f'o:{ORGANIZATION_SLUG}:p:{project.slug}:r:{resource_slug}' + print(f'Getting resource id: {resource_id}') + + try: + return self.tx_api.Resource.get(id=resource_id) + except exceptions.DoesNotExist as error: + print(f'Error: Resource not found: {resource_id}. Error: {error}') + raise + + def get_translations(self, lang_id, resource): + """ + Get a list of translations for a given language and resource. + """ + language = self.tx_api.Language.get(id=lang_id) + translations = self.tx_api.ResourceTranslation. \ + filter(resource=resource, language=language). \ + include('resource_string') + + return translations.all() + + def sync_translations(self, lang_id, main_resource, release_resource): + """ + Sync specific language translations into the release Transifex resource. + """ + print(' syncing', lang_id, '...') + translations_from_main_project = { + self.get_translation_id(translation): translation + for translation in self.get_translations(lang_id=lang_id, resource=main_resource) + } + + for release_translation in self.get_translations(lang_id=lang_id, resource=release_resource): + translation_id = self.get_translation_id(release_translation) + if translation_from_main_project := translations_from_main_project.get(translation_id): + self.sync_translation_entry( + translation_from_main_project=translation_from_main_project, + release_translation=release_translation, + ) + print(' finished', lang_id) + + def sync_translation_entry(self, translation_from_main_project, release_translation): + """ + Sync translation review from the main project to the release one. + + Return: + str: status code + - updated: if the entry was updated + - no-op: if the entry don't need any updates + - updated-dry-run: if the entry was updated in dry-run mode + """ + translation_id = self.get_translation_id(release_translation) + + updates = {} + + # Only update review status if translations are the same across projects + if translation_from_main_project.strings == release_translation.strings: + for attr in ['reviewed', 'proofread']: + # Reviews won't be deleted in the release project + if main_attr_value := getattr(translation_from_main_project, attr, None): + if main_attr_value != getattr(release_translation, attr, None): + updates[attr] = main_attr_value + else: + print(translation_id, 'has different translations will not update it') + return 'no-op' + + if updates: + print(translation_id, updates, '[Dry run]' if self.is_dry_run() else '') + if self.is_dry_run(): + return 'updated-dry-run' + else: + release_translation.save(**updates) + return 'updated' + + return 'no-op' + + def sync_tags(self, main_resource, release_resource): + """ + Sync tags from the main Transifex resource into the release Transifex resource. + + This process is language independent. + """ + main_resource_str = self.tx_api.ResourceString.filter(resource=main_resource) + release_resource_str = self.tx_api.ResourceString.filter(resource=release_resource) + + main_quick_lookup = {} + for item in main_resource_str.all(): + dict_item = item.to_dict() + main_quick_lookup[dict_item['attributes']['string_hash']] = dict_item['attributes']['tags'] + + for release_info in release_resource_str.all(): + main_tags = main_quick_lookup.get(release_info.string_hash) + release_tags = release_info.tags + + if main_tags is None: # in case of new changes are not synced yet + continue + if len(release_tags) == 0 and len(main_tags) == 0: # nothing to compare + continue + + if len(release_tags) != len(main_tags) or set(release_tags) != set(main_tags): + print(f' - found tag difference for {release_info.string_hash}. overwriting: {release_tags} with {main_tags}') + + if not self.is_dry_run(): + release_info.save(tags=main_tags) + + def get_translation_id(self, translation): + """ + Build a unique identifier for a translation entry. + """ + return f'context:{translation.resource_string.context}:key:{translation.resource_string.key}' + + def get_language_ids(self, project): + """ + Get a list of languages to sync translations for. + """ + if self.language: + # Limit to a specific language if specified + return [f'l:{self.language}'] + + languages = [ + lang.id + for lang in project.fetch('languages') + ] + print('languages', languages) + return languages + + def sync_pair_into_release_resource(self, main_resource, release_resource, language_ids): + """ + Sync translations from main openedx-translations project into openedx-translations-. + """ + print(f'Syncing {main_resource.name} --> {release_resource.name}...') + print(f'Syncing: {language_ids}') + print(f' - from: {self.get_resource_url(main_resource)}') + print(f' - to: {self.get_resource_url(release_resource)}') + + for lang_id in language_ids: + self.sync_translations( + lang_id=lang_id, + main_resource=main_resource, + release_resource=release_resource, + ) + + print('Syncing tags...') + self.sync_tags(main_resource, release_resource) + + print('-' * 80, '\n') + + def run(self): + """ + Run the script. + """ + main_project = self.get_transifex_project(project_slug=MAIN_PROJECT_SLUG) + release_project = self.get_transifex_project(project_slug=self.get_release_project_slug()) + + main_project_language_ids = self.get_language_ids(main_project) + release_project_language_ids = self.get_language_ids(release_project) + missing_languages = set(main_project_language_ids) - set(release_project_language_ids) + for lang_id in missing_languages: + missing_lang = self.tx_api.Resource.create(lang_id) + print('missing_lang', missing_lang) + + main_resources = main_project.fetch('resources') + pairs_list = [] + print('Verifying sync plan...') + for main_resource in main_resources: + if self.resource: + if self.resource.lower() != main_resource.slug.lower(): + # Limit to a specific language if specified + continue + + try: + release_resource = self.get_resource(release_project, main_resource.slug) + except exceptions.DoesNotExist as error: + print( + f'WARNING: Skipping resource {main_resource.slug} because it does not exist in ' + f'"{release_project.slug}". \nError: "{error}"' + ) + else: + print(f'Planning to sync "{main_resource.id}" --> "{release_resource.id}"') + pairs_list.append( + [main_resource, release_resource] + ) + + for main_resource, release_resource in pairs_list: + self.sync_pair_into_release_resource( + main_resource=main_resource, + release_resource=release_resource, + language_ids=main_project_language_ids, + ) + + +def main(): # pragma: no cover + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--dry-run', action='store_true', dest='dry_run', + help='Run the script without writing anything to Transifex.') + parser.add_argument('--resource', default='', dest='resource', + help='Resource slug e.g. "AudioXBlock" or "frontend-app-learning". ' + 'Leave empty to sync all resources.') + parser.add_argument('--language', default='', dest='language', + help='Language code e.g. "ar" or "zh_CN". Leave empty to sync all languages.') + parser.add_argument('release_name', + help='Open edX Release name in lower case .e.g redwood or zebrawood.') + argparse_args = parser.parse_args() + + command = Command( + tx_api=transifex_api, + environ=os.environ, + dry_run=argparse_args.dry_run, + language=argparse_args.language, + resource=argparse_args.resource, + release_name=argparse_args.release_name.lower(), + ) + command.run() + + +if __name__ == '__main__': + main() # pragma: no cover diff --git a/scripts/tests/response_data.py b/scripts/tests/response_data.py index 51b11fddc80..ff10c389f95 100644 --- a/scripts/tests/response_data.py +++ b/scripts/tests/response_data.py @@ -2,79 +2,63 @@ Holds dummy data for tests """ RESPONSE_GET_ORGANIZATION = { - 'data': [ - { - 'id': 'o:open-edx', - 'type': 'organizations', - 'attributes': { - 'name': 'Open edX', - 'slug': 'open-edx', - 'private': False - }, - 'relationships': { - 'projects': { - 'links': { - 'related': 'https://rest.api.transifex.com/projects?filter[organization]=o:open-edx' - } - }, - } - } - ], + 'data': [ + { + 'id': 'o:open-edx', + 'type': 'organizations', + 'attributes': { + 'name': 'Open edX', + 'slug': 'open-edx', + 'private': False + }, + 'relationships': { + } + } + ], } -RESPONSE_GET_PROJECTS = { - "data": [ - { - "id": "o:open-edx:p:openedx-translations", - "type": "projects", - "attributes": { - "slug": "openedx-translations", - "name": "openedx-translations", - "type": "file", - }, - "relationships": { - "source_language": { - "links": { - "related": "https://rest.api.transifex.com/languages/l:en" - }, - "data": { - "type": "languages", - "id": "l:en" - } +RESPONSE_GET_PROJECT = { + "data": { + "id": "o:open-edx:p:openedx-translations", + "type": "project", + "attributes": { + "slug": "openedx-translations", + "name": "openedx-translations", + "type": "file", }, - "languages": { - "links": { - "self": "https://rest.api.transifex.com/projects/o:open-edx:p:openedx-translations/relationships/languages", - "related": "https://rest.api.transifex.com/projects/o:open-edx:p:openedx-translations/languages" - } + "relationships": { }, - }, } - ], } RESPONSE_GET_LANGUAGE = { - "data": [ - { - "id": "l:ar", - "type": "languages", - "attributes": { - "code": "ar", - "name": "Arabic", - "rtl": True, - "plural_equation": "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5", - "plural_rules": { - "zero": "n is 0", - "one": "n is 1", - "two": "n is 2", - "many": "n mod 100 in 11..99", - "few": "n mod 100 in 3..10", - "other": "everything else" - } - }, - "links": { - "self": "https://rest.api.transifex.com/languages/l:ar" - } + "data": { + "id": "l:ar", + "type": "language", + "attributes": { + "code": "ar", + "name": "Arabic", + "rtl": True, + "plural_equation": "n==0 ? 0 : n==1 ? 1 : n==2 ? 2 : n%100>=3 && n%100<=10 ? 3 : n%100>=11 && n%100<=99 ? 4 : 5", + "plural_rules": { + "zero": "n is 0", + "one": "n is 1", + "two": "n is 2", + "many": "n mod 100 in 11..99", + "few": "n mod 100 in 3..10", + "other": "everything else" + } + }, + "links": { + "self": "https://rest.api.transifex.com/languages/l:ar" + }, + "relationships": { + }, } - ] +} + +RESPONSE_GET_LANGUAGES = { + 'data': [ + RESPONSE_GET_LANGUAGE['data'], + ] } diff --git a/scripts/tests/test_release_project_sync.py b/scripts/tests/test_release_project_sync.py new file mode 100644 index 00000000000..d79b8f2b94e --- /dev/null +++ b/scripts/tests/test_release_project_sync.py @@ -0,0 +1,260 @@ +""" +Tests for sync_translations.py +""" +from dataclasses import dataclass +import types +from typing import Union + +import responses + +from transifex.api import transifex_api, Project +from transifex.api.jsonapi import Resource +from transifex.api.jsonapi.auth import BearerAuthentication + +from . import response_data +from ..release_project_sync import ( + Command, + ORGANIZATION_SLUG, +) + +HOST = transifex_api.HOST + + +def sync_command(**kwargs): + command_args = { + 'tx_api': transifex_api, + 'dry_run': True, + 'language': '', + 'resource': '', + 'release_name': 'zebrawood', + 'environ': { + 'TX_API_TOKEN': 'dummy-token' + } + } + command_args.update(kwargs) + result = Command(**command_args) + result.tx_api.make_auth_headers = BearerAuthentication('dummy-token') + return result + + +@dataclass +class ResourceStringMock: + """ + String entry in Transifex. + + Mock class for the transifex.api.ResourceString class. + """ + key: str + context: str = '' + + +@dataclass +class ResourceTranslationMock: + """ + Translation for an entry in Transifex. + + Mock class for the transifex.api.ResourceTranslation class. + """ + resource_string: ResourceStringMock + strings: dict + reviewed: bool + proofread: bool + + _updates: dict = None # Last updates applied via `save()` + + def save(self, **updates): + """ + Mock ResourceTranslation.save() method. + """ + self._updates = updates + + @property + def updates(self): + """ + Return the last updates applied via `save()`. + """ + return self._updates + + @classmethod + def factory( + cls, + key='key', + context='', + translation: Union[str, None] = 'dummy translation', + **kwargs + ): + mock_kwargs = dict( + resource_string=ResourceStringMock( + key=key, + context=context + ), + strings={ + key: translation, + }, + reviewed=False, + proofread=False, + ) + + mock_kwargs.update(kwargs) + return cls(**mock_kwargs) + + +@responses.activate +def test_get_transifex_organization_project(): + """ + Verify that the get_transifex_project() method returns the correct data. + """ + command = sync_command() + + # Mocking responses + responses.add( + responses.GET, + HOST + f'/projects/o:open-edx:p:openedx-translations', + json=response_data.RESPONSE_GET_PROJECT, + status=200 + ) + + # Remove the make_auth_headers to verify later that transifex setup is called + delattr(command.tx_api, 'make_auth_headers') + + project = command.get_transifex_project('openedx-translations') + assert hasattr(command.tx_api, 'make_auth_headers') + assert isinstance(command.tx_api.make_auth_headers, BearerAuthentication) + assert isinstance(project, Project) + assert project.id == response_data.RESPONSE_GET_PROJECT["data"]['id'] + + +@responses.activate +def test_get_translations(): + """ + Verify that the get_translations() method returns the correct data. + """ + command = sync_command() + resource_id = f'{response_data.RESPONSE_GET_PROJECT["data"]["id"]}:r:ar' + + # Mocking responses + responses.add( + responses.GET, + HOST + f'/languages/l:ar', + json=response_data.RESPONSE_GET_LANGUAGE, + status=200 + ) + responses.add( + responses.GET, + HOST + f'/resource_translations?filter[resource]={resource_id}&filter[language]=l:ar&include=resource_string', + json=response_data.RESPONSE_GET_LANGUAGES, + status=200 + ) + + data = command.get_translations( + lang_id='l:ar', + resource=Resource(id=resource_id) + ) + assert isinstance(data, types.GeneratorType) + items = list(data) + assert len(items) == 1 + assert items[0].id == response_data.RESPONSE_GET_LANGUAGE['data']['id'] + + +def test_translations_entry_update_translation(): + """ + Test updating an entry from old project where `current_translation` is has outdated translation. + """ + command = sync_command(dry_run=False) + + translation_from_old_project = ResourceTranslationMock.factory( + key='test_key', + translation='same translation', + reviewed=True, + ) + + current_translation = ResourceTranslationMock.factory( + key='test_key', + translation='same translation', + ) + + status = command.sync_translation_entry( + translation_from_old_project, current_translation + ) + + assert status == 'updated' + assert current_translation.updates == { + 'reviewed': True, + } + + +def test_translations_entry_more_recent_review(): + """ + Verify that the release project reviews aren't removed even if the main project haven't had a review. + """ + command = sync_command(dry_run=False) + + translation_from_main_project = ResourceTranslationMock.factory( + key='test_key', + translation='one translation', + reviewed=False, + ) + + # Current translation is empty + release_translation = ResourceTranslationMock.factory( + key='test_key', + translation='more recent translation', + reviewed=True, + ) + + status = command.sync_translation_entry( + translation_from_main_project, release_translation + ) + + assert status == 'no-op' + assert not release_translation.updates, 'save() should not be called' + + +def test_translations_entry_dry_run(): + """ + Verify that --dry-run option skips the save() call. + """ + command = sync_command(dry_run=True) + + translation_from_old_project = ResourceTranslationMock.factory( + key='test_key', + translation='same translation', + reviewed=True, + ) + + current_translation = ResourceTranslationMock.factory( + key='test_key', + translation='same translation', + ) + + status = command.sync_translation_entry( + translation_from_old_project, current_translation + ) + + assert status == 'updated-dry-run' + assert not current_translation.updates, 'save() should not be called in --dry-run mode' + + +def test_translations_entry_different_translation(): + """ + Verify that different translations prevents reviews sync. + """ + command = sync_command(dry_run=False) + + translation_from_old_project = ResourceTranslationMock.factory( + key='test_key', + translation='some translation', + reviewed=True, + ) + + current_translation = ResourceTranslationMock.factory( + key='test_key', + translation='another translation', + ) + + status = command.sync_translation_entry( + translation_from_old_project, current_translation + ) + + assert status == 'no-op' + assert not current_translation.updates, 'save() should not be called in --dry-run mode'