From b03bf6349e7e7db222a3747507eab00de1776fd5 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 19 Sep 2022 08:12:01 +0300 Subject: [PATCH 01/24] Rewrite models --- cache_json/migrations/0001_initial.py | 56 ----------- github_manager/migrations/0001_initial.py | 7 +- github_manager/models.py | 1 + github_manager/tasks.py | 6 +- translate/admin.py | 4 +- translate/api.py | 113 ++++++++++++---------- translate/migrations/0001_initial.py | 46 +++++++-- translate/models.py | 43 ++++++-- translate/schemas.py | 6 +- translators/migrations/0001_initial.py | 10 +- translators/models.py | 1 + 11 files changed, 157 insertions(+), 136 deletions(-) delete mode 100644 cache_json/migrations/0001_initial.py diff --git a/cache_json/migrations/0001_initial.py b/cache_json/migrations/0001_initial.py deleted file mode 100644 index 0d9bc4c..0000000 --- a/cache_json/migrations/0001_initial.py +++ /dev/null @@ -1,56 +0,0 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Word", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("word", models.CharField(max_length=255, unique=True)), - ("is_checked", models.BooleanField(default=False)), - ], - options={ - "ordering": ["word"], - }, - ), - migrations.CreateModel( - name="Translate", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("translate", models.CharField(max_length=255)), - ("score", models.IntegerField(default=1)), - ( - "word", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="cache_json.word", - ), - ), - ], - ), - ] diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py index 8737487..5b6d1a4 100644 --- a/github_manager/migrations/0001_initial.py +++ b/github_manager/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-19 05:04 from django.db import migrations, models @@ -7,7 +7,9 @@ class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("translate", "0001_initial"), + ] operations = [ migrations.CreateModel( @@ -23,6 +25,7 @@ class Migration(migrations.Migration): ), ), ("prid", models.IntegerField()), + ("words", models.ManyToManyField(blank=True, to="translate.word")), ], ), ] diff --git a/github_manager/models.py b/github_manager/models.py index 6c46082..aeec00e 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -3,3 +3,4 @@ # Create your models here. class PullRequest(models.Model): prid = models.IntegerField() + words = models.ManyToManyField('translate.Word', blank=True) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 3dc40f4..e798e8e 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -1,11 +1,11 @@ -from translate.models import Word as TWord, Translate as TTranslate +# from translate.models import Word as TWord, Translate as TTranslate from .models import PullRequest from github import Github import json import os from pathlib import Path from django.conf import settings -from cache_json.models import Word, Translate +# from cache_json.models import Word, Translate account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) @@ -68,5 +68,5 @@ def push(_payload: dict): if local_json_file.read_text() == '{}': - check_for_update_json() + # check_for_update_json() print('The first data extraction process has been completed') \ No newline at end of file diff --git a/translate/admin.py b/translate/admin.py index 346242a..3b69e2a 100644 --- a/translate/admin.py +++ b/translate/admin.py @@ -1,8 +1,8 @@ from django.contrib import admin -from .models import Word, Translate +from .models import Word, UserTranslate # Register your models here. class TranslateAdminInline(admin.TabularInline): - model = Translate + model = UserTranslate class WordAdmin(admin.ModelAdmin): inlines = (TranslateAdminInline, ) diff --git a/translate/api.py b/translate/api.py index ff78521..4e29be5 100644 --- a/translate/api.py +++ b/translate/api.py @@ -1,59 +1,72 @@ -from difflib import SequenceMatcher -from typing import List -from django.shortcuts import get_object_or_404 +# from difflib import SequenceMatcher +# from typing import List +# from django.shortcuts import get_object_or_404 from ninja import Router -from . import schemas -from django.conf import settings -from .models import Translate, Word -from cache_json.models import Word as SourceWord -from translators.models import Translator -from ninja.errors import AuthenticationError -#This is for Arabic Only -import pyarabic.araby as araby +# from . import schemas +# from django.conf import settings +# from .models import Translate, Word +# from translators.models import Translator +# from ninja.errors import AuthenticationError +# #This is for Arabic Only +# import pyarabic.araby as araby -def do_before_check(word: str) -> str: - return araby.strip_harakat(araby.strip_tatweel(word)) + +# def do_before_check(word: str) -> str: +# return araby.strip_harakat(araby.strip_tatweel(word)) router = Router() -# @router.get("/", response={200: schemas.Word, 404: schemas.Error}) -# def get_nontranslate_words(request): -# words = SourceWord.objects.filter(translate=None).order_by('word') -# if words.count() > 0: -# return 200, words.first() # Get a random nontranslate word +# @router.post("/", response=schemas.Word) +# def recive_word(request, payload: schemas.WordIn): +# w: Word = get_object_or_404(Word, word=payload.word) +# try: +# translator: Translator = Translator.objects.get(uuid=payload.uuid) +# w.translators.add(translator) +# w.save() +# except Translator.DoesNotExist: +# raise AuthenticationError + + + + +# # @router.get("/", response={200: schemas.Word, 404: schemas.Error}) +# # def get_nontranslate_words(request): +# # words = SourceWord.objects.filter(translate=None).order_by('word') +# # if words.count() > 0: +# # return 200, words.first() # Get a random nontranslate word -# words = SourceWord.objects.filter(is_checked=False).order_by('word') -# if words.count() > 0: -# return 200, words.first() # Get a random word not checked word +# # words = SourceWord.objects.filter(is_checked=False).order_by('word') +# # if words.count() > 0: +# # return 200, words.first() # Get a random word not checked word +# # else: +# # return 404, {'messgae': 'there_are_no_nontranslate_words'} + +# @router.post("/", response=schemas.Word) +# def recive_word(request, payload: schemas.WordIn): +# w: Word = get_object_or_404(Word, word=payload.word) +# try: +# translator: Translator = Translator.objects.get(uuid=payload.uuid) +# w.translators.add(translator) +# w.save() +# except Translator.DoesNotExist: +# raise AuthenticationError +# user_translate = do_before_check(payload.translate.strip()) +# if user_translate == '': return w +# is_exists = (False, None) +# for t in w.translate_set.all(): +# similarity = SequenceMatcher(None, t.translate, user_translate) +# print(similarity.ratio()) +# print(similarity.quick_ratio()) +# if similarity.ratio() > 0.8: +# is_exists = (True, t) +# break +# if not is_exists[0]: +# t = Translate(word=w, translate=user_translate) +# t.save() # else: -# return 404, {'messgae': 'there_are_no_nontranslate_words'} - -@router.post("/", response=schemas.Word) -def recive_word(request, payload: schemas.WordIn): - w: Word = get_object_or_404(Word, word=payload.word) - try: - translator: Translator = Translator.objects.get(uuid=payload.uuid) - w.translators.add(translator) - w.save() - except Translator.DoesNotExist: - raise AuthenticationError - user_translate = do_before_check(payload.translate.strip()) - if user_translate == '': return w - is_exists = (False, None) - for t in w.translate_set.all(): - similarity = SequenceMatcher(None, t.translate, user_translate) - print(similarity.ratio()) - print(similarity.quick_ratio()) - if similarity.ratio() > 0.8: - is_exists = (True, t) - break - if not is_exists[0]: - t = Translate(word=w, translate=user_translate) - t.save() - else: - t: Translate = is_exists[1] - t.score += 1 - t.save() +# t: Translate = is_exists[1] +# t.score += 1 +# t.save() - return w \ No newline at end of file +# return w \ No newline at end of file diff --git a/translate/migrations/0001_initial.py b/translate/migrations/0001_initial.py index 53ea578..c22e2e5 100644 --- a/translate/migrations/0001_initial.py +++ b/translate/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-19 05:04 from django.db import migrations, models import django.db.models.deletion @@ -8,10 +8,7 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ("translators", "0001_initial"), - ("github_manager", "0001_initial"), - ] + dependencies = [] operations = [ migrations.CreateModel( @@ -27,15 +24,40 @@ class Migration(migrations.Migration): ), ), ("word", models.CharField(max_length=255, unique=True)), + ("is_checked", models.BooleanField(default=False)), + ], + options={ + "ordering": ["word"], + }, + ), + migrations.CreateModel( + name="UserTranslate", + fields=[ ( - "prs", - models.ManyToManyField(blank=True, to="github_manager.pullrequest"), + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("translate", models.CharField(max_length=255)), + ("score", models.PositiveIntegerField(default=0)), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="translate.word" + ), ), - ("translators", models.ManyToManyField(to="translators.translator")), ], + options={ + "ordering": ("score",), + "abstract": False, + }, ), migrations.CreateModel( - name="Translate", + name="SourceTranslate", fields=[ ( "id", @@ -47,7 +69,7 @@ class Migration(migrations.Migration): ), ), ("translate", models.CharField(max_length=255)), - ("score", models.IntegerField(default=1)), + ("is_default", models.BooleanField(default=False)), ( "word", models.ForeignKey( @@ -55,5 +77,9 @@ class Migration(migrations.Migration): ), ), ], + options={ + "ordering": ("is_default",), + "abstract": False, + }, ), ] diff --git a/translate/models.py b/translate/models.py index ba62d37..f195c73 100644 --- a/translate/models.py +++ b/translate/models.py @@ -1,20 +1,49 @@ from django.db import models + class Word(models.Model): word = models.CharField(max_length=255, unique=True) - translators = models.ManyToManyField("translators.Translator") - prs = models.ManyToManyField('github_manager.PullRequest', blank=True) - - def get_translates(self) -> list["Translate"]: + is_checked = models.BooleanField(default=False) + class Meta: + ordering = ['word'] + + def get_source_translates(self) -> list[str]: + return [translate.translate for translate in self.translate_set.all().order_by('score')] + + def get_users_translates(self) -> list[str]: return [translate.translate for translate in self.translate_set.all().order_by('score')] def __str__(self) -> str: return self.word -class Translate(models.Model): +class TranslateBase(models.Model): word = models.ForeignKey(Word, on_delete=models.CASCADE) translate = models.CharField(max_length=255) - score = models.IntegerField(default=1) - + + class Meta: + abstract = True + def __str__(self) -> str: return f'{self.word}-{self.translate}' + +class SourceTranslate(TranslateBase): + is_default = models.BooleanField(default=False) + + def save(self, *args, **kwargs) -> None: + if self.is_default: + try: + last_default_translate = SourceTranslate.objects.get(word=self.word, is_default=True) + last_default_translate.is_default = False + last_default_translate.save() + except SourceTranslate.DoesNotExist: + pass + return super().save(*args, **kwargs) + class Meta(TranslateBase.Meta): + ordering = ('is_default',) + +class UserTranslate(TranslateBase): + score = models.PositiveIntegerField(default=0) + class Meta(TranslateBase.Meta): + ordering = ('score',) + + diff --git a/translate/schemas.py b/translate/schemas.py index 911f465..02b66c3 100644 --- a/translate/schemas.py +++ b/translate/schemas.py @@ -1,9 +1,7 @@ from typing import List from ninja import ModelSchema, Schema, Field -from .models import (Word as WordModel) +from .models import Word as WordModel -class Error(Schema): - message: str class WordIn(Schema): uuid: str @@ -11,7 +9,7 @@ class WordIn(Schema): translate: str class Word(ModelSchema): - translates: List[str] = Field(..., alias='get_translates') + translates: List[str] = Field(..., alias='get_all_translates') class Config: model = WordModel model_fields = ['word'] \ No newline at end of file diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py index 10d2372..a87a341 100644 --- a/translators/migrations/0001_initial.py +++ b/translators/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.1.1 on 2022-09-17 18:03 +# Generated by Django 4.1.1 on 2022-09-19 05:04 from django.db import migrations, models import uuid @@ -8,7 +8,9 @@ class Migration(migrations.Migration): initial = True - dependencies = [] + dependencies = [ + ("translate", "0001_initial"), + ] operations = [ migrations.CreateModel( @@ -33,6 +35,10 @@ class Migration(migrations.Migration): models.IntegerField(verbose_name="Number of words to send"), ), ("send_time", models.TimeField()), + ( + "translated_words", + models.ManyToManyField(blank=True, to="translate.word"), + ), ], ), migrations.CreateModel( diff --git a/translators/models.py b/translators/models.py index 39f029e..831e91d 100644 --- a/translators/models.py +++ b/translators/models.py @@ -7,6 +7,7 @@ class Translator(models.Model): name = models.CharField(max_length=255) number_of_words = models.IntegerField('Number of words to send') send_time = models.TimeField() + translated_words = models.ManyToManyField('translate.Word', blank=True) def __str__(self) -> str: return self.name From 3caee2ddf436654c762771f0748a1f7cba1c1329 Mon Sep 17 00:00:00 2001 From: Karam Date: Tue, 20 Sep 2022 07:42:11 +0300 Subject: [PATCH 02/24] Refactoring the code; rename translate to translation --- Pipfile.lock | 18 ++-- cache_json/admin.py | 3 - cache_json/models.py | 22 ----- cache_json/views.py | 3 - github_manager/migrations/0001_initial.py | 31 ------- github_manager/tasks.py | 45 +--------- project/api.py | 8 +- project/settings.py | 52 ++++++------ translate/__init__.py | 0 translate/admin.py | 9 -- translate/api.py | 72 ---------------- translate/apps.py | 6 -- translate/migrations/0001_initial.py | 85 ------------------- translate/migrations/__init__.py | 0 translate/models.py | 49 ----------- translate/schemas.py | 15 ---- translate/tests.py | 3 - translate/views.py | 3 - {cache_json => translation}/__init__.py | 0 translation/admin.py | 19 +++++ translation/api.py | 31 +++++++ {cache_json => translation}/apps.py | 4 +- .../migrations/__init__.py | 0 translation/models.py | 81 ++++++++++++++++++ translation/schemas.py | 20 +++++ {cache_json => translation}/tests.py | 0 translation/utils.py | 46 ++++++++++ translators/api.py | 27 +++--- translators/migrations/0001_initial.py | 65 -------------- translators/models.py | 35 +++++--- translators/schemas.py | 12 +-- translators/tasks.py | 2 +- translators/views.py | 3 - 33 files changed, 282 insertions(+), 487 deletions(-) delete mode 100644 cache_json/admin.py delete mode 100644 cache_json/models.py delete mode 100644 cache_json/views.py delete mode 100644 github_manager/migrations/0001_initial.py delete mode 100644 translate/__init__.py delete mode 100644 translate/admin.py delete mode 100644 translate/api.py delete mode 100644 translate/apps.py delete mode 100644 translate/migrations/0001_initial.py delete mode 100644 translate/migrations/__init__.py delete mode 100644 translate/models.py delete mode 100644 translate/schemas.py delete mode 100644 translate/tests.py delete mode 100644 translate/views.py rename {cache_json => translation}/__init__.py (100%) create mode 100644 translation/admin.py create mode 100644 translation/api.py rename {cache_json => translation}/apps.py (60%) rename {cache_json => translation}/migrations/__init__.py (100%) create mode 100644 translation/models.py create mode 100644 translation/schemas.py rename {cache_json => translation}/tests.py (100%) create mode 100644 translation/utils.py delete mode 100644 translators/migrations/0001_initial.py delete mode 100644 translators/views.py diff --git a/Pipfile.lock b/Pipfile.lock index eb89931..827c251 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -21,7 +21,7 @@ "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1", "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.2.3" }, "asgiref": { @@ -45,7 +45,7 @@ "sha256:36973885b9542e6bd01dea287b2b4b3b21236307c56324fcc3f1160f2d655ed5", "sha256:e232343de1ab72c2aa521b625c80f699e356830fd0e2c620b465b304b17b0516" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.9.14" }, "cffi": { @@ -122,7 +122,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.1" }, "deprecated": { @@ -178,7 +178,7 @@ "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd", "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.0.9" }, "gitpython": { @@ -352,7 +352,7 @@ "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==1.5.0" }, "python-dateutil": { @@ -360,7 +360,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "redis": { @@ -384,7 +384,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smmap": { @@ -392,7 +392,7 @@ "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94", "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==5.0.0" }, "sqlparse": { @@ -609,7 +609,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "stack-data": { diff --git a/cache_json/admin.py b/cache_json/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/cache_json/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/cache_json/models.py b/cache_json/models.py deleted file mode 100644 index d468d96..0000000 --- a/cache_json/models.py +++ /dev/null @@ -1,22 +0,0 @@ -from django.db import models - -# Create your models here. -class Word(models.Model): - word = models.CharField(max_length=255, unique=True) - is_checked = models.BooleanField(default=False) - class Meta: - ordering = ['word'] - - def get_translates(self) -> list[str]: - return [translate.translate for translate in self.translate_set.all().order_by('score')] - - def __str__(self) -> str: - return self.word - -class Translate(models.Model): - word = models.ForeignKey(Word, on_delete=models.CASCADE) - translate = models.CharField(max_length=255) - score = models.IntegerField(default=1) - - def __str__(self) -> str: - return f'{self.word}-{self.translate}' diff --git a/cache_json/views.py b/cache_json/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/cache_json/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py deleted file mode 100644 index 5b6d1a4..0000000 --- a/github_manager/migrations/0001_initial.py +++ /dev/null @@ -1,31 +0,0 @@ -# Generated by Django 4.1.1 on 2022-09-19 05:04 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ("translate", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="PullRequest", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("prid", models.IntegerField()), - ("words", models.ManyToManyField(blank=True, to="translate.word")), - ], - ), - ] diff --git a/github_manager/tasks.py b/github_manager/tasks.py index e798e8e..9ee5190 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -5,7 +5,6 @@ import os from pathlib import Path from django.conf import settings -# from cache_json.models import Word, Translate account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) @@ -17,7 +16,7 @@ def check_for_update_json(): file = dictionary_repo.get_contents(os.environ.get('JSON_FILE')) - repo_file_json= json.loads(file.decoded_content) + repo_file_json = json.loads(file.decoded_content) old_file_json = json.loads(local_json_file.read_text()) if repo_file_json == old_file_json: return @@ -27,46 +26,8 @@ def check_for_update_json(): json.dump(repo_file_json, f, ensure_ascii=False, indent=2) update_source(repo_file_json) -def update_source(data: list[dict[str, str | list]]): - source_words = [] # store words to check if some words have been deleted - data_words = [] - - for word in data: - data_words.append(word['word'].strip()) - if word['is_checked']: - print('Skip word {} because it is checked'.format(word['name'])) - w = TWord.objects.get(word=word['word']) - w.delete() - continue - w: Word = Word.objects.get_or_create(word=word['word'])[0] - w.save() - for translate in word['translates']: - try: - Translate.objects.get(word=w, translate=translate) - except Translate.DoesNotExist: - print(f'new translation for word {w.word}') - t: Translate = Translate.objects.get_or_create(word=w, translate=translate)[0] - t.save() - - tw: TWord = TWord.objects.get_or_create(word=w.word)[0] - [t.delete() for t in tw.translate_set.all()] - [t.delete() for t in tw.prs.all()] - tt: TTranslate = TTranslate(word=tw, translate=translate) - tt.save() - source_words = [w.word for w in Word.objects.all()] - diff = list(set(data_words) - set(source_words)) - for word in diff: - w = Word.objects.get(word=word) - w.delete() - w = TWord.objects.get(word=word) - w.delete() def push(_payload: dict): - print('New Push Event') + print("New Push Event") check_for_update_json() - print('Finished Push Event') - - -if local_json_file.read_text() == '{}': - # check_for_update_json() - print('The first data extraction process has been completed') \ No newline at end of file + print("Finished Push Event") diff --git a/project/api.py b/project/api.py index 4afd3cc..ddd692f 100644 --- a/project/api.py +++ b/project/api.py @@ -1,14 +1,14 @@ from django.conf import settings from ninja import NinjaAPI -from translate.api import router as translate_router +from translation.api import router as translate_router from translators.api import router as translators_router from github_manager.webhook import router as github_manager_router api = NinjaAPI( - title='Torjoman Core API', - version='1.0', - description='This is the Torjoman Core API', + title='Torjoman Core API', + version='1.0', + description='This is the Torjoman Core API', ) api.add_router("/translate", translate_router) diff --git a/project/settings.py b/project/settings.py index 3236412..3feeb72 100644 --- a/project/settings.py +++ b/project/settings.py @@ -26,13 +26,15 @@ # See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env('SECRET_KEY') +SECRET_KEY = env("SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] -ALLOWED_HOSTS.append(env('HOST')) +ALLOWED_HOSTS = [ + "127.0.0.1", # TODO: remove me +] +ALLOWED_HOSTS.append(env("HOST")) # Application definition @@ -43,13 +45,12 @@ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - 'ninja', - 'django_q', - - 'cache_json.apps.CacheJsonConfig', - 'translate.apps.TranslateConfig', - 'translators.apps.TranslatorsConfig', - 'github_manager.apps.GithubManagerConfig', + "django_q", + "ninja", + + "translation.apps.TranslationConfig", + "translators.apps.TranslatorsConfig", + "github_manager.apps.GithubManagerConfig", ] MIDDLEWARE = [ @@ -88,14 +89,15 @@ DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': env("DATABASE_NAME"), - 'USER': env("DATABASE_USER"), - 'PASSWORD': env("DATABASE_PASSWORD"), - 'HOST': env("DATABASE_HOST"), - 'PORT': env("DATABASE_PORT"), - }} + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": env("DATABASE_NAME"), + "USER": env("DATABASE_USER"), + "PASSWORD": env("DATABASE_PASSWORD"), + "HOST": env("DATABASE_HOST"), + "PORT": env("DATABASE_PORT"), + } +} # Password validation @@ -133,7 +135,7 @@ # https://docs.djangoproject.com/en/4.1/howto/static-files/ STATIC_URL = "static/" -STATIC_ROOT = BASE_DIR / 'static/' +STATIC_ROOT = BASE_DIR / "static/" # Default primary key field type # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field @@ -143,14 +145,10 @@ Q_CLUSTER = { "name": "torjoman", - 'retry': 750, - 'timeout': 600, - 'redis': { - 'host': '127.0.0.1', - 'port': 6379, - 'db': 0 - } + "retry": 750, + "timeout": 600, + "redis": {"host": "127.0.0.1", "port": 6379, "db": 0}, } -GITHUB_WEBHOOK_KEY = env('GITHUB_WEBHOOK_KEY') \ No newline at end of file +GITHUB_WEBHOOK_KEY = env("GITHUB_WEBHOOK_KEY") diff --git a/translate/__init__.py b/translate/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/translate/admin.py b/translate/admin.py deleted file mode 100644 index 3b69e2a..0000000 --- a/translate/admin.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.contrib import admin -from .models import Word, UserTranslate -# Register your models here. -class TranslateAdminInline(admin.TabularInline): - model = UserTranslate - -class WordAdmin(admin.ModelAdmin): - inlines = (TranslateAdminInline, ) -admin.site.register(Word, WordAdmin) diff --git a/translate/api.py b/translate/api.py deleted file mode 100644 index 4e29be5..0000000 --- a/translate/api.py +++ /dev/null @@ -1,72 +0,0 @@ -# from difflib import SequenceMatcher -# from typing import List -# from django.shortcuts import get_object_or_404 -from ninja import Router -# from . import schemas -# from django.conf import settings -# from .models import Translate, Word -# from translators.models import Translator -# from ninja.errors import AuthenticationError - -# #This is for Arabic Only -# import pyarabic.araby as araby - - -# def do_before_check(word: str) -> str: -# return araby.strip_harakat(araby.strip_tatweel(word)) - -router = Router() - -# @router.post("/", response=schemas.Word) -# def recive_word(request, payload: schemas.WordIn): -# w: Word = get_object_or_404(Word, word=payload.word) -# try: -# translator: Translator = Translator.objects.get(uuid=payload.uuid) -# w.translators.add(translator) -# w.save() -# except Translator.DoesNotExist: -# raise AuthenticationError - - - - -# # @router.get("/", response={200: schemas.Word, 404: schemas.Error}) -# # def get_nontranslate_words(request): -# # words = SourceWord.objects.filter(translate=None).order_by('word') -# # if words.count() > 0: -# # return 200, words.first() # Get a random nontranslate word - -# # words = SourceWord.objects.filter(is_checked=False).order_by('word') -# # if words.count() > 0: -# # return 200, words.first() # Get a random word not checked word -# # else: -# # return 404, {'messgae': 'there_are_no_nontranslate_words'} - -# @router.post("/", response=schemas.Word) -# def recive_word(request, payload: schemas.WordIn): -# w: Word = get_object_or_404(Word, word=payload.word) -# try: -# translator: Translator = Translator.objects.get(uuid=payload.uuid) -# w.translators.add(translator) -# w.save() -# except Translator.DoesNotExist: -# raise AuthenticationError -# user_translate = do_before_check(payload.translate.strip()) -# if user_translate == '': return w -# is_exists = (False, None) -# for t in w.translate_set.all(): -# similarity = SequenceMatcher(None, t.translate, user_translate) -# print(similarity.ratio()) -# print(similarity.quick_ratio()) -# if similarity.ratio() > 0.8: -# is_exists = (True, t) -# break -# if not is_exists[0]: -# t = Translate(word=w, translate=user_translate) -# t.save() -# else: -# t: Translate = is_exists[1] -# t.score += 1 -# t.save() - -# return w \ No newline at end of file diff --git a/translate/apps.py b/translate/apps.py deleted file mode 100644 index 6c1c6c8..0000000 --- a/translate/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class TranslateConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "translate" diff --git a/translate/migrations/0001_initial.py b/translate/migrations/0001_initial.py deleted file mode 100644 index c22e2e5..0000000 --- a/translate/migrations/0001_initial.py +++ /dev/null @@ -1,85 +0,0 @@ -# Generated by Django 4.1.1 on 2022-09-19 05:04 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [] - - operations = [ - migrations.CreateModel( - name="Word", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("word", models.CharField(max_length=255, unique=True)), - ("is_checked", models.BooleanField(default=False)), - ], - options={ - "ordering": ["word"], - }, - ), - migrations.CreateModel( - name="UserTranslate", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("translate", models.CharField(max_length=255)), - ("score", models.PositiveIntegerField(default=0)), - ( - "word", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="translate.word" - ), - ), - ], - options={ - "ordering": ("score",), - "abstract": False, - }, - ), - migrations.CreateModel( - name="SourceTranslate", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("translate", models.CharField(max_length=255)), - ("is_default", models.BooleanField(default=False)), - ( - "word", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="translate.word" - ), - ), - ], - options={ - "ordering": ("is_default",), - "abstract": False, - }, - ), - ] diff --git a/translate/migrations/__init__.py b/translate/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/translate/models.py b/translate/models.py deleted file mode 100644 index f195c73..0000000 --- a/translate/models.py +++ /dev/null @@ -1,49 +0,0 @@ -from django.db import models - - -class Word(models.Model): - word = models.CharField(max_length=255, unique=True) - is_checked = models.BooleanField(default=False) - class Meta: - ordering = ['word'] - - def get_source_translates(self) -> list[str]: - return [translate.translate for translate in self.translate_set.all().order_by('score')] - - def get_users_translates(self) -> list[str]: - return [translate.translate for translate in self.translate_set.all().order_by('score')] - - def __str__(self) -> str: - return self.word - -class TranslateBase(models.Model): - word = models.ForeignKey(Word, on_delete=models.CASCADE) - translate = models.CharField(max_length=255) - - class Meta: - abstract = True - - def __str__(self) -> str: - return f'{self.word}-{self.translate}' - -class SourceTranslate(TranslateBase): - is_default = models.BooleanField(default=False) - - def save(self, *args, **kwargs) -> None: - if self.is_default: - try: - last_default_translate = SourceTranslate.objects.get(word=self.word, is_default=True) - last_default_translate.is_default = False - last_default_translate.save() - except SourceTranslate.DoesNotExist: - pass - return super().save(*args, **kwargs) - class Meta(TranslateBase.Meta): - ordering = ('is_default',) - -class UserTranslate(TranslateBase): - score = models.PositiveIntegerField(default=0) - class Meta(TranslateBase.Meta): - ordering = ('score',) - - diff --git a/translate/schemas.py b/translate/schemas.py deleted file mode 100644 index 02b66c3..0000000 --- a/translate/schemas.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import List -from ninja import ModelSchema, Schema, Field -from .models import Word as WordModel - - -class WordIn(Schema): - uuid: str - word: str - translate: str - -class Word(ModelSchema): - translates: List[str] = Field(..., alias='get_all_translates') - class Config: - model = WordModel - model_fields = ['word'] \ No newline at end of file diff --git a/translate/tests.py b/translate/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/translate/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/translate/views.py b/translate/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/translate/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/cache_json/__init__.py b/translation/__init__.py similarity index 100% rename from cache_json/__init__.py rename to translation/__init__.py diff --git a/translation/admin.py b/translation/admin.py new file mode 100644 index 0000000..f73c7df --- /dev/null +++ b/translation/admin.py @@ -0,0 +1,19 @@ +from django.contrib import admin +from .models import Word, UserTranslation, SourceTranslation + +# Register your models here. + + +class UserTranslationAdminInline(admin.TabularInline): + model = UserTranslation + + +class SourceTranslationAdminInline(admin.TabularInline): + model = SourceTranslation + + +class WordAdmin(admin.ModelAdmin): + inlines = (SourceTranslationAdminInline, UserTranslationAdminInline) + + +admin.site.register(Word, WordAdmin) diff --git a/translation/api.py b/translation/api.py new file mode 100644 index 0000000..fa8582e --- /dev/null +++ b/translation/api.py @@ -0,0 +1,31 @@ +from django.http import Http404 +from django.shortcuts import get_object_or_404 +from ninja import Router +from ninja.errors import AuthenticationError +from translators.models import Translator + +from . import schemas +from .models import Word +from .utils import modify_translation + +router = Router() + + +@router.post( + "/", + response={ + 200: Word, + 404: Http404, + }, +) +def receiving_translation(request, payload: schemas.WordIn): + w: Word = get_object_or_404(Word, word=payload.word) + try: + translator: Translator = Translator.objects.get(uuid=payload.uuid) + w.translators.add(translator) + w.save() + except Translator.DoesNotExist: + raise AuthenticationError + tranlation = modify_translation(payload.translation) + w.add_translation(tranlation) + return {"success": True} diff --git a/cache_json/apps.py b/translation/apps.py similarity index 60% rename from cache_json/apps.py rename to translation/apps.py index d30c37f..e950da2 100644 --- a/cache_json/apps.py +++ b/translation/apps.py @@ -1,6 +1,6 @@ from django.apps import AppConfig -class CacheJsonConfig(AppConfig): +class TranslationConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "cache_json" + name = "translation" diff --git a/cache_json/migrations/__init__.py b/translation/migrations/__init__.py similarity index 100% rename from cache_json/migrations/__init__.py rename to translation/migrations/__init__.py diff --git a/translation/models.py b/translation/models.py new file mode 100644 index 0000000..41d14e5 --- /dev/null +++ b/translation/models.py @@ -0,0 +1,81 @@ +from django.db import models +from .utils import get_close_match + + +class Word(models.Model): + word = models.CharField(max_length=255, unique=True) + is_checked = models.BooleanField(default=False) + + class Meta: + ordering = ["word"] + + def get_source_translations(self) -> list[str]: + return [translate.translate for translate in self.sourcetranslation_set.all()] + + def get_users_translations(self) -> list[str]: + return [translate.translate for translate in self.usertranslation_set.all()] + + def add_translation(self, translation: str) -> "UserTranslation": + """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. + + Args: + translation (str): the translation to compare and add/increase score. + + Returns: + UserTranslation: The new/similar translation object. + None: if the translation was empty. + """ + translation = translation.strip() + if translation == "": + return + match = get_close_match(translation, self.get_users_translations()) + if match: + t: UserTranslation = UserTranslation.objects.get( + word=self, translation=match + ) + t.score += 1 + t.save() + else: + t = UserTranslation(word=self, translation=translation) + t.save() + return t + + def __str__(self) -> str: + return self.word + + +class TranslationBase(models.Model): + word = models.ForeignKey(Word, on_delete=models.CASCADE) + translation = models.CharField(max_length=255) + + class Meta: + abstract = True + + def __str__(self) -> str: + return f"{self.word}-{self.translate}" + + +class SourceTranslation(TranslationBase): + is_default = models.BooleanField(default=False) + + def save(self, *args, **kwargs) -> None: + if self.is_default: + try: + last_default_translate = SourceTranslation.objects.get( + word=self.word, is_default=True + ) + last_default_translate.is_default = False + last_default_translate.save() + except SourceTranslation.DoesNotExist: + pass + return super().save(*args, **kwargs) + + class Meta(TranslationBase.Meta): + ordering = ("is_default", "translation") + + +class UserTranslation(TranslationBase): + score = models.PositiveIntegerField(default=0) + + class Meta(TranslationBase.Meta): + ordering = ("score", "translation") diff --git a/translation/schemas.py b/translation/schemas.py new file mode 100644 index 0000000..c787721 --- /dev/null +++ b/translation/schemas.py @@ -0,0 +1,20 @@ +from uuid import UUID +from ninja import Schema, ModelSchema, Field +from .models import Word, SourceTranslation, UserTranslation + + +class WordIn(ModelSchema): + by: UUID + translation: str + + class Config: + model = Word + model_fields = ["word"] + + +class Word(ModelSchema): + translations: list[str] = Field(..., alias="get_source_translations") + + class Config: + model = Word + model_fields = ["word"] diff --git a/cache_json/tests.py b/translation/tests.py similarity index 100% rename from cache_json/tests.py rename to translation/tests.py diff --git a/translation/utils.py b/translation/utils.py new file mode 100644 index 0000000..1901557 --- /dev/null +++ b/translation/utils.py @@ -0,0 +1,46 @@ +from difflib import SequenceMatcher +from heapq import nlargest as _nlargest +import pyarabic.araby as araby # Language specific module + +def get_close_match(word, possibilities, cutoff=0.8) -> (str | tuple[()]): + """Use SequenceMatcher to return a list of the indexes of the best + "good enough" matches. word is a sequence for which close matches + are desired (typically a string). + possibilities is a list of sequences against which to match word + (typically a list of strings). + Optional arg cutoff (default 0.8) is a float in [0, 1]. Possibilities + that don't score at least that similar to word are ignored. + """ + + if not 0.0 <= cutoff <= 1.0: + raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) + result = [] + s = SequenceMatcher() + s.set_seq2(word) + for idx, x in enumerate(possibilities): + s.set_seq1(x) + if s.real_quick_ratio() >= cutoff \ + and s.quick_ratio() >= cutoff \ + and s.ratio() >= cutoff: + result.append((s.ratio(), idx)) + + # Move the best scorers to head of list + result = _nlargest(1, result) + + # Strip scores for the best n matches + if result: + return possibilities[result[0][1]] + else: + return () + + +def modify_translation(translation: str) -> str: + """modify_translation Changes the translation before comparing it with saved translations + + Args: + translation (str): the translation to modify + + Returns: + str: the modified translation + """ + return araby.strip_harakat(araby.strip_tatweel(translation)) # remove the harakat and tatweel from the translation \ No newline at end of file diff --git a/translators/api.py b/translators/api.py index 17aba1b..fdcb071 100644 --- a/translators/api.py +++ b/translators/api.py @@ -1,16 +1,17 @@ from ninja import Router from . import schemas +from django.shortcuts import get_object_or_404 from .models import Translator, Platform +from django.http import Http404 router = Router() -@router.post("/register", response={200: schemas.Translator, 400: schemas.Error, 404: schemas.Error}) +@router.post("/register", response={200: schemas.Translator}) def register(request, payload: schemas.TranslatorRegister): - platforms: dict[str, Platform] = {f'{platform.name}': platform for platform in Platform.objects.all()} - platforms_names = list(platforms.keys()) - if payload.platform not in platforms_names: - return 400, schemas.Error(message='platform_not_exists') + platforms = Platform.get_all_platforms() + if payload.platform not in list(platforms.keys()): + return 422, translator = Translator(name=payload.name, number_of_words=payload.number_of_words, send_time=payload.send_time) translator.save() platform = platforms[payload.platform] @@ -18,17 +19,13 @@ def register(request, payload: schemas.TranslatorRegister): platform.save() return 200, translator - -@router.post("/login", response={200: schemas.Translator, 400: schemas.Error, 404: schemas.Error}) +@router.post("/login", response={200: schemas.Translator, 404: Http404}) def login(request, payload: schemas.TranslatorLogin): - try: translator = Translator.objects.get(uuid=payload.uuid) - except: return 404, schemas.Error(message='user_not_exists') - platforms: dict[str, Platform] = {f'{platform.name}': platform for platform in Platform.objects.all()} - platforms_names = list(platforms.keys()) - if payload.platform not in platforms_names: - return 400, schemas.Error(message='platform_not_exists') + translator = get_object_or_404(Translator, uuid=payload.uuid) + platforms = Platform.get_all_platforms() + if payload.platform not in list(platforms.keys()): + return 422, platform = platforms[payload.platform] platform.translators.add(translator) platform.save() - return 200, translator - + return 200, translator \ No newline at end of file diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py deleted file mode 100644 index a87a341..0000000 --- a/translators/migrations/0001_initial.py +++ /dev/null @@ -1,65 +0,0 @@ -# Generated by Django 4.1.1 on 2022-09-19 05:04 - -from django.db import migrations, models -import uuid - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ("translate", "0001_initial"), - ] - - operations = [ - migrations.CreateModel( - name="Translator", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField(default=uuid.uuid4, editable=False, unique=True), - ), - ("name", models.CharField(max_length=255)), - ( - "number_of_words", - models.IntegerField(verbose_name="Number of words to send"), - ), - ("send_time", models.TimeField()), - ( - "translated_words", - models.ManyToManyField(blank=True, to="translate.word"), - ), - ], - ), - migrations.CreateModel( - name="Platform", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=250)), - ("base_url", models.URLField()), - ("is_active", models.BooleanField(default=True)), - ( - "translators", - models.ManyToManyField(blank=True, to="translators.translator"), - ), - ], - ), - ] diff --git a/translators/models.py b/translators/models.py index 831e91d..6a10b91 100644 --- a/translators/models.py +++ b/translators/models.py @@ -3,20 +3,35 @@ # Create your models here. class Translator(models.Model): - uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) - name = models.CharField(max_length=255) - number_of_words = models.IntegerField('Number of words to send') - send_time = models.TimeField() - translated_words = models.ManyToManyField('translate.Word', blank=True) - - def __str__(self) -> str: - return self.name - + uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + name = models.CharField(max_length=255) + number_of_words = models.IntegerField("Number of words to send") + send_time = models.TimeField() + translated_words = models.ManyToManyField("translation.Word", blank=True) + + def get_platforms(self): + return [platform.name for platform in self.platform_set.all()] + + def __str__(self) -> str: + return self.name + class Platform(models.Model): name = models.CharField(max_length=250) base_url = models.URLField() translators = models.ManyToManyField(Translator, blank=True) - is_active = models.BooleanField(default=True) + is_active = models.BooleanField(default=True) # if True, torjoman will send untranslated words to this platform + + @classmethod + def get_all_platforms(cls) -> dict[str, "Platform"]: + """get_all_platforms get a dict of all platforms. + + Returns: + dict[str, Platform]: keys are platforms names, values are platforms instances + """ + return { + f'{platform.name}': platform + for platform in Platform.objects.all() + } def __str__(self) -> str: return self.name \ No newline at end of file diff --git a/translators/schemas.py b/translators/schemas.py index 1313cd4..e5c9d4d 100644 --- a/translators/schemas.py +++ b/translators/schemas.py @@ -3,21 +3,17 @@ from ninja import ModelSchema, Schema, Field from .models import (Translator as TranslatorModel) -class Error(Schema): - message: str - class TranslatorRegister(ModelSchema): platform: str class Config: model = TranslatorModel model_exclude = ['id'] - -class TranslatorLogin(Schema): - uuid: UUID +class TranslatorLogin(ModelSchema): platform: str - - + class Config: + model = TranslatorModel + model_fields = ['uuid'] class Translator(ModelSchema): class Config: diff --git a/translators/tasks.py b/translators/tasks.py index 24f6469..6dabf88 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -1,5 +1,5 @@ from .models import Translator -from translate.models import Word +from translation.models import Word from datetime import datetime import requests diff --git a/translators/views.py b/translators/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/translators/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. From a0090993c408c14f8b24b272ad7a4130a9872ec1 Mon Sep 17 00:00:00 2001 From: Karam Date: Tue, 20 Sep 2022 07:46:29 +0300 Subject: [PATCH 03/24] format code with black --- github_manager/admin.py | 3 +- github_manager/models.py | 4 +- github_manager/tasks.py | 35 +++++++++-------- github_manager/webhook.py | 57 +++++++++++++++------------- project/api.py | 6 +-- project/prepare-project.py | 5 ++- project/settings.py | 1 - project/urls.py | 4 +- translation/models.py | 78 +++++++++++++++++++------------------- translation/utils.py | 77 +++++++++++++++++++------------------ translators/admin.py | 3 +- translators/api.py | 43 +++++++++++---------- translators/models.py | 40 +++++++++---------- translators/schemas.py | 29 ++++++++------ translators/tasks.py | 42 ++++++++++---------- 15 files changed, 225 insertions(+), 202 deletions(-) diff --git a/github_manager/admin.py b/github_manager/admin.py index 6fcb822..5b522f8 100644 --- a/github_manager/admin.py +++ b/github_manager/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin from .models import PullRequest + # Register your models here. -admin.site.register(PullRequest) \ No newline at end of file +admin.site.register(PullRequest) diff --git a/github_manager/models.py b/github_manager/models.py index aeec00e..18afe92 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -2,5 +2,5 @@ # Create your models here. class PullRequest(models.Model): - prid = models.IntegerField() - words = models.ManyToManyField('translate.Word', blank=True) + prid = models.IntegerField() + words = models.ManyToManyField("translate.Word", blank=True) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 9ee5190..3b5b0ec 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -8,26 +8,29 @@ account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) -dictionary_repo = account.get_repo(os.environ.get('JSON_REPO')) -local_json_file: Path = (settings.BASE_DIR / 'local_dictionary.json').resolve() +dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) +local_json_file: Path = (settings.BASE_DIR / "local_dictionary.json").resolve() if not local_json_file.exists(): - raise Exception("Please run 'python3 project/prepare-project.py' before run Torjoman for the first time") + raise Exception( + "Please run 'python3 project/prepare-project.py' before run Torjoman for the first time" + ) + def check_for_update_json(): - file = dictionary_repo.get_contents(os.environ.get('JSON_FILE')) - repo_file_json = json.loads(file.decoded_content) - old_file_json = json.loads(local_json_file.read_text()) - if repo_file_json == old_file_json: - return - else: - print('Files Are not the same') - with open(str(local_json_file), 'w') as f: - json.dump(repo_file_json, f, ensure_ascii=False, indent=2) - update_source(repo_file_json) + file = dictionary_repo.get_contents(os.environ.get("JSON_FILE")) + repo_file_json = json.loads(file.decoded_content) + old_file_json = json.loads(local_json_file.read_text()) + if repo_file_json == old_file_json: + return + else: + print("Files Are not the same") + with open(str(local_json_file), "w") as f: + json.dump(repo_file_json, f, ensure_ascii=False, indent=2) + update_source(repo_file_json) def push(_payload: dict): - print("New Push Event") - check_for_update_json() - print("Finished Push Event") + print("New Push Event") + check_for_update_json() + print("Finished Push Event") diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 33685f3..424ac1e 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -11,30 +11,35 @@ router = Router() -@router.post('/') + +@router.post("/") def manage_webhooks(request): - # Verify the request signature - header_signature = request.META.get('HTTP_X_HUB_SIGNATURE') - if header_signature is None: - return HttpResponseForbidden('Permission denied.') - - sha_name, signature = header_signature.split('=') - if sha_name != 'sha1': - return HttpResponseServerError('Operation not supported.', status=501) - - mac = hmac.new(force_bytes(settings.GITHUB_WEBHOOK_KEY), msg=force_bytes(request.body), digestmod=sha1) - if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)): - return HttpResponseForbidden('Permission denied.') - - # If request reached this point we are in a good shape - # Process the GitHub events - event = request.META.get('HTTP_X_GITHUB_EVENT', 'ping') - - if event == 'ping': - return HttpResponse('pong') - elif event == 'push': - t = threading.Thread(target=push, args=(json.loads(request.body),)) - t.start() - return HttpResponse('success') - # In case we receive an event that's not ping or push - return HttpResponse(status=204) \ No newline at end of file + # Verify the request signature + header_signature = request.META.get("HTTP_X_HUB_SIGNATURE") + if header_signature is None: + return HttpResponseForbidden("Permission denied.") + + sha_name, signature = header_signature.split("=") + if sha_name != "sha1": + return HttpResponseServerError("Operation not supported.", status=501) + + mac = hmac.new( + force_bytes(settings.GITHUB_WEBHOOK_KEY), + msg=force_bytes(request.body), + digestmod=sha1, + ) + if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)): + return HttpResponseForbidden("Permission denied.") + + # If request reached this point we are in a good shape + # Process the GitHub events + event = request.META.get("HTTP_X_GITHUB_EVENT", "ping") + + if event == "ping": + return HttpResponse("pong") + elif event == "push": + t = threading.Thread(target=push, args=(json.loads(request.body),)) + t.start() + return HttpResponse("success") + # In case we receive an event that's not ping or push + return HttpResponse(status=204) diff --git a/project/api.py b/project/api.py index ddd692f..3aa41a9 100644 --- a/project/api.py +++ b/project/api.py @@ -6,9 +6,9 @@ api = NinjaAPI( - title='Torjoman Core API', - version='1.0', - description='This is the Torjoman Core API', + title="Torjoman Core API", + version="1.0", + description="This is the Torjoman Core API", ) api.add_router("/translate", translate_router) diff --git a/project/prepare-project.py b/project/prepare-project.py index bfaa0e1..0a0801b 100644 --- a/project/prepare-project.py +++ b/project/prepare-project.py @@ -1,7 +1,8 @@ from pathlib import Path + BASE_DIR = Path(__file__).resolve().parent.parent -local_json_file: Path = (BASE_DIR / 'local_dictionary.json').resolve() +local_json_file: Path = (BASE_DIR / "local_dictionary.json").resolve() local_json_file.touch() -local_json_file.write_text('{}') +local_json_file.write_text("{}") diff --git a/project/settings.py b/project/settings.py index 3feeb72..9defceb 100644 --- a/project/settings.py +++ b/project/settings.py @@ -47,7 +47,6 @@ "django.contrib.staticfiles", "django_q", "ninja", - "translation.apps.TranslationConfig", "translators.apps.TranslatorsConfig", "github_manager.apps.GithubManagerConfig", diff --git a/project/urls.py b/project/urls.py index fbd5a60..1d13aaa 100644 --- a/project/urls.py +++ b/project/urls.py @@ -3,6 +3,6 @@ from .api import api urlpatterns = [ - path('admin/', admin.site.urls), - path('api/', api.urls), + path("admin/", admin.site.urls), + path("api/", api.urls), ] diff --git a/translation/models.py b/translation/models.py index 41d14e5..198b756 100644 --- a/translation/models.py +++ b/translation/models.py @@ -3,45 +3,45 @@ class Word(models.Model): - word = models.CharField(max_length=255, unique=True) - is_checked = models.BooleanField(default=False) - - class Meta: - ordering = ["word"] - - def get_source_translations(self) -> list[str]: - return [translate.translate for translate in self.sourcetranslation_set.all()] - - def get_users_translations(self) -> list[str]: - return [translate.translate for translate in self.usertranslation_set.all()] - - def add_translation(self, translation: str) -> "UserTranslation": - """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. - - Args: - translation (str): the translation to compare and add/increase score. - - Returns: - UserTranslation: The new/similar translation object. - None: if the translation was empty. - """ - translation = translation.strip() - if translation == "": - return - match = get_close_match(translation, self.get_users_translations()) - if match: - t: UserTranslation = UserTranslation.objects.get( - word=self, translation=match - ) - t.score += 1 - t.save() - else: - t = UserTranslation(word=self, translation=translation) - t.save() - return t - - def __str__(self) -> str: - return self.word + word = models.CharField(max_length=255, unique=True) + is_checked = models.BooleanField(default=False) + + class Meta: + ordering = ["word"] + + def get_source_translations(self) -> list[str]: + return [translate.translate for translate in self.sourcetranslation_set.all()] + + def get_users_translations(self) -> list[str]: + return [translate.translate for translate in self.usertranslation_set.all()] + + def add_translation(self, translation: str) -> "UserTranslation": + """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. + + Args: + translation (str): the translation to compare and add/increase score. + + Returns: + UserTranslation: The new/similar translation object. + None: if the translation was empty. + """ + translation = translation.strip() + if translation == "": + return + match = get_close_match(translation, self.get_users_translations()) + if match: + t: UserTranslation = UserTranslation.objects.get( + word=self, translation=match + ) + t.score += 1 + t.save() + else: + t = UserTranslation(word=self, translation=translation) + t.save() + return t + + def __str__(self) -> str: + return self.word class TranslationBase(models.Model): diff --git a/translation/utils.py b/translation/utils.py index 1901557..c893df9 100644 --- a/translation/utils.py +++ b/translation/utils.py @@ -2,45 +2,50 @@ from heapq import nlargest as _nlargest import pyarabic.araby as araby # Language specific module + def get_close_match(word, possibilities, cutoff=0.8) -> (str | tuple[()]): - """Use SequenceMatcher to return a list of the indexes of the best - "good enough" matches. word is a sequence for which close matches - are desired (typically a string). - possibilities is a list of sequences against which to match word - (typically a list of strings). - Optional arg cutoff (default 0.8) is a float in [0, 1]. Possibilities - that don't score at least that similar to word are ignored. - """ - - if not 0.0 <= cutoff <= 1.0: - raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) - result = [] - s = SequenceMatcher() - s.set_seq2(word) - for idx, x in enumerate(possibilities): - s.set_seq1(x) - if s.real_quick_ratio() >= cutoff \ - and s.quick_ratio() >= cutoff \ - and s.ratio() >= cutoff: - result.append((s.ratio(), idx)) - - # Move the best scorers to head of list - result = _nlargest(1, result) - - # Strip scores for the best n matches - if result: - return possibilities[result[0][1]] - else: - return () + """Use SequenceMatcher to return a list of the indexes of the best + "good enough" matches. word is a sequence for which close matches + are desired (typically a string). + possibilities is a list of sequences against which to match word + (typically a list of strings). + Optional arg cutoff (default 0.8) is a float in [0, 1]. Possibilities + that don't score at least that similar to word are ignored. + """ + + if not 0.0 <= cutoff <= 1.0: + raise ValueError("cutoff must be in [0.0, 1.0]: %r" % (cutoff,)) + result = [] + s = SequenceMatcher() + s.set_seq2(word) + for idx, x in enumerate(possibilities): + s.set_seq1(x) + if ( + s.real_quick_ratio() >= cutoff + and s.quick_ratio() >= cutoff + and s.ratio() >= cutoff + ): + result.append((s.ratio(), idx)) + + # Move the best scorers to head of list + result = _nlargest(1, result) + + # Strip scores for the best n matches + if result: + return possibilities[result[0][1]] + else: + return () def modify_translation(translation: str) -> str: - """modify_translation Changes the translation before comparing it with saved translations + """modify_translation Changes the translation before comparing it with saved translations - Args: - translation (str): the translation to modify + Args: + translation (str): the translation to modify - Returns: - str: the modified translation - """ - return araby.strip_harakat(araby.strip_tatweel(translation)) # remove the harakat and tatweel from the translation \ No newline at end of file + Returns: + str: the modified translation + """ + return araby.strip_harakat( + araby.strip_tatweel(translation) + ) # remove the harakat and tatweel from the translation diff --git a/translators/admin.py b/translators/admin.py index 0885f1e..b201aeb 100644 --- a/translators/admin.py +++ b/translators/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin from .models import Translator, Platform + # Register your models here. admin.site.register(Translator) -admin.site.register(Platform) \ No newline at end of file +admin.site.register(Platform) diff --git a/translators/api.py b/translators/api.py index fdcb071..ae3db1d 100644 --- a/translators/api.py +++ b/translators/api.py @@ -5,27 +5,32 @@ from django.http import Http404 router = Router() - - + + @router.post("/register", response={200: schemas.Translator}) def register(request, payload: schemas.TranslatorRegister): - platforms = Platform.get_all_platforms() - if payload.platform not in list(platforms.keys()): - return 422, - translator = Translator(name=payload.name, number_of_words=payload.number_of_words, send_time=payload.send_time) - translator.save() - platform = platforms[payload.platform] - platform.translators.add(translator) - platform.save() - return 200, translator + platforms = Platform.get_all_platforms() + if payload.platform not in list(platforms.keys()): + return (422,) + translator = Translator( + name=payload.name, + number_of_words=payload.number_of_words, + send_time=payload.send_time, + ) + translator.save() + platform = platforms[payload.platform] + platform.translators.add(translator) + platform.save() + return 200, translator + @router.post("/login", response={200: schemas.Translator, 404: Http404}) def login(request, payload: schemas.TranslatorLogin): - translator = get_object_or_404(Translator, uuid=payload.uuid) - platforms = Platform.get_all_platforms() - if payload.platform not in list(platforms.keys()): - return 422, - platform = platforms[payload.platform] - platform.translators.add(translator) - platform.save() - return 200, translator \ No newline at end of file + translator = get_object_or_404(Translator, uuid=payload.uuid) + platforms = Platform.get_all_platforms() + if payload.platform not in list(platforms.keys()): + return (422,) + platform = platforms[payload.platform] + platform.translators.add(translator) + platform.save() + return 200, translator diff --git a/translators/models.py b/translators/models.py index 6a10b91..9ab146d 100644 --- a/translators/models.py +++ b/translators/models.py @@ -11,27 +11,27 @@ class Translator(models.Model): def get_platforms(self): return [platform.name for platform in self.platform_set.all()] - + def __str__(self) -> str: return self.name + class Platform(models.Model): - name = models.CharField(max_length=250) - base_url = models.URLField() - translators = models.ManyToManyField(Translator, blank=True) - is_active = models.BooleanField(default=True) # if True, torjoman will send untranslated words to this platform - - @classmethod - def get_all_platforms(cls) -> dict[str, "Platform"]: - """get_all_platforms get a dict of all platforms. - - Returns: - dict[str, Platform]: keys are platforms names, values are platforms instances - """ - return { - f'{platform.name}': platform - for platform in Platform.objects.all() - } - - def __str__(self) -> str: - return self.name \ No newline at end of file + name = models.CharField(max_length=250) + base_url = models.URLField() + translators = models.ManyToManyField(Translator, blank=True) + is_active = models.BooleanField( + default=True + ) # if True, torjoman will send untranslated words to this platform + + @classmethod + def get_all_platforms(cls) -> dict[str, "Platform"]: + """get_all_platforms get a dict of all platforms. + + Returns: + dict[str, Platform]: keys are platforms names, values are platforms instances + """ + return {f"{platform.name}": platform for platform in Platform.objects.all()} + + def __str__(self) -> str: + return self.name diff --git a/translators/schemas.py b/translators/schemas.py index e5c9d4d..1495c3a 100644 --- a/translators/schemas.py +++ b/translators/schemas.py @@ -1,21 +1,26 @@ from typing import List from uuid import UUID from ninja import ModelSchema, Schema, Field -from .models import (Translator as TranslatorModel) +from .models import Translator as TranslatorModel + class TranslatorRegister(ModelSchema): - platform: str - class Config: - model = TranslatorModel - model_exclude = ['id'] + platform: str + + class Config: + model = TranslatorModel + model_exclude = ["id"] + class TranslatorLogin(ModelSchema): - platform: str - class Config: - model = TranslatorModel - model_fields = ['uuid'] + platform: str + + class Config: + model = TranslatorModel + model_fields = ["uuid"] + class Translator(ModelSchema): - class Config: - model = TranslatorModel - model_exclude = ['id'] \ No newline at end of file + class Config: + model = TranslatorModel + model_exclude = ["id"] diff --git a/translators/tasks.py b/translators/tasks.py index 6dabf88..2b6f84b 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -3,26 +3,24 @@ from datetime import datetime import requests + def send_words(): - now = datetime.now() - # users = Translator.objects.filter(send_time=f"{now.hour}:{now.minute}") - users = Translator.objects.all() - print(f'Sending for {users.count()} users') - for user in users: - words: list[Word] = Word.objects.exclude(translators__in=[user]).order_by('word')[:user.number_of_words] - if words.count() < 1: continue - for platform in user.platform_set.filter(is_active=True): - data = { - 'uuid': str(user.uuid), - 'words': [ - { - 'word': word.word, - 'translates': word.get_translates() - } - for word in words - ] - } - res = requests.post( - f'{platform.base_url}/get_words', - json=data - ) + now = datetime.now() + # users = Translator.objects.filter(send_time=f"{now.hour}:{now.minute}") + users = Translator.objects.all() + print(f"Sending for {users.count()} users") + for user in users: + words: list[Word] = Word.objects.exclude(translators__in=[user]).order_by( + "word" + )[: user.number_of_words] + if words.count() < 1: + continue + for platform in user.platform_set.filter(is_active=True): + data = { + "uuid": str(user.uuid), + "words": [ + {"word": word.word, "translates": word.get_translates()} + for word in words + ], + } + res = requests.post(f"{platform.base_url}/get_words", json=data) From b2cf3df7baee094385151051905f917d92dd71b3 Mon Sep 17 00:00:00 2001 From: Karam Date: Wed, 21 Sep 2022 20:03:46 +0300 Subject: [PATCH 04/24] A partial fix for the slowness in the update source function & fix some errors --- Pipfile | 1 + Pipfile.lock | 22 ++++-- github_manager/migrations/0001_initial.py | 31 ++++++++ github_manager/models.py | 2 +- github_manager/tasks.py | 55 ++++++++++++-- translation/api.py | 5 +- translation/migrations/0001_initial.py | 87 +++++++++++++++++++++++ translation/models.py | 18 +++-- translators/api.py | 2 +- translators/migrations/0001_initial.py | 65 +++++++++++++++++ 10 files changed, 262 insertions(+), 26 deletions(-) create mode 100644 github_manager/migrations/0001_initial.py create mode 100644 translation/migrations/0001_initial.py create mode 100644 translators/migrations/0001_initial.py diff --git a/Pipfile b/Pipfile index 120f94c..717758c 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,7 @@ ipaddress = "*" [dev-packages] ipython = "*" +isort = "*" [requires] python_version = "3.10" diff --git a/Pipfile.lock b/Pipfile.lock index 827c251..5e7d168 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6167afb3d520ca03f00b201f05f4b46c1b068c8f014f71f2c9eb418a306b28ed" + "sha256": "03b1210b10ff76be1bc7f09f04f2ecc2e95925a5d9347938d87263ab398d94e5" }, "pipfile-spec": 6, "requires": { @@ -21,7 +21,7 @@ "sha256:3934b30ca1b9f292376d9db15b19446088d12ec58629bc3f0da28fd55fb633a1", "sha256:5a49ab92e3b7b71d96cd6bfcc4df14efefc9dfa96ea19045815914a6ab6b1fe2" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==1.2.3" }, "asgiref": { @@ -45,7 +45,7 @@ "sha256:36973885b9542e6bd01dea287b2b4b3b21236307c56324fcc3f1160f2d655ed5", "sha256:e232343de1ab72c2aa521b625c80f699e356830fd0e2c620b465b304b17b0516" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.9.14" }, "cffi": { @@ -122,7 +122,7 @@ "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.1" }, "deprecated": { @@ -178,7 +178,7 @@ "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd", "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==4.0.9" }, "gitpython": { @@ -352,7 +352,7 @@ "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==1.5.0" }, "python-dateutil": { @@ -392,7 +392,7 @@ "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94", "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==5.0.0" }, "sqlparse": { @@ -535,6 +535,14 @@ "index": "pypi", "version": "==8.5.0" }, + "isort": { + "hashes": [ + "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", + "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" + ], + "index": "pypi", + "version": "==5.10.1" + }, "jedi": { "hashes": [ "sha256:637c9635fcf47945ceb91cd7f320234a7be540ded6f3e99a50cb6febdfd1ba8d", diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py new file mode 100644 index 0000000..11245fd --- /dev/null +++ b/github_manager/migrations/0001_initial.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.1 on 2022-09-21 05:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("translation", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="PullRequest", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("prid", models.IntegerField()), + ("words", models.ManyToManyField(blank=True, to="translation.word")), + ], + ), + ] diff --git a/github_manager/models.py b/github_manager/models.py index 18afe92..9b48c4b 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -3,4 +3,4 @@ # Create your models here. class PullRequest(models.Model): prid = models.IntegerField() - words = models.ManyToManyField("translate.Word", blank=True) + words = models.ManyToManyField("translation.Word", blank=True) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 3b5b0ec..7064b75 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -1,11 +1,11 @@ -# from translate.models import Word as TWord, Translate as TTranslate from .models import PullRequest from github import Github import json import os from pathlib import Path from django.conf import settings - +import itertools +from translation.models import Word, SourceTranslation account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) @@ -19,15 +19,56 @@ def check_for_update_json(): file = dictionary_repo.get_contents(os.environ.get("JSON_FILE")) - repo_file_json = json.loads(file.decoded_content) - old_file_json = json.loads(local_json_file.read_text()) - if repo_file_json == old_file_json: + server = json.loads(file.decoded_content) + local = json.loads(local_json_file.read_text()) + if server == local: + print("There is no change") return else: print("Files Are not the same") + update_source(local, server) with open(str(local_json_file), "w") as f: - json.dump(repo_file_json, f, ensure_ascii=False, indent=2) - update_source(repo_file_json) + json.dump(server, f, ensure_ascii=False, indent=2) + + +def generate_dict_from_db() -> list[dict]: + return [ + { + "word": w.word, + "translations": w.get_source_translations, + "is_checked": w.is_checked, + } + for w in Word.objects.all() + ] + + +def update_source(local: list[dict], server: list[dict]): + rlocal = list(itertools.filterfalse(lambda x: x in local, server)) + rserver = list(itertools.filterfalse(lambda x: x in server, local)) + deleted_words: str = [ + x["word"] for x in rserver if x["word"] not in [i["word"] for i in rlocal] + ] + for word in deleted_words: + w: Word = Word.objects.get(word["word"]) + w.delete() + for word in rlocal: + w: Word = Word.objects.get_or_create(word=word["word"])[0] + if word["translations"]: + for translation in word["translations"]: + st: SourceTranslation = SourceTranslation.objects.get_or_create( + word=w, translation=translation + )[0] + st.save() + # Reset users translations + for translation in w.usertranslation_set.all(): + translation.delete() + deleted_translations = [ + SourceTranslation.objects.get(word=w, translation=t) + for t in w.get_source_translations + if t not in word["translations"] + ] + for translation in deleted_translations: + translation.delete() def push(_payload: dict): diff --git a/translation/api.py b/translation/api.py index fa8582e..bf609be 100644 --- a/translation/api.py +++ b/translation/api.py @@ -1,9 +1,7 @@ -from django.http import Http404 from django.shortcuts import get_object_or_404 from ninja import Router from ninja.errors import AuthenticationError from translators.models import Translator - from . import schemas from .models import Word from .utils import modify_translation @@ -14,8 +12,7 @@ @router.post( "/", response={ - 200: Word, - 404: Http404, + 200: schemas.Word, }, ) def receiving_translation(request, payload: schemas.WordIn): diff --git a/translation/migrations/0001_initial.py b/translation/migrations/0001_initial.py new file mode 100644 index 0000000..74f3a9c --- /dev/null +++ b/translation/migrations/0001_initial.py @@ -0,0 +1,87 @@ +# Generated by Django 4.1.1 on 2022-09-21 05:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Word", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("word", models.CharField(max_length=255, unique=True)), + ("is_checked", models.BooleanField(default=False)), + ], + options={ + "ordering": ["word"], + }, + ), + migrations.CreateModel( + name="UserTranslation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("translation", models.CharField(max_length=255)), + ("score", models.PositiveIntegerField(default=0)), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="translation.word", + ), + ), + ], + options={ + "ordering": ("score", "translation"), + "abstract": False, + }, + ), + migrations.CreateModel( + name="SourceTranslation", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("translation", models.CharField(max_length=255)), + ("is_default", models.BooleanField(default=False)), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="translation.word", + ), + ), + ], + options={ + "ordering": ("is_default", "translation"), + "abstract": False, + }, + ), + ] diff --git a/translation/models.py b/translation/models.py index 198b756..514a8e7 100644 --- a/translation/models.py +++ b/translation/models.py @@ -9,11 +9,17 @@ class Word(models.Model): class Meta: ordering = ["word"] + @property def get_source_translations(self) -> list[str]: - return [translate.translate for translate in self.sourcetranslation_set.all()] + return [ + translation.translation for translation in self.sourcetranslation_set.all() + ] + @property def get_users_translations(self) -> list[str]: - return [translate.translate for translate in self.usertranslation_set.all()] + return [ + translation.translation for translation in self.usertranslation_set.all() + ] def add_translation(self, translation: str) -> "UserTranslation": """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. @@ -52,7 +58,7 @@ class Meta: abstract = True def __str__(self) -> str: - return f"{self.word}-{self.translate}" + return f"{self.word}-{self.translation}" class SourceTranslation(TranslationBase): @@ -61,11 +67,11 @@ class SourceTranslation(TranslationBase): def save(self, *args, **kwargs) -> None: if self.is_default: try: - last_default_translate = SourceTranslation.objects.get( + last_default_translation = SourceTranslation.objects.get( word=self.word, is_default=True ) - last_default_translate.is_default = False - last_default_translate.save() + last_default_translation.is_default = False + last_default_translation.save() except SourceTranslation.DoesNotExist: pass return super().save(*args, **kwargs) diff --git a/translators/api.py b/translators/api.py index ae3db1d..15d1627 100644 --- a/translators/api.py +++ b/translators/api.py @@ -24,7 +24,7 @@ def register(request, payload: schemas.TranslatorRegister): return 200, translator -@router.post("/login", response={200: schemas.Translator, 404: Http404}) +@router.post("/login", response={200: schemas.Translator}) def login(request, payload: schemas.TranslatorLogin): translator = get_object_or_404(Translator, uuid=payload.uuid) platforms = Platform.get_all_platforms() diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py new file mode 100644 index 0000000..cd39dc8 --- /dev/null +++ b/translators/migrations/0001_initial.py @@ -0,0 +1,65 @@ +# Generated by Django 4.1.1 on 2022-09-21 05:55 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("translation", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Translator", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField(default=uuid.uuid4, editable=False, unique=True), + ), + ("name", models.CharField(max_length=255)), + ( + "number_of_words", + models.IntegerField(verbose_name="Number of words to send"), + ), + ("send_time", models.TimeField()), + ( + "translated_words", + models.ManyToManyField(blank=True, to="translation.word"), + ), + ], + ), + migrations.CreateModel( + name="Platform", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=250)), + ("base_url", models.URLField()), + ("is_active", models.BooleanField(default=True)), + ( + "translators", + models.ManyToManyField(blank=True, to="translators.translator"), + ), + ], + ), + ] From 0d69296e0f624c0aefa8768c6fab9e6470d10534 Mon Sep 17 00:00:00 2001 From: Karam Date: Wed, 21 Sep 2022 20:14:36 +0300 Subject: [PATCH 05/24] Sort imports using isort --- github_manager/admin.py | 1 + github_manager/models.py | 1 + github_manager/tasks.py | 11 +++++++---- github_manager/webhook.py | 12 +++++++----- project/api.py | 4 ++-- project/settings.py | 1 + project/urls.py | 1 + translation/admin.py | 3 ++- translation/api.py | 2 ++ translation/migrations/0001_initial.py | 2 +- translation/models.py | 1 + translation/schemas.py | 6 ++++-- translation/utils.py | 1 + translators/admin.py | 3 ++- translators/api.py | 7 ++++--- translators/migrations/0001_initial.py | 3 ++- translators/models.py | 2 ++ translators/schemas.py | 4 +++- translators/tasks.py | 7 +++++-- 19 files changed, 49 insertions(+), 23 deletions(-) diff --git a/github_manager/admin.py b/github_manager/admin.py index 5b522f8..e09eb06 100644 --- a/github_manager/admin.py +++ b/github_manager/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from .models import PullRequest # Register your models here. diff --git a/github_manager/models.py b/github_manager/models.py index 9b48c4b..5d272d5 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -1,5 +1,6 @@ from django.db import models + # Create your models here. class PullRequest(models.Model): prid = models.IntegerField() diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 7064b75..4a5d095 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -1,11 +1,14 @@ -from .models import PullRequest -from github import Github +import itertools import json import os from pathlib import Path + from django.conf import settings -import itertools -from translation.models import Word, SourceTranslation +from github import Github + +from translation.models import SourceTranslation, Word + +from .models import PullRequest account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 424ac1e..34f369a 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -1,13 +1,15 @@ import hmac -from hashlib import sha1 import json -from ninja import Router +import threading +from hashlib import sha1 + from django.conf import settings -from django.http import HttpResponse, HttpResponseForbidden, HttpResponseServerError +from django.http import (HttpResponse, HttpResponseForbidden, + HttpResponseServerError) from django.utils.encoding import force_bytes -from .tasks import push -import threading +from ninja import Router +from .tasks import push router = Router() diff --git a/project/api.py b/project/api.py index 3aa41a9..d7a1be5 100644 --- a/project/api.py +++ b/project/api.py @@ -1,9 +1,9 @@ from django.conf import settings from ninja import NinjaAPI + +from github_manager.webhook import router as github_manager_router from translation.api import router as translate_router from translators.api import router as translators_router -from github_manager.webhook import router as github_manager_router - api = NinjaAPI( title="Torjoman Core API", diff --git a/project/settings.py b/project/settings.py index 9defceb..0f4e8c7 100644 --- a/project/settings.py +++ b/project/settings.py @@ -11,6 +11,7 @@ """ from pathlib import Path + import environ env = environ.Env() diff --git a/project/urls.py b/project/urls.py index 1d13aaa..9c3a2b3 100644 --- a/project/urls.py +++ b/project/urls.py @@ -1,5 +1,6 @@ from django.contrib import admin from django.urls import path + from .api import api urlpatterns = [ diff --git a/translation/admin.py b/translation/admin.py index f73c7df..8944e8c 100644 --- a/translation/admin.py +++ b/translation/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin -from .models import Word, UserTranslation, SourceTranslation + +from .models import SourceTranslation, UserTranslation, Word # Register your models here. diff --git a/translation/api.py b/translation/api.py index bf609be..438dad7 100644 --- a/translation/api.py +++ b/translation/api.py @@ -1,7 +1,9 @@ from django.shortcuts import get_object_or_404 from ninja import Router from ninja.errors import AuthenticationError + from translators.models import Translator + from . import schemas from .models import Word from .utils import modify_translation diff --git a/translation/migrations/0001_initial.py b/translation/migrations/0001_initial.py index 74f3a9c..2a069d2 100644 --- a/translation/migrations/0001_initial.py +++ b/translation/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 4.1.1 on 2022-09-21 05:55 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/translation/models.py b/translation/models.py index 514a8e7..98bb726 100644 --- a/translation/models.py +++ b/translation/models.py @@ -1,4 +1,5 @@ from django.db import models + from .utils import get_close_match diff --git a/translation/schemas.py b/translation/schemas.py index c787721..4874b8e 100644 --- a/translation/schemas.py +++ b/translation/schemas.py @@ -1,6 +1,8 @@ from uuid import UUID -from ninja import Schema, ModelSchema, Field -from .models import Word, SourceTranslation, UserTranslation + +from ninja import Field, ModelSchema, Schema + +from .models import SourceTranslation, UserTranslation, Word class WordIn(ModelSchema): diff --git a/translation/utils.py b/translation/utils.py index c893df9..80b8119 100644 --- a/translation/utils.py +++ b/translation/utils.py @@ -1,5 +1,6 @@ from difflib import SequenceMatcher from heapq import nlargest as _nlargest + import pyarabic.araby as araby # Language specific module diff --git a/translators/admin.py b/translators/admin.py index b201aeb..50cd49a 100644 --- a/translators/admin.py +++ b/translators/admin.py @@ -1,5 +1,6 @@ from django.contrib import admin -from .models import Translator, Platform + +from .models import Platform, Translator # Register your models here. admin.site.register(Translator) diff --git a/translators/api.py b/translators/api.py index 15d1627..7dab2d3 100644 --- a/translators/api.py +++ b/translators/api.py @@ -1,8 +1,9 @@ +from django.http import Http404 +from django.shortcuts import get_object_or_404 from ninja import Router + from . import schemas -from django.shortcuts import get_object_or_404 -from .models import Translator, Platform -from django.http import Http404 +from .models import Platform, Translator router = Router() diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py index cd39dc8..7f6655f 100644 --- a/translators/migrations/0001_initial.py +++ b/translators/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 4.1.1 on 2022-09-21 05:55 -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): diff --git a/translators/models.py b/translators/models.py index 9ab146d..5534f87 100644 --- a/translators/models.py +++ b/translators/models.py @@ -1,6 +1,8 @@ import uuid + from django.db import models + # Create your models here. class Translator(models.Model): uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) diff --git a/translators/schemas.py b/translators/schemas.py index 1495c3a..299e5c4 100644 --- a/translators/schemas.py +++ b/translators/schemas.py @@ -1,6 +1,8 @@ from typing import List from uuid import UUID -from ninja import ModelSchema, Schema, Field + +from ninja import Field, ModelSchema, Schema + from .models import Translator as TranslatorModel diff --git a/translators/tasks.py b/translators/tasks.py index 2b6f84b..ece862c 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -1,8 +1,11 @@ -from .models import Translator -from translation.models import Word from datetime import datetime + import requests +from translation.models import Word + +from .models import Translator + def send_words(): now = datetime.now() From baa099f3513dec7a165175cd942f27bbd472d8db Mon Sep 17 00:00:00 2001 From: Karam Date: Thu, 22 Sep 2022 09:12:59 +0300 Subject: [PATCH 06/24] fix UserTranslation ordering --- translation/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translation/models.py b/translation/models.py index 98bb726..7160f68 100644 --- a/translation/models.py +++ b/translation/models.py @@ -78,7 +78,7 @@ def save(self, *args, **kwargs) -> None: return super().save(*args, **kwargs) class Meta(TranslationBase.Meta): - ordering = ("is_default", "translation") + ordering = ("-is_default", "translation") class UserTranslation(TranslationBase): From fec7fdb2ef717c35ea91f08e9c8c76a7c5daee02 Mon Sep 17 00:00:00 2001 From: Karam Date: Thu, 22 Sep 2022 09:14:02 +0300 Subject: [PATCH 07/24] Add Moderator Model in github_manager --- github_manager/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/github_manager/models.py b/github_manager/models.py index 5d272d5..7440757 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -5,3 +5,15 @@ class PullRequest(models.Model): prid = models.IntegerField() words = models.ManyToManyField("translation.Word", blank=True) + +class Moderator(models.Model): + username = models.CharField(max_length=255) + + @property + def get_all_moderators(self) -> list[str]: + """get_all_moderators Return a list of moderators usernames + + Returns: + list[str]: The list of moderators usernames + """ + return [mod.username for mod in Moderator.objects.all()] \ No newline at end of file From 75a3c75711b552921f9de75ca7c063739eebede5 Mon Sep 17 00:00:00 2001 From: Karam Date: Thu, 22 Sep 2022 09:27:46 +0300 Subject: [PATCH 08/24] Clearify push function name --- github_manager/tasks.py | 2 +- github_manager/webhook.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 4a5d095..8c5311b 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -74,7 +74,7 @@ def update_source(local: list[dict], server: list[dict]): translation.delete() -def push(_payload: dict): +def handle_push(_payload: dict): print("New Push Event") check_for_update_json() print("Finished Push Event") diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 34f369a..69c0b27 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -40,7 +40,7 @@ def manage_webhooks(request): if event == "ping": return HttpResponse("pong") elif event == "push": - t = threading.Thread(target=push, args=(json.loads(request.body),)) + t = threading.Thread(target=handle_push, args=(json.loads(request.body),)) t.start() return HttpResponse("success") # In case we receive an event that's not ping or push From 98a4a940deafe795e445ed293a10fc5758403a13 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 26 Sep 2022 07:21:23 +0300 Subject: [PATCH 09/24] Edit PR & Word relationship to ForeignKey --- github_manager/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_manager/models.py b/github_manager/models.py index 7440757..03a3c38 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -4,7 +4,7 @@ # Create your models here. class PullRequest(models.Model): prid = models.IntegerField() - words = models.ManyToManyField("translation.Word", blank=True) + words = models.ForeignKey("translation.Word") class Moderator(models.Model): username = models.CharField(max_length=255) From b1ba201772bce96aa9fd1f0936a9bd7c230bf1ba Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 26 Sep 2022 07:21:58 +0300 Subject: [PATCH 10/24] Edit json example format --- json-example.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json-example.json b/json-example.json index 763730f..d16ea58 100644 --- a/json-example.json +++ b/json-example.json @@ -2,14 +2,14 @@ { "word": "Hello", "is_checked": false, - "translates": [ + "translation": [ "Hola" ] }, { "word": "Ping", "is_checked": true, - "translates": [ + "translation": [ "Pong" ] } From 4b72774905f8e26db3ae990e40bde78e3164867e Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 26 Sep 2022 08:30:08 +0300 Subject: [PATCH 11/24] Fix type annotation --- github_manager/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 8c5311b..4cff453 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -48,11 +48,11 @@ def generate_dict_from_db() -> list[dict]: def update_source(local: list[dict], server: list[dict]): rlocal = list(itertools.filterfalse(lambda x: x in local, server)) rserver = list(itertools.filterfalse(lambda x: x in server, local)) - deleted_words: str = [ + deleted_words: list[str] = [ x["word"] for x in rserver if x["word"] not in [i["word"] for i in rlocal] ] for word in deleted_words: - w: Word = Word.objects.get(word["word"]) + w: Word = Word.objects.get(word) w.delete() for word in rlocal: w: Word = Word.objects.get_or_create(word=word["word"])[0] From 0eb5b6b481bba1df2d62a4ef7e39378b073f62bf Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 26 Sep 2022 08:31:07 +0300 Subject: [PATCH 12/24] Fix deleted words error on get word objects --- github_manager/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 4cff453..b1d9d83 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -52,7 +52,7 @@ def update_source(local: list[dict], server: list[dict]): x["word"] for x in rserver if x["word"] not in [i["word"] for i in rlocal] ] for word in deleted_words: - w: Word = Word.objects.get(word) + w: Word = Word.objects.get(word=word) w.delete() for word in rlocal: w: Word = Word.objects.get_or_create(word=word["word"])[0] From 2c29dc4c295af581f144615d6bb93c84ea5c108c Mon Sep 17 00:00:00 2001 From: Karam Date: Tue, 27 Sep 2022 06:43:47 +0300 Subject: [PATCH 13/24] Fix: reverse ordering in translation --- translation/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translation/models.py b/translation/models.py index 7160f68..ec55609 100644 --- a/translation/models.py +++ b/translation/models.py @@ -85,4 +85,4 @@ class UserTranslation(TranslationBase): score = models.PositiveIntegerField(default=0) class Meta(TranslationBase.Meta): - ordering = ("score", "translation") + ordering = ("-score", "translation") From e2bcef1d240e461f62e4cb24cb9c2e4f7a9045f4 Mon Sep 17 00:00:00 2001 From: Karam Date: Fri, 30 Sep 2022 11:59:21 +0300 Subject: [PATCH 14/24] Change prid to number in PullRequest --- github_manager/migrations/0001_initial.py | 28 ++++++++++++++++++++--- github_manager/models.py | 4 ++-- translation/migrations/0001_initial.py | 8 +++---- translators/migrations/0001_initial.py | 5 ++-- 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py index 11245fd..32136b4 100644 --- a/github_manager/migrations/0001_initial.py +++ b/github_manager/migrations/0001_initial.py @@ -1,6 +1,7 @@ -# Generated by Django 4.1.1 on 2022-09-21 05:55 +# Generated by Django 4.1.1 on 2022-09-30 08:32 from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -12,6 +13,21 @@ class Migration(migrations.Migration): ] operations = [ + migrations.CreateModel( + name="Moderator", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("username", models.CharField(max_length=255)), + ], + ), migrations.CreateModel( name="PullRequest", fields=[ @@ -24,8 +40,14 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), - ("prid", models.IntegerField()), - ("words", models.ManyToManyField(blank=True, to="translation.word")), + ("number", models.IntegerField()), + ( + "word", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="translation.word", + ), + ), ], ), ] diff --git a/github_manager/models.py b/github_manager/models.py index 03a3c38..e89f02f 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -3,8 +3,8 @@ # Create your models here. class PullRequest(models.Model): - prid = models.IntegerField() - words = models.ForeignKey("translation.Word") + number = models.IntegerField() + word = models.ForeignKey("translation.Word", on_delete=models.CASCADE) class Moderator(models.Model): username = models.CharField(max_length=255) diff --git a/translation/migrations/0001_initial.py b/translation/migrations/0001_initial.py index 2a069d2..c4ae3e1 100644 --- a/translation/migrations/0001_initial.py +++ b/translation/migrations/0001_initial.py @@ -1,7 +1,7 @@ -# Generated by Django 4.1.1 on 2022-09-21 05:55 +# Generated by Django 4.1.1 on 2022-09-30 08:32 -import django.db.models.deletion from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -53,7 +53,7 @@ class Migration(migrations.Migration): ), ], options={ - "ordering": ("score", "translation"), + "ordering": ("-score", "translation"), "abstract": False, }, ), @@ -80,7 +80,7 @@ class Migration(migrations.Migration): ), ], options={ - "ordering": ("is_default", "translation"), + "ordering": ("-is_default", "translation"), "abstract": False, }, ), diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py index 7f6655f..c123e10 100644 --- a/translators/migrations/0001_initial.py +++ b/translators/migrations/0001_initial.py @@ -1,8 +1,7 @@ -# Generated by Django 4.1.1 on 2022-09-21 05:55 - -import uuid +# Generated by Django 4.1.1 on 2022-09-30 08:32 from django.db import migrations, models +import uuid class Migration(migrations.Migration): From b914a54f2c946744c349599e6115b24120715525 Mon Sep 17 00:00:00 2001 From: Karam Date: Sun, 2 Oct 2022 11:33:26 +0300 Subject: [PATCH 15/24] fix: Send words that are in pullrequest --- translators/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translators/tasks.py b/translators/tasks.py index ece862c..6472369 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -13,7 +13,7 @@ def send_words(): users = Translator.objects.all() print(f"Sending for {users.count()} users") for user in users: - words: list[Word] = Word.objects.exclude(translators__in=[user]).order_by( + words: list[Word] = Word.objects.filter(pullrequest=None).exclude(translators__in=[user]).order_by( "word" )[: user.number_of_words] if words.count() < 1: From fde545f15b2efdc6716d5aa126e504091c1f6419 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:23:38 +0300 Subject: [PATCH 16/24] Add Moderator to django admin --- github_manager/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/github_manager/admin.py b/github_manager/admin.py index e09eb06..980bf19 100644 --- a/github_manager/admin.py +++ b/github_manager/admin.py @@ -1,6 +1,7 @@ from django.contrib import admin -from .models import PullRequest +from .models import Moderator, PullRequest # Register your models here. admin.site.register(PullRequest) +admin.site.register(Moderator) From 5ca63342f79b0fb7d65ea1196ca9d3c9fbc4d70b Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:25:53 +0300 Subject: [PATCH 17/24] Fix: get_all_moderators function --- github_manager/models.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/github_manager/models.py b/github_manager/models.py index e89f02f..d40ffe2 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -9,11 +9,6 @@ class PullRequest(models.Model): class Moderator(models.Model): username = models.CharField(max_length=255) - @property - def get_all_moderators(self) -> list[str]: - """get_all_moderators Return a list of moderators usernames - - Returns: - list[str]: The list of moderators usernames - """ - return [mod.username for mod in Moderator.objects.all()] \ No newline at end of file + @classmethod + def get_all_moderators(cls) -> list[str]: + return [mod.username for mod in cls.objects.all()] \ No newline at end of file From b26b63e8b869913ffadc1f09816dd7eb65f572b6 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:28:01 +0300 Subject: [PATCH 18/24] remove localhost from ALLOWED_HOSTS --- project/settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/project/settings.py b/project/settings.py index 0f4e8c7..5d2f678 100644 --- a/project/settings.py +++ b/project/settings.py @@ -32,9 +32,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [ - "127.0.0.1", # TODO: remove me -] +ALLOWED_HOSTS = [] ALLOWED_HOSTS.append(env("HOST")) # Application definition From c48cf648b91fcf3a325483ad42cc0a4135982843 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:42:38 +0300 Subject: [PATCH 19/24] Update generate_dict_from_db Function take now word argument\nWhen loop over words in db if the word is same as the passed one, users translations will be used --- github_manager/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index b1d9d83..1bd3124 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -34,11 +34,11 @@ def check_for_update_json(): json.dump(server, f, ensure_ascii=False, indent=2) -def generate_dict_from_db() -> list[dict]: +def generate_dict_from_db(word: Word) -> list[dict]: return [ { "word": w.word, - "translations": w.get_source_translations, + "translations": w.get_users_translations if w.word == word.word else w.get_source_translations, "is_checked": w.is_checked, } for w in Word.objects.all() From b116c6666aa522219d04665b8adfd5dbf78fa80f Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:48:25 +0300 Subject: [PATCH 20/24] github_manager: Handle pull request events and add make_pull_request function --- github_manager/messages.json | 8 ++++++ github_manager/tasks.py | 54 ++++++++++++++++++++++++++++++++++-- github_manager/webhook.py | 25 +++++++++-------- 3 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 github_manager/messages.json diff --git a/github_manager/messages.json b/github_manager/messages.json new file mode 100644 index 0000000..6020ee7 --- /dev/null +++ b/github_manager/messages.json @@ -0,0 +1,8 @@ +{ + "pull_request": { + "title": "Change '{word}' Translation to '{translation}'", + "head": "'{word}' has reached the minimum translations limit\nThe New default Translation is: '{translation}'\n\nThese are the 5 most commoun translations:\n", + "other_translation": "{index}. '{translation}'", + "footer": "To change the default translation, write: /set-default NumberOfTranslation\nTo delete Specific translation, write: /delete NumberOfTranslation" + } +} \ No newline at end of file diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 1bd3124..97ab121 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -6,11 +6,13 @@ from django.conf import settings from github import Github -from translation.models import SourceTranslation, Word - -from .models import PullRequest +from translation.models import SourceTranslation, UserTranslation, Word +from django.db.models import Count +from .models import PullRequest, Moderator +import re account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) +account_id = account.get_user().id dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) local_json_file: Path = (settings.BASE_DIR / "local_dictionary.json").resolve() @@ -78,3 +80,49 @@ def handle_push(_payload: dict): print("New Push Event") check_for_update_json() print("Finished Push Event") + +def handle_pull_request(payload: dict): + # Verify whether the request is a pull request and if its owner is our account + if issue:=payload.get('issue'): + if not issue.get('pull_request'): + pass + elif pr:=payload.get('pull_request'): + if pr['user']['id'] != account_id: return + match payload['action']: + case 'closed': + PullRequest.objects.get(number=payload['number']).delete() + word: str = re.search(r"'(?P.+)'\s", payload['issue']['title']).group('word') + [ref for ref in dictionary_repo.get_git_refs() if ref.ref == f'refs/heads/Torjoman-{word.word}'][0].delete() + + +def make_pull_request(): + """make_pull_request Get Words that have at least 5 translations and make a pull request + """ + words: list[Word] = Word.objects.annotate(num_usertranslation=Count('usertranslation')).filter(num_usertranslation__gt=3, pullrequest=None) + if not words: return + with open(settings.BASE_DIR / 'github_manager' / 'messages.json') as f: + messages = json.load(f)['pull_request'] + source_branch = dictionary_repo.get_branch(dictionary_repo.default_branch) + for word in words: + try: + [ref for ref in dictionary_repo.get_git_refs() if ref.ref == f'refs/heads/Torjoman-{word.word}'][0].delete() + except: + pass + dictionary_repo.create_git_ref(f'refs/heads/Torjoman-{word.word}', source_branch.commit.sha) + f = dictionary_repo.get_contents(os.environ.get('JSON_FILE'), ref=f"Torjoman-{word.word}") + dictionary_repo.update_file( + os.environ.get('JSON_FILE'), + f'Update {word.word} Translation To {word.usertranslation_set.first()}', + json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), + f.sha, branch = f"Torjoman-{word.word}" + ) + body = messages['head'].format(word=word.word, translation=word.get_users_translations[0]) + for index, translation in enumerate(word.get_users_translations[:5]): + body += messages['other_translation'].format(index=index+1, translation=translation) + '\n' + body += '\n' + messages['footer'] + pull = dictionary_repo.create_pull( + title=messages['title'].format(word=word.word, translation=word.get_users_translations[0]), + body=body, + head=f"Torjoman-{word.word}", base='main' + ) + PullRequest.objects.create(number=pull.number, word=word) \ No newline at end of file diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 69c0b27..2c36666 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -9,7 +9,7 @@ from django.utils.encoding import force_bytes from ninja import Router -from .tasks import push +from .tasks import handle_pr_comments, handle_push, handle_pull_request router = Router() @@ -32,16 +32,17 @@ def manage_webhooks(request): ) if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)): return HttpResponseForbidden("Permission denied.") - - # If request reached this point we are in a good shape - # Process the GitHub events event = request.META.get("HTTP_X_GITHUB_EVENT", "ping") - - if event == "ping": - return HttpResponse("pong") - elif event == "push": - t = threading.Thread(target=handle_push, args=(json.loads(request.body),)) - t.start() - return HttpResponse("success") - # In case we receive an event that's not ping or push + match event: + case 'ping': + return HttpResponse("pong") + case "push": + t = threading.Thread(target=handle_push, args=(json.loads(request.body),)) + t.start() + return HttpResponse("success") + case "pull_request": + t = threading.Thread(target=handle_pull_request, args=(json.loads(request.body),)) + t.start() + return HttpResponse("success") + return HttpResponse(status=204) From 015639eebf4f252680877fafd74bfdd8d1a50ef4 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 12:49:58 +0300 Subject: [PATCH 21/24] github_manager: handle pull request comments events --- github_manager/tasks.py | 53 +++++++++++++++++++++++++++++++++++++++ github_manager/webhook.py | 5 +++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 97ab121..15393a5 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -95,6 +95,59 @@ def handle_pull_request(payload: dict): [ref for ref in dictionary_repo.get_git_refs() if ref.ref == f'refs/heads/Torjoman-{word.word}'][0].delete() +def handle_pr_comments(payload): + number = payload['issue']['number'] + if not payload['comment']['user']['login'] in Moderator.get_all_moderators(): + return + r = re.search(r'^/(?P.+) (?P\d+)$', payload['comment']['body']) + word: str = re.search(r"^'(?P.+)'\s", payload['issue']['body']).group('word') + translations: dict[str, str] = {i: t for i, t in re.findall(r"(?P\d+)\. '(?P.+?)'", payload['issue']['body'])} + print(word, translations) + match r.group('command'): + case 'set-default': + w: Word = Word.objects.get(word=word) + tr: UserTranslation = UserTranslation.objects.get( + word=w, + translation=translations[r.group('translation')] + ) + tr.score = w.usertranslation_set.first().score + 1 + tr.save() + update_pull_request(number, w) + case _: + pass + +def update_json_file(word: Word, translations: list[str]) -> str: + """update_json_file Update word translations and return json as string. + + Args: + word (Word): The word to update. + translations (list[str]): List of translations. Order is important. + + Returns: + str: updated json file content as string + """ + +def update_pull_request(number, word): + with open(settings.BASE_DIR / 'github_manager' / 'messages.json') as f: + messages = json.load(f)['pull_request'] + pull = dictionary_repo.get_pull(number) + f = dictionary_repo.get_contents(os.environ.get('JSON_FILE'), ref=f"Torjoman-{word.word}") + dictionary_repo.update_file( + os.environ.get('JSON_FILE'), + f'Update {word.word} Translation To {word.usertranslation_set.first()}', + json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), + f.sha, branch = f"Torjoman-{word.word}" + ) + body = messages['head'].format(word=word.word, translation=word.get_users_translations[0]) + for index, translation in enumerate(word.get_users_translations[:5]): + body += messages['other_translation'].format(index=index+1, translation=translation) + '\n' + body += '\n' + messages['footer'] + pull.edit( + title=messages['title'].format(word=word.word, translation=word.get_users_translations[0]), + body=body, + ) + + def make_pull_request(): """make_pull_request Get Words that have at least 5 translations and make a pull request """ diff --git a/github_manager/webhook.py b/github_manager/webhook.py index 2c36666..dca110c 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -44,5 +44,8 @@ def manage_webhooks(request): t = threading.Thread(target=handle_pull_request, args=(json.loads(request.body),)) t.start() return HttpResponse("success") - + case "issue_comment": + t = threading.Thread(target=handle_pr_comments, args=(json.loads(request.body),)) + t.start() + return HttpResponse("success") return HttpResponse(status=204) From e81fc2876b40ebd4eb05b9487fdaf6032de34342 Mon Sep 17 00:00:00 2001 From: Karam Date: Mon, 3 Oct 2022 13:44:00 +0300 Subject: [PATCH 22/24] use black & isort --- github_manager/migrations/0001_initial.py | 2 +- github_manager/models.py | 5 +- github_manager/tasks.py | 156 ++++++++++++++-------- github_manager/webhook.py | 17 ++- translation/migrations/0001_initial.py | 2 +- translators/migrations/0001_initial.py | 3 +- translators/tasks.py | 8 +- 7 files changed, 127 insertions(+), 66 deletions(-) diff --git a/github_manager/migrations/0001_initial.py b/github_manager/migrations/0001_initial.py index 32136b4..6201896 100644 --- a/github_manager/migrations/0001_initial.py +++ b/github_manager/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 4.1.1 on 2022-09-30 08:32 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/github_manager/models.py b/github_manager/models.py index d40ffe2..bfd9992 100644 --- a/github_manager/models.py +++ b/github_manager/models.py @@ -6,9 +6,10 @@ class PullRequest(models.Model): number = models.IntegerField() word = models.ForeignKey("translation.Word", on_delete=models.CASCADE) + class Moderator(models.Model): username = models.CharField(max_length=255) - + @classmethod def get_all_moderators(cls) -> list[str]: - return [mod.username for mod in cls.objects.all()] \ No newline at end of file + return [mod.username for mod in cls.objects.all()] diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 15393a5..3c4758a 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -1,16 +1,17 @@ import itertools import json import os +import re from pathlib import Path from django.conf import settings +from django.db.models import Count from github import Github from translation.models import SourceTranslation, UserTranslation, Word -from django.db.models import Count -from .models import PullRequest, Moderator -import re +from .models import Moderator, PullRequest + account = Github(os.environ.get("GITHUB_ACCOUNT_TOKEN")) account_id = account.get_user().id dictionary_repo = account.get_repo(os.environ.get("JSON_REPO")) @@ -40,7 +41,9 @@ def generate_dict_from_db(word: Word) -> list[dict]: return [ { "word": w.word, - "translations": w.get_users_translations if w.word == word.word else w.get_source_translations, + "translations": w.get_users_translations + if w.word == word.word + else w.get_source_translations, "is_checked": w.is_checked, } for w in Word.objects.all() @@ -81,34 +84,48 @@ def handle_push(_payload: dict): check_for_update_json() print("Finished Push Event") + def handle_pull_request(payload: dict): # Verify whether the request is a pull request and if its owner is our account - if issue:=payload.get('issue'): - if not issue.get('pull_request'): + if issue := payload.get("issue"): + if not issue.get("pull_request"): pass - elif pr:=payload.get('pull_request'): - if pr['user']['id'] != account_id: return - match payload['action']: - case 'closed': - PullRequest.objects.get(number=payload['number']).delete() - word: str = re.search(r"'(?P.+)'\s", payload['issue']['title']).group('word') - [ref for ref in dictionary_repo.get_git_refs() if ref.ref == f'refs/heads/Torjoman-{word.word}'][0].delete() - + elif pr := payload.get("pull_request"): + if pr["user"]["id"] != account_id: + return + match payload["action"]: + case "closed": + PullRequest.objects.get(number=payload["number"]).delete() + word: str = re.search(r"'(?P.+)'\s", payload["issue"]["title"]).group( + "word" + ) + [ + ref + for ref in dictionary_repo.get_git_refs() + if ref.ref == f"refs/heads/Torjoman-{word.word}" + ][0].delete() + def handle_pr_comments(payload): - number = payload['issue']['number'] - if not payload['comment']['user']['login'] in Moderator.get_all_moderators(): + number = payload["issue"]["number"] + if not payload["comment"]["user"]["login"] in Moderator.get_all_moderators(): return - r = re.search(r'^/(?P.+) (?P\d+)$', payload['comment']['body']) - word: str = re.search(r"^'(?P.+)'\s", payload['issue']['body']).group('word') - translations: dict[str, str] = {i: t for i, t in re.findall(r"(?P\d+)\. '(?P.+?)'", payload['issue']['body'])} + r = re.search( + r"^/(?P.+) (?P\d+)$", payload["comment"]["body"] + ) + word: str = re.search(r"^'(?P.+)'\s", payload["issue"]["body"]).group("word") + translations: dict[str, str] = { + i: t + for i, t in re.findall( + r"(?P\d+)\. '(?P.+?)'", payload["issue"]["body"] + ) + } print(word, translations) - match r.group('command'): - case 'set-default': + match r.group("command"): + case "set-default": w: Word = Word.objects.get(word=word) tr: UserTranslation = UserTranslation.objects.get( - word=w, - translation=translations[r.group('translation')] + word=w, translation=translations[r.group("translation")] ) tr.score = w.usertranslation_set.first().score + 1 tr.save() @@ -116,6 +133,7 @@ def handle_pr_comments(payload): case _: pass + def update_json_file(word: Word, translations: list[str]) -> str: """update_json_file Update word translations and return json as string. @@ -127,55 +145,89 @@ def update_json_file(word: Word, translations: list[str]) -> str: str: updated json file content as string """ + def update_pull_request(number, word): - with open(settings.BASE_DIR / 'github_manager' / 'messages.json') as f: - messages = json.load(f)['pull_request'] + with open(settings.BASE_DIR / "github_manager" / "messages.json") as f: + messages = json.load(f)["pull_request"] pull = dictionary_repo.get_pull(number) - f = dictionary_repo.get_contents(os.environ.get('JSON_FILE'), ref=f"Torjoman-{word.word}") + f = dictionary_repo.get_contents( + os.environ.get("JSON_FILE"), ref=f"Torjoman-{word.word}" + ) dictionary_repo.update_file( - os.environ.get('JSON_FILE'), - f'Update {word.word} Translation To {word.usertranslation_set.first()}', + os.environ.get("JSON_FILE"), + f"Update {word.word} Translation To {word.usertranslation_set.first()}", json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), - f.sha, branch = f"Torjoman-{word.word}" + f.sha, + branch=f"Torjoman-{word.word}", + ) + body = messages["head"].format( + word=word.word, translation=word.get_users_translations[0] ) - body = messages['head'].format(word=word.word, translation=word.get_users_translations[0]) for index, translation in enumerate(word.get_users_translations[:5]): - body += messages['other_translation'].format(index=index+1, translation=translation) + '\n' - body += '\n' + messages['footer'] + body += ( + messages["other_translation"].format( + index=index + 1, translation=translation + ) + + "\n" + ) + body += "\n" + messages["footer"] pull.edit( - title=messages['title'].format(word=word.word, translation=word.get_users_translations[0]), + title=messages["title"].format( + word=word.word, translation=word.get_users_translations[0] + ), body=body, ) - + def make_pull_request(): - """make_pull_request Get Words that have at least 5 translations and make a pull request - """ - words: list[Word] = Word.objects.annotate(num_usertranslation=Count('usertranslation')).filter(num_usertranslation__gt=3, pullrequest=None) - if not words: return - with open(settings.BASE_DIR / 'github_manager' / 'messages.json') as f: - messages = json.load(f)['pull_request'] + """make_pull_request Get Words that have at least 5 translations and make a pull request""" + words: list[Word] = Word.objects.annotate( + num_usertranslation=Count("usertranslation") + ).filter(num_usertranslation__gt=3, pullrequest=None) + if not words: + return + with open(settings.BASE_DIR / "github_manager" / "messages.json") as f: + messages = json.load(f)["pull_request"] source_branch = dictionary_repo.get_branch(dictionary_repo.default_branch) for word in words: try: - [ref for ref in dictionary_repo.get_git_refs() if ref.ref == f'refs/heads/Torjoman-{word.word}'][0].delete() + [ + ref + for ref in dictionary_repo.get_git_refs() + if ref.ref == f"refs/heads/Torjoman-{word.word}" + ][0].delete() except: pass - dictionary_repo.create_git_ref(f'refs/heads/Torjoman-{word.word}', source_branch.commit.sha) - f = dictionary_repo.get_contents(os.environ.get('JSON_FILE'), ref=f"Torjoman-{word.word}") + dictionary_repo.create_git_ref( + f"refs/heads/Torjoman-{word.word}", source_branch.commit.sha + ) + f = dictionary_repo.get_contents( + os.environ.get("JSON_FILE"), ref=f"Torjoman-{word.word}" + ) dictionary_repo.update_file( - os.environ.get('JSON_FILE'), - f'Update {word.word} Translation To {word.usertranslation_set.first()}', + os.environ.get("JSON_FILE"), + f"Update {word.word} Translation To {word.usertranslation_set.first()}", json.dumps(generate_dict_from_db(word), indent=4, ensure_ascii=False), - f.sha, branch = f"Torjoman-{word.word}" + f.sha, + branch=f"Torjoman-{word.word}", + ) + body = messages["head"].format( + word=word.word, translation=word.get_users_translations[0] ) - body = messages['head'].format(word=word.word, translation=word.get_users_translations[0]) for index, translation in enumerate(word.get_users_translations[:5]): - body += messages['other_translation'].format(index=index+1, translation=translation) + '\n' - body += '\n' + messages['footer'] + body += ( + messages["other_translation"].format( + index=index + 1, translation=translation + ) + + "\n" + ) + body += "\n" + messages["footer"] pull = dictionary_repo.create_pull( - title=messages['title'].format(word=word.word, translation=word.get_users_translations[0]), + title=messages["title"].format( + word=word.word, translation=word.get_users_translations[0] + ), body=body, - head=f"Torjoman-{word.word}", base='main' + head=f"Torjoman-{word.word}", + base="main", ) - PullRequest.objects.create(number=pull.number, word=word) \ No newline at end of file + PullRequest.objects.create(number=pull.number, word=word) diff --git a/github_manager/webhook.py b/github_manager/webhook.py index dca110c..2ae1172 100644 --- a/github_manager/webhook.py +++ b/github_manager/webhook.py @@ -4,12 +4,13 @@ from hashlib import sha1 from django.conf import settings -from django.http import (HttpResponse, HttpResponseForbidden, - HttpResponseServerError) +from django.http import ( + HttpResponse, HttpResponseForbidden, HttpResponseServerError, +) from django.utils.encoding import force_bytes from ninja import Router -from .tasks import handle_pr_comments, handle_push, handle_pull_request +from .tasks import handle_pr_comments, handle_pull_request, handle_push router = Router() @@ -34,18 +35,22 @@ def manage_webhooks(request): return HttpResponseForbidden("Permission denied.") event = request.META.get("HTTP_X_GITHUB_EVENT", "ping") match event: - case 'ping': + case "ping": return HttpResponse("pong") case "push": t = threading.Thread(target=handle_push, args=(json.loads(request.body),)) t.start() return HttpResponse("success") case "pull_request": - t = threading.Thread(target=handle_pull_request, args=(json.loads(request.body),)) + t = threading.Thread( + target=handle_pull_request, args=(json.loads(request.body),) + ) t.start() return HttpResponse("success") case "issue_comment": - t = threading.Thread(target=handle_pr_comments, args=(json.loads(request.body),)) + t = threading.Thread( + target=handle_pr_comments, args=(json.loads(request.body),) + ) t.start() return HttpResponse("success") return HttpResponse(status=204) diff --git a/translation/migrations/0001_initial.py b/translation/migrations/0001_initial.py index c4ae3e1..4d5cc6e 100644 --- a/translation/migrations/0001_initial.py +++ b/translation/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 4.1.1 on 2022-09-30 08:32 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/translators/migrations/0001_initial.py b/translators/migrations/0001_initial.py index c123e10..622cc0f 100644 --- a/translators/migrations/0001_initial.py +++ b/translators/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 4.1.1 on 2022-09-30 08:32 -from django.db import migrations, models import uuid +from django.db import migrations, models + class Migration(migrations.Migration): diff --git a/translators/tasks.py b/translators/tasks.py index 6472369..81d5b0e 100644 --- a/translators/tasks.py +++ b/translators/tasks.py @@ -13,9 +13,11 @@ def send_words(): users = Translator.objects.all() print(f"Sending for {users.count()} users") for user in users: - words: list[Word] = Word.objects.filter(pullrequest=None).exclude(translators__in=[user]).order_by( - "word" - )[: user.number_of_words] + words: list[Word] = ( + Word.objects.filter(pullrequest=None) + .exclude(translators__in=[user]) + .order_by("word")[: user.number_of_words] + ) if words.count() < 1: continue for platform in user.platform_set.filter(is_active=True): From 3824e8fd28b88ff55f547597919bb7e5f4c05679 Mon Sep 17 00:00:00 2001 From: Karam Date: Wed, 5 Oct 2022 12:15:15 +0300 Subject: [PATCH 23/24] fix: deleted words error if the local json empty --- github_manager/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_manager/tasks.py b/github_manager/tasks.py index 3c4758a..4a4f2f6 100644 --- a/github_manager/tasks.py +++ b/github_manager/tasks.py @@ -54,7 +54,7 @@ def update_source(local: list[dict], server: list[dict]): rlocal = list(itertools.filterfalse(lambda x: x in local, server)) rserver = list(itertools.filterfalse(lambda x: x in server, local)) deleted_words: list[str] = [ - x["word"] for x in rserver if x["word"] not in [i["word"] for i in rlocal] + x["word"] for x in rserver if x and x["word"] not in [i["word"] for i in rlocal] ] for word in deleted_words: w: Word = Word.objects.get(word=word) From b159417fd1b29970b590f8778f81c0e7227c545d Mon Sep 17 00:00:00 2001 From: Karam Date: Fri, 14 Oct 2022 11:43:14 +0300 Subject: [PATCH 24/24] some improvements --- translation/models.py | 11 +++-------- translators/api.py | 10 +++------- translators/models.py | 4 ++-- translators/schemas.py | 7 +++++++ 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/translation/models.py b/translation/models.py index ec55609..e7b5511 100644 --- a/translation/models.py +++ b/translation/models.py @@ -12,15 +12,11 @@ class Meta: @property def get_source_translations(self) -> list[str]: - return [ - translation.translation for translation in self.sourcetranslation_set.all() - ] + return self.sourcetranslation_set.objects.values_list("translation") @property def get_users_translations(self) -> list[str]: - return [ - translation.translation for translation in self.usertranslation_set.all() - ] + return self.usertranslation_set.objects.values_list("translation") def add_translation(self, translation: str) -> "UserTranslation": """add_translation Add new `UserTranslation` to word and increase score if similar translation was found. @@ -32,8 +28,7 @@ def add_translation(self, translation: str) -> "UserTranslation": UserTranslation: The new/similar translation object. None: if the translation was empty. """ - translation = translation.strip() - if translation == "": + if (translation := translation.strip()) == "": return match = get_close_match(translation, self.get_users_translations()) if match: diff --git a/translators/api.py b/translators/api.py index 7dab2d3..739751c 100644 --- a/translators/api.py +++ b/translators/api.py @@ -11,13 +11,9 @@ @router.post("/register", response={200: schemas.Translator}) def register(request, payload: schemas.TranslatorRegister): platforms = Platform.get_all_platforms() - if payload.platform not in list(platforms.keys()): + if payload.platform not in platforms: return (422,) - translator = Translator( - name=payload.name, - number_of_words=payload.number_of_words, - send_time=payload.send_time, - ) + translator = payload.to_model() translator.save() platform = platforms[payload.platform] platform.translators.add(translator) @@ -29,7 +25,7 @@ def register(request, payload: schemas.TranslatorRegister): def login(request, payload: schemas.TranslatorLogin): translator = get_object_or_404(Translator, uuid=payload.uuid) platforms = Platform.get_all_platforms() - if payload.platform not in list(platforms.keys()): + if payload.platform not in platforms.keys(): return (422,) platform = platforms[payload.platform] platform.translators.add(translator) diff --git a/translators/models.py b/translators/models.py index 5534f87..7edf148 100644 --- a/translators/models.py +++ b/translators/models.py @@ -12,7 +12,7 @@ class Translator(models.Model): translated_words = models.ManyToManyField("translation.Word", blank=True) def get_platforms(self): - return [platform.name for platform in self.platform_set.all()] + return self.platform_set.objects.values_list("name") def __str__(self) -> str: return self.name @@ -33,7 +33,7 @@ def get_all_platforms(cls) -> dict[str, "Platform"]: Returns: dict[str, Platform]: keys are platforms names, values are platforms instances """ - return {f"{platform.name}": platform for platform in Platform.objects.all()} + return {platform.name: platform for platform in Platform.objects.all()} def __str__(self) -> str: return self.name diff --git a/translators/schemas.py b/translators/schemas.py index 299e5c4..942760a 100644 --- a/translators/schemas.py +++ b/translators/schemas.py @@ -12,6 +12,13 @@ class TranslatorRegister(ModelSchema): class Config: model = TranslatorModel model_exclude = ["id"] + + def to_model(self) -> TranslatorModel: + return TranslatorModel.objects.create( + name=self.name, + number_of_words=self.number_of_words, + send_time=self.send_time, + ) class TranslatorLogin(ModelSchema):