From de70e2ea5730018efedf97d55c1b9c9bf24c05f0 Mon Sep 17 00:00:00 2001 From: maeriil Date: Sat, 14 Oct 2023 00:27:01 -0400 Subject: [PATCH] layout changes --- backend/README.md | 15 ++ backend/example.py | 19 ++ backend/examples/image_conversion_example.py | 19 ++ backend/{ => src}/api.py | 0 .../core/__init__.py} | 0 backend/src/core/imagetranslation.py | 234 ++++++++++++++++++ backend/src/modules/__init__.py | 0 backend/{ => src}/modules/coreimage.py | 0 backend/{ => src}/modules/fontdetection.py | 4 +- backend/{ => src}/modules/textdetection.py | 4 +- backend/{ => src}/modules/textinsertion.py | 2 +- backend/{ => src}/modules/textsection.py | 13 +- backend/src/modules/translators/__init__.py | 0 .../modules/translators/amazon_translator.py | 0 .../modules/translators/google_translator.py | 0 .../translators/unofficial_google_trans.py | 0 .../utilities/fonts/Wild-Words-Roman.ttf | Bin backend/src/utilities/helpers/__init__.py | 0 .../utilities/helpers/imageHelpers.py | 0 .../utilities/helpers/stringHelpers.py | 0 20 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 backend/README.md create mode 100644 backend/example.py create mode 100644 backend/examples/image_conversion_example.py rename backend/{ => src}/api.py (100%) rename backend/{modules/translators/amazon_translator.py => src/core/__init__.py} (100%) create mode 100644 backend/src/core/imagetranslation.py create mode 100644 backend/src/modules/__init__.py rename backend/{ => src}/modules/coreimage.py (100%) rename backend/{ => src}/modules/fontdetection.py (97%) rename backend/{ => src}/modules/textdetection.py (97%) rename backend/{ => src}/modules/textinsertion.py (99%) rename backend/{ => src}/modules/textsection.py (98%) create mode 100644 backend/src/modules/translators/__init__.py create mode 100644 backend/src/modules/translators/amazon_translator.py rename backend/{ => src}/modules/translators/google_translator.py (100%) rename backend/{ => src}/modules/translators/unofficial_google_trans.py (100%) rename backend/{ => src}/utilities/fonts/Wild-Words-Roman.ttf (100%) create mode 100644 backend/src/utilities/helpers/__init__.py rename backend/{ => src}/utilities/helpers/imageHelpers.py (100%) rename backend/{ => src}/utilities/helpers/stringHelpers.py (100%) diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..0e6aace --- /dev/null +++ b/backend/README.md @@ -0,0 +1,15 @@ +# Aoriil Backend + +# Project Description + +# Installation + +# Example Usage + +# Dev setup + +# Change Log + +# License + +# Contributing \ No newline at end of file diff --git a/backend/example.py b/backend/example.py new file mode 100644 index 0000000..0a8de63 --- /dev/null +++ b/backend/example.py @@ -0,0 +1,19 @@ +""" + +""" + +import cv2 + +import src.core.imagetranslation as imagetranslation + + +image_path = "images/japanese4.png" +image = cv2.imread(image_path) + + +translated_image = imagetranslation.translate(image) + +cv2.imshow("Original image ", image) +cv2.imshow("Translated Image ", translated_image) +cv2.waitKey() +cv2.destroyAllWindows() diff --git a/backend/examples/image_conversion_example.py b/backend/examples/image_conversion_example.py new file mode 100644 index 0000000..e193904 --- /dev/null +++ b/backend/examples/image_conversion_example.py @@ -0,0 +1,19 @@ +""" + +""" + +import cv2 + +import backend.src.core.imagetranslation as imagetranslation + + +image_path = "images/japanese4.png" +image = cv2.imread(image_path) + + +translated_image = imagetranslation.translate(image) + +cv2.imshow("Original image ", image) +cv2.imshow("Translated Image ", translated_image) +cv2.waitKey() +cv2.destroyAllWindows() diff --git a/backend/api.py b/backend/src/api.py similarity index 100% rename from backend/api.py rename to backend/src/api.py diff --git a/backend/modules/translators/amazon_translator.py b/backend/src/core/__init__.py similarity index 100% rename from backend/modules/translators/amazon_translator.py rename to backend/src/core/__init__.py diff --git a/backend/src/core/imagetranslation.py b/backend/src/core/imagetranslation.py new file mode 100644 index 0000000..3526834 --- /dev/null +++ b/backend/src/core/imagetranslation.py @@ -0,0 +1,234 @@ +""" +This module, imagetranslation, contains the main function that will translate +the image to the destination language. + +TODO Currently, the only supported destination lang is English. We want to be +able to support multiple destination languages in future + +TODO Currently, the only supported images are of the following source languages + - japanese + - korean + - mandarin + +We want to be able to support multiple languages in future. + +""" + +import cv2 +import numpy as np +import pytesseract + +# Modules used +from src.modules.coreimage import calc_max_gap_dist, replace_image_section +from src.modules.fontdetection import calculate_font_size +from src.modules.textdetection import get_sections +from src.modules.textinsertion import manage_text +from src.modules.textsection import ( + should_merge_sections, + textsection, + parenttextsection, +) + + +# Translation APIs +from src.modules.translators.unofficial_google_trans import ( + translate_to_destination_lang, +) + + +# Helper utilities used +from src.utilities.helpers.imageHelpers import ( + crop_image, + convert_cv2_to_pil, + convert_pil_to_cv2, + calculate_box_width, + calculate_box_height, +) +from src.utilities.helpers.stringHelpers import remove_trailing_whitespace + + +# TODO: Support for multiple other source languages and destination languages +# When this is added, make sure to remove default value from src_lang but no +# need to remove default value from dest_lang +# TODO: Need to support robust error handling incase of some failures. As of +# this commit, there virtually exists no error handling... +def translate( + image: np.array, + src_lang: str = "japanese", + dest_lang: str = "english", + translation_service: str = "python-googletrans", + show_borders: bool = False, + save_image: bool = False, +) -> np.array: + """ + The image to translate its content from source lang to destination lang + + Parameters + ---------- + image : np.array + The image to translate its content + + src_lang : str, optional + The image's primary language to translate from. Default is Japanese + + dest_lang : str, optional + The language to translate all image's content to. Default is English + + translation_service : str, optional + The translation API to use. Default is python.googletrans which is free + + show_borders : bool, optional + Draws borders around detected text regions if True. Default is False + + save_image : bool, optional + Saves the translated images to the ./results folder. If the folder does + not exists, it will create one at the root folder. Default is False + + Returns + ------- + np.array + The translated image + """ + + # General config sections + # TODO: Validate the source language before we do any processing + + # TODO: Validate the destination language before we do any processing + + # TODO: Validate if the provided translation service is supported + + # TODO: Validate if proper API credentials are provided in .configs file + + # Start Processing Section + pytesseract_config = r"--oem 3 --psm 5 -l jpn_vert" + + # TODO: if language is part of vertical text, we MUST rotate it: + image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE) + + easyocr_sections = get_sections(image, source_lang=src_lang) + + text_sections = [] + for x_min, x_max, y_min, y_max in easyocr_sections: + text_sections.append(textsection("", [x_min, x_max, y_min, y_max], 12)) + + MAXIMUM_GAP_DIST = calc_max_gap_dist(image=image) + merged_sections = [] + + for section in text_sections: + if len(merged_sections) == 0: + merged_sections.append(parenttextsection(section)) + continue + + is_section_merged = False + for msection in merged_sections: + if should_merge_sections( + msection, section, MAXIMUM_GAP_DIST, image + ): + is_section_merged = True + msection.add_section(section) + break + if not is_section_merged: + merged_sections.append(parenttextsection(section)) + + destination_image = image.copy() + for section in merged_sections: + cropped_section = crop_image( + destination_image, section.start_pos, section.end_pos + ) + + # TODO: Change the way we mask the original text to remove it from + # the image. Right now, once we have the merged section, we simply + # cover it entirely by white and add a black background on the outside + # so that when we pass it to cv2.inpaint, it "attempts" to inpaint it + # However, based of examples, this doesnt look as good and thus needs + # to be revamped.... + border_size = 1 + mask = np.full( + ( + section.height - border_size * 2, + section.width - border_size * 2, + 1, + ), + 255, + np.uint8, + ) # white img + mask = cv2.copyMakeBorder( + mask, + top=border_size, + bottom=border_size, + left=border_size, + right=border_size, + borderType=cv2.BORDER_CONSTANT, + value=(0, 0, 0), + ) + processed_cropped_section = cv2.inpaint( + cropped_section, mask, 7, cv2.INPAINT_NS + ) + + replace_image_section( + destination_image, processed_cropped_section, section.start_pos + ) + + # TODO: if language is part of vertical text, we MUST rotate it: + rotate_img_height, rotate_img_width, _ = destination_image.shape + destination_image = cv2.rotate( + destination_image, cv2.ROTATE_90_COUNTERCLOCKWISE + ) + image = cv2.rotate(image, cv2.ROTATE_90_COUNTERCLOCKWISE) + + destination_image_pil = convert_cv2_to_pil(destination_image) + for section in merged_sections: + # TODO: if language is part of vertical text, we MUST rotate it + start_pos = [ + section.start_pos[1], + rotate_img_width - section.end_pos[0], + ] + end_pos = [section.end_pos[1], rotate_img_width - section.start_pos[0]] + + cropped_section = crop_image(image, start_pos, end_pos) + + # TODO: if language is part of vertical text, we must use diff config + current_text = pytesseract.image_to_string( + cropped_section, config=pytesseract_config + ) + + current_text = remove_trailing_whitespace(current_text).replace( + " ", "" + ) + + if current_text == "": + continue + + scale = round(rotate_img_height / 1000) + if scale == 0: + scale = 1 + + text_font_size = calculate_font_size( + current_text, start_pos, end_pos, scale=scale + ) + + translated_text = "" + if translation_service == "python-googletrans": + translated_text = translate_to_destination_lang( + current_text, src_lang + ) + box = [ + start_pos, + [end_pos[0], start_pos[1]], + end_pos, + [start_pos[0], end_pos[1]], + ] + box_width = calculate_box_width(box) + box_height = calculate_box_height(box) + + destination_image_pil = manage_text( + destination_image_pil, + translated_text, + start_pos, + box_width, + box_height, + text_font_size, + ) + + destination_image = convert_pil_to_cv2(destination_image_pil) + return destination_image diff --git a/backend/src/modules/__init__.py b/backend/src/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/modules/coreimage.py b/backend/src/modules/coreimage.py similarity index 100% rename from backend/modules/coreimage.py rename to backend/src/modules/coreimage.py diff --git a/backend/modules/fontdetection.py b/backend/src/modules/fontdetection.py similarity index 97% rename from backend/modules/fontdetection.py rename to backend/src/modules/fontdetection.py index b411f08..34cea45 100644 --- a/backend/modules/fontdetection.py +++ b/backend/src/modules/fontdetection.py @@ -22,7 +22,7 @@ # TODO: We want to specify paths to font file outside of this file, in a # different location. This is an low priority enhancement -DEFAULT_FONT_PATH = "utilities/fonts/Wild-Words-Roman.ttf" +DEFAULT_FONT_PATH = "src/utilities/fonts/Wild-Words-Roman.ttf" # TODO: What should be the default font size used on most mangas? Is 12 @@ -122,7 +122,7 @@ def calculate_font_size( font_path = DEFAULT_FONT_PATH if font_name == "wild-words": - font_path = "utilities/fonts/Wild-Words-Roman.ttf" + font_path = "src/utilities/fonts/Wild-Words-Roman.ttf" left = MINIMUM_FONT_SIZE * scale right = MAXIMUM_FONT_SIZE * scale diff --git a/backend/modules/textdetection.py b/backend/src/modules/textdetection.py similarity index 97% rename from backend/modules/textdetection.py rename to backend/src/modules/textdetection.py index 5234a19..a4a0f1a 100644 --- a/backend/modules/textdetection.py +++ b/backend/src/modules/textdetection.py @@ -9,7 +9,7 @@ import numpy as np import easyocr -import utilities.helpers.imageHelpers as imgutil +from src.utilities.helpers.imageHelpers import unpack_box # TODO: Should this be initalized here or on the main file? Find better place @@ -46,7 +46,7 @@ def draw_text_borders(image: np.array, borders_list: list) -> np.array: border_pixel = 1 for box, _, _ in borders_list: - (tl, _, br, _) = imgutil.unpack_box(box) + (tl, _, br, _) = unpack_box(box) cv2.rectangle( image, tl, br, color=border_color, thickness=border_pixel diff --git a/backend/modules/textinsertion.py b/backend/src/modules/textinsertion.py similarity index 99% rename from backend/modules/textinsertion.py rename to backend/src/modules/textinsertion.py index 62ee6bf..cbaf4aa 100644 --- a/backend/modules/textinsertion.py +++ b/backend/src/modules/textinsertion.py @@ -309,7 +309,7 @@ def manage_text( box_width: int, box_height: int, font_size: int = 12, - font_path: str = "utilities/fonts/Wild-Words-Roman.ttf", + font_path: str = "src/utilities/fonts/Wild-Words-Roman.ttf", padding: int = 2, line_height: int = LINE_HEIGHT_DEFAULT_VAL, vertical_text: bool = False, diff --git a/backend/modules/textsection.py b/backend/src/modules/textsection.py similarity index 98% rename from backend/modules/textsection.py rename to backend/src/modules/textsection.py index 2e9125b..81d3d73 100644 --- a/backend/modules/textsection.py +++ b/backend/src/modules/textsection.py @@ -9,8 +9,7 @@ """ -import utilities.helpers.imageHelpers as imgutil - +from src.utilities.helpers.imageHelpers import unpack_box import cv2 import math import numpy as np @@ -603,9 +602,7 @@ def is_section_overlap(section1: list, section2: list) -> bool: True if a section overlaps another, False otherwise """ - section1_tl, section1_tr, section1_br, section1_bl = imgutil.unpack_box( - section1 - ) + section1_tl, section1_tr, section1_br, section1_bl = unpack_box(section1) is_inside_section2 = ( is_point_in_rectangle(section1_tl, section2) @@ -617,9 +614,7 @@ def is_section_overlap(section1: list, section2: list) -> bool: if is_inside_section2: return True - section2_tl, section2_tr, section2_br, section2_bl = imgutil.unpack_box( - section2 - ) + section2_tl, section2_tr, section2_br, section2_bl = unpack_box(section2) is_inside_section1 = ( is_point_in_rectangle(section2_tl, section1) @@ -652,7 +647,7 @@ def is_point_in_rectangle(point: list, rectangle: list) -> bool: True if a point lies in the rectangle, False otherwise """ - (tl, _, br, _) = imgutil.unpack_box(rectangle) + (tl, _, br, _) = unpack_box(rectangle) x1, y1, x2, y2 = tl[0], tl[1], br[0], br[1] x, y = point[0], point[1] diff --git a/backend/src/modules/translators/__init__.py b/backend/src/modules/translators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/modules/translators/amazon_translator.py b/backend/src/modules/translators/amazon_translator.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/modules/translators/google_translator.py b/backend/src/modules/translators/google_translator.py similarity index 100% rename from backend/modules/translators/google_translator.py rename to backend/src/modules/translators/google_translator.py diff --git a/backend/modules/translators/unofficial_google_trans.py b/backend/src/modules/translators/unofficial_google_trans.py similarity index 100% rename from backend/modules/translators/unofficial_google_trans.py rename to backend/src/modules/translators/unofficial_google_trans.py diff --git a/backend/utilities/fonts/Wild-Words-Roman.ttf b/backend/src/utilities/fonts/Wild-Words-Roman.ttf similarity index 100% rename from backend/utilities/fonts/Wild-Words-Roman.ttf rename to backend/src/utilities/fonts/Wild-Words-Roman.ttf diff --git a/backend/src/utilities/helpers/__init__.py b/backend/src/utilities/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/utilities/helpers/imageHelpers.py b/backend/src/utilities/helpers/imageHelpers.py similarity index 100% rename from backend/utilities/helpers/imageHelpers.py rename to backend/src/utilities/helpers/imageHelpers.py diff --git a/backend/utilities/helpers/stringHelpers.py b/backend/src/utilities/helpers/stringHelpers.py similarity index 100% rename from backend/utilities/helpers/stringHelpers.py rename to backend/src/utilities/helpers/stringHelpers.py