From c4db7ca10023c6236f49f3f6e63e9a250c8b98d1 Mon Sep 17 00:00:00 2001 From: Scott Hale Date: Thu, 3 Oct 2024 01:18:19 +0100 Subject: [PATCH] CV2-5412: no newlines in text for FastText (#458) * no newlines in text for FastText * quick test with newlines --------- Co-authored-by: computermacgyver Co-authored-by: Devin Gaffney --- app/main/lib/langid.py | 3 ++- app/test/test_langid.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/main/lib/langid.py b/app/main/lib/langid.py index 4a060582..b125f042 100644 --- a/app/main/lib/langid.py +++ b/app/main/lib/langid.py @@ -1,6 +1,7 @@ # 3rd party langid providers from flask import current_app as app import json +import re from google.cloud import translate_v2 as translate # import requests # Used for MicrosoftLangidProvider @@ -87,7 +88,7 @@ class FastTextLangidProvider: fasttext_model = fasttext.load_model("extra/fasttext_language_id/lid.176.ftz") @staticmethod def langid(text): - prediction = list(FastTextLangidProvider.fasttext_model.predict(text)) + prediction = list(FastTextLangidProvider.fasttext_model.predict(re.sub("[\n\r]"," ",text,re.MULTILINE))) # prediction is a list of tuples, e.g., [('__label__en',), array([0.22517213])] language = prediction[0][0].split("__")[-1] diff --git a/app/test/test_langid.py b/app/test/test_langid.py index 21664515..395f2031 100644 --- a/app/test/test_langid.py +++ b/app/test/test_langid.py @@ -14,6 +14,7 @@ class TestLangidBlueprint(BaseTestCase): TESTS = [ { 'fasttext': 'hi', 'cld3': 'hi', 'microsoft': 'hi', 'google': 'hi', 'text': 'नमस्ते मेरा नाम करीम है' }, + { 'fasttext': 'hi', 'cld3': 'hi', 'microsoft': 'hi', 'google': 'hi', 'text': 'नमस्ते मेरा नाम\n\n करीम है' }, { 'fasttext': None, 'cld3': 'hi-Latn', 'microsoft': 'en', 'google': ['hi', 'hi-Latn'], 'text': 'namaste mera naam Karim hai' }, { 'fasttext': 'mr', 'cld3': 'mr', 'microsoft': 'hi', 'google': 'mr', 'text': 'हॅलो माझे नाव करीम आहे' }, { 'fasttext': 'bn', 'cld3': 'bn', 'microsoft': 'bn', 'google': 'bn', 'text': 'হ্যালো আমার নাম কারিম' },