Merge remote-tracking branch 'origin'

meedan · Oct 3, 2024 · f871a21 · f871a21
2 parents f537582 + 4ea9708
commit f871a21
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/app/main/lib/langid.py b/app/main/lib/langid.py
@@ -1,7 +1,6 @@
 # 3rd party langid providers
 from flask import current_app as app
 import json
-import re
 
 from google.cloud import translate_v2 as translate
 # import requests # Used for MicrosoftLangidProvider
@@ -88,7 +87,7 @@ class FastTextLangidProvider:
   fasttext_model = fasttext.load_model("extra/fasttext_language_id/lid.176.ftz")
   @staticmethod
   def langid(text):
-    prediction = list(FastTextLangidProvider.fasttext_model.predict(re.sub("[\n\r]"," ",text,re.MULTILINE)))
+    prediction = list(FastTextLangidProvider.fasttext_model.predict(text.replace("\n"," ")))
     # prediction is a list of tuples, e.g., [('__label__en',), array([0.22517213])]
 
     language = prediction[0][0].split("__")[-1]