no newlines in text for FastText

meedan · Oct 2, 2024 · 61963fd · 61963fd
1 parent c7fcdb2
commit 61963fd
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/app/main/lib/langid.py b/app/main/lib/langid.py
@@ -1,6 +1,7 @@
 # 3rd party langid providers
 from flask import current_app as app
 import json
+import re
 
 from google.cloud import translate_v2 as translate
 # import requests # Used for MicrosoftLangidProvider
@@ -87,7 +88,7 @@ class FastTextLangidProvider:
   fasttext_model = fasttext.load_model("extra/fasttext_language_id/lid.176.ftz")
   @staticmethod
   def langid(text):
-    prediction = list(FastTextLangidProvider.fasttext_model.predict(text))
+    prediction = list(FastTextLangidProvider.fasttext_model.predict(re.sub("[\n\r]"," ",text,re.MULTILINE)))
     # prediction is a list of tuples, e.g., [('__label__en',), array([0.22517213])]
 
     language = prediction[0][0].split("__")[-1]