fix(textprocessing): improve English tokenizer

guo-yong-zhi · Oct 10, 2024 · 43ec72f · 43ec72f
1 parent e57c419
commit 43ec72f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/textprocessing.jl b/src/textprocessing.jl
@@ -50,7 +50,7 @@ function tokenizer(text::AbstractString, regexp=r"\w+")
     [text[i] for i in findall(regexp, text)]
 end
 
-function tokenizer_eng(text::AbstractString, regexp=r"\w[\w']*")
+function tokenizer_eng(text::AbstractString, regexp=r"\b\w+(?:'\w+)*\b")
     indices = findall(regexp, text)
     [endswith(text[i], "'s") ? text[i][1:prevind(text[i], end, 2)] : text[i] for i in indices]
 end