From d3aaa1fa6796c44aa7f2e857bbffc2f3aa2ecc51 Mon Sep 17 00:00:00 2001
From: huali <huali@juanpi.com>
Date: Thu, 7 Jan 2016 17:26:31 +0800
Subject: [PATCH] =?UTF-8?q?*=20=E6=90=9C=E7=B4=A2=E5=BC=95=E6=93=8E?=
 =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A0=B9=E6=8D=AE=E5=86=85=E5=AE=B9=E8=8E=B7?=
 =?UTF-8?q?=E5=8F=96=E5=88=86=E8=AF=8D=E7=BB=93=E6=9E=9C=E6=8E=A5=E5=8F=A3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 engine/engine.go | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/engine/engine.go b/engine/engine.go
index 509ffa4..db7d9ed 100644
--- a/engine/engine.go
+++ b/engine/engine.go
@@ -277,6 +277,19 @@ func (engine *Engine) FlushIndex() {
 	}
 }
 
+// 获取文本的分词结果
+func (engine *Engine) Tokens(text []byte) (tokens []string) {
+	querySegments := engine.segmenter.Segment(text)
+	for _, s := range querySegments {
+		token := s.Token().Text()
+		if !engine.stopTokens.IsStopToken(token) {
+			tokens = append(tokens, token)
+		}
+	}
+
+	return tokens
+}
+
 // 查找满足搜索条件的文档，此函数线程安全
 func (engine *Engine) Search(request types.SearchRequest) (output types.SearchResponse) {
 	if !engine.initialized {
@@ -296,13 +309,7 @@ func (engine *Engine) Search(request types.SearchRequest) (output types.SearchRe
 	// 收集关键词
 	tokens := []string{}
 	if request.Text != "" {
-		querySegments := engine.segmenter.Segment([]byte(request.Text))
-		for _, s := range querySegments {
-			token := s.Token().Text()
-			if !engine.stopTokens.IsStopToken(token) {
-				tokens = append(tokens, s.Token().Text())
-			}
-		}
+		tokens = engine.Tokens([]byte(request.Text))
 	} else {
 		for _, t := range request.Tokens {
 			tokens = append(tokens, t)