gibheer/bleve
0
0
Fork 0
Code

change unicode tokenizer to use direct segmenter api

Browse Source
This commit is contained in:
Marty Schoch 2015-01-12 17:57:45 -05:00
parent 603c3af8bb
commit 0a4844f9d0
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
analysis/tokenizers/unicode/unicode.go
View File

@ -10,8 +10,6 @@
package unicode
import (
"bytes"
"github.com/blevesearch/segment"
"github.com/blevesearch/bleve/analysis"
@ -31,7 +29,7 @@ func (rt *UnicodeTokenizer) Tokenize(input []byte) analysis.TokenStream {
rv := make(analysis.TokenStream, 0)
segmenter := segment.NewWordSegmenter(bytes.NewReader(input))
segmenter := segment.NewWordSegmenterDirect(input)
start := 0
pos := 1
for segmenter.Segment() {