gibheer/bleve
0
0
Fork 0
Code

Merge pull request #238 from ikawaha/ja-morph-analyzer

Browse Source 
fix compliation with the latest changes to kagome
This commit is contained in:
Marty Schoch 2015-09-28 17:05:46 -04:00
commit c3a4fab911
1 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
analysis/language/ja/ja_morph_kagome.go
View File

@ -13,23 +13,27 @@ import (
"github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/registry"
"github.com/ikawaha/kagome" "github.com/ikawaha/kagome/tokenizer"
) )
const TokenizerName = "kagome" const TokenizerName = "kagome"
type KagomeMorphTokenizer struct { type KagomeMorphTokenizer struct {
tok *kagome.Tokenizer tok tokenizer.Tokenizer
}
func init() {
_ = tokenizer.SysDic() // prepare system dictionary
} }
func NewKagomeMorphTokenizer() *KagomeMorphTokenizer { func NewKagomeMorphTokenizer() *KagomeMorphTokenizer {
return &KagomeMorphTokenizer{ return &KagomeMorphTokenizer{
tok: kagome.NewTokenizer(), tok: tokenizer.New(),
} }
} }
func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTokenizer { func NewKagomeMorphTokenizerWithUserDic(userdic tokenizer.UserDic) *KagomeMorphTokenizer {
k := kagome.NewTokenizer() k := tokenizer.New()
k.SetUserDic(userdic) k.SetUserDic(userdic)
return &KagomeMorphTokenizer{ return &KagomeMorphTokenizer{
tok: k, tok: k,
@ -38,7 +42,7 @@ func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTok
func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream { func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream {
var ( var (
morphs []kagome.Token morphs []tokenizer.Token
prevstart int prevstart int
) )
@ -47,7 +51,7 @@ func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream {
return rv return rv
} }
morphs = t.tok.Tokenize(string(input)) morphs = t.tok.Analyze(string(input), tokenizer.Search)
for i, m := range morphs { for i, m := range morphs {
if m.Surface == "EOS" || m.Surface == "BOS" { if m.Surface == "EOS" || m.Surface == "BOS" {