changed whitespace tokenizer to work better on cjk input
now it will return each cjk character as a separate token this will pair well with a cjk bigram filter for indexing
This commit is contained in:
parent
933d99c576
commit
6b4c86b35a
@ -12,6 +12,7 @@ package regexp_tokenizer
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/analysis"
|
"github.com/blevesearch/bleve/analysis"
|
||||||
"github.com/blevesearch/bleve/registry"
|
"github.com/blevesearch/bleve/registry"
|
||||||
@ -19,6 +20,8 @@ import (
|
|||||||
|
|
||||||
const Name = "regexp"
|
const Name = "regexp"
|
||||||
|
|
||||||
|
var IdeographRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}`)
|
||||||
|
|
||||||
type RegexpTokenizer struct {
|
type RegexpTokenizer struct {
|
||||||
r *regexp.Regexp
|
r *regexp.Regexp
|
||||||
}
|
}
|
||||||
@ -33,12 +36,13 @@ func (rt *RegexpTokenizer) Tokenize(input []byte) analysis.TokenStream {
|
|||||||
matches := rt.r.FindAllIndex(input, -1)
|
matches := rt.r.FindAllIndex(input, -1)
|
||||||
rv := make(analysis.TokenStream, len(matches))
|
rv := make(analysis.TokenStream, len(matches))
|
||||||
for i, match := range matches {
|
for i, match := range matches {
|
||||||
|
matchBytes := input[match[0]:match[1]]
|
||||||
token := analysis.Token{
|
token := analysis.Token{
|
||||||
Term: input[match[0]:match[1]],
|
Term: matchBytes,
|
||||||
Start: match[0],
|
Start: match[0],
|
||||||
End: match[1],
|
End: match[1],
|
||||||
Position: i + 1,
|
Position: i + 1,
|
||||||
Type: analysis.AlphaNumeric,
|
Type: detectTokenType(matchBytes),
|
||||||
}
|
}
|
||||||
rv[i] = &token
|
rv[i] = &token
|
||||||
}
|
}
|
||||||
@ -60,3 +64,14 @@ func RegexpTokenizerConstructor(config map[string]interface{}, cache *registry.C
|
|||||||
func init() {
|
func init() {
|
||||||
registry.RegisterTokenizer(Name, RegexpTokenizerConstructor)
|
registry.RegisterTokenizer(Name, RegexpTokenizerConstructor)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func detectTokenType(termBytes []byte) analysis.TokenType {
|
||||||
|
if IdeographRegexp.Match(termBytes) {
|
||||||
|
return analysis.Ideographic
|
||||||
|
}
|
||||||
|
_, err := strconv.ParseFloat(string(termBytes), 64)
|
||||||
|
if err == nil {
|
||||||
|
return analysis.Numeric
|
||||||
|
}
|
||||||
|
return analysis.AlphaNumeric
|
||||||
|
}
|
||||||
|
@ -19,7 +19,7 @@ import (
|
|||||||
|
|
||||||
func TestBoundary(t *testing.T) {
|
func TestBoundary(t *testing.T) {
|
||||||
|
|
||||||
wordRegex := regexp.MustCompile(`\w+`)
|
wordRegex := regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}|\w+`)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
input []byte
|
input []byte
|
||||||
@ -44,6 +44,60 @@ func TestBoundary(t *testing.T) {
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
[]byte("こんにちは世界"),
|
||||||
|
analysis.TokenStream{
|
||||||
|
{
|
||||||
|
Start: 0,
|
||||||
|
End: 3,
|
||||||
|
Term: []byte("こ"),
|
||||||
|
Position: 1,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 3,
|
||||||
|
End: 6,
|
||||||
|
Term: []byte("ん"),
|
||||||
|
Position: 2,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 6,
|
||||||
|
End: 9,
|
||||||
|
Term: []byte("に"),
|
||||||
|
Position: 3,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 9,
|
||||||
|
End: 12,
|
||||||
|
Term: []byte("ち"),
|
||||||
|
Position: 4,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 12,
|
||||||
|
End: 15,
|
||||||
|
Term: []byte("は"),
|
||||||
|
Position: 5,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 15,
|
||||||
|
End: 18,
|
||||||
|
Term: []byte("世"),
|
||||||
|
Position: 6,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Start: 18,
|
||||||
|
End: 21,
|
||||||
|
Term: []byte("界"),
|
||||||
|
Position: 7,
|
||||||
|
Type: analysis.Ideographic,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
[]byte(""),
|
[]byte(""),
|
||||||
analysis.TokenStream{},
|
analysis.TokenStream{},
|
||||||
|
@ -19,7 +19,7 @@ import (
|
|||||||
|
|
||||||
const Name = "whitespace"
|
const Name = "whitespace"
|
||||||
|
|
||||||
var whitespaceTokenizerRegexp = regexp.MustCompile(`\w+`)
|
var whitespaceTokenizerRegexp = regexp.MustCompile(`\p{Han}|\p{Hangul}|\p{Hiragana}|\p{Katakana}|\w+`)
|
||||||
|
|
||||||
func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
|
func TokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
|
||||||
return regexp_tokenizer.NewRegexpTokenizer(whitespaceTokenizerRegexp), nil
|
return regexp_tokenizer.NewRegexpTokenizer(whitespaceTokenizerRegexp), nil
|
||||||
|
@ -37,7 +37,7 @@ type Token struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *Token) String() string {
|
func (t *Token) String() string {
|
||||||
return fmt.Sprintf("Start: %d End: %d Position: %d Token: %s", t.Start, t.End, t.Position, string(t.Term))
|
return fmt.Sprintf("Start: %d End: %d Position: %d Token: %s Type: %d", t.Start, t.End, t.Position, string(t.Term), t.Type)
|
||||||
}
|
}
|
||||||
|
|
||||||
type TokenStream []*Token
|
type TokenStream []*Token
|
||||||
|
Loading…
Reference in New Issue
Block a user