Merge pull request #524 from mschoch/fix523
fix edge ngram output when side=Back and input token len=max
This commit is contained in:
commit
d48d2b6c68
|
@ -55,7 +55,7 @@ func (s *EdgeNgramFilter) Filter(input analysis.TokenStream) analysis.TokenStrea
|
||||||
// index of the starting rune for this token
|
// index of the starting rune for this token
|
||||||
for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {
|
for ngramSize := s.minLength; ngramSize <= s.maxLength; ngramSize++ {
|
||||||
// build an ngram of this size starting at i
|
// build an ngram of this size starting at i
|
||||||
if i-ngramSize > 0 {
|
if i-ngramSize >= 0 {
|
||||||
ngramTerm := analysis.BuildTermFromRunes(runes[i-ngramSize : i])
|
ngramTerm := analysis.BuildTermFromRunes(runes[i-ngramSize : i])
|
||||||
token := analysis.Token{
|
token := analysis.Token{
|
||||||
Position: token.Position,
|
Position: token.Position,
|
||||||
|
|
|
@ -135,6 +135,48 @@ func TestEdgeNgramFilter(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
side: BACK,
|
||||||
|
min: 3,
|
||||||
|
max: 5,
|
||||||
|
input: analysis.TokenStream{
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Beryl"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
output: analysis.TokenStream{
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("ryl"),
|
||||||
|
},
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("eryl"),
|
||||||
|
},
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Beryl"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
side: FRONT,
|
||||||
|
min: 3,
|
||||||
|
max: 5,
|
||||||
|
input: analysis.TokenStream{
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Beryl"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
output: analysis.TokenStream{
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Ber"),
|
||||||
|
},
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Bery"),
|
||||||
|
},
|
||||||
|
&analysis.Token{
|
||||||
|
Term: []byte("Beryl"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
|
|
Loading…
Reference in New Issue
Block a user