3eb63a887b
stop words can be loaded from files/bytes, closes #19 stop words loaded for large list of languages, closes #20 defined language specific analyzers for as much as possible right now, closes #21 opened new issues for some of the remaining gaps
54 lines
1.6 KiB
Go
54 lines
1.6 KiB
Go
// Copyright (c) 2014 Couchbase, Inc.
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
|
|
// except in compliance with the License. You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
// Unless required by applicable law or agreed to in writing, software distributed under the
|
|
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
|
// either express or implied. See the License for the specific language governing permissions
|
|
// and limitations under the License.
|
|
package stop_words_filter
|
|
|
|
import (
|
|
"github.com/couchbaselabs/bleve/analysis"
|
|
)
|
|
|
|
// var DEFAULT_STOP_WORDS []string = []string{
|
|
// "a", "an", "and", "are", "as", "at", "be", "but", "by",
|
|
// "for", "if", "in", "into", "is", "it",
|
|
// "no", "not", "of", "on", "or", "such",
|
|
// "that", "the", "their", "then", "there", "these",
|
|
// "they", "this", "to", "was", "will", "with",
|
|
// }
|
|
|
|
type StopWordsFilter struct {
|
|
stopWords StopWordsMap
|
|
}
|
|
|
|
func NewStopWordsFilter(stopWordsMap StopWordsMap) *StopWordsFilter {
|
|
return &StopWordsFilter{
|
|
stopWords: stopWordsMap,
|
|
}
|
|
}
|
|
|
|
func (f *StopWordsFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
|
|
rv := make(analysis.TokenStream, 0)
|
|
|
|
for _, token := range input {
|
|
word := string(token.Term)
|
|
_, isStopWord := f.stopWords[word]
|
|
if !isStopWord {
|
|
rv = append(rv, token)
|
|
}
|
|
}
|
|
|
|
return rv
|
|
}
|
|
|
|
// func buildStopWordMap(words []string) map[string]bool {
|
|
// rv := make(map[string]bool, len(words))
|
|
// for _, word := range words {
|
|
// rv[word] = true
|
|
// }
|
|
// return rv
|
|
// }
|