gibheer
/
bleve
0
0
Fork 0
Code

change registry cache implementation to allow concurrent use 

previously we just used a Go builtin map
this was not safe for concurrent read/write and upon upgrading
to Go 1.6 we were notified of the problem

fixes #349
This commit is contained in:
Marty Schoch 2016-03-07 16:05:34 -05:00
parent e51f4d5450
commit e00577f265
12 changed files with 369 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
index_test.go
View File

@ -1496,3 +1496,15 @@ func TestSearchTimeout(t *testing.T) {
t.Fatalf("exected %v, got: %v", context.Canceled, err)
}
}
// TestConfigCache exposes a concurrent map write with go 1.6
func TestConfigCache(t *testing.T) {
for i := 0; i < 100; i++ {
go func() {
_, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
if err != nil {
t.Error(err)
}
}()
}
}

49
registry/analyzer.go
View File

@ -25,40 +25,47 @@ func RegisterAnalyzer(name string, constructor AnalyzerConstructor) {
type AnalyzerConstructor func(config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error)
type AnalyzerRegistry map[string]AnalyzerConstructor
type AnalyzerCache map[string]*analysis.Analyzer
func (c AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (*analysis.Analyzer, error) {
analyzer, cached := c[name]
if cached {
return analyzer, nil
type AnalyzerCache struct {
*ConcurrentCache
}
func NewAnalyzerCache() *AnalyzerCache {
return &AnalyzerCache{
NewConcurrentCache(),
}
analyzerConstructor, registered := analyzers[name]
}
func AnalyzerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := analyzers[name]
if !registered {
return nil, fmt.Errorf("no analyzer with name or type '%s' registered", name)
}
analyzer, err := analyzerConstructor(nil, cache)
analyzer, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building analyzer: %v", err)
}
c[name] = analyzer
return analyzer, nil
}
func (c AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("analyzer named '%s' already defined", name)
}
analyzerConstructor, registered := analyzers[typ]
if !registered {
return nil, fmt.Errorf("no analyzer type '%s' registered", typ)
}
analyzer, err := analyzerConstructor(config, cache)
func (c *AnalyzerCache) AnalyzerNamed(name string, cache *Cache) (*analysis.Analyzer, error) {
item, err := c.ItemNamed(name, cache, AnalyzerBuild)
if err != nil {
return nil, fmt.Errorf("error building analyzer: %v", err)
return nil, err
}
c[name] = analyzer
return analyzer, nil
return item.(*analysis.Analyzer), nil
}
func (c *AnalyzerCache) DefineAnalyzer(name string, typ string, config map[string]interface{}, cache *Cache) (*analysis.Analyzer, error) {
item, err := c.DefineItem(name, typ, config, cache, AnalyzerBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("analyzer named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(*analysis.Analyzer), nil
}
func AnalyzerTypesAndInstances() ([]string, []string) {

82
registry/cache.go Normal file
View File

@ -0,0 +1,82 @@
// Copyright (c) 2016 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package registry
import (
"fmt"
"sync"
)
var ErrAlreadyDefined = fmt.Errorf("item already defined")
type CacheBuild func(name string, config map[string]interface{}, cache *Cache) (interface{}, error)
type ConcurrentCache struct {
mutex sync.RWMutex
data map[string]interface{}
}
func NewConcurrentCache() *ConcurrentCache {
return &ConcurrentCache{
data: make(map[string]interface{}),
}
}
func (c *ConcurrentCache) ItemNamed(name string, cache *Cache, build CacheBuild) (interface{}, error) {
c.mutex.RLock()
item, cached := c.data[name]
if cached {
c.mutex.RUnlock()
return item, nil
}
// give up read lock
c.mutex.RUnlock()
// try to build it
newItem, err := build(name, nil, cache)
if err != nil {
return nil, err
}
// acquire write lock
c.mutex.Lock()
defer c.mutex.Unlock()
// check again because it could have been created while trading locks
item, cached = c.data[name]
if cached {
return item, nil
}
c.data[name] = newItem
return newItem, nil
}
func (c *ConcurrentCache) DefineItem(name string, typ string, config map[string]interface{}, cache *Cache, build CacheBuild) (interface{}, error) {
c.mutex.RLock()
_, cached := c.data[name]
if cached {
c.mutex.RUnlock()
return nil, ErrAlreadyDefined
}
// give up read lock so others lookups can proceed
c.mutex.RUnlock()
// really not there, try to build it
newItem, err := build(typ, config, cache)
if err != nil {
return nil, err
}
// now we've built it, acquire lock
c.mutex.Lock()
defer c.mutex.Unlock()
// check again because it could have been created while trading locks
_, cached = c.data[name]
if cached {
return nil, ErrAlreadyDefined
}
c.data[name] = newItem
return newItem, nil
}

49
registry/char_filter.go
View File

@ -25,40 +25,47 @@ func RegisterCharFilter(name string, constructor CharFilterConstructor) {
type CharFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.CharFilter, error)
type CharFilterRegistry map[string]CharFilterConstructor
type CharFilterCache map[string]analysis.CharFilter
func (c CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) {
charFilter, cached := c[name]
if cached {
return charFilter, nil
type CharFilterCache struct {
*ConcurrentCache
}
func NewCharFilterCache() *CharFilterCache {
return &CharFilterCache{
NewConcurrentCache(),
}
charFilterConstructor, registered := charFilters[name]
}
func CharFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := charFilters[name]
if !registered {
return nil, fmt.Errorf("no char filter with name or type '%s' registered", name)
}
charFilter, err := charFilterConstructor(nil, cache)
charFilter, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building char filter: %v", err)
}
c[name] = charFilter
return charFilter, nil
}
func (c CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("char filter named '%s' already defined", name)
}
charFilterConstructor, registered := charFilters[typ]
if !registered {
return nil, fmt.Errorf("no char filter type '%s' registered", typ)
}
charFilter, err := charFilterConstructor(config, cache)
func (c *CharFilterCache) CharFilterNamed(name string, cache *Cache) (analysis.CharFilter, error) {
item, err := c.ItemNamed(name, cache, CharFilterBuild)
if err != nil {
return nil, fmt.Errorf("error building char filter: %v", err)
return nil, err
}
c[name] = charFilter
return charFilter, nil
return item.(analysis.CharFilter), nil
}
func (c *CharFilterCache) DefineCharFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.CharFilter, error) {
item, err := c.DefineItem(name, typ, config, cache, CharFilterBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("char filter named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(analysis.CharFilter), nil
}
func CharFilterTypesAndInstances() ([]string, []string) {

57
registry/datetime_parser.go
View File

@ -25,42 +25,49 @@ func RegisterDateTimeParser(name string, constructor DateTimeParserConstructor)
type DateTimeParserConstructor func(config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error)
type DateTimeParserRegistry map[string]DateTimeParserConstructor
type DateTimeParserCache map[string]analysis.DateTimeParser
func (c DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) {
dateTimeParser, cached := c[name]
if cached {
return dateTimeParser, nil
type DateTimeParserCache struct {
*ConcurrentCache
}
func NewDateTimeParserCache() *DateTimeParserCache {
return &DateTimeParserCache{
NewConcurrentCache(),
}
dateTimeParserConstructor, registered := dateTimeParsers[name]
}
func DateTimeParserBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := dateTimeParsers[name]
if !registered {
return nil, fmt.Errorf("no date time parser with name or type '%s' registered", name)
}
dateTimeParser, err := dateTimeParserConstructor(nil, cache)
if err != nil {
return nil, fmt.Errorf("error building date time parse: %v", err)
}
c[name] = dateTimeParser
return dateTimeParser, nil
}
func (c DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("date time parser named '%s' already defined", name)
}
dateTimeParserConstructor, registered := dateTimeParsers[typ]
if !registered {
return nil, fmt.Errorf("no date time parser type '%s' registered", typ)
}
dateTimeParser, err := dateTimeParserConstructor(config, cache)
dateTimeParser, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building date time parser: %v", err)
}
c[name] = dateTimeParser
return dateTimeParser, nil
}
func (c *DateTimeParserCache) DateTimeParserNamed(name string, cache *Cache) (analysis.DateTimeParser, error) {
item, err := c.ItemNamed(name, cache, DateTimeParserBuild)
if err != nil {
return nil, err
}
return item.(analysis.DateTimeParser), nil
}
func (c *DateTimeParserCache) DefineDateTimeParser(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.DateTimeParser, error) {
item, err := c.DefineItem(name, typ, config, cache, DateTimeParserBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("date time parser named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(analysis.DateTimeParser), nil
}
func DateTimeParserTypesAndInstances() ([]string, []string) {
emptyConfig := map[string]interface{}{}
emptyCache := NewCache()

49
registry/fragment_formatter.go
View File

@ -25,40 +25,47 @@ func RegisterFragmentFormatter(name string, constructor FragmentFormatterConstru
type FragmentFormatterConstructor func(config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error)
type FragmentFormatterRegistry map[string]FragmentFormatterConstructor
type FragmentFormatterCache map[string]highlight.FragmentFormatter
func (c FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) {
fragmentFormatter, cached := c[name]
if cached {
return fragmentFormatter, nil
type FragmentFormatterCache struct {
*ConcurrentCache
}
func NewFragmentFormatterCache() *FragmentFormatterCache {
return &FragmentFormatterCache{
NewConcurrentCache(),
}
fragmentFormatterConstructor, registered := fragmentFormatters[name]
}
func FragmentFormatterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := fragmentFormatters[name]
if !registered {
return nil, fmt.Errorf("no fragment formatter with name or type '%s' registered", name)
}
fragmentFormatter, err := fragmentFormatterConstructor(nil, cache)
fragmentFormatter, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building fragment formatter: %v", err)
}
c[name] = fragmentFormatter
return fragmentFormatter, nil
}
func (c FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("fragment formatter named '%s' already defined", name)
}
fragmentFormatterConstructor, registered := fragmentFormatters[typ]
if !registered {
return nil, fmt.Errorf("no fragment formatter type '%s' registered", typ)
}
fragmentFormatter, err := fragmentFormatterConstructor(config, cache)
func (c *FragmentFormatterCache) FragmentFormatterNamed(name string, cache *Cache) (highlight.FragmentFormatter, error) {
item, err := c.ItemNamed(name, cache, FragmentFormatterBuild)
if err != nil {
return nil, fmt.Errorf("error building fragment formatter: %v", err)
return nil, err
}
c[name] = fragmentFormatter
return fragmentFormatter, nil
return item.(highlight.FragmentFormatter), nil
}
func (c *FragmentFormatterCache) DefineFragmentFormatter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.FragmentFormatter, error) {
item, err := c.DefineItem(name, typ, config, cache, FragmentFormatterBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("fragment formatter named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(highlight.FragmentFormatter), nil
}
func FragmentFormatterTypesAndInstances() ([]string, []string) {

49
registry/fragmenter.go
View File

@ -25,40 +25,47 @@ func RegisterFragmenter(name string, constructor FragmenterConstructor) {
type FragmenterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error)
type FragmenterRegistry map[string]FragmenterConstructor
type FragmenterCache map[string]highlight.Fragmenter
func (c FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) {
fragmenter, cached := c[name]
if cached {
return fragmenter, nil
type FragmenterCache struct {
*ConcurrentCache
}
func NewFragmenterCache() *FragmenterCache {
return &FragmenterCache{
NewConcurrentCache(),
}
fragmenterConstructor, registered := fragmenters[name]
}
func FragmenterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := fragmenters[name]
if !registered {
return nil, fmt.Errorf("no fragmenter with name or type '%s' registered", name)
}
fragmenter, err := fragmenterConstructor(nil, cache)
fragmenter, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building fragmenter: %v", err)
}
c[name] = fragmenter
return fragmenter, nil
}
func (c FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("fragmenter named '%s' already defined", name)
}
fragmenterConstructor, registered := fragmenters[typ]
if !registered {
return nil, fmt.Errorf("no fragmenter type '%s' registered", typ)
}
fragmenter, err := fragmenterConstructor(config, cache)
func (c *FragmenterCache) FragmenterNamed(name string, cache *Cache) (highlight.Fragmenter, error) {
item, err := c.ItemNamed(name, cache, FragmenterBuild)
if err != nil {
return nil, fmt.Errorf("error building fragmenter: %v", err)
return nil, err
}
c[name] = fragmenter
return fragmenter, nil
return item.(highlight.Fragmenter), nil
}
func (c *FragmenterCache) DefineFragmenter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Fragmenter, error) {
item, err := c.DefineItem(name, typ, config, cache, FragmenterBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("fragmenter named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(highlight.Fragmenter), nil
}
func FragmenterTypesAndInstances() ([]string, []string) {

49
registry/highlighter.go
View File

@ -25,40 +25,47 @@ func RegisterHighlighter(name string, constructor HighlighterConstructor) {
type HighlighterConstructor func(config map[string]interface{}, cache *Cache) (highlight.Highlighter, error)
type HighlighterRegistry map[string]HighlighterConstructor
type HighlighterCache map[string]highlight.Highlighter
func (c HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) {
highlighter, cached := c[name]
if cached {
return highlighter, nil
type HighlighterCache struct {
*ConcurrentCache
}
func NewHighlighterCache() *HighlighterCache {
return &HighlighterCache{
NewConcurrentCache(),
}
highlighterConstructor, registered := highlighters[name]
}
func HighlighterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := highlighters[name]
if !registered {
return nil, fmt.Errorf("no highlighter with name or type '%s' registered", name)
}
highlighter, err := highlighterConstructor(nil, cache)
highlighter, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building highlighter: %v", err)
}
c[name] = highlighter
return highlighter, nil
}
func (c HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("highlighter named '%s' already defined", name)
}
highlighterConstructor, registered := highlighters[typ]
if !registered {
return nil, fmt.Errorf("no highlighter type '%s' registered", typ)
}
highlighter, err := highlighterConstructor(config, cache)
func (c *HighlighterCache) HighlighterNamed(name string, cache *Cache) (highlight.Highlighter, error) {
item, err := c.ItemNamed(name, cache, HighlighterBuild)
if err != nil {
return nil, fmt.Errorf("error building highlighter: %v", err)
return nil, err
}
c[name] = highlighter
return highlighter, nil
return item.(highlight.Highlighter), nil
}
func (c *HighlighterCache) DefineHighlighter(name string, typ string, config map[string]interface{}, cache *Cache) (highlight.Highlighter, error) {
item, err := c.DefineItem(name, typ, config, cache, HighlighterBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("highlighter named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(highlight.Highlighter), nil
}
func HighlighterTypesAndInstances() ([]string, []string) {

36
registry/registry.go
View File

@ -35,28 +35,28 @@ var analyzers = make(AnalyzerRegistry, 0)
var dateTimeParsers = make(DateTimeParserRegistry, 0)
type Cache struct {
CharFilters CharFilterCache
Tokenizers TokenizerCache
TokenMaps TokenMapCache
TokenFilters TokenFilterCache
Analyzers AnalyzerCache
DateTimeParsers DateTimeParserCache
FragmentFormatters FragmentFormatterCache
Fragmenters FragmenterCache
Highlighters HighlighterCache
CharFilters *CharFilterCache
Tokenizers *TokenizerCache
TokenMaps *TokenMapCache
TokenFilters *TokenFilterCache
Analyzers *AnalyzerCache
DateTimeParsers *DateTimeParserCache
FragmentFormatters *FragmentFormatterCache
Fragmenters *FragmenterCache
Highlighters *HighlighterCache
}
func NewCache() *Cache {
return &Cache{
CharFilters: make(CharFilterCache, 0),
Tokenizers: make(TokenizerCache, 0),
TokenMaps: make(TokenMapCache, 0),
TokenFilters: make(TokenFilterCache, 0),
Analyzers: make(AnalyzerCache, 0),
DateTimeParsers: make(DateTimeParserCache, 0),
FragmentFormatters: make(FragmentFormatterCache, 0),
Fragmenters: make(FragmenterCache, 0),
Highlighters: make(HighlighterCache, 0),
CharFilters: NewCharFilterCache(),
Tokenizers: NewTokenizerCache(),
TokenMaps: NewTokenMapCache(),
TokenFilters: NewTokenFilterCache(),
Analyzers: NewAnalyzerCache(),
DateTimeParsers: NewDateTimeParserCache(),
FragmentFormatters: NewFragmentFormatterCache(),
Fragmenters: NewFragmenterCache(),
Highlighters: NewHighlighterCache(),
}
}

49
registry/token_filter.go
View File

@ -25,40 +25,47 @@ func RegisterTokenFilter(name string, constructor TokenFilterConstructor) {
type TokenFilterConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error)
type TokenFilterRegistry map[string]TokenFilterConstructor
type TokenFilterCache map[string]analysis.TokenFilter
func (c TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) {
tokenFilter, cached := c[name]
if cached {
return tokenFilter, nil
type TokenFilterCache struct {
*ConcurrentCache
}
func NewTokenFilterCache() *TokenFilterCache {
return &TokenFilterCache{
NewConcurrentCache(),
}
tokenFilterConstructor, registered := tokenFilters[name]
}
func TokenFilterBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := tokenFilters[name]
if !registered {
return nil, fmt.Errorf("no token filter with name or type '%s' registered", name)
}
tokenFilter, err := tokenFilterConstructor(nil, cache)
tokenFilter, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building token filter: %v", err)
}
c[name] = tokenFilter
return tokenFilter, nil
}
func (c TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("token filter named '%s' already defined", name)
}
tokenFilterConstructor, registered := tokenFilters[typ]
if !registered {
return nil, fmt.Errorf("no token filter type '%s' registered", typ)
}
tokenFilter, err := tokenFilterConstructor(config, cache)
func (c *TokenFilterCache) TokenFilterNamed(name string, cache *Cache) (analysis.TokenFilter, error) {
item, err := c.ItemNamed(name, cache, TokenFilterBuild)
if err != nil {
return nil, fmt.Errorf("error building token filter: %v", err)
return nil, err
}
c[name] = tokenFilter
return tokenFilter, nil
return item.(analysis.TokenFilter), nil
}
func (c *TokenFilterCache) DefineTokenFilter(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenFilter, error) {
item, err := c.DefineItem(name, typ, config, cache, TokenFilterBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("token filter named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(analysis.TokenFilter), nil
}
func TokenFilterTypesAndInstances() ([]string, []string) {

49
registry/token_maps.go
View File

@ -25,40 +25,47 @@ func RegisterTokenMap(name string, constructor TokenMapConstructor) {
type TokenMapConstructor func(config map[string]interface{}, cache *Cache) (analysis.TokenMap, error)
type TokenMapRegistry map[string]TokenMapConstructor
type TokenMapCache map[string]analysis.TokenMap
func (c TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) {
tokenMap, cached := c[name]
if cached {
return tokenMap, nil
type TokenMapCache struct {
*ConcurrentCache
}
func NewTokenMapCache() *TokenMapCache {
return &TokenMapCache{
NewConcurrentCache(),
}
tokenMapConstructor, registered := tokenMaps[name]
}
func TokenMapBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := tokenMaps[name]
if !registered {
return nil, fmt.Errorf("no token map with name or type '%s' registered", name)
}
tokenMap, err := tokenMapConstructor(nil, cache)
tokenMap, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building token map: %v", err)
}
c[name] = tokenMap
return tokenMap, nil
}
func (c TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("token map named '%s' already defined", name)
}
tokenMapConstructor, registered := tokenMaps[typ]
if !registered {
return nil, fmt.Errorf("no token map type '%s' registered", typ)
}
tokenMap, err := tokenMapConstructor(config, cache)
func (c *TokenMapCache) TokenMapNamed(name string, cache *Cache) (analysis.TokenMap, error) {
item, err := c.ItemNamed(name, cache, TokenMapBuild)
if err != nil {
return nil, fmt.Errorf("error building token map: %v", err)
return nil, err
}
c[name] = tokenMap
return tokenMap, nil
return item.(analysis.TokenMap), nil
}
func (c *TokenMapCache) DefineTokenMap(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.TokenMap, error) {
item, err := c.DefineItem(name, typ, config, cache, TokenMapBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("token map named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(analysis.TokenMap), nil
}
func TokenMapTypesAndInstances() ([]string, []string) {

51
registry/tokenizer.go
View File

@ -25,40 +25,47 @@ func RegisterTokenizer(name string, constructor TokenizerConstructor) {
type TokenizerConstructor func(config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error)
type TokenizerRegistry map[string]TokenizerConstructor
type TokenizerCache map[string]analysis.Tokenizer
func (c TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) {
tokenizer, cached := c[name]
if cached {
return tokenizer, nil
type TokenizerCache struct {
*ConcurrentCache
}
func NewTokenizerCache() *TokenizerCache {
return &TokenizerCache{
NewConcurrentCache(),
}
tokenizerConstructor, registered := tokenizers[name]
}
func TokenizerBuild(name string, config map[string]interface{}, cache *Cache) (interface{}, error) {
cons, registered := tokenizers[name]
if !registered {
return nil, fmt.Errorf("no tokenizer with name or type '%s' registered", name)
}
tokenizer, err := tokenizerConstructor(nil, cache)
tokenizer, err := cons(config, cache)
if err != nil {
return nil, fmt.Errorf("error building tokenizer '%s': %v", name, err)
return nil, fmt.Errorf("error building tokenizer: %v", err)
}
c[name] = tokenizer
return tokenizer, nil
}
func (c TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) {
_, cached := c[name]
if cached {
return nil, fmt.Errorf("tokenizer named '%s' already defined", name)
}
tokenizerConstructor, registered := tokenizers[typ]
if !registered {
return nil, fmt.Errorf("no tokenizer type '%s' registered", typ)
}
tokenizer, err := tokenizerConstructor(config, cache)
func (c *TokenizerCache) TokenizerNamed(name string, cache *Cache) (analysis.Tokenizer, error) {
item, err := c.ItemNamed(name, cache, TokenizerBuild)
if err != nil {
return nil, fmt.Errorf("error building tokenizer '%s': %v", name, err)
return nil, err
}
c[name] = tokenizer
return tokenizer, nil
return item.(analysis.Tokenizer), nil
}
func (c *TokenizerCache) DefineTokenizer(name string, typ string, config map[string]interface{}, cache *Cache) (analysis.Tokenizer, error) {
item, err := c.DefineItem(name, typ, config, cache, TokenizerBuild)
if err != nil {
if err == ErrAlreadyDefined {
return nil, fmt.Errorf("tokenizer named '%s' already defined", name)
} else {
return nil, err
}
}
return item.(analysis.Tokenizer), nil
}
func TokenizerTypesAndInstances() ([]string, []string) {