gibheer
/
bleve
0
0
Fork 0
Code

moved byte array converts into the analysis package

This commit is contained in:
Marty Schoch 2014-08-29 19:23:21 -04:00
parent 9487a46188
commit 7bfad18d40
8 changed files with 143 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
analysis/byte_array_converters/ignore/ignore_byte_array_converter.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package ignore_byte_array_converter
import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
type IgnoreByteArrayConverter struct{}
func NewIgnoreByteArrayConverter() *IgnoreByteArrayConverter {
return &IgnoreByteArrayConverter{}
}
func (c *IgnoreByteArrayConverter) Convert(in []byte) (interface{}, error) {
return nil, nil
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewIgnoreByteArrayConverter(), nil
}
func init() {
registry.RegisterByteArrayConverter("ignore", Constructor)
}

29
byte_array_converter.go → analysis/byte_array_converters/json/json_byte_array_converter.go
View File

@ -7,26 +7,15 @@
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package bleve
package json_byte_array_converter
import (
"encoding/json"
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
type ByteArrayConverter interface {
Convert([]byte) (interface{}, error)
}
type StringByteArrayConverter struct{}
func NewStringByteArrayConverter() *StringByteArrayConverter {
return &StringByteArrayConverter{}
}
func (c *StringByteArrayConverter) Convert(in []byte) (interface{}, error) {
return string(in), nil
}
type JSONByteArrayConverter struct{}
func NewJSONByteArrayConverter() *JSONByteArrayConverter {
@ -42,12 +31,10 @@ func (c *JSONByteArrayConverter) Convert(in []byte) (interface{}, error) {
return rv, nil
}
type IgnoreByteArrayConverter struct{}
func NewIgnoreByteArrayConverter() *IgnoreByteArrayConverter {
return &IgnoreByteArrayConverter{}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewJSONByteArrayConverter(), nil
}
func (c *IgnoreByteArrayConverter) Convert(in []byte) (interface{}, error) {
return nil, nil
func init() {
registry.RegisterByteArrayConverter("json", Constructor)
}

33
analysis/byte_array_converters/string/string_byte_array_conveter.go Normal file
View File

@ -0,0 +1,33 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package string_byte_array_converter
import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
type StringByteArrayConverter struct{}
func NewStringByteArrayConverter() *StringByteArrayConverter {
return &StringByteArrayConverter{}
}
func (c *StringByteArrayConverter) Convert(in []byte) (interface{}, error) {
return string(in), nil
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewStringByteArrayConverter(), nil
}
func init() {
registry.RegisterByteArrayConverter("string", Constructor)
}

4
analysis/type.go
View File

@ -74,3 +74,7 @@ var INVALID_DATETIME = fmt.Errorf("unable to parse datetime with any of the layo
type DateTimeParser interface {
ParseDateTime(string) (time.Time, error)
}
type ByteArrayConverter interface {
Convert([]byte) (interface{}, error)
}

18
config.go
View File

@ -80,6 +80,11 @@ import (
// kv stores
_ "github.com/blevesearch/bleve/index/store/boltdb"
_ "github.com/blevesearch/bleve/index/store/inmem"
// byte array converters
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/ignore"
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/json"
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/string"
)
var bleveExpVar = expvar.NewMap("bleve")
@ -89,10 +94,9 @@ type HighlightConfig struct {
}
type configuration struct {
Highlight *HighlightConfig
DefaultHighlighter *string
ByteArrayConverters map[string]ByteArrayConverter
DefaultKVStore string
Highlight *HighlightConfig
DefaultHighlighter *string
DefaultKVStore string
}
func newConfiguration() *configuration {
@ -100,7 +104,6 @@ func newConfiguration() *configuration {
Highlight: &HighlightConfig{
Highlighters: make(map[string]search.Highlighter),
},
ByteArrayConverters: make(map[string]ByteArrayConverter),
}
}
@ -112,11 +115,6 @@ func init() {
// build the default configuration
Config = newConfiguration()
// register byte array converters
Config.ByteArrayConverters["string"] = NewStringByteArrayConverter()
Config.ByteArrayConverters["json"] = NewJSONByteArrayConverter()
Config.ByteArrayConverters["ignore"] = NewIgnoreByteArrayConverter()
// register ansi highlighter
Config.Highlight.Highlighters["ansi"] = search.NewSimpleHighlighter()

19
mapping_index.go
View File

@ -211,13 +211,20 @@ func (im *IndexMapping) MapDocument(doc *document.Document, data interface{}) er
// see if the top level object is a byte array, and possibly run through conveter
byteArrayData, ok := data.([]byte)
if ok {
byteArrayConverter, valid := Config.ByteArrayConverters[im.ByteArrayConverter]
if valid {
convertedData, err := byteArrayConverter.Convert(byteArrayData)
if err != nil {
return err
byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter)
if byteArrayConverterConstructor != nil {
byteArrayConverter, err := byteArrayConverterConstructor(nil, nil)
if err == nil {
convertedData, err := byteArrayConverter.Convert(byteArrayData)
if err != nil {
return err
}
data = convertedData
} else {
log.Printf("error creating byte array converter: %v", err)
}
data = convertedData
} else {
log.Printf("no byte array converter named: %s", im.ByteArrayConverter)
}
}

31
registry/byte_array_converter.go Normal file
View File

@ -0,0 +1,31 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package registry
import (
"fmt"
"github.com/blevesearch/bleve/analysis"
)
func RegisterByteArrayConverter(name string, constructor ByteArrayConverterConstructor) {
_, exists := byteArrayConverters[name]
if exists {
panic(fmt.Errorf("attempted to register duplicate byte array converter named '%s'", name))
}
byteArrayConverters[name] = constructor
}
type ByteArrayConverterConstructor func(config map[string]interface{}, cache *Cache) (analysis.ByteArrayConverter, error)
type ByteArrayConverterRegistry map[string]ByteArrayConverterConstructor
func ByteArrayConverterByName(name string) ByteArrayConverterConstructor {
return byteArrayConverters[name]
}

13
registry/registry.go
View File

@ -17,6 +17,8 @@ import (
var stores = make(KVStoreRegistry, 0)
var byteArrayConverters = make(ByteArrayConverterRegistry, 0)
// analysis
var charFilters = make(CharFilterRegistry, 0)
var tokenizers = make(TokenizerRegistry, 0)
@ -170,4 +172,15 @@ func PrintRegistry() {
fmt.Printf("\t%s\n", name)
}
fmt.Println()
sorted = make(sort.StringSlice, 0, len(byteArrayConverters))
for name, _ := range byteArrayConverters {
sorted = append(sorted, name)
}
sorted.Sort()
fmt.Printf("Byte Array Converters:\n")
for _, name := range sorted {
fmt.Printf("\t%s\n", name)
}
fmt.Println()
}