gibheer
/
bleve
0
0
Fork 0
Code

remove byte_array_conveters 

fixes #392
fixes #100
This commit is contained in:
Marty Schoch 2016-07-01 10:21:41 -04:00
parent 63f2eb6740
commit 9089de251f
8 changed files with 17 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
analysis/byte_array_converters/ignore/ignore_byte_array_converter.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package ignore_byte_array_converter
import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
const Name = "ignore"
type IgnoreByteArrayConverter struct{}
func NewIgnoreByteArrayConverter() *IgnoreByteArrayConverter {
return &IgnoreByteArrayConverter{}
}
func (c *IgnoreByteArrayConverter) Convert(in []byte) (interface{}, error) {
return nil, nil
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewIgnoreByteArrayConverter(), nil
}
func init() {
registry.RegisterByteArrayConverter(Name, Constructor)
}

42
analysis/byte_array_converters/json/json_byte_array_converter.go
View File

@ -1,42 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package json_byte_array_converter
import (
"encoding/json"
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
const Name = "json"
type JSONByteArrayConverter struct{}
func NewJSONByteArrayConverter() *JSONByteArrayConverter {
return &JSONByteArrayConverter{}
}
func (c *JSONByteArrayConverter) Convert(in []byte) (interface{}, error) {
var rv map[string]interface{}
err := json.Unmarshal(in, &rv)
if err != nil {
return nil, err
}
return rv, nil
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewJSONByteArrayConverter(), nil
}
func init() {
registry.RegisterByteArrayConverter(Name, Constructor)
}

35
analysis/byte_array_converters/string/string_byte_array_conveter.go
View File

@ -1,35 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package string_byte_array_converter
import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/registry"
)
const Name = "string"
type StringByteArrayConverter struct{}
func NewStringByteArrayConverter() *StringByteArrayConverter {
return &StringByteArrayConverter{}
}
func (c *StringByteArrayConverter) Convert(in []byte) (interface{}, error) {
return string(in), nil
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (analysis.ByteArrayConverter, error) {
return NewStringByteArrayConverter(), nil
}
func init() {
registry.RegisterByteArrayConverter(Name, Constructor)
}

5
config/config.go
View File

@ -92,9 +92,4 @@ import (
// index types
_ "github.com/blevesearch/bleve/index/upside_down"
// byte array converters
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/ignore"
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/json"
_ "github.com/blevesearch/bleve/analysis/byte_array_converters/string"
)

11
http/doc_index.go
View File

@ -10,6 +10,7 @@
package http
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
@ -60,7 +61,15 @@ func (h *DocIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
err = index.Index(docID, requestBody)
// parse request body as json
var doc interface{}
err = json.Unmarshal(requestBody, &doc)
if err != nil {
showError(w, req, fmt.Sprintf("error parsing request body as JSON: %v", err), 400)
return
}
err = index.Index(docID, doc)
if err != nil {
showError(w, req, fmt.Sprintf("error indexing document '%s': %v", docID, err), 500)
return

8
index_test.go
View File

@ -23,7 +23,6 @@ import (
"golang.org/x/net/context"
"encoding/json"
"strconv"
"github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer"
@ -1242,12 +1241,7 @@ func TestDateTimeFieldMappingIssue287(t *testing.T) {
for i := 0; i < 3; i++ {
d := doc{now.Add(time.Duration((i - 3)) * time.Hour)}
docJson, err := json.Marshal(d)
if err != nil {
t.Fatal(err)
}
err = index.Index(strconv.FormatInt(int64(i), 10), docJson)
err = index.Index(strconv.FormatInt(int64(i), 10), d)
if err != nil {
t.Fatal(err)
}

30
mapping_index.go
View File

@ -15,7 +15,6 @@ import (
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer"
"github.com/blevesearch/bleve/analysis/byte_array_converters/json"
"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/registry"
@ -28,7 +27,6 @@ const defaultType = "_default"
const defaultField = "_all"
const defaultAnalyzer = standard_analyzer.Name
const defaultDateTimeParser = datetime_optional.Name
const defaultByteArrayConverter = json_byte_array_converter.Name
type customAnalysis struct {
CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"`
@ -129,7 +127,6 @@ type IndexMapping struct {
DefaultAnalyzer string `json:"default_analyzer"`
DefaultDateTimeParser string `json:"default_datetime_parser"`
DefaultField string `json:"default_field"`
ByteArrayConverter string `json:"byte_array_converter"`
StoreDynamic bool `json:"store_dynamic"`
IndexDynamic bool `json:"index_dynamic"`
CustomAnalysis *customAnalysis `json:"analysis,omitempty"`
@ -234,7 +231,6 @@ func NewIndexMapping() *IndexMapping {
DefaultAnalyzer: defaultAnalyzer,
DefaultDateTimeParser: defaultDateTimeParser,
DefaultField: defaultField,
ByteArrayConverter: defaultByteArrayConverter,
IndexDynamic: IndexDynamic,
StoreDynamic: StoreDynamic,
CustomAnalysis: newCustomAnalysis(),
@ -296,7 +292,6 @@ func (im *IndexMapping) UnmarshalJSON(data []byte) error {
im.DefaultAnalyzer = defaultAnalyzer
im.DefaultDateTimeParser = defaultDateTimeParser
im.DefaultField = defaultField
im.ByteArrayConverter = defaultByteArrayConverter
im.DefaultMapping = NewDocumentMapping()
im.TypeMapping = make(map[string]*DocumentMapping)
im.StoreDynamic = StoreDynamic
@ -335,11 +330,6 @@ func (im *IndexMapping) UnmarshalJSON(data []byte) error {
if err != nil {
return err
}
case "byte_array_converter":
err := json.Unmarshal(v, &im.ByteArrayConverter)
if err != nil {
return err
}
case "default_mapping":
err := json.Unmarshal(v, &im.DefaultMapping)
if err != nil {
@ -394,26 +384,6 @@ func (im *IndexMapping) determineType(data interface{}) string {
}
func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error {
// see if the top level object is a byte array, and possibly run through a converter
byteArrayData, ok := data.([]byte)
if ok {
byteArrayConverterConstructor := registry.ByteArrayConverterByName(im.ByteArrayConverter)
if byteArrayConverterConstructor != nil {
byteArrayConverter, err := byteArrayConverterConstructor(nil, nil)
if err == nil {
convertedData, err := byteArrayConverter.Convert(byteArrayData)
if err != nil {
return err
}
data = convertedData
} else {
logger.Printf("error creating byte array converter: %v", err)
}
} else {
logger.Printf("no byte array converter named: %s", im.ByteArrayConverter)
}
}
docType := im.determineType(data)
docMapping := im.mappingForType(docType)
walkContext := im.newWalkContext(doc, docMapping)

7
test/integration_test.go
View File

@ -121,10 +121,15 @@ func runTestDir(t *testing.T, dir, datasetName string) {
t.Errorf("error reading data file: %v", err)
return
}
var fileDoc interface{}
err = json.Unmarshal(fileBytes, &fileDoc)
if err != nil {
t.Errorf("error parsing data file as json: %v", err)
}
filename := fi.Name()
ext := filepath.Ext(filename)
id := filename[0 : len(filename)-len(ext)]
err = index.Index(id, fileBytes)
err = index.Index(id, fileDoc)
if err != nil {
t.Errorf("error indexing data: %v", err)
return