0
0
Fork 0

refactor mapping to inteface and move into separate package

the index mapping contains some relatively messy logic
and the top-level bleve package only cares about a relatively
small portion of this
the motivation for this change is to codify the part that the
top-level bleve package cares about into an interface
then move all the details into its own package

NOTE: the top-level bleve package still has hard dependency on
the actual implementation (for now) because it must deserialize
mappings from JSON and simply assumes it is this one instance.
this is seen as OK for now, and this issue could be revisited
in a future change.  moving the logic into a separate package
is seen as a simplification of top-level bleve, even though
we still depend on the one particular implementation.
This commit is contained in:
Marty Schoch 2016-09-18 09:33:18 -04:00
parent 97393d0273
commit 79cc39a67e
37 changed files with 348 additions and 212 deletions

View File

@ -15,11 +15,12 @@ import (
"testing" "testing"
"time" "time"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/highlight/highlighters/ansi" "github.com/blevesearch/bleve/search/highlight/highlighters/ansi"
) )
var mapping *IndexMapping var indexMapping mapping.IndexMapping
var example_index Index var example_index Index
var err error var err error
@ -43,8 +44,8 @@ func TestMain(m *testing.M) {
} }
func ExampleNew() { func ExampleNew() {
mapping = NewIndexMapping() indexMapping = NewIndexMapping()
example_index, err = New("path_to_index", mapping) example_index, err = New("path_to_index", indexMapping)
if err != nil { if err != nil {
panic(err) panic(err)
} }

View File

@ -13,7 +13,7 @@ import (
"fmt" "fmt"
"net/http" "net/http"
"github.com/blevesearch/bleve" "github.com/blevesearch/bleve/mapping"
) )
type GetIndexHandler struct { type GetIndexHandler struct {
@ -42,9 +42,9 @@ func (h *GetIndexHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} }
rv := struct { rv := struct {
Status string `json:"status"` Status string `json:"status"`
Name string `json:"name"` Name string `json:"name"`
Mapping *bleve.IndexMapping `json:"mapping"` Mapping mapping.IndexMapping `json:"mapping"`
}{ }{
Status: "ok", Status: "ok",
Name: indexName, Name: indexName,

View File

@ -13,6 +13,7 @@ import (
"github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/mapping"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
@ -35,7 +36,7 @@ func (b *Batch) Index(id string, data interface{}) error {
return ErrorEmptyID return ErrorEmptyID
} }
doc := document.NewDocument(id) doc := document.NewDocument(id)
err := b.index.Mapping().mapDocument(doc, data) err := b.index.Mapping().MapDocument(doc, data)
if err != nil { if err != nil {
return err return err
} }
@ -92,11 +93,6 @@ func (b *Batch) Reset() {
// assigns string paths to its fields or values then applies field mappings on // assigns string paths to its fields or values then applies field mappings on
// them. // them.
// //
// If the value is a []byte, the indexer attempts to convert it to something
// else using the ByteArrayConverter registered as
// IndexMapping.ByteArrayConverter. By default, it interprets the value as a
// JSON payload and unmarshals it to map[string]interface{}.
//
// The DocumentMapping used to index a value is deduced by the following rules: // The DocumentMapping used to index a value is deduced by the following rules:
// 1) If value implements Classifier interface, resolve the mapping from Type(). // 1) If value implements Classifier interface, resolve the mapping from Type().
// 2) If value has a string field or value at IndexMapping.TypeField. // 2) If value has a string field or value at IndexMapping.TypeField.
@ -178,7 +174,7 @@ type Index interface {
Close() error Close() error
Mapping() *IndexMapping Mapping() mapping.IndexMapping
Stats() *IndexStat Stats() *IndexStat
StatsMap() map[string]interface{} StatsMap() map[string]interface{}
@ -197,16 +193,10 @@ type Index interface {
Advanced() (index.Index, store.KVStore, error) Advanced() (index.Index, store.KVStore, error)
} }
// A Classifier is an interface describing any object
// which knows how to identify its own type.
type Classifier interface {
Type() string
}
// New index at the specified path, must not exist. // New index at the specified path, must not exist.
// The provided mapping will be used for all // The provided mapping will be used for all
// Index/Search operations. // Index/Search operations.
func New(path string, mapping *IndexMapping) (Index, error) { func New(path string, mapping mapping.IndexMapping) (Index, error) {
return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
} }
@ -215,7 +205,7 @@ func New(path string, mapping *IndexMapping) (Index, error) {
// and will be lost once closed. // and will be lost once closed.
// The provided mapping will be used for all // The provided mapping will be used for all
// Index/Search operations. // Index/Search operations.
func NewMemOnly(mapping *IndexMapping) (Index, error) { func NewMemOnly(mapping mapping.IndexMapping) (Index, error) {
return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
} }
@ -227,7 +217,7 @@ func NewMemOnly(mapping *IndexMapping) (Index, error) {
// The specified kvstore implementation will be used // The specified kvstore implementation will be used
// and the provided kvconfig will be passed to its // and the provided kvconfig will be passed to its
// constructor. // constructor.
func NewUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) { func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) {
return newIndexUsing(path, mapping, indexType, kvstore, kvconfig) return newIndexUsing(path, mapping, indexType, kvstore, kvconfig)
} }

View File

@ -19,6 +19,7 @@ import (
"github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -259,7 +260,7 @@ func (i *indexAliasImpl) Close() error {
return nil return nil
} }
func (i *indexAliasImpl) Mapping() *IndexMapping { func (i *indexAliasImpl) Mapping() mapping.IndexMapping {
i.mutex.RLock() i.mutex.RLock()
defer i.mutex.RUnlock() defer i.mutex.RUnlock()

View File

@ -11,6 +11,7 @@ import (
"github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/numeric_util" "github.com/blevesearch/bleve/numeric_util"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -1269,7 +1270,7 @@ func (i *stubIndex) Close() error {
return i.err return i.err
} }
func (i *stubIndex) Mapping() *IndexMapping { func (i *stubIndex) Mapping() mapping.IndexMapping {
return nil return nil
} }

View File

@ -23,6 +23,7 @@ import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/index/upside_down"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/registry"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/collectors" "github.com/blevesearch/bleve/search/collectors"
@ -35,7 +36,7 @@ type indexImpl struct {
name string name string
meta *indexMeta meta *indexMeta
i index.Index i index.Index
m *IndexMapping m mapping.IndexMapping
mutex sync.RWMutex mutex sync.RWMutex
open bool open bool
stats *IndexStat stats *IndexStat
@ -49,7 +50,7 @@ func indexStorePath(path string) string {
return path + string(os.PathSeparator) + storePath return path + string(os.PathSeparator) + storePath
} }
func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
// first validate the mapping // first validate the mapping
err := mapping.Validate() err := mapping.Validate()
if err != nil { if err != nil {
@ -183,7 +184,7 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde
return nil, err return nil, err
} }
var im IndexMapping var im *mapping.IndexMappingImpl
err = json.Unmarshal(mappingBytes, &im) err = json.Unmarshal(mappingBytes, &im)
if err != nil { if err != nil {
return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes))
@ -202,7 +203,7 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde
return rv, err return rv, err
} }
rv.m = &im rv.m = im
indexStats.Register(rv) indexStats.Register(rv)
return rv, err return rv, err
} }
@ -219,7 +220,7 @@ func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) {
// Mapping returns the IndexMapping in use by this // Mapping returns the IndexMapping in use by this
// Index. // Index.
func (i *indexImpl) Mapping() *IndexMapping { func (i *indexImpl) Mapping() mapping.IndexMapping {
return i.m return i.m
} }
@ -239,7 +240,7 @@ func (i *indexImpl) Index(id string, data interface{}) (err error) {
} }
doc := document.NewDocument(id) doc := document.NewDocument(id)
err = i.m.mapDocument(doc, data) err = i.m.MapDocument(doc, data)
if err != nil { if err != nil {
return return
} }
@ -387,7 +388,7 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
} else if facetRequest.DateTimeRanges != nil { } else if facetRequest.DateTimeRanges != nil {
// build date range facet // build date range facet
facetBuilder := facets.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size) facetBuilder := facets.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
dateTimeParser := i.m.dateTimeParserNamed(i.m.DefaultDateTimeParser) dateTimeParser := i.m.DateTimeParserNamed("")
for _, dr := range facetRequest.DateTimeRanges { for _, dr := range facetRequest.DateTimeRanges {
dr.ParseDates(dateTimeParser) dr.ParseDates(dateTimeParser)
facetBuilder.AddRange(dr.Name, dr.Start, dr.End) facetBuilder.AddRange(dr.Name, dr.Start, dr.End)

View File

@ -29,6 +29,7 @@ import (
"github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer" "github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/index/store/null" "github.com/blevesearch/bleve/index/store/null"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -365,7 +366,7 @@ func (s *slowQuery) SetField(f string) Query {
return s.actual.SetField(f) return s.actual.SetField(f)
} }
func (s *slowQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (s *slowQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
time.Sleep(s.delay) time.Sleep(s.delay)
return s.actual.Searcher(i, m, explain) return s.actual.Searcher(i, m, explain)
} }

56
mapping.go Normal file
View File

@ -0,0 +1,56 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package bleve
import "github.com/blevesearch/bleve/mapping"
// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules
func NewIndexMapping() *mapping.IndexMappingImpl {
return mapping.NewIndexMapping()
}
// NewDocumentMapping returns a new document mapping
// with all the default values.
func NewDocumentMapping() *mapping.DocumentMapping {
return mapping.NewDocumentMapping()
}
// NewDocumentStaticMapping returns a new document
// mapping that will not automatically index parts
// of a document without an explicit mapping.
func NewDocumentStaticMapping() *mapping.DocumentMapping {
return mapping.NewDocumentStaticMapping()
}
// NewDocumentDisabledMapping returns a new document
// mapping that will not perform any indexing.
func NewDocumentDisabledMapping() *mapping.DocumentMapping {
return mapping.NewDocumentDisabledMapping()
}
// NewTextFieldMapping returns a default field mapping for text
func NewTextFieldMapping() *mapping.FieldMapping {
return mapping.NewTextFieldMapping()
}
// NewNumericFieldMapping returns a default field mapping for numbers
func NewNumericFieldMapping() *mapping.FieldMapping {
return mapping.NewNumericFieldMapping()
}
// NewDateTimeFieldMapping returns a default field mapping for dates
func NewDateTimeFieldMapping() *mapping.FieldMapping {
return mapping.NewDateTimeFieldMapping()
}
// NewBooleanFieldMapping returns a default field mapping for booleans
func NewBooleanFieldMapping() *mapping.FieldMapping {
return mapping.NewBooleanFieldMapping()
}

94
mapping/analysis.go Normal file
View File

@ -0,0 +1,94 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package mapping
type customAnalysis struct {
CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"`
Tokenizers map[string]map[string]interface{} `json:"tokenizers,omitempty"`
TokenMaps map[string]map[string]interface{} `json:"token_maps,omitempty"`
TokenFilters map[string]map[string]interface{} `json:"token_filters,omitempty"`
Analyzers map[string]map[string]interface{} `json:"analyzers,omitempty"`
DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"`
}
func (c *customAnalysis) registerAll(i *IndexMappingImpl) error {
for name, config := range c.CharFilters {
_, err := i.cache.DefineCharFilter(name, config)
if err != nil {
return err
}
}
if len(c.Tokenizers) > 0 {
// put all the names in map tracking work to do
todo := map[string]struct{}{}
for name := range c.Tokenizers {
todo[name] = struct{}{}
}
registered := 1
errs := []error{}
// as long as we keep making progress, keep going
for len(todo) > 0 && registered > 0 {
registered = 0
errs = []error{}
for name := range todo {
config := c.Tokenizers[name]
_, err := i.cache.DefineTokenizer(name, config)
if err != nil {
errs = append(errs, err)
} else {
delete(todo, name)
registered++
}
}
}
if len(errs) > 0 {
return errs[0]
}
}
for name, config := range c.TokenMaps {
_, err := i.cache.DefineTokenMap(name, config)
if err != nil {
return err
}
}
for name, config := range c.TokenFilters {
_, err := i.cache.DefineTokenFilter(name, config)
if err != nil {
return err
}
}
for name, config := range c.Analyzers {
_, err := i.cache.DefineAnalyzer(name, config)
if err != nil {
return err
}
}
for name, config := range c.DateTimeParsers {
_, err := i.cache.DefineDateTimeParser(name, config)
if err != nil {
return err
}
}
return nil
}
func newCustomAnalysis() *customAnalysis {
rv := customAnalysis{
CharFilters: make(map[string]map[string]interface{}),
Tokenizers: make(map[string]map[string]interface{}),
TokenMaps: make(map[string]map[string]interface{}),
TokenFilters: make(map[string]map[string]interface{}),
Analyzers: make(map[string]map[string]interface{}),
DateTimeParsers: make(map[string]map[string]interface{}),
}
return &rv
}

View File

@ -7,7 +7,7 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package bleve package mapping
import ( import (
"encoding/json" "encoding/json"
@ -405,7 +405,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
// automatic indexing behavior // automatic indexing behavior
// first see if it can be parsed by the default date parser // first see if it can be parsed by the default date parser
dateTimeParser := context.im.dateTimeParserNamed(context.im.DefaultDateTimeParser) dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser)
if dateTimeParser != nil { if dateTimeParser != nil {
parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)
if err != nil { if err != nil {

View File

@ -7,7 +7,7 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package bleve package mapping
import ( import (
"encoding/json" "encoding/json"
@ -61,7 +61,7 @@ func NewTextFieldMapping() *FieldMapping {
} }
} }
func newTextFieldMappingDynamic(im *IndexMapping) *FieldMapping { func newTextFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {
rv := NewTextFieldMapping() rv := NewTextFieldMapping()
rv.Store = im.StoreDynamic rv.Store = im.StoreDynamic
rv.Index = im.IndexDynamic rv.Index = im.IndexDynamic
@ -78,7 +78,7 @@ func NewNumericFieldMapping() *FieldMapping {
} }
} }
func newNumericFieldMappingDynamic(im *IndexMapping) *FieldMapping { func newNumericFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {
rv := NewNumericFieldMapping() rv := NewNumericFieldMapping()
rv.Store = im.StoreDynamic rv.Store = im.StoreDynamic
rv.Index = im.IndexDynamic rv.Index = im.IndexDynamic
@ -95,7 +95,7 @@ func NewDateTimeFieldMapping() *FieldMapping {
} }
} }
func newDateTimeFieldMappingDynamic(im *IndexMapping) *FieldMapping { func newDateTimeFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {
rv := NewDateTimeFieldMapping() rv := NewDateTimeFieldMapping()
rv.Store = im.StoreDynamic rv.Store = im.StoreDynamic
rv.Index = im.IndexDynamic rv.Index = im.IndexDynamic
@ -112,7 +112,7 @@ func NewBooleanFieldMapping() *FieldMapping {
} }
} }
func newBooleanFieldMappingDynamic(im *IndexMapping) *FieldMapping { func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping {
rv := NewBooleanFieldMapping() rv := NewBooleanFieldMapping()
rv.Store = im.StoreDynamic rv.Store = im.StoreDynamic
rv.Index = im.IndexDynamic rv.Index = im.IndexDynamic
@ -150,7 +150,7 @@ func (fm *FieldMapping) processString(propertyValueString string, pathString str
if fm.DateFormat != "" { if fm.DateFormat != "" {
dateTimeFormat = fm.DateFormat dateTimeFormat = fm.DateFormat
} }
dateTimeParser := context.im.dateTimeParserNamed(dateTimeFormat) dateTimeParser := context.im.DateTimeParserNamed(dateTimeFormat)
if dateTimeParser != nil { if dateTimeParser != nil {
parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString) parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)
if err == nil { if err == nil {
@ -211,7 +211,7 @@ func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *a
analyzerName = context.im.DefaultAnalyzer analyzerName = context.im.DefaultAnalyzer
} }
} }
return context.im.analyzerNamed(analyzerName) return context.im.AnalyzerNamed(analyzerName)
} }
func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string {

View File

@ -7,7 +7,7 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package bleve package mapping
import ( import (
"encoding/json" "encoding/json"
@ -28,98 +28,14 @@ const defaultField = "_all"
const defaultAnalyzer = standard_analyzer.Name const defaultAnalyzer = standard_analyzer.Name
const defaultDateTimeParser = datetime_optional.Name const defaultDateTimeParser = datetime_optional.Name
type customAnalysis struct { // An IndexMappingImpl controls how objects are placed
CharFilters map[string]map[string]interface{} `json:"char_filters,omitempty"`
Tokenizers map[string]map[string]interface{} `json:"tokenizers,omitempty"`
TokenMaps map[string]map[string]interface{} `json:"token_maps,omitempty"`
TokenFilters map[string]map[string]interface{} `json:"token_filters,omitempty"`
Analyzers map[string]map[string]interface{} `json:"analyzers,omitempty"`
DateTimeParsers map[string]map[string]interface{} `json:"date_time_parsers,omitempty"`
}
func (c *customAnalysis) registerAll(i *IndexMapping) error {
for name, config := range c.CharFilters {
_, err := i.cache.DefineCharFilter(name, config)
if err != nil {
return err
}
}
if len(c.Tokenizers) > 0 {
// put all the names in map tracking work to do
todo := map[string]struct{}{}
for name := range c.Tokenizers {
todo[name] = struct{}{}
}
registered := 1
errs := []error{}
// as long as we keep making progress, keep going
for len(todo) > 0 && registered > 0 {
registered = 0
errs = []error{}
for name := range todo {
config := c.Tokenizers[name]
_, err := i.cache.DefineTokenizer(name, config)
if err != nil {
errs = append(errs, err)
} else {
delete(todo, name)
registered++
}
}
}
if len(errs) > 0 {
return errs[0]
}
}
for name, config := range c.TokenMaps {
_, err := i.cache.DefineTokenMap(name, config)
if err != nil {
return err
}
}
for name, config := range c.TokenFilters {
_, err := i.cache.DefineTokenFilter(name, config)
if err != nil {
return err
}
}
for name, config := range c.Analyzers {
_, err := i.cache.DefineAnalyzer(name, config)
if err != nil {
return err
}
}
for name, config := range c.DateTimeParsers {
_, err := i.cache.DefineDateTimeParser(name, config)
if err != nil {
return err
}
}
return nil
}
func newCustomAnalysis() *customAnalysis {
rv := customAnalysis{
CharFilters: make(map[string]map[string]interface{}),
Tokenizers: make(map[string]map[string]interface{}),
TokenMaps: make(map[string]map[string]interface{}),
TokenFilters: make(map[string]map[string]interface{}),
Analyzers: make(map[string]map[string]interface{}),
DateTimeParsers: make(map[string]map[string]interface{}),
}
return &rv
}
// An IndexMapping controls how objects are placed
// into an index. // into an index.
// First the type of the object is determined. // First the type of the object is determined.
// Once the type is know, the appropriate // Once the type is know, the appropriate
// DocumentMapping is selected by the type. // DocumentMapping is selected by the type.
// If no mapping was determined for that type, // If no mapping was determined for that type,
// a DefaultMapping will be used. // a DefaultMapping will be used.
type IndexMapping struct { type IndexMappingImpl struct {
TypeMapping map[string]*DocumentMapping `json:"types,omitempty"` TypeMapping map[string]*DocumentMapping `json:"types,omitempty"`
DefaultMapping *DocumentMapping `json:"default_mapping"` DefaultMapping *DocumentMapping `json:"default_mapping"`
TypeField string `json:"type_field"` TypeField string `json:"type_field"`
@ -134,7 +50,7 @@ type IndexMapping struct {
} }
// AddCustomCharFilter defines a custom char filter for use in this mapping // AddCustomCharFilter defines a custom char filter for use in this mapping
func (im *IndexMapping) AddCustomCharFilter(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomCharFilter(name string, config map[string]interface{}) error {
_, err := im.cache.DefineCharFilter(name, config) _, err := im.cache.DefineCharFilter(name, config)
if err != nil { if err != nil {
return err return err
@ -144,7 +60,7 @@ func (im *IndexMapping) AddCustomCharFilter(name string, config map[string]inter
} }
// AddCustomTokenizer defines a custom tokenizer for use in this mapping // AddCustomTokenizer defines a custom tokenizer for use in this mapping
func (im *IndexMapping) AddCustomTokenizer(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomTokenizer(name string, config map[string]interface{}) error {
_, err := im.cache.DefineTokenizer(name, config) _, err := im.cache.DefineTokenizer(name, config)
if err != nil { if err != nil {
return err return err
@ -154,7 +70,7 @@ func (im *IndexMapping) AddCustomTokenizer(name string, config map[string]interf
} }
// AddCustomTokenMap defines a custom token map for use in this mapping // AddCustomTokenMap defines a custom token map for use in this mapping
func (im *IndexMapping) AddCustomTokenMap(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomTokenMap(name string, config map[string]interface{}) error {
_, err := im.cache.DefineTokenMap(name, config) _, err := im.cache.DefineTokenMap(name, config)
if err != nil { if err != nil {
return err return err
@ -164,7 +80,7 @@ func (im *IndexMapping) AddCustomTokenMap(name string, config map[string]interfa
} }
// AddCustomTokenFilter defines a custom token filter for use in this mapping // AddCustomTokenFilter defines a custom token filter for use in this mapping
func (im *IndexMapping) AddCustomTokenFilter(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomTokenFilter(name string, config map[string]interface{}) error {
_, err := im.cache.DefineTokenFilter(name, config) _, err := im.cache.DefineTokenFilter(name, config)
if err != nil { if err != nil {
return err return err
@ -202,7 +118,7 @@ func (im *IndexMapping) AddCustomTokenFilter(name string, config map[string]inte
// ... // ...
// }, // },
// }) // })
func (im *IndexMapping) AddCustomAnalyzer(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomAnalyzer(name string, config map[string]interface{}) error {
_, err := im.cache.DefineAnalyzer(name, config) _, err := im.cache.DefineAnalyzer(name, config)
if err != nil { if err != nil {
return err return err
@ -212,7 +128,7 @@ func (im *IndexMapping) AddCustomAnalyzer(name string, config map[string]interfa
} }
// AddCustomDateTimeParser defines a custom date time parser for use in this mapping // AddCustomDateTimeParser defines a custom date time parser for use in this mapping
func (im *IndexMapping) AddCustomDateTimeParser(name string, config map[string]interface{}) error { func (im *IndexMappingImpl) AddCustomDateTimeParser(name string, config map[string]interface{}) error {
_, err := im.cache.DefineDateTimeParser(name, config) _, err := im.cache.DefineDateTimeParser(name, config)
if err != nil { if err != nil {
return err return err
@ -222,8 +138,8 @@ func (im *IndexMapping) AddCustomDateTimeParser(name string, config map[string]i
} }
// NewIndexMapping creates a new IndexMapping that will use all the default indexing rules // NewIndexMapping creates a new IndexMapping that will use all the default indexing rules
func NewIndexMapping() *IndexMapping { func NewIndexMapping() *IndexMappingImpl {
return &IndexMapping{ return &IndexMappingImpl{
TypeMapping: make(map[string]*DocumentMapping), TypeMapping: make(map[string]*DocumentMapping),
DefaultMapping: NewDocumentMapping(), DefaultMapping: NewDocumentMapping(),
TypeField: defaultTypeField, TypeField: defaultTypeField,
@ -240,7 +156,7 @@ func NewIndexMapping() *IndexMapping {
// Validate will walk the entire structure ensuring the following // Validate will walk the entire structure ensuring the following
// explicitly named and default analyzers can be built // explicitly named and default analyzers can be built
func (im *IndexMapping) Validate() error { func (im *IndexMappingImpl) Validate() error {
_, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer) _, err := im.cache.AnalyzerNamed(im.DefaultAnalyzer)
if err != nil { if err != nil {
return err return err
@ -263,11 +179,11 @@ func (im *IndexMapping) Validate() error {
} }
// AddDocumentMapping sets a custom document mapping for the specified type // AddDocumentMapping sets a custom document mapping for the specified type
func (im *IndexMapping) AddDocumentMapping(doctype string, dm *DocumentMapping) { func (im *IndexMappingImpl) AddDocumentMapping(doctype string, dm *DocumentMapping) {
im.TypeMapping[doctype] = dm im.TypeMapping[doctype] = dm
} }
func (im *IndexMapping) mappingForType(docType string) *DocumentMapping { func (im *IndexMappingImpl) mappingForType(docType string) *DocumentMapping {
docMapping := im.TypeMapping[docType] docMapping := im.TypeMapping[docType]
if docMapping == nil { if docMapping == nil {
docMapping = im.DefaultMapping docMapping = im.DefaultMapping
@ -276,7 +192,7 @@ func (im *IndexMapping) mappingForType(docType string) *DocumentMapping {
} }
// UnmarshalJSON offers custom unmarshaling with optional strict validation // UnmarshalJSON offers custom unmarshaling with optional strict validation
func (im *IndexMapping) UnmarshalJSON(data []byte) error { func (im *IndexMappingImpl) UnmarshalJSON(data []byte) error {
var tmp map[string]json.RawMessage var tmp map[string]json.RawMessage
err := json.Unmarshal(data, &tmp) err := json.Unmarshal(data, &tmp)
@ -367,7 +283,7 @@ func (im *IndexMapping) UnmarshalJSON(data []byte) error {
return nil return nil
} }
func (im *IndexMapping) determineType(data interface{}) string { func (im *IndexMappingImpl) determineType(data interface{}) string {
// first see if the object implements Classifier // first see if the object implements Classifier
classifier, ok := data.(Classifier) classifier, ok := data.(Classifier)
if ok { if ok {
@ -383,7 +299,7 @@ func (im *IndexMapping) determineType(data interface{}) string {
return im.DefaultType return im.DefaultType
} }
func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) error { func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}) error {
docType := im.determineType(data) docType := im.determineType(data)
docMapping := im.mappingForType(docType) docMapping := im.mappingForType(docType)
walkContext := im.newWalkContext(doc, docMapping) walkContext := im.newWalkContext(doc, docMapping)
@ -403,12 +319,12 @@ func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) er
type walkContext struct { type walkContext struct {
doc *document.Document doc *document.Document
im *IndexMapping im *IndexMappingImpl
dm *DocumentMapping dm *DocumentMapping
excludedFromAll []string excludedFromAll []string
} }
func (im *IndexMapping) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext { func (im *IndexMappingImpl) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext {
return &walkContext{ return &walkContext{
doc: doc, doc: doc,
im: im, im: im,
@ -422,7 +338,7 @@ func (im *IndexMapping) newWalkContext(doc *document.Document, dm *DocumentMappi
// provided path, if one exists and it has an explicit analyzer // provided path, if one exists and it has an explicit analyzer
// that is returned // that is returned
// nil should be an acceptable return value meaning we don't know // nil should be an acceptable return value meaning we don't know
func (im *IndexMapping) analyzerNameForPath(path string) string { func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string {
// first we look for explicit mapping on the field // first we look for explicit mapping on the field
for _, docMapping := range im.TypeMapping { for _, docMapping := range im.TypeMapping {
analyzerName := docMapping.analyzerNameForPath(path) analyzerName := docMapping.analyzerNameForPath(path)
@ -452,7 +368,7 @@ func (im *IndexMapping) analyzerNameForPath(path string) string {
return im.DefaultAnalyzer return im.DefaultAnalyzer
} }
func (im *IndexMapping) analyzerNamed(name string) *analysis.Analyzer { func (im *IndexMappingImpl) AnalyzerNamed(name string) *analysis.Analyzer {
analyzer, err := im.cache.AnalyzerNamed(name) analyzer, err := im.cache.AnalyzerNamed(name)
if err != nil { if err != nil {
logger.Printf("error using analyzer named: %s", name) logger.Printf("error using analyzer named: %s", name)
@ -461,7 +377,10 @@ func (im *IndexMapping) analyzerNamed(name string) *analysis.Analyzer {
return analyzer return analyzer
} }
func (im *IndexMapping) dateTimeParserNamed(name string) analysis.DateTimeParser { func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimeParser {
if name == "" {
name = im.DefaultDateTimeParser
}
dateTimeParser, err := im.cache.DateTimeParserNamed(name) dateTimeParser, err := im.cache.DateTimeParserNamed(name)
if err != nil { if err != nil {
logger.Printf("error using datetime parser named: %s", name) logger.Printf("error using datetime parser named: %s", name)
@ -470,7 +389,7 @@ func (im *IndexMapping) dateTimeParserNamed(name string) analysis.DateTimeParser
return dateTimeParser return dateTimeParser
} }
func (im *IndexMapping) datetimeParserNameForPath(path string) string { func (im *IndexMappingImpl) datetimeParserNameForPath(path string) string {
// first we look for explicit mapping on the field // first we look for explicit mapping on the field
for _, docMapping := range im.TypeMapping { for _, docMapping := range im.TypeMapping {
@ -487,7 +406,7 @@ func (im *IndexMapping) datetimeParserNameForPath(path string) string {
return im.DefaultDateTimeParser return im.DefaultDateTimeParser
} }
func (im *IndexMapping) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) { func (im *IndexMappingImpl) AnalyzeText(analyzerName string, text []byte) (analysis.TokenStream, error) {
analyzer, err := im.cache.AnalyzerNamed(analyzerName) analyzer, err := im.cache.AnalyzerNamed(analyzerName)
if err != nil { if err != nil {
return nil, err return nil, err
@ -496,6 +415,12 @@ func (im *IndexMapping) AnalyzeText(analyzerName string, text []byte) (analysis.
} }
// FieldAnalyzer returns the name of the analyzer used on a field. // FieldAnalyzer returns the name of the analyzer used on a field.
func (im *IndexMapping) FieldAnalyzer(field string) string { func (im *IndexMappingImpl) FieldAnalyzer(field string) string {
return im.analyzerNameForPath(field) return im.AnalyzerNameForPath(field)
}
// wrapper to satisfy new interface
func (im *IndexMappingImpl) DefaultSearchField() string {
return im.DefaultField
} }

44
mapping/mapping.go Normal file
View File

@ -0,0 +1,44 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package mapping
import (
"io/ioutil"
"log"
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/document"
)
// A Classifier is an interface describing any object
// which knows how to identify its own type.
type Classifier interface {
Type() string
}
var logger = log.New(ioutil.Discard, "bleve mapping ", log.LstdFlags)
// SetLog sets the logger used for logging
// by default log messages are sent to ioutil.Discard
func SetLog(l *log.Logger) {
logger = l
}
type IndexMapping interface {
MapDocument(doc *document.Document, data interface{}) error
Validate() error
DateTimeParserNamed(name string) analysis.DateTimeParser
DefaultSearchField() string
AnalyzerNameForPath(path string) string
AnalyzerNamed(name string) *analysis.Analyzer
}

View File

@ -7,7 +7,7 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package bleve package mapping
import ( import (
"encoding/json" "encoding/json"
@ -46,7 +46,7 @@ var mappingSource = []byte(`{
"default_type": "_default" "default_type": "_default"
}`) }`)
func buildMapping() *IndexMapping { func buildMapping() IndexMapping {
nameFieldMapping := NewTextFieldMapping() nameFieldMapping := NewTextFieldMapping()
nameFieldMapping.Name = "name" nameFieldMapping.Name = "name"
nameFieldMapping.Analyzer = "standard" nameFieldMapping.Analyzer = "standard"
@ -66,7 +66,7 @@ func buildMapping() *IndexMapping {
func TestUnmarshalMappingJSON(t *testing.T) { func TestUnmarshalMappingJSON(t *testing.T) {
mapping := buildMapping() mapping := buildMapping()
var indexMapping IndexMapping var indexMapping IndexMappingImpl
err := json.Unmarshal(mappingSource, &indexMapping) err := json.Unmarshal(mappingSource, &indexMapping)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -88,7 +88,7 @@ func TestMappingStructWithJSONTags(t *testing.T) {
} }
doc := document.NewDocument("1") doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x) err := mapping.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -128,7 +128,7 @@ func TestMappingStructWithJSONTagsOneDisabled(t *testing.T) {
} }
doc := document.NewDocument("1") doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x) err := mapping.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -168,7 +168,7 @@ func TestMappingStructWithPointerToString(t *testing.T) {
} }
doc := document.NewDocument("1") doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x) err := mapping.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -200,7 +200,7 @@ func TestMappingJSONWithNull(t *testing.T) {
} }
doc := document.NewDocument("1") doc := document.NewDocument("1")
err = mapping.mapDocument(doc, jsondoc) err = mapping.MapDocument(doc, jsondoc)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -243,17 +243,17 @@ func TestMappingForPath(t *testing.T) {
mapping := NewIndexMapping() mapping := NewIndexMapping()
mapping.AddDocumentMapping("a", docMappingA) mapping.AddDocumentMapping("a", docMappingA)
analyzerName := mapping.analyzerNameForPath("name") analyzerName := mapping.AnalyzerNameForPath("name")
if analyzerName != enFieldMapping.Analyzer { if analyzerName != enFieldMapping.Analyzer {
t.Errorf("expected '%s' got '%s'", enFieldMapping.Analyzer, analyzerName) t.Errorf("expected '%s' got '%s'", enFieldMapping.Analyzer, analyzerName)
} }
analyzerName = mapping.analyzerNameForPath("nameCustom") analyzerName = mapping.AnalyzerNameForPath("nameCustom")
if analyzerName != customMapping.Analyzer { if analyzerName != customMapping.Analyzer {
t.Errorf("expected '%s' got '%s'", customMapping.Analyzer, analyzerName) t.Errorf("expected '%s' got '%s'", customMapping.Analyzer, analyzerName)
} }
analyzerName = mapping.analyzerNameForPath("child.desc") analyzerName = mapping.AnalyzerNameForPath("child.desc")
if analyzerName != customFieldX.Analyzer { if analyzerName != customFieldX.Analyzer {
t.Errorf("expected '%s' got '%s'", customFieldX.Analyzer, analyzerName) t.Errorf("expected '%s' got '%s'", customFieldX.Analyzer, analyzerName)
} }
@ -345,7 +345,7 @@ func TestEnablingDisablingStoringDynamicFields(t *testing.T) {
} }
doc := document.NewDocument("x") doc := document.NewDocument("x")
mapping := NewIndexMapping() mapping := NewIndexMapping()
err := mapping.mapDocument(doc, data) err := mapping.MapDocument(doc, data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -363,7 +363,7 @@ func TestEnablingDisablingStoringDynamicFields(t *testing.T) {
mapping = NewIndexMapping() mapping = NewIndexMapping()
doc = document.NewDocument("y") doc = document.NewDocument("y")
err = mapping.mapDocument(doc, data) err = mapping.MapDocument(doc, data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -377,7 +377,7 @@ func TestEnablingDisablingStoringDynamicFields(t *testing.T) {
mapping = NewIndexMapping() mapping = NewIndexMapping()
mapping.StoreDynamic = true mapping.StoreDynamic = true
doc = document.NewDocument("y") doc = document.NewDocument("y")
err = mapping.mapDocument(doc, data) err = mapping.MapDocument(doc, data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -405,7 +405,7 @@ func TestMappingBool(t *testing.T) {
} }
doc := document.NewDocument("1") doc := document.NewDocument("1")
err := mapping.mapDocument(doc, x) err := mapping.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -441,7 +441,7 @@ func TestDisableDefaultMapping(t *testing.T) {
} }
doc := document.NewDocument("x") doc := document.NewDocument("x")
err := indexMapping.mapDocument(doc, data) err := indexMapping.MapDocument(doc, data)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
} }
@ -526,7 +526,7 @@ func TestInvalidIndexMappingStrict(t *testing.T) {
mappingBytes := []byte(`{"typeField":"type","default_field":"all"}`) mappingBytes := []byte(`{"typeField":"type","default_field":"all"}`)
// first unmarhsal it without strict // first unmarhsal it without strict
var im IndexMapping var im IndexMappingImpl
err := json.Unmarshal(mappingBytes, &im) err := json.Unmarshal(mappingBytes, &im)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -590,7 +590,7 @@ func TestMappingBug353(t *testing.T) {
mapping.DefaultMapping.AddSubDocumentMapping("Other", otherMapping) mapping.DefaultMapping.AddSubDocumentMapping("Other", otherMapping)
doc := document.NewDocument("x") doc := document.NewDocument("x")
err = mapping.mapDocument(doc, data) err = mapping.MapDocument(doc, data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -636,7 +636,7 @@ func TestAnonymousStructFields(t *testing.T) {
doc := document.NewDocument("1") doc := document.NewDocument("1")
m := NewIndexMapping() m := NewIndexMapping()
err := m.mapDocument(doc, x) err := m.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -676,7 +676,7 @@ func TestAnonymousStructFields(t *testing.T) {
} }
doc2 := document.NewDocument("2") doc2 := document.NewDocument("2")
err = m.mapDocument(doc2, y) err = m.MapDocument(doc2, y)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -712,7 +712,7 @@ func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) {
doc := document.NewDocument("1") doc := document.NewDocument("1")
m := NewIndexMapping() m := NewIndexMapping()
err := m.mapDocument(doc, x) err := m.MapDocument(doc, x)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -749,7 +749,7 @@ func TestMappingPrimitives(t *testing.T) {
m := NewIndexMapping() m := NewIndexMapping()
for _, test := range tests { for _, test := range tests {
doc := document.NewDocument("x") doc := document.NewDocument("x")
err := m.mapDocument(doc, test.data) err := m.MapDocument(doc, test.data)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -7,7 +7,7 @@
// either express or implied. See the License for the specific language governing permissions // either express or implied. See the License for the specific language governing permissions
// and limitations under the License. // and limitations under the License.
package bleve package mapping
import ( import (
"reflect" "reflect"

View File

@ -14,6 +14,7 @@ import (
"fmt" "fmt"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -24,7 +25,7 @@ type Query interface {
SetBoost(b float64) Query SetBoost(b float64) Query
Field() string Field() string
SetField(f string) Query SetField(f string) Query
Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error)
Validate() error Validate() error
} }
@ -251,7 +252,7 @@ func ParseQuery(input []byte) (Query, error) {
// expandQuery traverses the input query tree and returns a new tree where // expandQuery traverses the input query tree and returns a new tree where
// query string queries have been expanded into base queries. Returned tree may // query string queries have been expanded into base queries. Returned tree may
// reference queries from the input tree or new queries. // reference queries from the input tree or new queries.
func expandQuery(m *IndexMapping, query Query) (Query, error) { func expandQuery(m mapping.IndexMapping, query Query) (Query, error) {
var expand func(query Query) (Query, error) var expand func(query Query) (Query, error)
var expandSlice func(queries []Query) ([]Query, error) var expandSlice func(queries []Query) ([]Query, error)
@ -326,7 +327,7 @@ func expandQuery(m *IndexMapping, query Query) (Query, error) {
// DumpQuery returns a string representation of the query tree, where query // DumpQuery returns a string representation of the query tree, where query
// string queries have been expanded into base queries. The output format is // string queries have been expanded into base queries. The output format is
// meant for debugging purpose and may change in the future. // meant for debugging purpose and may change in the future.
func DumpQuery(m *IndexMapping, query Query) (string, error) { func DumpQuery(m mapping.IndexMapping, query Query) (string, error) {
q, err := expandQuery(m, query) q, err := expandQuery(m, query)
if err != nil { if err != nil {
return "", err return "", err

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -47,10 +48,10 @@ func (q *boolFieldQuery) SetField(f string) Query {
return q return q
} }
func (q *boolFieldQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *boolFieldQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
term := "F" term := "F"
if q.Bool { if q.Bool {

View File

@ -14,6 +14,7 @@ import (
"fmt" "fmt"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -95,7 +96,7 @@ func (q *booleanQuery) SetBoost(b float64) Query {
return q return q
} }
func (q *booleanQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *booleanQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
var err error var err error
var mustNotSearcher search.Searcher var mustNotSearcher search.Searcher
if q.MustNot != nil { if q.MustNot != nil {

View File

@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -45,7 +46,7 @@ func (q *conjunctionQuery) AddQuery(aq Query) *conjunctionQuery {
return q return q
} }
func (q *conjunctionQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *conjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
ss := make([]search.Searcher, len(q.Conjuncts)) ss := make([]search.Searcher, len(q.Conjuncts))
for in, conjunct := range q.Conjuncts { for in, conjunct := range q.Conjuncts {
var err error var err error

View File

@ -14,6 +14,7 @@ import (
"math" "math"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/numeric_util" "github.com/blevesearch/bleve/numeric_util"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
@ -71,7 +72,7 @@ func (q *dateRangeQuery) SetField(f string) Query {
return q return q
} }
func (q *dateRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *dateRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
min, max, err := q.parseEndpoints() min, max, err := q.parseEndpoints()
if err != nil { if err != nil {
@ -80,7 +81,7 @@ func (q *dateRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
return searchers.NewNumericRangeSearcher(i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal, explain) return searchers.NewNumericRangeSearcher(i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal, explain)

View File

@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -65,7 +66,7 @@ func (q *disjunctionQuery) SetMin(m float64) Query {
return q return q
} }
func (q *disjunctionQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *disjunctionQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
ss := make([]search.Searcher, len(q.Disjuncts)) ss := make([]search.Searcher, len(q.Disjuncts))
for in, disjunct := range q.Disjuncts { for in, disjunct := range q.Disjuncts {
var err error var err error

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -47,7 +48,7 @@ func (q *docIDQuery) SetField(f string) Query {
return q return q
} }
func (q *docIDQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *docIDQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
return searchers.NewDocIDSearcher(i, q.IDs, q.BoostVal, explain) return searchers.NewDocIDSearcher(i, q.IDs, q.BoostVal, explain)
} }

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -75,10 +76,10 @@ func (q *fuzzyQuery) SetPrefix(p int) Query {
return q return q
} }
func (q *fuzzyQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *fuzzyQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
return searchers.NewFuzzySearcher(i, q.Term, q.PrefixVal, q.FuzzinessVal, field, q.BoostVal, explain) return searchers.NewFuzzySearcher(i, q.Term, q.PrefixVal, q.FuzzinessVal, field, q.BoostVal, explain)
} }

View File

@ -14,6 +14,7 @@ import (
"fmt" "fmt"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -145,20 +146,20 @@ func (q *matchQuery) SetOperator(operator MatchQueryOperator) Query {
return q return q
} }
func (q *matchQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *matchQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
analyzerName := "" analyzerName := ""
if q.Analyzer != "" { if q.Analyzer != "" {
analyzerName = q.Analyzer analyzerName = q.Analyzer
} else { } else {
analyzerName = m.analyzerNameForPath(field) analyzerName = m.AnalyzerNameForPath(field)
} }
analyzer := m.analyzerNamed(analyzerName) analyzer := m.AnalyzerNamed(analyzerName)
if analyzer == nil { if analyzer == nil {
return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer)

View File

@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -38,7 +39,7 @@ func (q *matchAllQuery) SetBoost(b float64) Query {
return q return q
} }
func (q *matchAllQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *matchAllQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
return searchers.NewMatchAllSearcher(i, q.BoostVal, explain) return searchers.NewMatchAllSearcher(i, q.BoostVal, explain)
} }

View File

@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -38,7 +39,7 @@ func (q *matchNoneQuery) SetBoost(b float64) Query {
return q return q
} }
func (q *matchNoneQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *matchNoneQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
return searchers.NewMatchNoneSearcher(i) return searchers.NewMatchNoneSearcher(i)
} }

View File

@ -14,6 +14,7 @@ import (
"github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -57,19 +58,19 @@ func (q *matchPhraseQuery) SetField(f string) Query {
return q return q
} }
func (q *matchPhraseQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *matchPhraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
analyzerName := "" analyzerName := ""
if q.Analyzer != "" { if q.Analyzer != "" {
analyzerName = q.Analyzer analyzerName = q.Analyzer
} else { } else {
analyzerName = m.analyzerNameForPath(field) analyzerName = m.AnalyzerNameForPath(field)
} }
analyzer := m.analyzerNamed(analyzerName) analyzer := m.AnalyzerNamed(analyzerName)
if analyzer == nil { if analyzer == nil {
return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer) return nil, fmt.Errorf("no analyzer named '%s' registered", q.Analyzer)
} }

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -65,10 +66,10 @@ func (q *numericRangeQuery) SetField(f string) Query {
return q return q
} }
func (q *numericRangeQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *numericRangeQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
return searchers.NewNumericRangeSearcher(i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal, explain) return searchers.NewNumericRangeSearcher(i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal, explain)
} }

View File

@ -13,6 +13,7 @@ import (
"encoding/json" "encoding/json"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -54,7 +55,7 @@ func (q *phraseQuery) SetBoost(b float64) Query {
return q return q
} }
func (q *phraseQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *phraseQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
conjunctionQuery := NewConjunctionQuery(q.termQueries) conjunctionQuery := NewConjunctionQuery(q.termQueries)
conjunctionSearcher, err := conjunctionQuery.Searcher(i, m, explain) conjunctionSearcher, err := conjunctionQuery.Searcher(i, m, explain)

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -49,10 +50,10 @@ func (q *prefixQuery) SetField(f string) Query {
return q return q
} }
func (q *prefixQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *prefixQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
return searchers.NewTermPrefixSearcher(i, q.Prefix, field, q.BoostVal, explain) return searchers.NewTermPrefixSearcher(i, q.Prefix, field, q.BoostVal, explain)
} }

View File

@ -14,6 +14,7 @@ import (
"strings" "strings"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -53,10 +54,10 @@ func (q *regexpQuery) SetField(f string) Query {
return q return q
} }
func (q *regexpQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *regexpQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
err := q.compile() err := q.compile()
if err != nil { if err != nil {

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
) )
@ -38,7 +39,7 @@ func (q *queryStringQuery) SetBoost(b float64) Query {
return q return q
} }
func (q *queryStringQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *queryStringQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
newQuery, err := parseQuerySyntax(q.Query) newQuery, err := parseQuerySyntax(q.Query)
if err != nil { if err != nil {
return nil, err return nil, err

View File

@ -13,6 +13,8 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"github.com/blevesearch/bleve/mapping"
) )
func TestQuerySyntaxParserValid(t *testing.T) { func TestQuerySyntaxParserValid(t *testing.T) {
@ -23,7 +25,7 @@ func TestQuerySyntaxParserValid(t *testing.T) {
tests := []struct { tests := []struct {
input string input string
result Query result Query
mapping *IndexMapping mapping mapping.IndexMapping
}{ }{
{ {
input: "test", input: "test",

View File

@ -11,6 +11,7 @@ package bleve
import ( import (
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -48,10 +49,10 @@ func (q *termQuery) SetField(f string) Query {
return q return q
} }
func (q *termQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *termQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
return searchers.NewTermSearcher(i, q.Term, field, q.BoostVal, explain) return searchers.NewTermSearcher(i, q.Term, field, q.BoostVal, explain)
} }

View File

@ -268,7 +268,7 @@ func TestQueryValidate(t *testing.T) {
} }
func TestDumpQuery(t *testing.T) { func TestDumpQuery(t *testing.T) {
mapping := &IndexMapping{} mapping := NewIndexMapping()
q := NewQueryStringQuery("+water -light beer") q := NewQueryStringQuery("+water -light beer")
s, err := DumpQuery(mapping, q) s, err := DumpQuery(mapping, q)
if err != nil { if err != nil {

View File

@ -14,6 +14,7 @@ import (
"strings" "strings"
"github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index"
"github.com/blevesearch/bleve/mapping"
"github.com/blevesearch/bleve/search" "github.com/blevesearch/bleve/search"
"github.com/blevesearch/bleve/search/searchers" "github.com/blevesearch/bleve/search/searchers"
) )
@ -74,10 +75,10 @@ func (q *wildcardQuery) SetField(f string) Query {
return q return q
} }
func (q *wildcardQuery) Searcher(i index.IndexReader, m *IndexMapping, explain bool) (search.Searcher, error) { func (q *wildcardQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, explain bool) (search.Searcher, error) {
field := q.FieldVal field := q.FieldVal
if q.FieldVal == "" { if q.FieldVal == "" {
field = m.DefaultField field = m.DefaultSearchField()
} }
if q.compiled == nil { if q.compiled == nil {
var err error var err error

View File

@ -24,6 +24,7 @@ import (
// we must explicitly include any functionality we plan on testing // we must explicitly include any functionality we plan on testing
_ "github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer" _ "github.com/blevesearch/bleve/analysis/analyzers/keyword_analyzer"
"github.com/blevesearch/bleve/mapping"
// allow choosing alternate kvstores // allow choosing alternate kvstores
_ "github.com/blevesearch/bleve/config" _ "github.com/blevesearch/bleve/config"
@ -77,7 +78,7 @@ func runTestDir(t *testing.T, dir, datasetName string) {
t.Errorf("error reading mapping: %v", err) t.Errorf("error reading mapping: %v", err)
return return
} }
var mapping bleve.IndexMapping var mapping mapping.IndexMappingImpl
err = json.Unmarshal(mappingBytes, &mapping) err = json.Unmarshal(mappingBytes, &mapping)
if err != nil { if err != nil {
t.Errorf("error unmarshalling mapping: %v", err) t.Errorf("error unmarshalling mapping: %v", err)