gibheer
/
bleve
0
0
Fork 0
Code

continued refactoring of the mapping code 

also renamed some constant that didnt follow go convetions
This commit is contained in:
Marty Schoch 2014-09-03 13:02:10 -04:00
parent a151bda2ad
commit 8e6c8e5644
14 changed files with 202 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
document/field_composite.go
View File

@ -13,7 +13,7 @@ import (
"github.com/blevesearch/bleve/analysis"
)
const DEFAULT_COMPOSITE_INDEXING_OPTIONS = INDEX_FIELD
const DefaultCompositeIndexingOptions = IndexField
type CompositeField struct {
name string
@ -26,7 +26,7 @@ type CompositeField struct {
}
func NewCompositeField(name string, defaultInclude bool, include []string, exclude []string) *CompositeField {
return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DEFAULT_COMPOSITE_INDEXING_OPTIONS)
return NewCompositeFieldWithIndexingOptions(name, defaultInclude, include, exclude, DefaultCompositeIndexingOptions)
}
func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField {

12
document/field_datetime.go
View File

@ -18,8 +18,8 @@ import (
"github.com/blevesearch/bleve/numeric_util"
)
const DEFAULT_DATETIME_INDEXING_OPTIONS = STORE_FIELD | INDEX_FIELD
const DEFAULT_DATETIME_PRECISION_STEP uint = 4
const DefaultDateTimeIndexingOptions = StoreField | IndexField
const DefaultDateTimePrecisionStep uint = 4
var MinTimeRepresentable = time.Unix(0, math.MinInt64)
var MaxTimeRepresentable = time.Unix(0, math.MaxInt64)
@ -56,7 +56,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
original, err := n.value.Int64()
if err == nil {
shift := DEFAULT_PRECISION_STEP
shift := DefaultDateTimePrecisionStep
for shift < 64 {
shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
if err != nil {
@ -70,7 +70,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
Type: analysis.DateTime,
}
tokens = append(tokens, &token)
shift += DEFAULT_PRECISION_STEP
shift += DefaultDateTimePrecisionStep
}
}
@ -100,12 +100,12 @@ func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byt
name: name,
arrayPositions: arrayPositions,
value: value,
options: DEFAULT_DATETIME_INDEXING_OPTIONS,
options: DefaultDateTimeIndexingOptions,
}
}
func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*DateTimeField, error) {
return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DEFAULT_DATETIME_INDEXING_OPTIONS)
return NewDateTimeFieldWithIndexingOptions(name, arrayPositions, dt, DefaultDateTimeIndexingOptions)
}
func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) {

12
document/field_numeric.go
View File

@ -16,9 +16,9 @@ import (
"github.com/blevesearch/bleve/numeric_util"
)
const DEFAULT_NUMERIC_INDEXING_OPTIONS = STORE_FIELD | INDEX_FIELD
const DefaultNumericIndexingOptions = StoreField | IndexField
const DEFAULT_PRECISION_STEP uint = 4
const DefaultPrecisionStep uint = 4
type NumericField struct {
name string
@ -52,7 +52,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) {
original, err := n.value.Int64()
if err == nil {
shift := DEFAULT_PRECISION_STEP
shift := DefaultPrecisionStep
for shift < 64 {
shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
if err != nil {
@ -66,7 +66,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) {
Type: analysis.Numeric,
}
tokens = append(tokens, &token)
shift += DEFAULT_PRECISION_STEP
shift += DefaultPrecisionStep
}
}
@ -96,12 +96,12 @@ func NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte
name: name,
arrayPositions: arrayPositions,
value: value,
options: DEFAULT_NUMERIC_INDEXING_OPTIONS,
options: DefaultNumericIndexingOptions,
}
}
func NewNumericField(name string, arrayPositions []uint64, number float64) *NumericField {
return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DEFAULT_NUMERIC_INDEXING_OPTIONS)
return NewNumericFieldWithIndexingOptions(name, arrayPositions, number, DefaultNumericIndexingOptions)
}
func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField {

6
document/field_text.go
View File

@ -15,7 +15,7 @@ import (
"github.com/blevesearch/bleve/analysis"
)
const DEFAULT_TEXT_INDEXING_OPTIONS = INDEX_FIELD
const DefaultTextIndexingOptions = IndexField
type TextField struct {
name string
@ -66,7 +66,7 @@ func (t *TextField) GoString() string {
}
func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField {
return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DEFAULT_TEXT_INDEXING_OPTIONS)
return NewTextFieldWithIndexingOptions(name, arrayPositions, value, DefaultTextIndexingOptions)
}
func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField {
@ -82,7 +82,7 @@ func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte
return &TextField{
name: name,
arrayPositions: arrayPositions,
options: DEFAULT_TEXT_INDEXING_OPTIONS,
options: DefaultTextIndexingOptions,
analyzer: analyzer,
value: value,
}

12
document/indexing_options.go
View File

@ -12,21 +12,21 @@ package document
type IndexingOptions int
const (
INDEX_FIELD IndexingOptions = 1 << iota
STORE_FIELD
INCLUDE_TERM_VECTORS
IndexField IndexingOptions = 1 << iota
StoreField
IncludeTermVectors
)
func (o IndexingOptions) IsIndexed() bool {
return o&INDEX_FIELD != 0
return o&IndexField != 0
}
func (o IndexingOptions) IsStored() bool {
return o&STORE_FIELD != 0
return o&StoreField != 0
}
func (o IndexingOptions) IncludeTermVectors() bool {
return o&INCLUDE_TERM_VECTORS != 0
return o&IncludeTermVectors != 0
}
func (o IndexingOptions) String() string {

10
document/indexing_options_test.go
View File

@ -21,31 +21,31 @@ func TestIndexingOptions(t *testing.T) {
includeTermVectors bool
}{
{
options: INDEX_FIELD | STORE_FIELD | INCLUDE_TERM_VECTORS,
options: IndexField | StoreField | IncludeTermVectors,
isIndexed: true,
isStored: true,
includeTermVectors: true,
},
{
options: INDEX_FIELD | INCLUDE_TERM_VECTORS,
options: IndexField | IncludeTermVectors,
isIndexed: true,
isStored: false,
includeTermVectors: true,
},
{
options: STORE_FIELD | INCLUDE_TERM_VECTORS,
options: StoreField | IncludeTermVectors,
isIndexed: false,
isStored: true,
includeTermVectors: true,
},
{
options: INDEX_FIELD,
options: IndexField,
isIndexed: true,
isStored: false,
includeTermVectors: false,
},
{
options: STORE_FIELD,
options: StoreField,
isIndexed: false,
isStored: true,
includeTermVectors: false,

16
index/upside_down/dump_test.go
View File

@ -39,9 +39,9 @@ func TestDump(t *testing.T) {
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.INDEX_FIELD|document.STORE_FIELD))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD)
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
@ -52,9 +52,9 @@ func TestDump(t *testing.T) {
}
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.INDEX_FIELD|document.STORE_FIELD))
dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD)
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
@ -77,7 +77,7 @@ func TestDump(t *testing.T) {
// 16 numeric terms
// 16 date terms
// 3 stored fields
expectedDocRowCount := int(1 + (2 * (64 / document.DEFAULT_PRECISION_STEP)) + 3)
expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3)
docRowCount := 0
docRows := idx.DumpDoc("1")
for _ = range docRows {
@ -103,7 +103,7 @@ func TestDump(t *testing.T) {
// 2 text term row count (2 different text terms)
// 16 numeric term row counts (shared for both docs, same numeric value)
// 16 date term row counts (shared for both docs, same date value)
expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DEFAULT_PRECISION_STEP))))
expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep))))
allRowCount := 0
allRows := idx.DumpAll()
for _ = range allRows {

4
index/upside_down/field_reader_test.go
View File

@ -40,8 +40,8 @@ func TestIndexFieldReader(t *testing.T) {
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS, testAnalyzer))
doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS, testAnalyzer))
doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)

4
index/upside_down/reader_test.go
View File

@ -41,7 +41,7 @@ func TestIndexReader(t *testing.T) {
doc = document.NewDocument("2")
doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS, testAnalyzer))
doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -183,7 +183,7 @@ func TestIndexDocIdReader(t *testing.T) {
doc = document.NewDocument("2")
doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test")))
doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)

46
index/upside_down/upside_down_test.go
View File

@ -302,7 +302,7 @@ func TestIndexInsertWithStore(t *testing.T) {
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -484,9 +484,9 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
}
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.INDEX_FIELD|document.STORE_FIELD))
df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD)
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
df, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
@ -513,7 +513,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
// 16 for numeric term counts
// 16 for date term counts
// 1 for the back index entry
expectedLength := uint64(1 + 3 + 1 + (64 / document.DEFAULT_PRECISION_STEP) + (64 / document.DEFAULT_PRECISION_STEP) + 3 + 1 + (64 / document.DEFAULT_PRECISION_STEP) + (64 / document.DEFAULT_PRECISION_STEP) + 1)
expectedLength := uint64(1 + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 3 + 1 + (64 / document.DefaultPrecisionStep) + (64 / document.DefaultPrecisionStep) + 1)
rowCount := idx.rowCount()
if rowCount != expectedLength {
t.Errorf("expected %d rows, got: %d", expectedLength, rowCount)
@ -561,8 +561,8 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
// now update the document, but omit one of the fields
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testup"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 36.99, document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -629,9 +629,9 @@ func TestIndexInsertFields(t *testing.T) {
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.INDEX_FIELD|document.STORE_FIELD))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.INDEX_FIELD|document.STORE_FIELD)
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
if err != nil {
t.Error(err)
}
@ -668,9 +668,9 @@ func TestIndexUpdateComposites(t *testing.T) {
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.INDEX_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -691,9 +691,9 @@ func TestIndexUpdateComposites(t *testing.T) {
// now lets update it
doc = document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.INDEX_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("testupdated"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("misterupdated"), document.IndexField|document.StoreField))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -737,8 +737,8 @@ func TestIndexFieldsMisc(t *testing.T) {
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.INDEX_FIELD|document.STORE_FIELD))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -774,9 +774,9 @@ func TestIndexTermReaderCompositeFields(t *testing.T) {
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewCompositeFieldWithIndexingOptions("_all", true, nil, nil, document.IndexField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)
@ -814,8 +814,8 @@ func TestIndexDocumentFieldTerms(t *testing.T) {
defer idx.Close()
doc := document.NewDocument("1")
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.INDEX_FIELD|document.STORE_FIELD|document.INCLUDE_TERM_VECTORS))
doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField|document.IncludeTermVectors))
doc.AddField(document.NewTextFieldWithIndexingOptions("title", []uint64{}, []byte("mister"), document.IndexField|document.StoreField|document.IncludeTermVectors))
err = idx.Update(doc)
if err != nil {
t.Errorf("Error updating index: %v", err)

82
mapping_document.go
View File

@ -12,11 +12,9 @@ package bleve
import (
"encoding/json"
"fmt"
"log"
"reflect"
"time"
"github.com/blevesearch/bleve/document"
"github.com/blevesearch/bleve/registry"
)
@ -257,34 +255,7 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
if subDocMapping != nil {
// index by explicit mapping
for _, fieldMapping := range subDocMapping.Fields {
fieldName := getFieldName(pathString, path, fieldMapping)
options := fieldMapping.Options()
if *fieldMapping.Type == "text" {
analyzer := context.im.analyzerNamed(*fieldMapping.Analyzer)
field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer)
context.doc.AddField(field)
if fieldMapping.IncludeInAll != nil && !*fieldMapping.IncludeInAll {
context.excludedFromAll = append(context.excludedFromAll, fieldName)
}
} else if *fieldMapping.Type == "datetime" {
dateTimeFormat := context.im.DefaultDateTimeParser
if fieldMapping.DateFormat != nil {
dateTimeFormat = *fieldMapping.DateFormat
}
dateTimeParser := context.im.dateTimeParserNamed(dateTimeFormat)
if dateTimeParser != nil {
parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)
if err != nil {
field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, parsedDateTime, options)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
}
}
}
fieldMapping.processString(propertyValueString, pathString, path, indexes, context)
}
} else {
// automatic indexing behavior
@ -294,23 +265,13 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
if dateTimeParser != nil {
parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)
if err != nil {
// index as plain text
options := document.STORE_FIELD | document.INDEX_FIELD | document.INCLUDE_TERM_VECTORS
analyzerName := dm.defaultAnalyzerName(path)
if analyzerName == "" {
analyzerName = context.im.DefaultAnalyzer
}
analyzer := context.im.analyzerNamed(analyzerName)
field := document.NewTextFieldCustom(pathString, indexes, []byte(propertyValueString), options, analyzer)
context.doc.AddField(field)
// index as text
fieldMapping := defaultTextFieldMapping()
fieldMapping.processString(propertyValueString, pathString, path, indexes, context)
} else {
// index as datetime
field, err := document.NewDateTimeField(pathString, indexes, parsedDateTime)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
fieldMapping := defaultDateTimeFieldMapping()
fieldMapping.processTime(parsedDateTime, pathString, path, indexes, context)
}
}
}
@ -319,17 +280,12 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
if subDocMapping != nil {
// index by explicit mapping
for _, fieldMapping := range subDocMapping.Fields {
fieldName := getFieldName(pathString, path, fieldMapping)
if *fieldMapping.Type == "number" {
options := fieldMapping.Options()
field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options)
context.doc.AddField(field)
}
fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)
}
} else {
// automatic indexing behavior
field := document.NewNumericField(pathString, indexes, propertyValFloat)
context.doc.AddField(field)
fieldMapping := defaultNumericFieldMapping()
fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context)
}
case reflect.Struct:
switch property := property.(type) {
@ -338,25 +294,11 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string,
if subDocMapping != nil {
// index by explicit mapping
for _, fieldMapping := range subDocMapping.Fields {
fieldName := getFieldName(pathString, path, fieldMapping)
if *fieldMapping.Type == "datetime" {
options := fieldMapping.Options()
field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, property, options)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
}
fieldMapping.processTime(property, pathString, path, indexes, context)
}
} else {
// automatic indexing behavior
field, err := document.NewDateTimeField(pathString, indexes, property)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
fieldMapping := defaultDateTimeFieldMapping()
fieldMapping.processTime(property, pathString, path, indexes, context)
}
default:
dm.walkDocument(property, path, indexes, context)

127
mapping_field.go
View File

@ -10,6 +10,10 @@
package bleve
import (
"log"
"time"
"github.com/blevesearch/bleve/analysis"
"github.com/blevesearch/bleve/document"
)
@ -40,17 +44,128 @@ func NewFieldMapping(name, typ, analyzer string, store, index bool, includeTermV
}
}
func defaultNumericFieldMapping() *FieldMapping {
typ := "number"
store := true
index := true
return &FieldMapping{
Type: &typ,
Store: &store,
Index: &index,
}
}
func defaultDateTimeFieldMapping() *FieldMapping {
typ := "datetime"
store := true
index := true
return &FieldMapping{
Type: &typ,
Store: &store,
Index: &index,
}
}
func defaultTextFieldMapping() *FieldMapping {
typ := "text"
store := true
index := true
include := true
return &FieldMapping{
Type: &typ,
Store: &store,
Index: &index,
IncludeTermVectors: &include,
}
}
// Options returns the indexing options for this field.
func (fm *FieldMapping) Options() document.IndexingOptions {
var rv document.IndexingOptions
if *fm.Store {
rv |= document.STORE_FIELD
if fm.Store != nil && *fm.Store {
rv |= document.StoreField
}
if *fm.Index {
rv |= document.INDEX_FIELD
if fm.Index != nil && *fm.Index {
rv |= document.IndexField
}
if *fm.IncludeTermVectors {
rv |= document.INCLUDE_TERM_VECTORS
if fm.IncludeTermVectors != nil && *fm.IncludeTermVectors {
rv |= document.IncludeTermVectors
}
return rv
}
func (fm *FieldMapping) processString(propertyValueString string, pathString string, path []string, indexes []uint64, context *walkContext) {
fieldName := getFieldName(pathString, path, fm)
options := fm.Options()
if *fm.Type == "text" {
analyzer := fm.analyzerForField(path, context)
field := document.NewTextFieldCustom(fieldName, indexes, []byte(propertyValueString), options, analyzer)
context.doc.AddField(field)
if fm.IncludeInAll != nil && !*fm.IncludeInAll {
context.excludedFromAll = append(context.excludedFromAll, fieldName)
}
} else if *fm.Type == "datetime" {
dateTimeFormat := context.im.DefaultDateTimeParser
if fm.DateFormat != nil {
dateTimeFormat = *fm.DateFormat
}
dateTimeParser := context.im.dateTimeParserNamed(dateTimeFormat)
if dateTimeParser != nil {
parsedDateTime, err := dateTimeParser.ParseDateTime(propertyValueString)
if err != nil {
field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, parsedDateTime, options)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
}
}
}
}
func (fm *FieldMapping) processFloat64(propertyValFloat float64, pathString string, path []string, indexes []uint64, context *walkContext) {
fieldName := getFieldName(pathString, path, fm)
if *fm.Type == "number" {
options := fm.Options()
field := document.NewNumericFieldWithIndexingOptions(fieldName, indexes, propertyValFloat, options)
context.doc.AddField(field)
}
}
func (fm *FieldMapping) processTime(propertyValueTime time.Time, pathString string, path []string, indexes []uint64, context *walkContext) {
fieldName := getFieldName(pathString, path, fm)
if *fm.Type == "datetime" {
options := fm.Options()
field, err := document.NewDateTimeFieldWithIndexingOptions(fieldName, indexes, propertyValueTime, options)
if err == nil {
context.doc.AddField(field)
} else {
log.Printf("could not build date %v", err)
}
}
}
func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) *analysis.Analyzer {
analyzerName := context.dm.defaultAnalyzerName(path)
if analyzerName == "" {
analyzerName = context.im.DefaultAnalyzer
}
if fm.Analyzer != nil && *fm.Analyzer != "" {
analyzerName = *fm.Analyzer
}
return context.im.analyzerNamed(analyzerName)
}
func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string {
fieldName := pathString
if fieldMapping.Name != nil && *fieldMapping.Name != "" {
parentName := ""
if len(path) > 1 {
parentName = encodePath(path[:len(path)-1]) + pathSeparator
}
fieldName = parentName + *fieldMapping.Name
}
return fieldName
}

20
mapping_index.go
View File

@ -351,13 +351,13 @@ func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) er
docType := im.determineType(data)
docMapping := im.mappingForType(docType)
walkContext := im.newWalkContext(doc)
walkContext := im.newWalkContext(doc, docMapping)
docMapping.walkDocument(data, []string{}, []uint64{}, walkContext)
// see if the _all field was disabled
allMapping := docMapping.documentMappingForPath("_all")
if allMapping == nil || (allMapping.Enabled != false) {
field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.INDEX_FIELD|document.INCLUDE_TERM_VECTORS)
field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, document.IndexField|document.IncludeTermVectors)
doc.AddField(field)
}
@ -367,13 +367,15 @@ func (im *IndexMapping) mapDocument(doc *document.Document, data interface{}) er
type walkContext struct {
doc *document.Document
im *IndexMapping
dm *DocumentMapping
excludedFromAll []string
}
func (im *IndexMapping) newWalkContext(doc *document.Document) *walkContext {
func (im *IndexMapping) newWalkContext(doc *document.Document, dm *DocumentMapping) *walkContext {
return &walkContext{
doc: doc,
im: im,
dm: dm,
excludedFromAll: []string{},
}
}
@ -443,15 +445,3 @@ func (im *IndexMapping) datetimeParserNameForPath(path string) string {
return im.DefaultDateTimeParser
}
func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string {
fieldName := pathString
if fieldMapping.Name != nil && *fieldMapping.Name != "" {
parentName := ""
if len(path) > 1 {
parentName = encodePath(path[:len(path)-1]) + pathSeparator
}
fieldName = parentName + *fieldMapping.Name
}
return fieldName
}

2
search/searchers/base_test.go
View File

@ -37,7 +37,7 @@ var testAnalyzer = &analysis.Analyzer{
}
// sets up some mock data used in many tests in this package
var twoDocIndexDescIndexingOptions = document.DEFAULT_TEXT_INDEXING_OPTIONS | document.INCLUDE_TERM_VECTORS
var twoDocIndexDescIndexingOptions = document.DefaultTextIndexingOptions | document.IncludeTermVectors
var twoDocIndexDocs = []*document.Document{
// must have 4/4 beer