2014-07-30 18:30:38 +02:00
|
|
|
// Copyright (c) 2014 Couchbase, Inc.
|
2016-10-02 16:13:14 +02:00
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
2014-08-29 20:18:36 +02:00
|
|
|
|
2014-07-30 18:30:38 +02:00
|
|
|
package bleve
|
|
|
|
|
|
|
|
import (
|
2014-08-20 22:58:20 +02:00
|
|
|
"encoding/json"
|
2014-07-30 18:30:38 +02:00
|
|
|
"fmt"
|
2014-08-20 22:58:20 +02:00
|
|
|
"os"
|
2014-08-25 15:06:53 +02:00
|
|
|
"sync"
|
2014-10-02 20:12:22 +02:00
|
|
|
"sync/atomic"
|
2014-08-06 19:52:20 +02:00
|
|
|
"time"
|
2014-07-30 18:30:38 +02:00
|
|
|
|
2016-03-02 22:55:02 +01:00
|
|
|
"golang.org/x/net/context"
|
|
|
|
|
2014-08-28 21:38:57 +02:00
|
|
|
"github.com/blevesearch/bleve/document"
|
|
|
|
"github.com/blevesearch/bleve/index"
|
|
|
|
"github.com/blevesearch/bleve/index/store"
|
2016-09-30 17:30:17 +02:00
|
|
|
"github.com/blevesearch/bleve/index/upsidedown"
|
2016-09-18 15:33:18 +02:00
|
|
|
"github.com/blevesearch/bleve/mapping"
|
2014-08-28 21:38:57 +02:00
|
|
|
"github.com/blevesearch/bleve/registry"
|
|
|
|
"github.com/blevesearch/bleve/search"
|
2016-10-01 23:20:59 +02:00
|
|
|
"github.com/blevesearch/bleve/search/collector"
|
|
|
|
"github.com/blevesearch/bleve/search/facet"
|
2016-04-28 20:12:33 +02:00
|
|
|
"github.com/blevesearch/bleve/search/highlight"
|
2014-07-30 18:30:38 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
type indexImpl struct {
|
2014-08-25 15:06:53 +02:00
|
|
|
path string
|
2015-12-06 20:01:03 +01:00
|
|
|
name string
|
2014-08-25 15:06:53 +02:00
|
|
|
meta *indexMeta
|
|
|
|
i index.Index
|
2016-09-18 15:33:18 +02:00
|
|
|
m mapping.IndexMapping
|
2014-08-25 15:06:53 +02:00
|
|
|
mutex sync.RWMutex
|
|
|
|
open bool
|
2014-10-02 20:12:22 +02:00
|
|
|
stats *IndexStat
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
const storePath = "store"
|
|
|
|
|
|
|
|
var mappingInternalKey = []byte("_mapping")
|
|
|
|
|
|
|
|
func indexStorePath(path string) string {
|
|
|
|
return path + string(os.PathSeparator) + storePath
|
|
|
|
}
|
|
|
|
|
2016-09-18 15:33:18 +02:00
|
|
|
func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
|
2014-08-20 22:58:20 +02:00
|
|
|
// first validate the mapping
|
2016-03-28 23:14:41 +02:00
|
|
|
err := mapping.Validate()
|
2014-08-14 03:14:47 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2015-01-06 23:19:46 +01:00
|
|
|
if kvconfig == nil {
|
|
|
|
kvconfig = map[string]interface{}{}
|
|
|
|
}
|
|
|
|
|
2016-07-30 03:27:23 +02:00
|
|
|
if kvstore == "" {
|
|
|
|
return nil, fmt.Errorf("bleve not configured for file based indexing")
|
|
|
|
}
|
|
|
|
|
2014-08-20 22:58:20 +02:00
|
|
|
rv := indexImpl{
|
2016-03-08 00:37:46 +01:00
|
|
|
path: path,
|
|
|
|
name: path,
|
|
|
|
m: mapping,
|
|
|
|
meta: newIndexMeta(indexType, kvstore, kvconfig),
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
2016-03-08 00:37:46 +01:00
|
|
|
rv.stats = &IndexStat{i: &rv}
|
2014-12-18 18:43:12 +01:00
|
|
|
// at this point there is hope that we can be successful, so save index meta
|
BREAKING CHANGE - new method to create memory only index
Previously bleve allowed you to create a memory-only index by
simply passing "" as the path argument to the New() method.
This was not clear when reading the code, and led to some
problematic error cases as well.
Now, to create a memory-only index one should use the
NewMemOnly() method. Passing "" as the path argument
to the New() method will now return os.ErrInvalid.
Advanced users calling NewUsing() can create disk-based or
memory-only indexes, but the change here is that pass ""
as the path argument no longer defaults you into getting
a memory-only index. Instead, the KV store is selected
manually, just as it is for the disk-based solutions.
Here is an example use of the NewUsing() method to create
a memory-only index:
NewUsing("", indexMapping, Config.DefaultIndexType,
Config.DefaultMemKVStore, nil)
Config.DefaultMemKVStore is just a new default value
added to the configuration, it currently points to
gtreap.Name (which could have been used directly
instead for more control)
closes #427
2016-09-27 20:05:55 +02:00
|
|
|
if path != "" {
|
|
|
|
err = rv.meta.Save(path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
kvconfig["create_if_missing"] = true
|
|
|
|
kvconfig["error_if_exists"] = true
|
|
|
|
kvconfig["path"] = indexStorePath(path)
|
|
|
|
} else {
|
|
|
|
kvconfig["path"] = ""
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
|
|
|
|
2014-12-18 18:43:12 +01:00
|
|
|
// open the index
|
2015-09-02 19:12:08 +02:00
|
|
|
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
|
|
|
if indexTypeConstructor == nil {
|
|
|
|
return nil, ErrorUnknownIndexType
|
|
|
|
}
|
|
|
|
|
2015-09-23 20:25:47 +02:00
|
|
|
rv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue)
|
2015-09-02 19:12:08 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-08-20 22:58:20 +02:00
|
|
|
err = rv.i.Open()
|
2014-07-30 18:30:38 +02:00
|
|
|
if err != nil {
|
2015-09-23 20:25:47 +02:00
|
|
|
if err == index.ErrorUnknownStorageType {
|
|
|
|
return nil, ErrorUnknownStorageType
|
|
|
|
}
|
2014-07-30 18:30:38 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
2014-08-20 22:58:20 +02:00
|
|
|
|
|
|
|
// now persist the mapping
|
|
|
|
mappingBytes, err := json.Marshal(mapping)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-08-25 15:06:53 +02:00
|
|
|
|
|
|
|
// mark the index as open
|
|
|
|
rv.mutex.Lock()
|
|
|
|
defer rv.mutex.Unlock()
|
|
|
|
rv.open = true
|
2015-12-06 20:01:03 +01:00
|
|
|
indexStats.Register(&rv)
|
2014-08-20 22:58:20 +02:00
|
|
|
return &rv, nil
|
|
|
|
}
|
|
|
|
|
2015-03-11 06:28:51 +01:00
|
|
|
func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {
|
|
|
|
rv = &indexImpl{
|
2016-03-08 00:37:46 +01:00
|
|
|
path: path,
|
|
|
|
name: path,
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
2016-03-08 00:37:46 +01:00
|
|
|
rv.stats = &IndexStat{i: rv}
|
2015-03-11 06:28:51 +01:00
|
|
|
|
2014-08-29 21:19:02 +02:00
|
|
|
rv.meta, err = openIndexMeta(path)
|
2014-08-20 22:58:20 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2016-01-14 23:46:27 +01:00
|
|
|
// backwards compatibility if index type is missing
|
2015-09-25 18:56:42 +02:00
|
|
|
if rv.meta.IndexType == "" {
|
2016-09-30 17:30:17 +02:00
|
|
|
rv.meta.IndexType = upsidedown.Name
|
2015-09-25 18:56:42 +02:00
|
|
|
}
|
|
|
|
|
2015-01-06 23:19:46 +01:00
|
|
|
storeConfig := rv.meta.Config
|
|
|
|
if storeConfig == nil {
|
|
|
|
storeConfig = map[string]interface{}{}
|
|
|
|
}
|
|
|
|
|
|
|
|
storeConfig["path"] = indexStorePath(path)
|
|
|
|
storeConfig["create_if_missing"] = false
|
|
|
|
storeConfig["error_if_exists"] = false
|
|
|
|
for rck, rcv := range runtimeConfig {
|
|
|
|
storeConfig[rck] = rcv
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
|
|
|
|
2014-12-18 18:43:12 +01:00
|
|
|
// open the index
|
2015-09-02 19:12:08 +02:00
|
|
|
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
|
|
|
if indexTypeConstructor == nil {
|
|
|
|
return nil, ErrorUnknownIndexType
|
|
|
|
}
|
|
|
|
|
2015-09-23 20:25:47 +02:00
|
|
|
rv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue)
|
2015-09-02 19:12:08 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-08-20 22:58:20 +02:00
|
|
|
err = rv.i.Open()
|
|
|
|
if err != nil {
|
2015-09-23 20:25:47 +02:00
|
|
|
if err == index.ErrorUnknownStorageType {
|
|
|
|
return nil, ErrorUnknownStorageType
|
|
|
|
}
|
2014-08-20 22:58:20 +02:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// now load the mapping
|
2014-10-31 14:40:23 +01:00
|
|
|
indexReader, err := rv.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if cerr := indexReader.Close(); cerr != nil && err == nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
2014-09-12 23:21:35 +02:00
|
|
|
mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
|
2014-07-30 18:30:38 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-08-20 22:58:20 +02:00
|
|
|
|
2016-09-18 15:33:18 +02:00
|
|
|
var im *mapping.IndexMappingImpl
|
2014-08-20 22:58:20 +02:00
|
|
|
err = json.Unmarshal(mappingBytes, &im)
|
|
|
|
if err != nil {
|
2016-03-11 22:36:43 +01:00
|
|
|
return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes))
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
|
|
|
|
2014-08-25 15:06:53 +02:00
|
|
|
// mark the index as open
|
|
|
|
rv.mutex.Lock()
|
|
|
|
defer rv.mutex.Unlock()
|
|
|
|
rv.open = true
|
|
|
|
|
2014-08-20 22:58:20 +02:00
|
|
|
// validate the mapping
|
2016-03-28 23:14:41 +02:00
|
|
|
err = im.Validate()
|
2014-08-20 22:58:20 +02:00
|
|
|
if err != nil {
|
2014-08-25 15:06:53 +02:00
|
|
|
// note even if the mapping is invalid
|
|
|
|
// we still return an open usable index
|
2015-03-11 06:28:51 +01:00
|
|
|
return rv, err
|
2014-08-20 22:58:20 +02:00
|
|
|
}
|
|
|
|
|
2016-09-18 15:33:18 +02:00
|
|
|
rv.m = im
|
2015-12-06 20:01:03 +01:00
|
|
|
indexStats.Register(rv)
|
2015-03-11 06:28:51 +01:00
|
|
|
return rv, err
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2014-12-27 22:23:46 +01:00
|
|
|
// Advanced returns implementation internals
|
|
|
|
// necessary ONLY for advanced usage.
|
|
|
|
func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) {
|
2016-02-02 17:54:18 +01:00
|
|
|
s, err := i.i.Advanced()
|
|
|
|
if err != nil {
|
|
|
|
return nil, nil, err
|
|
|
|
}
|
|
|
|
return i.i, s, nil
|
2014-12-27 22:23:46 +01:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Mapping returns the IndexMapping in use by this
|
|
|
|
// Index.
|
2016-09-18 15:33:18 +02:00
|
|
|
func (i *indexImpl) Mapping() mapping.IndexMapping {
|
2014-08-25 15:06:53 +02:00
|
|
|
return i.m
|
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Index the object with the specified identifier.
|
|
|
|
// The IndexMapping for this index will determine
|
|
|
|
// how the object is indexed.
|
2015-09-21 20:20:27 +02:00
|
|
|
func (i *indexImpl) Index(id string, data interface{}) (err error) {
|
2015-09-28 23:00:08 +02:00
|
|
|
if id == "" {
|
|
|
|
return ErrorEmptyID
|
|
|
|
}
|
|
|
|
|
2014-09-24 14:13:14 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
2014-08-25 15:06:53 +02:00
|
|
|
|
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
|
|
|
|
2014-07-30 18:30:38 +02:00
|
|
|
doc := document.NewDocument(id)
|
2016-09-18 15:33:18 +02:00
|
|
|
err = i.m.MapDocument(doc, data)
|
2014-07-30 18:30:38 +02:00
|
|
|
if err != nil {
|
2015-09-29 19:02:56 +02:00
|
|
|
return
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
err = i.i.Update(doc)
|
2015-09-21 20:20:27 +02:00
|
|
|
return
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Delete entries for the specified identifier from
|
|
|
|
// the index.
|
2015-09-21 20:20:27 +02:00
|
|
|
func (i *indexImpl) Delete(id string) (err error) {
|
2015-09-28 23:00:08 +02:00
|
|
|
if id == "" {
|
|
|
|
return ErrorEmptyID
|
|
|
|
}
|
|
|
|
|
2014-09-24 14:13:14 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
2014-08-25 15:06:53 +02:00
|
|
|
|
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
|
|
|
|
2015-09-21 20:20:27 +02:00
|
|
|
err = i.i.Delete(id)
|
|
|
|
return
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Batch executes multiple Index and Delete
|
|
|
|
// operations at the same time. There are often
|
|
|
|
// significant performance benefits when performing
|
|
|
|
// operations in a batch.
|
2014-10-31 14:40:23 +01:00
|
|
|
func (i *indexImpl) Batch(b *Batch) error {
|
2014-09-24 14:13:14 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
2014-08-25 15:06:53 +02:00
|
|
|
|
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
|
|
|
|
2015-03-03 19:18:20 +01:00
|
|
|
return i.i.Batch(b.internal)
|
2014-08-11 22:27:18 +02:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Document is used to find the values of all the
|
|
|
|
// stored fields for a document in the index. These
|
|
|
|
// stored fields are put back into a Document object
|
|
|
|
// and returned.
|
2015-03-11 06:28:51 +01:00
|
|
|
func (i *indexImpl) Document(id string) (doc *document.Document, err error) {
|
2014-08-25 15:06:53 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return nil, ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
2014-10-31 14:40:23 +01:00
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if cerr := indexReader.Close(); err == nil && cerr != nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
doc, err = indexReader.Document(id)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return doc, nil
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// DocCount returns the number of documents in the
|
|
|
|
// index.
|
2016-09-13 18:40:01 +02:00
|
|
|
func (i *indexImpl) DocCount() (count uint64, err error) {
|
2014-08-25 15:06:53 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
|
|
|
if !i.open {
|
2014-10-31 14:40:23 +01:00
|
|
|
return 0, ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
|
|
|
|
2016-09-13 18:40:01 +02:00
|
|
|
// open a reader for this search
|
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
return 0, fmt.Errorf("error opening index reader %v", err)
|
|
|
|
}
|
|
|
|
defer func() {
|
|
|
|
if cerr := indexReader.Close(); err == nil && cerr != nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
count, err = indexReader.DocCount()
|
|
|
|
return
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Search executes a search request operation.
|
|
|
|
// Returns a SearchResult object or an error.
|
2015-03-11 06:28:51 +01:00
|
|
|
func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) {
|
2016-03-02 22:55:02 +01:00
|
|
|
return i.SearchInContext(context.Background(), req)
|
|
|
|
}
|
|
|
|
|
|
|
|
// SearchInContext executes a search request operation within the provided
|
|
|
|
// Context. Returns a SearchResult object or an error.
|
|
|
|
func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) {
|
2014-08-25 15:06:53 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
2014-10-02 20:12:22 +02:00
|
|
|
searchStart := time.Now()
|
|
|
|
|
2014-08-25 15:06:53 +02:00
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return nil, ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
|
|
|
|
2016-10-02 16:29:39 +02:00
|
|
|
collector := collector.NewTopNCollector(req.Size, req.From, req.Sort)
|
2014-09-12 23:21:35 +02:00
|
|
|
|
|
|
|
// open a reader for this search
|
2014-10-31 14:40:23 +01:00
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
2015-01-16 20:07:47 +01:00
|
|
|
return nil, fmt.Errorf("error opening index reader %v", err)
|
2014-10-31 14:40:23 +01:00
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if cerr := indexReader.Close(); err == nil && cerr != nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
2014-09-12 23:21:35 +02:00
|
|
|
|
|
|
|
searcher, err := req.Query.Searcher(indexReader, i.m, req.Explain)
|
2014-07-30 18:30:38 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if serr := searcher.Close(); err == nil && serr != nil {
|
|
|
|
err = serr
|
|
|
|
}
|
|
|
|
}()
|
2014-08-11 17:03:29 +02:00
|
|
|
|
|
|
|
if req.Facets != nil {
|
2014-09-12 23:21:35 +02:00
|
|
|
facetsBuilder := search.NewFacetsBuilder(indexReader)
|
2014-08-11 17:03:29 +02:00
|
|
|
for facetName, facetRequest := range req.Facets {
|
|
|
|
if facetRequest.NumericRanges != nil {
|
|
|
|
// build numeric range facet
|
2016-10-02 16:29:39 +02:00
|
|
|
facetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)
|
2014-08-11 17:03:29 +02:00
|
|
|
for _, nr := range facetRequest.NumericRanges {
|
|
|
|
facetBuilder.AddRange(nr.Name, nr.Min, nr.Max)
|
|
|
|
}
|
|
|
|
facetsBuilder.Add(facetName, facetBuilder)
|
|
|
|
} else if facetRequest.DateTimeRanges != nil {
|
|
|
|
// build date range facet
|
2016-10-02 16:29:39 +02:00
|
|
|
facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
|
2016-09-18 15:33:18 +02:00
|
|
|
dateTimeParser := i.m.DateTimeParserNamed("")
|
2014-08-11 17:03:29 +02:00
|
|
|
for _, dr := range facetRequest.DateTimeRanges {
|
2014-08-14 03:14:47 +02:00
|
|
|
dr.ParseDates(dateTimeParser)
|
2014-08-11 17:03:29 +02:00
|
|
|
facetBuilder.AddRange(dr.Name, dr.Start, dr.End)
|
|
|
|
}
|
|
|
|
facetsBuilder.Add(facetName, facetBuilder)
|
|
|
|
} else {
|
|
|
|
// build terms facet
|
2016-10-02 16:29:39 +02:00
|
|
|
facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
|
2014-08-11 17:03:29 +02:00
|
|
|
facetsBuilder.Add(facetName, facetBuilder)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
collector.SetFacetsBuilder(facetsBuilder)
|
|
|
|
}
|
|
|
|
|
2016-07-31 19:46:18 +02:00
|
|
|
err = collector.Collect(ctx, searcher, indexReader)
|
2014-07-30 18:30:38 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
hits := collector.Results()
|
|
|
|
|
2016-04-28 20:12:33 +02:00
|
|
|
var highlighter highlight.Highlighter
|
|
|
|
|
2014-07-30 18:30:38 +02:00
|
|
|
if req.Highlight != nil {
|
|
|
|
// get the right highlighter
|
2016-04-28 20:12:33 +02:00
|
|
|
highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
|
2014-09-01 17:15:38 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2014-07-30 18:30:38 +02:00
|
|
|
if req.Highlight.Style != nil {
|
2014-09-01 17:15:38 +02:00
|
|
|
highlighter, err = Config.Cache.HighlighterNamed(*req.Highlight.Style)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
}
|
2014-09-01 17:15:38 +02:00
|
|
|
if highlighter == nil {
|
|
|
|
return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
|
|
|
|
}
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
|
|
|
|
2016-04-28 20:12:33 +02:00
|
|
|
for _, hit := range hits {
|
|
|
|
if len(req.Fields) > 0 || highlighter != nil {
|
2014-09-12 23:21:35 +02:00
|
|
|
doc, err := indexReader.Document(hit.ID)
|
2016-02-01 18:31:26 +01:00
|
|
|
if err == nil && doc != nil {
|
2016-04-28 20:12:33 +02:00
|
|
|
if len(req.Fields) > 0 {
|
|
|
|
for _, f := range req.Fields {
|
|
|
|
for _, docF := range doc.Fields {
|
|
|
|
if f == "*" || docF.Name() == f {
|
|
|
|
var value interface{}
|
|
|
|
switch docF := docF.(type) {
|
|
|
|
case *document.TextField:
|
|
|
|
value = string(docF.Value())
|
|
|
|
case *document.NumericField:
|
|
|
|
num, err := docF.Number()
|
|
|
|
if err == nil {
|
|
|
|
value = num
|
|
|
|
}
|
|
|
|
case *document.DateTimeField:
|
|
|
|
datetime, err := docF.DateTime()
|
|
|
|
if err == nil {
|
|
|
|
value = datetime.Format(time.RFC3339)
|
|
|
|
}
|
|
|
|
case *document.BooleanField:
|
|
|
|
boolean, err := docF.Boolean()
|
|
|
|
if err == nil {
|
|
|
|
value = boolean
|
|
|
|
}
|
2014-08-06 19:52:20 +02:00
|
|
|
}
|
2016-04-28 20:12:33 +02:00
|
|
|
if value != nil {
|
|
|
|
hit.AddFieldValue(docF.Name(), value)
|
2016-01-12 02:18:03 +01:00
|
|
|
}
|
2014-08-06 19:52:20 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-04-28 20:12:33 +02:00
|
|
|
if highlighter != nil {
|
|
|
|
highlightFields := req.Highlight.Fields
|
|
|
|
if highlightFields == nil {
|
|
|
|
// add all fields with matches
|
|
|
|
highlightFields = make([]string, 0, len(hit.Locations))
|
|
|
|
for k := range hit.Locations {
|
|
|
|
highlightFields = append(highlightFields, k)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for _, hf := range highlightFields {
|
|
|
|
highlighter.BestFragmentsInField(hit, doc, hf, 1)
|
|
|
|
}
|
|
|
|
}
|
2016-02-01 18:31:26 +01:00
|
|
|
} else if doc == nil {
|
|
|
|
// unexpected case, a doc ID that was found as a search hit
|
|
|
|
// was unable to be found during document lookup
|
2016-02-03 16:23:49 +01:00
|
|
|
return nil, ErrorIndexReadInconsistency
|
2014-08-06 19:52:20 +02:00
|
|
|
}
|
|
|
|
}
|
2015-12-08 19:55:04 +01:00
|
|
|
if i.name != "" {
|
|
|
|
hit.Index = i.name
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-10-02 20:12:22 +02:00
|
|
|
atomic.AddUint64(&i.stats.searches, 1)
|
2014-12-29 04:34:16 +01:00
|
|
|
searchDuration := time.Since(searchStart)
|
|
|
|
atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))
|
|
|
|
|
2016-10-10 20:34:32 +02:00
|
|
|
if Config.SlowSearchLogThreshold > 0 &&
|
|
|
|
searchDuration > Config.SlowSearchLogThreshold {
|
2014-12-29 04:44:03 +01:00
|
|
|
logger.Printf("slow search took %s - %v", searchDuration, req)
|
2014-12-29 04:34:16 +01:00
|
|
|
}
|
2014-10-02 20:12:22 +02:00
|
|
|
|
2014-07-30 18:30:38 +02:00
|
|
|
return &SearchResult{
|
2016-02-22 22:50:40 +01:00
|
|
|
Status: &SearchStatus{
|
|
|
|
Total: 1,
|
|
|
|
Failed: 0,
|
|
|
|
Successful: 1,
|
|
|
|
Errors: make(map[string]error),
|
|
|
|
},
|
2014-07-30 18:30:38 +02:00
|
|
|
Request: req,
|
|
|
|
Hits: hits,
|
|
|
|
Total: collector.Total(),
|
|
|
|
MaxScore: collector.MaxScore(),
|
2015-02-19 07:41:40 +01:00
|
|
|
Took: searchDuration,
|
2014-08-11 17:03:29 +02:00
|
|
|
Facets: collector.FacetResults(),
|
2014-07-30 18:30:38 +02:00
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2014-08-31 16:55:22 +02:00
|
|
|
// Fields returns the name of all the fields this
|
|
|
|
// Index has operated on.
|
2015-03-11 06:28:51 +01:00
|
|
|
func (i *indexImpl) Fields() (fields []string, err error) {
|
2014-08-25 15:06:53 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
|
|
|
if !i.open {
|
2014-09-02 20:14:05 +02:00
|
|
|
return nil, ErrorIndexClosed
|
2014-08-25 15:06:53 +02:00
|
|
|
}
|
2014-09-12 23:21:35 +02:00
|
|
|
|
2014-10-31 14:40:23 +01:00
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if cerr := indexReader.Close(); err == nil && cerr != nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
fields, err = indexReader.Fields()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return fields, nil
|
2014-07-30 20:29:26 +02:00
|
|
|
}
|
|
|
|
|
2015-03-10 21:22:19 +01:00
|
|
|
func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) {
|
|
|
|
i.mutex.RLock()
|
|
|
|
|
|
|
|
if !i.open {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
fieldDict, err := indexReader.FieldDict(field)
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &indexImplFieldDict{
|
|
|
|
index: i,
|
|
|
|
indexReader: indexReader,
|
|
|
|
fieldDict: fieldDict,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
|
|
|
|
i.mutex.RLock()
|
|
|
|
|
|
|
|
if !i.open {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm)
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &indexImplFieldDict{
|
|
|
|
index: i,
|
|
|
|
indexReader: indexReader,
|
|
|
|
fieldDict: fieldDict,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {
|
|
|
|
i.mutex.RLock()
|
|
|
|
|
|
|
|
if !i.open {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
|
|
|
indexReader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix)
|
|
|
|
if err != nil {
|
|
|
|
i.mutex.RUnlock()
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &indexImplFieldDict{
|
|
|
|
index: i,
|
|
|
|
indexReader: indexReader,
|
|
|
|
fieldDict: fieldDict,
|
|
|
|
}, nil
|
|
|
|
}
|
|
|
|
|
2014-10-31 14:40:23 +01:00
|
|
|
func (i *indexImpl) Close() error {
|
2014-08-25 15:06:53 +02:00
|
|
|
i.mutex.Lock()
|
|
|
|
defer i.mutex.Unlock()
|
|
|
|
|
2016-04-20 17:43:14 +02:00
|
|
|
indexStats.UnRegister(i)
|
|
|
|
|
2014-08-25 15:06:53 +02:00
|
|
|
i.open = false
|
2014-10-31 14:40:23 +01:00
|
|
|
return i.i.Close()
|
2014-07-30 18:30:38 +02:00
|
|
|
}
|
2014-10-02 20:12:22 +02:00
|
|
|
|
|
|
|
func (i *indexImpl) Stats() *IndexStat {
|
|
|
|
return i.stats
|
|
|
|
}
|
2014-10-22 22:03:55 +02:00
|
|
|
|
2016-03-08 00:37:46 +01:00
|
|
|
func (i *indexImpl) StatsMap() map[string]interface{} {
|
|
|
|
return i.stats.statsMap()
|
|
|
|
}
|
|
|
|
|
2015-03-11 06:28:51 +01:00
|
|
|
func (i *indexImpl) GetInternal(key []byte) (val []byte, err error) {
|
2014-10-22 22:03:55 +02:00
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
2015-04-04 00:05:24 +02:00
|
|
|
if !i.open {
|
|
|
|
return nil, ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
2014-10-31 14:40:23 +01:00
|
|
|
reader, err := i.i.Reader()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2015-03-11 06:28:51 +01:00
|
|
|
defer func() {
|
|
|
|
if cerr := reader.Close(); err == nil && cerr != nil {
|
|
|
|
err = cerr
|
|
|
|
}
|
|
|
|
}()
|
2014-10-22 22:03:55 +02:00
|
|
|
|
2015-03-11 06:28:51 +01:00
|
|
|
val, err = reader.GetInternal(key)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
return val, nil
|
2014-10-22 22:03:55 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
func (i *indexImpl) SetInternal(key, val []byte) error {
|
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
2015-04-04 00:05:24 +02:00
|
|
|
if !i.open {
|
|
|
|
return ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
2014-10-22 22:03:55 +02:00
|
|
|
return i.i.SetInternal(key, val)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *indexImpl) DeleteInternal(key []byte) error {
|
|
|
|
i.mutex.RLock()
|
|
|
|
defer i.mutex.RUnlock()
|
|
|
|
|
2015-04-04 00:05:24 +02:00
|
|
|
if !i.open {
|
|
|
|
return ErrorIndexClosed
|
|
|
|
}
|
|
|
|
|
2014-10-22 22:03:55 +02:00
|
|
|
return i.i.DeleteInternal(key)
|
|
|
|
}
|
2015-03-03 19:18:20 +01:00
|
|
|
|
|
|
|
// NewBatch creates a new empty batch.
|
|
|
|
func (i *indexImpl) NewBatch() *Batch {
|
|
|
|
return &Batch{
|
|
|
|
index: i,
|
|
|
|
internal: index.NewBatch(),
|
|
|
|
}
|
|
|
|
}
|
2015-03-10 21:22:19 +01:00
|
|
|
|
2015-12-06 20:01:03 +01:00
|
|
|
func (i *indexImpl) Name() string {
|
|
|
|
return i.name
|
|
|
|
}
|
|
|
|
|
|
|
|
func (i *indexImpl) SetName(name string) {
|
|
|
|
indexStats.UnRegister(i)
|
|
|
|
i.name = name
|
|
|
|
indexStats.Register(i)
|
|
|
|
}
|
|
|
|
|
2015-03-10 21:22:19 +01:00
|
|
|
type indexImplFieldDict struct {
|
|
|
|
index *indexImpl
|
|
|
|
indexReader index.IndexReader
|
|
|
|
fieldDict index.FieldDict
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *indexImplFieldDict) Next() (*index.DictEntry, error) {
|
|
|
|
return f.fieldDict.Next()
|
|
|
|
}
|
|
|
|
|
|
|
|
func (f *indexImplFieldDict) Close() error {
|
|
|
|
defer f.index.mutex.RUnlock()
|
|
|
|
err := f.fieldDict.Close()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return f.indexReader.Close()
|
|
|
|
}
|