Merge pull request #452 from mschoch/memonly
BREAKING CHANGE - new method to create memory only index
This commit is contained in:
commit
1077cb6012
|
@ -17,6 +17,7 @@ import (
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
|
"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
|
||||||
"github.com/blevesearch/bleve/index"
|
"github.com/blevesearch/bleve/index"
|
||||||
|
"github.com/blevesearch/bleve/index/store/gtreap"
|
||||||
"github.com/blevesearch/bleve/index/upside_down"
|
"github.com/blevesearch/bleve/index/upside_down"
|
||||||
"github.com/blevesearch/bleve/registry"
|
"github.com/blevesearch/bleve/registry"
|
||||||
"github.com/blevesearch/bleve/search/highlight/highlighters/html"
|
"github.com/blevesearch/bleve/search/highlight/highlighters/html"
|
||||||
|
@ -28,6 +29,7 @@ type configuration struct {
|
||||||
Cache *registry.Cache
|
Cache *registry.Cache
|
||||||
DefaultHighlighter string
|
DefaultHighlighter string
|
||||||
DefaultKVStore string
|
DefaultKVStore string
|
||||||
|
DefaultMemKVStore string
|
||||||
DefaultIndexType string
|
DefaultIndexType string
|
||||||
QueryDateTimeParser string
|
QueryDateTimeParser string
|
||||||
SlowSearchLogThreshold time.Duration
|
SlowSearchLogThreshold time.Duration
|
||||||
|
@ -60,6 +62,9 @@ func init() {
|
||||||
// default kv store
|
// default kv store
|
||||||
Config.DefaultKVStore = ""
|
Config.DefaultKVStore = ""
|
||||||
|
|
||||||
|
// default mem only kv store
|
||||||
|
Config.DefaultMemKVStore = gtreap.Name
|
||||||
|
|
||||||
// default index
|
// default index
|
||||||
Config.DefaultIndexType = upside_down.Name
|
Config.DefaultIndexType = upside_down.Name
|
||||||
|
|
||||||
|
|
9
index.go
9
index.go
|
@ -210,6 +210,15 @@ func New(path string, mapping *IndexMapping) (Index, error) {
|
||||||
return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
|
return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewMemOnly creates a memory-only index.
|
||||||
|
// The contents of the index is NOT persisted,
|
||||||
|
// and will be lost once closed.
|
||||||
|
// The provided mapping will be used for all
|
||||||
|
// Index/Search operations.
|
||||||
|
func NewMemOnly(mapping *IndexMapping) (Index, error) {
|
||||||
|
return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
|
||||||
|
}
|
||||||
|
|
||||||
// NewUsing creates index at the specified path,
|
// NewUsing creates index at the specified path,
|
||||||
// which must not already exist.
|
// which must not already exist.
|
||||||
// The provided mapping will be used for all
|
// The provided mapping will be used for all
|
||||||
|
|
|
@ -21,6 +21,7 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/index/store"
|
"github.com/blevesearch/bleve/index/store"
|
||||||
"github.com/blevesearch/bleve/registry"
|
"github.com/blevesearch/bleve/registry"
|
||||||
|
@ -46,6 +47,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("must specify path")
|
return nil, fmt.Errorf("must specify path")
|
||||||
}
|
}
|
||||||
|
if path == "" {
|
||||||
|
return nil, os.ErrInvalid
|
||||||
|
}
|
||||||
|
|
||||||
bucket, ok := config["bucket"].(string)
|
bucket, ok := config["bucket"].(string)
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|
|
@ -12,6 +12,7 @@ package goleveldb
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/index/store"
|
"github.com/blevesearch/bleve/index/store"
|
||||||
"github.com/blevesearch/bleve/registry"
|
"github.com/blevesearch/bleve/registry"
|
||||||
|
@ -41,6 +42,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("must specify path")
|
return nil, fmt.Errorf("must specify path")
|
||||||
}
|
}
|
||||||
|
if path == "" {
|
||||||
|
return nil, os.ErrInvalid
|
||||||
|
}
|
||||||
|
|
||||||
opts, err := applyConfig(&opt.Options{}, config)
|
opts, err := applyConfig(&opt.Options{}, config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -17,6 +17,8 @@ package gtreap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/blevesearch/bleve/index/store"
|
"github.com/blevesearch/bleve/index/store"
|
||||||
|
@ -42,6 +44,14 @@ func itemCompare(a, b interface{}) int {
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
|
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
|
||||||
|
path, ok := config["path"].(string)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("must specify path")
|
||||||
|
}
|
||||||
|
if path != "" {
|
||||||
|
return nil, os.ErrInvalid
|
||||||
|
}
|
||||||
|
|
||||||
rv := Store{
|
rv := Store{
|
||||||
t: gtreap.NewTreap(itemCompare),
|
t: gtreap.NewTreap(itemCompare),
|
||||||
mo: mo,
|
mo: mo,
|
||||||
|
|
|
@ -20,7 +20,9 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
|
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
|
||||||
rv, err := New(mo, nil)
|
rv, err := New(mo, map[string]interface{}{
|
||||||
|
"path": "",
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -36,6 +36,7 @@ func TestMetricsStore(t *testing.T) {
|
||||||
|
|
||||||
s, err = New(nil, map[string]interface{}{
|
s, err = New(nil, map[string]interface{}{
|
||||||
"kvStoreName_actual": gtreap.Name,
|
"kvStoreName_actual": gtreap.Name,
|
||||||
|
"path": "",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
@ -74,6 +75,7 @@ func TestMetricsStore(t *testing.T) {
|
||||||
func TestErrors(t *testing.T) {
|
func TestErrors(t *testing.T) {
|
||||||
s, err := New(nil, map[string]interface{}{
|
s, err := New(nil, map[string]interface{}{
|
||||||
"kvStoreName_actual": gtreap.Name,
|
"kvStoreName_actual": gtreap.Name,
|
||||||
|
"path": "",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|
|
@ -9,7 +9,10 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
|
func open(t *testing.T, mo store.MergeOperator) store.KVStore {
|
||||||
rv, err := New(mo, map[string]interface{}{"kvStoreName_actual": gtreap.Name})
|
rv, err := New(mo, map[string]interface{}{
|
||||||
|
"kvStoreName_actual": gtreap.Name,
|
||||||
|
"path": "",
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -421,6 +421,9 @@ func InitMossStore(config map[string]interface{}, options moss.CollectionOptions
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
|
return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
|
||||||
}
|
}
|
||||||
|
if path == "" {
|
||||||
|
return nil, nil, nil, nil, os.ErrInvalid
|
||||||
|
}
|
||||||
|
|
||||||
err := os.MkdirAll(path, 0700)
|
err := os.MkdirAll(path, 0700)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -22,7 +22,6 @@ import (
|
||||||
"github.com/blevesearch/bleve/document"
|
"github.com/blevesearch/bleve/document"
|
||||||
"github.com/blevesearch/bleve/index"
|
"github.com/blevesearch/bleve/index"
|
||||||
"github.com/blevesearch/bleve/index/store"
|
"github.com/blevesearch/bleve/index/store"
|
||||||
"github.com/blevesearch/bleve/index/store/gtreap"
|
|
||||||
"github.com/blevesearch/bleve/index/upside_down"
|
"github.com/blevesearch/bleve/index/upside_down"
|
||||||
"github.com/blevesearch/bleve/registry"
|
"github.com/blevesearch/bleve/registry"
|
||||||
"github.com/blevesearch/bleve/search"
|
"github.com/blevesearch/bleve/search"
|
||||||
|
@ -50,50 +49,6 @@ func indexStorePath(path string) string {
|
||||||
return path + string(os.PathSeparator) + storePath
|
return path + string(os.PathSeparator) + storePath
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) {
|
|
||||||
rv := indexImpl{
|
|
||||||
path: "",
|
|
||||||
name: "mem",
|
|
||||||
m: mapping,
|
|
||||||
meta: newIndexMeta(indexType, gtreap.Name, nil),
|
|
||||||
}
|
|
||||||
|
|
||||||
rv.stats = &IndexStat{i: &rv}
|
|
||||||
|
|
||||||
// open the index
|
|
||||||
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
|
||||||
if indexTypeConstructor == nil {
|
|
||||||
return nil, ErrorUnknownIndexType
|
|
||||||
}
|
|
||||||
|
|
||||||
var err error
|
|
||||||
rv.i, err = indexTypeConstructor(rv.meta.Storage, nil, Config.analysisQueue)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
err = rv.i.Open()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// now persist the mapping
|
|
||||||
mappingBytes, err := json.Marshal(mapping)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// mark the index as open
|
|
||||||
rv.mutex.Lock()
|
|
||||||
defer rv.mutex.Unlock()
|
|
||||||
rv.open = true
|
|
||||||
indexStats.Register(&rv)
|
|
||||||
return &rv, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
|
func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
|
||||||
// first validate the mapping
|
// first validate the mapping
|
||||||
err := mapping.Validate()
|
err := mapping.Validate()
|
||||||
|
@ -101,10 +56,6 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if path == "" {
|
|
||||||
return newMemIndex(indexType, mapping)
|
|
||||||
}
|
|
||||||
|
|
||||||
if kvconfig == nil {
|
if kvconfig == nil {
|
||||||
kvconfig = map[string]interface{}{}
|
kvconfig = map[string]interface{}{}
|
||||||
}
|
}
|
||||||
|
@ -121,13 +72,17 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
|
||||||
}
|
}
|
||||||
rv.stats = &IndexStat{i: &rv}
|
rv.stats = &IndexStat{i: &rv}
|
||||||
// at this point there is hope that we can be successful, so save index meta
|
// at this point there is hope that we can be successful, so save index meta
|
||||||
err = rv.meta.Save(path)
|
if path != "" {
|
||||||
if err != nil {
|
err = rv.meta.Save(path)
|
||||||
return nil, err
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
kvconfig["create_if_missing"] = true
|
||||||
|
kvconfig["error_if_exists"] = true
|
||||||
|
kvconfig["path"] = indexStorePath(path)
|
||||||
|
} else {
|
||||||
|
kvconfig["path"] = ""
|
||||||
}
|
}
|
||||||
kvconfig["create_if_missing"] = true
|
|
||||||
kvconfig["error_if_exists"] = true
|
|
||||||
kvconfig["path"] = indexStorePath(path)
|
|
||||||
|
|
||||||
// open the index
|
// open the index
|
||||||
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
|
||||||
|
|
|
@ -287,7 +287,7 @@ func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
|
||||||
|
|
||||||
func TestInMemIndex(t *testing.T) {
|
func TestInMemIndex(t *testing.T) {
|
||||||
|
|
||||||
index, err := New("", NewIndexMapping())
|
index, err := NewMemOnly(NewIndexMapping())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -298,7 +298,7 @@ func TestInMemIndex(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClosedIndex(t *testing.T) {
|
func TestClosedIndex(t *testing.T) {
|
||||||
index, err := New("", NewIndexMapping())
|
index, err := NewMemOnly(NewIndexMapping())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -1685,7 +1685,7 @@ func TestBug408(t *testing.T) {
|
||||||
indexMapping := NewIndexMapping()
|
indexMapping := NewIndexMapping()
|
||||||
indexMapping.DefaultMapping = docMapping
|
indexMapping.DefaultMapping = docMapping
|
||||||
|
|
||||||
index, err := New("", indexMapping)
|
index, err := NewMemOnly(indexMapping)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -26,7 +26,11 @@ var twoDocIndex index.Index //= upside_down.NewUpsideDownCouch(inmem.MustOpen())
|
||||||
func init() {
|
func init() {
|
||||||
analysisQueue := index.NewAnalysisQueue(1)
|
analysisQueue := index.NewAnalysisQueue(1)
|
||||||
var err error
|
var err error
|
||||||
twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
|
twoDocIndex, err = upside_down.NewUpsideDownCouch(
|
||||||
|
gtreap.Name,
|
||||||
|
map[string]interface{}{
|
||||||
|
"path": "",
|
||||||
|
}, analysisQueue)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,12 @@ import (
|
||||||
|
|
||||||
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
|
func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
|
||||||
analysisQueue := index.NewAnalysisQueue(1)
|
analysisQueue := index.NewAnalysisQueue(1)
|
||||||
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
|
i, err := upside_down.NewUpsideDownCouch(
|
||||||
|
gtreap.Name,
|
||||||
|
map[string]interface{}{
|
||||||
|
"path": "",
|
||||||
|
},
|
||||||
|
analysisQueue)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,12 @@ func TestTermSearcher(t *testing.T) {
|
||||||
var queryExplain = true
|
var queryExplain = true
|
||||||
|
|
||||||
analysisQueue := index.NewAnalysisQueue(1)
|
analysisQueue := index.NewAnalysisQueue(1)
|
||||||
i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
|
i, err := upside_down.NewUpsideDownCouch(
|
||||||
|
gtreap.Name,
|
||||||
|
map[string]interface{}{
|
||||||
|
"path": "",
|
||||||
|
},
|
||||||
|
analysisQueue)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user