From fb0f4bbecd20a83d911364fb2c498e48abe7a6f7 Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 27 Sep 2016 14:05:55 -0400 Subject: [PATCH] BREAKING CHANGE - new method to create memory only index Previously bleve allowed you to create a memory-only index by simply passing "" as the path argument to the New() method. This was not clear when reading the code, and led to some problematic error cases as well. Now, to create a memory-only index one should use the NewMemOnly() method. Passing "" as the path argument to the New() method will now return os.ErrInvalid. Advanced users calling NewUsing() can create disk-based or memory-only indexes, but the change here is that pass "" as the path argument no longer defaults you into getting a memory-only index. Instead, the KV store is selected manually, just as it is for the disk-based solutions. Here is an example use of the NewUsing() method to create a memory-only index: NewUsing("", indexMapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) Config.DefaultMemKVStore is just a new default value added to the configuration, it currently points to gtreap.Name (which could have been used directly instead for more control) closes #427 --- config.go | 5 +++ index.go | 9 ++++ index/store/boltdb/store.go | 4 ++ index/store/goleveldb/store.go | 4 ++ index/store/gtreap/store.go | 10 +++++ index/store/gtreap/store_test.go | 4 +- index/store/metrics/metrics_test.go | 2 + index/store/metrics/store_test.go | 5 ++- index/store/moss/lower.go | 3 ++ index_impl.go | 65 +++++---------------------- index_test.go | 6 +-- search/searchers/base_test.go | 6 ++- search/searchers/search_docid_test.go | 7 ++- search/searchers/search_term_test.go | 7 ++- 14 files changed, 74 insertions(+), 63 deletions(-) diff --git a/config.go b/config.go index d3877363..2a4476c6 100644 --- a/config.go +++ b/config.go @@ -17,6 +17,7 @@ import ( "github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional" "github.com/blevesearch/bleve/index" + "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search/highlight/highlighters/html" @@ -28,6 +29,7 @@ type configuration struct { Cache *registry.Cache DefaultHighlighter string DefaultKVStore string + DefaultMemKVStore string DefaultIndexType string QueryDateTimeParser string SlowSearchLogThreshold time.Duration @@ -60,6 +62,9 @@ func init() { // default kv store Config.DefaultKVStore = "" + // default mem only kv store + Config.DefaultMemKVStore = gtreap.Name + // default index Config.DefaultIndexType = upside_down.Name diff --git a/index.go b/index.go index 0584431e..df708c62 100644 --- a/index.go +++ b/index.go @@ -210,6 +210,15 @@ func New(path string, mapping *IndexMapping) (Index, error) { return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil) } +// NewMemOnly creates a memory-only index. +// The contents of the index is NOT persisted, +// and will be lost once closed. +// The provided mapping will be used for all +// Index/Search operations. +func NewMemOnly(mapping *IndexMapping) (Index, error) { + return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil) +} + // NewUsing creates index at the specified path, // which must not already exist. // The provided mapping will be used for all diff --git a/index/store/boltdb/store.go b/index/store/boltdb/store.go index f236c525..fa893834 100644 --- a/index/store/boltdb/store.go +++ b/index/store/boltdb/store.go @@ -21,6 +21,7 @@ import ( "bytes" "encoding/json" "fmt" + "os" "github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/registry" @@ -46,6 +47,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, if !ok { return nil, fmt.Errorf("must specify path") } + if path == "" { + return nil, os.ErrInvalid + } bucket, ok := config["bucket"].(string) if !ok { diff --git a/index/store/goleveldb/store.go b/index/store/goleveldb/store.go index 4ca2e9e6..889cc2d8 100644 --- a/index/store/goleveldb/store.go +++ b/index/store/goleveldb/store.go @@ -12,6 +12,7 @@ package goleveldb import ( "bytes" "fmt" + "os" "github.com/blevesearch/bleve/index/store" "github.com/blevesearch/bleve/registry" @@ -41,6 +42,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, if !ok { return nil, fmt.Errorf("must specify path") } + if path == "" { + return nil, os.ErrInvalid + } opts, err := applyConfig(&opt.Options{}, config) if err != nil { diff --git a/index/store/gtreap/store.go b/index/store/gtreap/store.go index 7b0048f2..0a1b8252 100644 --- a/index/store/gtreap/store.go +++ b/index/store/gtreap/store.go @@ -17,6 +17,8 @@ package gtreap import ( "bytes" + "fmt" + "os" "sync" "github.com/blevesearch/bleve/index/store" @@ -42,6 +44,14 @@ func itemCompare(a, b interface{}) int { } func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) { + path, ok := config["path"].(string) + if !ok { + return nil, fmt.Errorf("must specify path") + } + if path != "" { + return nil, os.ErrInvalid + } + rv := Store{ t: gtreap.NewTreap(itemCompare), mo: mo, diff --git a/index/store/gtreap/store_test.go b/index/store/gtreap/store_test.go index c42502c1..9dff1b6e 100644 --- a/index/store/gtreap/store_test.go +++ b/index/store/gtreap/store_test.go @@ -20,7 +20,9 @@ import ( ) func open(t *testing.T, mo store.MergeOperator) store.KVStore { - rv, err := New(mo, nil) + rv, err := New(mo, map[string]interface{}{ + "path": "", + }) if err != nil { t.Fatal(err) } diff --git a/index/store/metrics/metrics_test.go b/index/store/metrics/metrics_test.go index 7cd9bd2e..449e618a 100644 --- a/index/store/metrics/metrics_test.go +++ b/index/store/metrics/metrics_test.go @@ -36,6 +36,7 @@ func TestMetricsStore(t *testing.T) { s, err = New(nil, map[string]interface{}{ "kvStoreName_actual": gtreap.Name, + "path": "", }) if err != nil { t.Fatal(err) @@ -74,6 +75,7 @@ func TestMetricsStore(t *testing.T) { func TestErrors(t *testing.T) { s, err := New(nil, map[string]interface{}{ "kvStoreName_actual": gtreap.Name, + "path": "", }) if err != nil { t.Fatal(err) diff --git a/index/store/metrics/store_test.go b/index/store/metrics/store_test.go index 91667e8c..27381ade 100644 --- a/index/store/metrics/store_test.go +++ b/index/store/metrics/store_test.go @@ -9,7 +9,10 @@ import ( ) func open(t *testing.T, mo store.MergeOperator) store.KVStore { - rv, err := New(mo, map[string]interface{}{"kvStoreName_actual": gtreap.Name}) + rv, err := New(mo, map[string]interface{}{ + "kvStoreName_actual": gtreap.Name, + "path": "", + }) if err != nil { t.Fatal(err) } diff --git a/index/store/moss/lower.go b/index/store/moss/lower.go index 8d4afc00..4e1c63ed 100644 --- a/index/store/moss/lower.go +++ b/index/store/moss/lower.go @@ -421,6 +421,9 @@ func InitMossStore(config map[string]interface{}, options moss.CollectionOptions if !ok { return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config") } + if path == "" { + return nil, nil, nil, nil, os.ErrInvalid + } err := os.MkdirAll(path, 0700) if err != nil { diff --git a/index_impl.go b/index_impl.go index 4b54f673..aae26a0b 100644 --- a/index_impl.go +++ b/index_impl.go @@ -22,7 +22,6 @@ import ( "github.com/blevesearch/bleve/document" "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/store" - "github.com/blevesearch/bleve/index/store/gtreap" "github.com/blevesearch/bleve/index/upside_down" "github.com/blevesearch/bleve/registry" "github.com/blevesearch/bleve/search" @@ -50,50 +49,6 @@ func indexStorePath(path string) string { return path + string(os.PathSeparator) + storePath } -func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) { - rv := indexImpl{ - path: "", - name: "mem", - m: mapping, - meta: newIndexMeta(indexType, gtreap.Name, nil), - } - - rv.stats = &IndexStat{i: &rv} - - // open the index - indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) - if indexTypeConstructor == nil { - return nil, ErrorUnknownIndexType - } - - var err error - rv.i, err = indexTypeConstructor(rv.meta.Storage, nil, Config.analysisQueue) - if err != nil { - return nil, err - } - err = rv.i.Open() - if err != nil { - return nil, err - } - - // now persist the mapping - mappingBytes, err := json.Marshal(mapping) - if err != nil { - return nil, err - } - err = rv.i.SetInternal(mappingInternalKey, mappingBytes) - if err != nil { - return nil, err - } - - // mark the index as open - rv.mutex.Lock() - defer rv.mutex.Unlock() - rv.open = true - indexStats.Register(&rv) - return &rv, nil -} - func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) { // first validate the mapping err := mapping.Validate() @@ -101,10 +56,6 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore return nil, err } - if path == "" { - return newMemIndex(indexType, mapping) - } - if kvconfig == nil { kvconfig = map[string]interface{}{} } @@ -121,13 +72,17 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore } rv.stats = &IndexStat{i: &rv} // at this point there is hope that we can be successful, so save index meta - err = rv.meta.Save(path) - if err != nil { - return nil, err + if path != "" { + err = rv.meta.Save(path) + if err != nil { + return nil, err + } + kvconfig["create_if_missing"] = true + kvconfig["error_if_exists"] = true + kvconfig["path"] = indexStorePath(path) + } else { + kvconfig["path"] = "" } - kvconfig["create_if_missing"] = true - kvconfig["error_if_exists"] = true - kvconfig["path"] = indexStorePath(path) // open the index indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType) diff --git a/index_test.go b/index_test.go index 89a640f4..9fdeb175 100644 --- a/index_test.go +++ b/index_test.go @@ -287,7 +287,7 @@ func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) { func TestInMemIndex(t *testing.T) { - index, err := New("", NewIndexMapping()) + index, err := NewMemOnly(NewIndexMapping()) if err != nil { t.Fatal(err) } @@ -298,7 +298,7 @@ func TestInMemIndex(t *testing.T) { } func TestClosedIndex(t *testing.T) { - index, err := New("", NewIndexMapping()) + index, err := NewMemOnly(NewIndexMapping()) if err != nil { t.Fatal(err) } @@ -1685,7 +1685,7 @@ func TestBug408(t *testing.T) { indexMapping := NewIndexMapping() indexMapping.DefaultMapping = docMapping - index, err := New("", indexMapping) + index, err := NewMemOnly(indexMapping) if err != nil { t.Fatal(err) } diff --git a/search/searchers/base_test.go b/search/searchers/base_test.go index 54363ee9..b4729a6c 100644 --- a/search/searchers/base_test.go +++ b/search/searchers/base_test.go @@ -26,7 +26,11 @@ var twoDocIndex index.Index //= upside_down.NewUpsideDownCouch(inmem.MustOpen()) func init() { analysisQueue := index.NewAnalysisQueue(1) var err error - twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) + twoDocIndex, err = upside_down.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, analysisQueue) if err != nil { panic(err) } diff --git a/search/searchers/search_docid_test.go b/search/searchers/search_docid_test.go index 56b2da97..e2d72605 100644 --- a/search/searchers/search_docid_test.go +++ b/search/searchers/search_docid_test.go @@ -21,7 +21,12 @@ import ( func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) { analysisQueue := index.NewAnalysisQueue(1) - i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) + i, err := upside_down.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) if err != nil { t.Fatal(err) } diff --git a/search/searchers/search_term_test.go b/search/searchers/search_term_test.go index db8638d6..3811663a 100644 --- a/search/searchers/search_term_test.go +++ b/search/searchers/search_term_test.go @@ -28,7 +28,12 @@ func TestTermSearcher(t *testing.T) { var queryExplain = true analysisQueue := index.NewAnalysisQueue(1) - i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue) + i, err := upside_down.NewUpsideDownCouch( + gtreap.Name, + map[string]interface{}{ + "path": "", + }, + analysisQueue) if err != nil { t.Fatal(err) }