BREAKING CHANGE - new method to create memory only index

Previously bleve allowed you to create a memory-only index by
simply passing "" as the path argument to the New() method.

This was not clear when reading the code, and led to some
problematic error cases as well.

Now, to create a memory-only index one should use the
NewMemOnly() method.  Passing "" as the path argument
to the New() method will now return os.ErrInvalid.

Advanced users calling NewUsing() can create disk-based or
memory-only indexes, but the change here is that pass ""
as the path argument no longer defaults you into getting
a memory-only index.  Instead, the KV store is selected
manually, just as it is for the disk-based solutions.

Here is an example use of the NewUsing() method to create
a memory-only index:

NewUsing("", indexMapping, Config.DefaultIndexType,
         Config.DefaultMemKVStore, nil)

Config.DefaultMemKVStore is just a new default value
added to the configuration, it currently points to
gtreap.Name (which could have been used directly
instead for more control)

closes #427

config.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
 	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store/gtreap"
 	"github.com/blevesearch/bleve/index/upside_down"
 	"github.com/blevesearch/bleve/registry"
 	"github.com/blevesearch/bleve/search/highlight/highlighters/html"
@@ -28,6 +29,7 @@ type configuration struct {
 	Cache                  *registry.Cache
 	DefaultHighlighter     string
 	DefaultKVStore         string
+	DefaultMemKVStore      string
 	DefaultIndexType       string
 	QueryDateTimeParser    string
 	SlowSearchLogThreshold time.Duration
@@ -60,6 +62,9 @@ func init() {
 	// default kv store
 	Config.DefaultKVStore = ""
 
+	// default mem only kv store
+	Config.DefaultMemKVStore = gtreap.Name
+
 	// default index
 	Config.DefaultIndexType = upside_down.Name
 

index.go

@@ -210,6 +210,15 @@ func New(path string, mapping *IndexMapping) (Index, error) {
 	return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
 }
 
+// NewMemOnly creates a memory-only index.
+// The contents of the index is NOT persisted,
+// and will be lost once closed.
+// The provided mapping will be used for all
+// Index/Search operations.
+func NewMemOnly(mapping *IndexMapping) (Index, error) {
+	return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
+}
+
 // NewUsing creates index at the specified path,
 // which must not already exist.
 // The provided mapping will be used for all

index/store/boltdb/store.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/blevesearch/bleve/index/store"
 	"github.com/blevesearch/bleve/registry"
@@ -46,6 +47,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 	if !ok {
 		return nil, fmt.Errorf("must specify path")
 	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
 
 	bucket, ok := config["bucket"].(string)
 	if !ok {

index/store/goleveldb/store.go

@@ -12,6 +12,7 @@ package goleveldb
 import (
 	"bytes"
 	"fmt"
+	"os"
 
 	"github.com/blevesearch/bleve/index/store"
 	"github.com/blevesearch/bleve/registry"
@@ -41,6 +42,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 	if !ok {
 		return nil, fmt.Errorf("must specify path")
 	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
 
 	opts, err := applyConfig(&opt.Options{}, config)
 	if err != nil {

index/store/gtreap/store.go

@@ -17,6 +17,8 @@ package gtreap
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"sync"
 
 	"github.com/blevesearch/bleve/index/store"
@@ -42,6 +44,14 @@ func itemCompare(a, b interface{}) int {
 }
 
 func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path != "" {
+		return nil, os.ErrInvalid
+	}
+
 	rv := Store{
 		t:  gtreap.NewTreap(itemCompare),
 		mo: mo,

index/store/gtreap/store_test.go

@@ -20,7 +20,9 @@ import (
 )
 
 func open(t *testing.T, mo store.MergeOperator) store.KVStore {
-	rv, err := New(mo, nil)
+	rv, err := New(mo, map[string]interface{}{
+		"path": "",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}

index/store/metrics/metrics_test.go

@@ -36,6 +36,7 @@ func TestMetricsStore(t *testing.T) {
 
 	s, err = New(nil, map[string]interface{}{
 		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -74,6 +75,7 @@ func TestMetricsStore(t *testing.T) {
 func TestErrors(t *testing.T) {
 	s, err := New(nil, map[string]interface{}{
 		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
 	})
 	if err != nil {
 		t.Fatal(err)

index/store/metrics/store_test.go

@@ -9,7 +9,10 @@ import (
 )
 
 func open(t *testing.T, mo store.MergeOperator) store.KVStore {
-	rv, err := New(mo, map[string]interface{}{"kvStoreName_actual": gtreap.Name})
+	rv, err := New(mo, map[string]interface{}{
+		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}

index/store/moss/lower.go

@@ -421,6 +421,9 @@ func InitMossStore(config map[string]interface{}, options moss.CollectionOptions
 	if !ok {
 		return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
 	}
+	if path == "" {
+		return nil, nil, nil, nil, os.ErrInvalid
+	}
 
 	err := os.MkdirAll(path, 0700)
 	if err != nil {

index_impl.go

@@ -22,7 +22,6 @@ import (
 	"github.com/blevesearch/bleve/document"
 	"github.com/blevesearch/bleve/index"
 	"github.com/blevesearch/bleve/index/store"
-	"github.com/blevesearch/bleve/index/store/gtreap"
 	"github.com/blevesearch/bleve/index/upside_down"
 	"github.com/blevesearch/bleve/registry"
 	"github.com/blevesearch/bleve/search"
@@ -50,50 +49,6 @@ func indexStorePath(path string) string {
 	return path + string(os.PathSeparator) + storePath
 }
 
-func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) {
-	rv := indexImpl{
-		path: "",
-		name: "mem",
-		m:    mapping,
-		meta: newIndexMeta(indexType, gtreap.Name, nil),
-	}
-
-	rv.stats = &IndexStat{i: &rv}
-
-	// open the index
-	indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
-	if indexTypeConstructor == nil {
-		return nil, ErrorUnknownIndexType
-	}
-
-	var err error
-	rv.i, err = indexTypeConstructor(rv.meta.Storage, nil, Config.analysisQueue)
-	if err != nil {
-		return nil, err
-	}
-	err = rv.i.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	// now persist the mapping
-	mappingBytes, err := json.Marshal(mapping)
-	if err != nil {
-		return nil, err
-	}
-	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	// mark the index as open
-	rv.mutex.Lock()
-	defer rv.mutex.Unlock()
-	rv.open = true
-	indexStats.Register(&rv)
-	return &rv, nil
-}
-
 func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
 	// first validate the mapping
 	err := mapping.Validate()
@@ -101,10 +56,6 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
 		return nil, err
 	}
 
-	if path == "" {
-		return newMemIndex(indexType, mapping)
-	}
-
 	if kvconfig == nil {
 		kvconfig = map[string]interface{}{}
 	}
@@ -121,13 +72,17 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
 	}
 	rv.stats = &IndexStat{i: &rv}
 	// at this point there is hope that we can be successful, so save index meta
-	err = rv.meta.Save(path)
-	if err != nil {
-		return nil, err
+	if path != "" {
+		err = rv.meta.Save(path)
+		if err != nil {
+			return nil, err
+		}
+		kvconfig["create_if_missing"] = true
+		kvconfig["error_if_exists"] = true
+		kvconfig["path"] = indexStorePath(path)
+	} else {
+		kvconfig["path"] = ""
 	}
-	kvconfig["create_if_missing"] = true
-	kvconfig["error_if_exists"] = true
-	kvconfig["path"] = indexStorePath(path)
 
 	// open the index
 	indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)

index_test.go

@@ -287,7 +287,7 @@ func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
 
 func TestInMemIndex(t *testing.T) {
 
-	index, err := New("", NewIndexMapping())
+	index, err := NewMemOnly(NewIndexMapping())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -298,7 +298,7 @@ func TestInMemIndex(t *testing.T) {
 }
 
 func TestClosedIndex(t *testing.T) {
-	index, err := New("", NewIndexMapping())
+	index, err := NewMemOnly(NewIndexMapping())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1685,7 +1685,7 @@ func TestBug408(t *testing.T) {
 	indexMapping := NewIndexMapping()
 	indexMapping.DefaultMapping = docMapping
 
-	index, err := New("", indexMapping)
+	index, err := NewMemOnly(indexMapping)
 	if err != nil {
 		t.Fatal(err)
 	}

search/searchers/base_test.go

@@ -26,7 +26,11 @@ var twoDocIndex index.Index //= upside_down.NewUpsideDownCouch(inmem.MustOpen())
 func init() {
 	analysisQueue := index.NewAnalysisQueue(1)
 	var err error
-	twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	twoDocIndex, err = upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		}, analysisQueue)
 	if err != nil {
 		panic(err)
 	}

search/searchers/search_docid_test.go

@@ -21,7 +21,12 @@ import (
 
 func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
 	analysisQueue := index.NewAnalysisQueue(1)
-	i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	i, err := upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		},
+		analysisQueue)
 	if err != nil {
 		t.Fatal(err)
 	}

search/searchers/search_term_test.go

@@ -28,7 +28,12 @@ func TestTermSearcher(t *testing.T) {
 	var queryExplain = true
 
 	analysisQueue := index.NewAnalysisQueue(1)
-	i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	i, err := upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		},
+		analysisQueue)
 	if err != nil {
 		t.Fatal(err)
 	}