Merge pull request #452 from mschoch/memonly

BREAKING CHANGE - new method to create memory only index

config.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/blevesearch/bleve/analysis/datetime_parsers/datetime_optional"
 	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store/gtreap"
 	"github.com/blevesearch/bleve/index/upside_down"
 	"github.com/blevesearch/bleve/registry"
 	"github.com/blevesearch/bleve/search/highlight/highlighters/html"
@@ -28,6 +29,7 @@ type configuration struct {
 	Cache                  *registry.Cache
 	DefaultHighlighter     string
 	DefaultKVStore         string
+	DefaultMemKVStore      string
 	DefaultIndexType       string
 	QueryDateTimeParser    string
 	SlowSearchLogThreshold time.Duration
@@ -60,6 +62,9 @@ func init() {
 	// default kv store
 	Config.DefaultKVStore = ""
 
+	// default mem only kv store
+	Config.DefaultMemKVStore = gtreap.Name
+
 	// default index
 	Config.DefaultIndexType = upside_down.Name
 

index.go

@@ -210,6 +210,15 @@ func New(path string, mapping *IndexMapping) (Index, error) {
 	return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
 }
 
+// NewMemOnly creates a memory-only index.
+// The contents of the index is NOT persisted,
+// and will be lost once closed.
+// The provided mapping will be used for all
+// Index/Search operations.
+func NewMemOnly(mapping *IndexMapping) (Index, error) {
+	return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
+}
+
 // NewUsing creates index at the specified path,
 // which must not already exist.
 // The provided mapping will be used for all

index/store/boltdb/store.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/blevesearch/bleve/index/store"
 	"github.com/blevesearch/bleve/registry"
@@ -46,6 +47,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 	if !ok {
 		return nil, fmt.Errorf("must specify path")
 	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
 
 	bucket, ok := config["bucket"].(string)
 	if !ok {

index/store/goleveldb/store.go

@@ -12,6 +12,7 @@ package goleveldb
 import (
 	"bytes"
 	"fmt"
+	"os"
 
 	"github.com/blevesearch/bleve/index/store"
 	"github.com/blevesearch/bleve/registry"
@@ -41,6 +42,9 @@ func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore,
 	if !ok {
 		return nil, fmt.Errorf("must specify path")
 	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
 
 	opts, err := applyConfig(&opt.Options{}, config)
 	if err != nil {

index/store/gtreap/store.go

@@ -17,6 +17,8 @@ package gtreap
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"sync"
 
 	"github.com/blevesearch/bleve/index/store"
@@ -42,6 +44,14 @@ func itemCompare(a, b interface{}) int {
 }
 
 func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path != "" {
+		return nil, os.ErrInvalid
+	}
+
 	rv := Store{
 		t:  gtreap.NewTreap(itemCompare),
 		mo: mo,

index/store/gtreap/store_test.go

@@ -20,7 +20,9 @@ import (
 )
 
 func open(t *testing.T, mo store.MergeOperator) store.KVStore {
-	rv, err := New(mo, nil)
+	rv, err := New(mo, map[string]interface{}{
+		"path": "",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}

index/store/metrics/metrics_test.go

@@ -36,6 +36,7 @@ func TestMetricsStore(t *testing.T) {
 
 	s, err = New(nil, map[string]interface{}{
 		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -74,6 +75,7 @@ func TestMetricsStore(t *testing.T) {
 func TestErrors(t *testing.T) {
 	s, err := New(nil, map[string]interface{}{
 		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
 	})
 	if err != nil {
 		t.Fatal(err)

index/store/metrics/store_test.go

@@ -9,7 +9,10 @@ import (
 )
 
 func open(t *testing.T, mo store.MergeOperator) store.KVStore {
-	rv, err := New(mo, map[string]interface{}{"kvStoreName_actual": gtreap.Name})
+	rv, err := New(mo, map[string]interface{}{
+		"kvStoreName_actual": gtreap.Name,
+		"path":               "",
+	})
 	if err != nil {
 		t.Fatal(err)
 	}

index/store/moss/lower.go

@@ -421,6 +421,9 @@ func InitMossStore(config map[string]interface{}, options moss.CollectionOptions
 	if !ok {
 		return nil, nil, nil, nil, fmt.Errorf("lower: missing path for InitMossStore config")
 	}
+	if path == "" {
+		return nil, nil, nil, nil, os.ErrInvalid
+	}
 
 	err := os.MkdirAll(path, 0700)
 	if err != nil {

index_impl.go

@@ -22,7 +22,6 @@ import (
 	"github.com/blevesearch/bleve/document"
 	"github.com/blevesearch/bleve/index"
 	"github.com/blevesearch/bleve/index/store"
-	"github.com/blevesearch/bleve/index/store/gtreap"
 	"github.com/blevesearch/bleve/index/upside_down"
 	"github.com/blevesearch/bleve/registry"
 	"github.com/blevesearch/bleve/search"
@@ -50,50 +49,6 @@ func indexStorePath(path string) string {
 	return path + string(os.PathSeparator) + storePath
 }
 
-func newMemIndex(indexType string, mapping *IndexMapping) (*indexImpl, error) {
-	rv := indexImpl{
-		path: "",
-		name: "mem",
-		m:    mapping,
-		meta: newIndexMeta(indexType, gtreap.Name, nil),
-	}
-
-	rv.stats = &IndexStat{i: &rv}
-
-	// open the index
-	indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
-	if indexTypeConstructor == nil {
-		return nil, ErrorUnknownIndexType
-	}
-
-	var err error
-	rv.i, err = indexTypeConstructor(rv.meta.Storage, nil, Config.analysisQueue)
-	if err != nil {
-		return nil, err
-	}
-	err = rv.i.Open()
-	if err != nil {
-		return nil, err
-	}
-
-	// now persist the mapping
-	mappingBytes, err := json.Marshal(mapping)
-	if err != nil {
-		return nil, err
-	}
-	err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
-	if err != nil {
-		return nil, err
-	}
-
-	// mark the index as open
-	rv.mutex.Lock()
-	defer rv.mutex.Unlock()
-	rv.open = true
-	indexStats.Register(&rv)
-	return &rv, nil
-}
-
 func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
 	// first validate the mapping
 	err := mapping.Validate()
@@ -101,10 +56,6 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
 		return nil, err
 	}
 
-	if path == "" {
-		return newMemIndex(indexType, mapping)
-	}
-
 	if kvconfig == nil {
 		kvconfig = map[string]interface{}{}
 	}
@@ -121,13 +72,17 @@ func newIndexUsing(path string, mapping *IndexMapping, indexType string, kvstore
 	}
 	rv.stats = &IndexStat{i: &rv}
 	// at this point there is hope that we can be successful, so save index meta
-	err = rv.meta.Save(path)
-	if err != nil {
-		return nil, err
+	if path != "" {
+		err = rv.meta.Save(path)
+		if err != nil {
+			return nil, err
+		}
+		kvconfig["create_if_missing"] = true
+		kvconfig["error_if_exists"] = true
+		kvconfig["path"] = indexStorePath(path)
+	} else {
+		kvconfig["path"] = ""
 	}
-	kvconfig["create_if_missing"] = true
-	kvconfig["error_if_exists"] = true
-	kvconfig["path"] = indexStorePath(path)
 
 	// open the index
 	indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)

index_test.go

@@ -287,7 +287,7 @@ func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
 
 func TestInMemIndex(t *testing.T) {
 
-	index, err := New("", NewIndexMapping())
+	index, err := NewMemOnly(NewIndexMapping())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -298,7 +298,7 @@ func TestInMemIndex(t *testing.T) {
 }
 
 func TestClosedIndex(t *testing.T) {
-	index, err := New("", NewIndexMapping())
+	index, err := NewMemOnly(NewIndexMapping())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1685,7 +1685,7 @@ func TestBug408(t *testing.T) {
 	indexMapping := NewIndexMapping()
 	indexMapping.DefaultMapping = docMapping
 
-	index, err := New("", indexMapping)
+	index, err := NewMemOnly(indexMapping)
 	if err != nil {
 		t.Fatal(err)
 	}

search/searchers/base_test.go

@@ -26,7 +26,11 @@ var twoDocIndex index.Index //= upside_down.NewUpsideDownCouch(inmem.MustOpen())
 func init() {
 	analysisQueue := index.NewAnalysisQueue(1)
 	var err error
-	twoDocIndex, err = upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	twoDocIndex, err = upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		}, analysisQueue)
 	if err != nil {
 		panic(err)
 	}

search/searchers/search_docid_test.go

@@ -21,7 +21,12 @@ import (
 
 func testDocIDSearcher(t *testing.T, indexed, searched, wanted []string) {
 	analysisQueue := index.NewAnalysisQueue(1)
-	i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	i, err := upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		},
+		analysisQueue)
 	if err != nil {
 		t.Fatal(err)
 	}

search/searchers/search_term_test.go

@@ -28,7 +28,12 @@ func TestTermSearcher(t *testing.T) {
 	var queryExplain = true
 
 	analysisQueue := index.NewAnalysisQueue(1)
-	i, err := upside_down.NewUpsideDownCouch(gtreap.Name, nil, analysisQueue)
+	i, err := upside_down.NewUpsideDownCouch(
+		gtreap.Name,
+		map[string]interface{}{
+			"path": "",
+		},
+		analysisQueue)
 	if err != nil {
 		t.Fatal(err)
 	}