gibheer
/
bleve
0
0
Fork 0
Code

overhauled top-level New/Open API 

New is now used to create new indexes
Open is used to open existing indexes
calls to Open no longer specify a mapping because the mapping
is serialized and stored along with the index
This commit is contained in:
Marty Schoch 2014-08-20 16:58:20 -04:00
parent a347b818d1
commit 27f001bc14
23 changed files with 491 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -7,7 +7,7 @@
.DS_Store
/analysis/token_filters/cld2/cld2-read-only
/examples/bleve_index_json/bleve_index_json
/examples/bleve_index_json/index/
/examples/bleve_index_json/index.bleve/
/examples/bleve_query/bleve_query
/examples/beer-search/beer-search
/examples/beer-search/beer-search.bleve

5
README.md
View File

@ -26,13 +26,12 @@ Discuss usage and development of bleve in the [google group](https://groups.goog
}
mapping := bleve.NewIndexMapping()
index, _ := bleve.Open("example.bleve", mapping)
index, _ := bleve.New("example.bleve", mapping)
index.Index(message)
## Querying
mapping := bleve.NewIndexMapping()
index, _ := bleve.Open("example.bleve", mapping)
index, _ := bleve.Open("example.bleve")
query := bleve.NewSyntaxQuery("bleve")
searchRequest := bleve.NewSearchRequest(query)
searchResult, _ := index.Search(searchRequest)

19
config.go
View File

@ -80,6 +80,10 @@ import (
_ "github.com/couchbaselabs/bleve/analysis/language/sv"
_ "github.com/couchbaselabs/bleve/analysis/language/th"
_ "github.com/couchbaselabs/bleve/analysis/language/tr"
// kv stores
_ "github.com/couchbaselabs/bleve/index/store/inmem"
_ "github.com/couchbaselabs/bleve/index/store/leveldb"
)
var bleveExpVar = expvar.NewMap("bleve")
@ -89,13 +93,10 @@ type HighlightConfig struct {
}
type Configuration struct {
DefaultAnalyzer *string
Highlight *HighlightConfig
DefaultHighlighter *string
CreateIfMissing bool
DefaultDateTimeFormat *string
DefaultField *string
ByteArrayConverters map[string]ByteArrayConverter
Highlight *HighlightConfig
DefaultHighlighter *string
ByteArrayConverters map[string]ByteArrayConverter
DefaultKVStore string
}
func NewConfiguration() *Configuration {
@ -133,8 +134,8 @@ func init() {
htmlHighlighterName := "html"
Config.DefaultHighlighter = &htmlHighlighterName
// default CreateIfMissing to true
Config.CreateIfMissing = true
// default kv store
Config.DefaultKVStore = "leveldb"
bootDuration := time.Since(bootStart)
bleveExpVar.Add("bootDuration", int64(bootDuration))

14
error.go
View File

@ -9,13 +9,16 @@
package bleve
const (
ERROR_INDEX_EXISTS Error = iota
ERROR_INDEX_DOES_NOT_EXIST
ERROR_INDEX_PATH_EXISTS Error = iota
ERROR_INDEX_PATH_DOES_NOT_EXIST
ERROR_INDEX_META_MISSING
ERROR_INDEX_META_CORRUPT
ERROR_DISJUNCTION_FEWER_THAN_MIN_CLAUSES
ERROR_BOOLEAN_QUERY_NEEDS_MUST_OR_SHOULD
ERROR_NUMERIC_QUERY_NO_BOUNDS
ERROR_PHRASE_QUERY_NO_TERMS
ERROR_UNKNOWN_QUERY_TYPE
ERROR_UNKNOWN_STORAGE_TYPE
)
type Error int
@ -25,11 +28,14 @@ func (e Error) Error() string {
}
var errorMessages = map[int]string{
int(ERROR_INDEX_EXISTS): "cannot create new index, it already exists",
int(ERROR_INDEX_DOES_NOT_EXIST): "cannot open index, it does not exist",
int(ERROR_INDEX_PATH_EXISTS): "cannot create new index, path already exists",
int(ERROR_INDEX_PATH_DOES_NOT_EXIST): "cannot open index, path does not exist",
int(ERROR_INDEX_META_MISSING): "cannot open index, metadata missing",
int(ERROR_INDEX_META_CORRUPT): "cannot open index, metadata corrupt",
int(ERROR_DISJUNCTION_FEWER_THAN_MIN_CLAUSES): "disjunction query has fewer than the minimum number of clauses to satisfy",
int(ERROR_BOOLEAN_QUERY_NEEDS_MUST_OR_SHOULD): "boolean query must contain at least one must or should clause",
int(ERROR_NUMERIC_QUERY_NO_BOUNDS): "numeric range query must specify min or max",
int(ERROR_PHRASE_QUERY_NO_TERMS): "phrase query must contain at least one term",
int(ERROR_UNKNOWN_QUERY_TYPE): "unknown query type",
int(ERROR_UNKNOWN_STORAGE_TYPE): "unkown storage type",
}

53
examples/beer-search/main.go
View File

@ -14,21 +14,17 @@ import (
"io/ioutil"
"log"
"net/http"
"os"
"path/filepath"
"runtime/pprof"
"time"
"github.com/couchbaselabs/bleve"
bleveHttp "github.com/couchbaselabs/bleve/http"
)
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write mem profile to file")
var batchSize = flag.Int("batchSize", 100, "batch size for indexing")
var bindAddr = flag.String("addr", ":8094", "http listen address")
var jsonDir = flag.String("jsonDir", "../../samples/beer-sample/", "json directory")
var indexDir = flag.String("indexDir", "beer-search.bleve", "index directory")
var indexPath = flag.String("index", "beer-search.bleve", "index path")
var staticEtag = flag.String("staticEtag", "", "A static etag value.")
var staticPath = flag.String("static", "static/", "Path to the static content")
@ -36,43 +32,26 @@ func main() {
flag.Parse()
// create cpu profile if requested
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
}
// create a mapping
indexMapping := buildIndexMapping()
// open the index
beerIndex, err := bleve.Open(*indexDir, indexMapping)
if err != nil {
log.Fatal(err)
}
beerIndex, err := bleve.Open(*indexPath)
if err == bleve.ERROR_INDEX_PATH_DOES_NOT_EXIST {
log.Printf("Creating new index...")
// create a mapping
indexMapping := buildIndexMapping()
beerIndex, err = bleve.New(*indexPath, indexMapping)
// index data in the background
go func() {
err = indexBeer(beerIndex)
if err != nil {
log.Fatal(err)
}
if *cpuprofile != "" {
pprof.StopCPUProfile()
log.Printf("closing cpu profile")
}
if *memprofile != "" {
f, err := os.Create(*memprofile)
// index data in the background
go func() {
err = indexBeer(beerIndex)
if err != nil {
log.Fatal(err)
}
pprof.WriteHeapProfile(f)
log.Printf("mem profile written")
}
}()
}()
} else if err != nil {
log.Fatal(err)
} else {
log.Printf("Opening existing index...")
}
// create a router to serve static files
router := staticFileRouter()

2
examples/beer-search/main_test.go
View File

@ -13,7 +13,7 @@ func TestBeerSearchAll(t *testing.T) {
defer os.RemoveAll("beer-search-test.bleve")
mapping := buildIndexMapping()
index, err := bleve.Open("beer-search-test.bleve", mapping)
index, err := bleve.New("beer-search-test.bleve", mapping)
if err != nil {
t.Fatal(err)
}

4
examples/bleve_index_json/main.go
View File

@ -17,7 +17,7 @@ import (
)
var jsonDir = flag.String("jsonDir", "json", "json directory")
var indexDir = flag.String("indexDir", "index", "index directory")
var indexPath = flag.String("index", "index.bleve", "index path")
func main() {
@ -27,7 +27,7 @@ func main() {
mapping := bleve.NewIndexMapping()
// open the index
index, err := bleve.Open(*indexDir, mapping)
index, err := bleve.New(*indexPath, mapping)
if err != nil {
log.Fatal(err)
}

14
examples/bleve_query/main.go
View File

@ -18,7 +18,7 @@ import (
)
var field = flag.String("field", "_all", "field to query")
var indexDir = flag.String("indexDir", "index", "index directory")
var indexPath = flag.String("index", "", "index path")
var limit = flag.Int("limit", 10, "limit to first N results")
var skip = flag.Int("skip", 0, "skip the first N results")
var explain = flag.Bool("explain", false, "explain scores")
@ -28,18 +28,16 @@ func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("specify index to query")
}
if flag.NArg() < 1 {
log.Fatal("Specify search query")
}
// don't create an index if it doesn't exist
bleve.Config.CreateIfMissing = false
// create a new default mapping
mapping := bleve.NewIndexMapping()
// open index
index, err := bleve.Open(*indexDir, mapping)
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}

10
index.go
View File

@ -50,8 +50,14 @@ type Classifier interface {
Type() string
}
// Open the index at the specified path, and create it if it does not exist.
// New index at the specified path, must not exist.
// The provided mapping will be used for all Index/Search operations.
func Open(path string, mapping *IndexMapping) (Index, error) {
func New(path string, mapping *IndexMapping) (Index, error) {
return newIndex(path, mapping)
}
// Open index at the specified path, must exist.
// The mapping used when it was created will be used for all Index/Search operations.
func Open(path string) (Index, error) {
return openIndex(path)
}

11
index/store/inmem/store.go
View File

@ -10,9 +10,12 @@ package inmem
import (
"github.com/couchbaselabs/bleve/index/store"
"github.com/couchbaselabs/bleve/registry"
"github.com/ryszard/goskiplist/skiplist"
)
const Name = "mem"
type InMemStore struct {
list *skiplist.SkipList
}
@ -68,3 +71,11 @@ func (i *InMemStore) Iterator(key []byte) store.KVIterator {
func (i *InMemStore) NewBatch() store.KVBatch {
return newInMemBatch(i)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
return Open()
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}

30
index/store/leveldb/store.go
View File

@ -9,23 +9,29 @@
package leveldb
import (
"fmt"
"github.com/couchbaselabs/bleve/index/store"
"github.com/couchbaselabs/bleve/registry"
"github.com/jmhodges/levigo"
)
const Name = "leveldb"
type LevelDBStore struct {
path string
opts *levigo.Options
db *levigo.DB
}
func Open(path string, createIfMissing bool) (*LevelDBStore, error) {
func Open(path string, createIfMissing bool, errorIfExists bool) (*LevelDBStore, error) {
rv := LevelDBStore{
path: path,
}
opts := levigo.NewOptions()
opts.SetCreateIfMissing(createIfMissing)
opts.SetErrorIfExists(errorIfExists)
rv.opts = opts
var err error
@ -67,3 +73,25 @@ func (ldbs *LevelDBStore) Iterator(key []byte) store.KVIterator {
func (ldbs *LevelDBStore) NewBatch() store.KVBatch {
return newLevelDBBatch(ldbs)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
createIfMissing := false
cim, ok := config["create_if_missing"].(bool)
if ok {
createIfMissing = cim
}
errorIfExists := true
eie, ok := config["error_if_exists"].(bool)
if ok {
errorIfExists = eie
}
return Open(path, createIfMissing, errorIfExists)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}

2
index/store/leveldb/store_test.go
View File

@ -16,7 +16,7 @@ import (
)
func TestLevelDBStore(t *testing.T) {
s, err := Open("test", true)
s, err := Open("test", true, true)
if err != nil {
t.Fatal(err)
}

2
index/upside_down/benchmark_leveldb_test.go
View File

@ -16,7 +16,7 @@ import (
)
func BenchmarkLevelDBIndexing(b *testing.B) {
s, err := leveldb.Open("test", true)
s, err := leveldb.Open("test", true, true)
if err != nil {
b.Fatal(err)
}

2
index/upside_down/dump_test.go
View File

@ -20,7 +20,7 @@ import (
func TestDump(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}

2
index/upside_down/field_reader_test.go
View File

@ -20,7 +20,7 @@ import (
func TestIndexFieldReader(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {

35
index/upside_down/upside_down_test.go
View File

@ -29,7 +29,7 @@ var testAnalyzer = &analysis.Analyzer{
func TestIndexOpenReopen(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -52,7 +52,10 @@ func TestIndexOpenReopen(t *testing.T) {
// now close it
idx.Close()
store, err = leveldb.Open("test", true)
store, err = leveldb.Open("test", true, false)
if err != nil {
t.Fatalf("error opening store: %v", err)
}
idx = NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -66,7 +69,7 @@ func TestIndexOpenReopen(t *testing.T) {
func TestIndexInsert(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -104,7 +107,7 @@ func TestIndexInsert(t *testing.T) {
func TestIndexInsertThenDelete(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -172,7 +175,7 @@ func TestIndexInsertThenDelete(t *testing.T) {
func TestIndexInsertThenUpdate(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -221,7 +224,7 @@ func TestIndexInsertThenUpdate(t *testing.T) {
func TestIndexInsertMultiple(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -255,7 +258,7 @@ func TestIndexInsertMultiple(t *testing.T) {
// close and reopen and and one more to testing counting works correctly
idx.Close()
store, err = leveldb.Open("test", true)
store, err = leveldb.Open("test", true, false)
idx = NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -280,7 +283,7 @@ func TestIndexInsertMultiple(t *testing.T) {
func TestIndexInsertWithStore(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -337,7 +340,7 @@ func TestIndexInsertWithStore(t *testing.T) {
func TestIndexInternalCRUD(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -388,7 +391,7 @@ func TestIndexInternalCRUD(t *testing.T) {
func TestIndexBatch(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
idx := NewUpsideDownCouch(store)
err = idx.Open()
if err != nil {
@ -462,7 +465,7 @@ func TestIndexBatch(t *testing.T) {
func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -613,7 +616,7 @@ func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) {
func TestIndexInsertFields(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -652,7 +655,7 @@ func TestIndexInsertFields(t *testing.T) {
func TestIndexUpdateComposites(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -721,7 +724,7 @@ func TestIndexUpdateComposites(t *testing.T) {
func TestIndexFieldsMisc(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -758,7 +761,7 @@ func TestIndexFieldsMisc(t *testing.T) {
func TestIndexTermReaderCompositeFields(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}
@ -798,7 +801,7 @@ func TestIndexTermReaderCompositeFields(t *testing.T) {
func TestIndexDocumentFieldTerms(t *testing.T) {
defer os.RemoveAll("test")
store, err := leveldb.Open("test", true)
store, err := leveldb.Open("test", true, true)
if err != nil {
t.Error(err)
}

165
index_impl.go
View File

@ -9,44 +9,183 @@
package bleve
import (
"encoding/json"
"fmt"
"os"
"time"
"github.com/couchbaselabs/bleve/document"
"github.com/couchbaselabs/bleve/index"
"github.com/couchbaselabs/bleve/index/store"
"github.com/couchbaselabs/bleve/index/store/leveldb"
"github.com/couchbaselabs/bleve/index/upside_down"
"github.com/couchbaselabs/bleve/registry"
"github.com/couchbaselabs/bleve/search"
)
type indexImpl struct {
s store.KVStore
i index.Index
m *IndexMapping
path string
meta *indexMeta
s store.KVStore
i index.Index
m *IndexMapping
}
const storePath = "store"
var mappingInternalKey = []byte("_mapping")
func indexStorePath(path string) string {
return path + string(os.PathSeparator) + storePath
}
func newMemIndex(mapping *IndexMapping) (*indexImpl, error) {
rv := indexImpl{
path: "",
m: mapping,
meta: NewIndexMeta("mem"),
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ERROR_UNKNOWN_STORAGE_TYPE
}
// now open the store
var err error
rv.s, err = storeConstructor(nil)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
return &rv, nil
}
func newIndex(path string, mapping *IndexMapping) (*indexImpl, error) {
// start by validating the index mapping
// first validate the mapping
err := mapping.Validate()
if err != nil {
return nil, err
}
store, err := leveldb.Open(path, Config.CreateIfMissing)
if path == "" {
return newMemIndex(mapping)
}
rv := indexImpl{
path: path,
m: mapping,
meta: NewIndexMeta(Config.DefaultKVStore),
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ERROR_UNKNOWN_STORAGE_TYPE
}
// at this point there hope we can be successful, so save index meta
err = rv.meta.Save(path)
if err != nil {
return nil, err
}
idx := upside_down.NewUpsideDownCouch(store)
err = idx.Open()
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": true,
"error_if_exists": true,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
return &indexImpl{
s: store,
i: idx,
m: mapping,
}, nil
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now persist the mapping
mappingBytes, err := json.Marshal(mapping)
if err != nil {
return nil, err
}
err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
if err != nil {
return nil, err
}
return &rv, nil
}
func openIndex(path string) (*indexImpl, error) {
rv := indexImpl{
path: path,
}
var err error
rv.meta, err = OpenIndexMeta(path)
if err != nil {
return nil, err
}
storeConstructor := registry.KVStoreConstructorByName(rv.meta.Storage)
if storeConstructor == nil {
return nil, ERROR_UNKNOWN_STORAGE_TYPE
}
storeConfig := map[string]interface{}{
"path": indexStorePath(path),
"create_if_missing": false,
"error_if_exists": false,
}
// now open the store
rv.s, err = storeConstructor(storeConfig)
if err != nil {
return nil, err
}
// open open the index
rv.i = upside_down.NewUpsideDownCouch(rv.s)
err = rv.i.Open()
if err != nil {
return nil, err
}
// now load the mapping
mappingBytes, err := rv.i.GetInternal(mappingInternalKey)
if err != nil {
return nil, err
}
var im IndexMapping
err = json.Unmarshal(mappingBytes, &im)
if err != nil {
return nil, err
}
// validate the mapping
err = im.Validate()
if err != nil {
return nil, err
}
rv.m = &im
return &rv, nil
}
func (i *indexImpl) Index(id string, data interface{}) error {

73
index_meta.go Normal file
View File

@ -0,0 +1,73 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package bleve
import (
"encoding/json"
"io/ioutil"
"os"
)
const metaFilename = "index_meta.json"
type indexMeta struct {
Storage string `json:"storage"`
}
func NewIndexMeta(storage string) *indexMeta {
return &indexMeta{Storage: storage}
}
func OpenIndexMeta(path string) (*indexMeta, error) {
if _, err := os.Stat(path); os.IsNotExist(err) {
return nil, ERROR_INDEX_PATH_DOES_NOT_EXIST
}
indexMetaPath := indexMetaPath(path)
metaBytes, err := ioutil.ReadFile(indexMetaPath)
if err != nil {
return nil, ERROR_INDEX_META_MISSING
}
var im indexMeta
err = json.Unmarshal(metaBytes, &im)
if err != nil {
return nil, ERROR_INDEX_META_CORRUPT
}
return &im, nil
}
func (i *indexMeta) Save(path string) error {
indexMetaPath := indexMetaPath(path)
// ensure any necessary parent directories exist
err := os.Mkdir(path, 0700)
if err != nil {
return ERROR_INDEX_PATH_EXISTS
}
metaBytes, err := json.Marshal(i)
if err != nil {
return err
}
indexMetaFile, err := os.OpenFile(indexMetaPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
if err != nil {
if os.IsExist(err) {
return ERROR_INDEX_PATH_EXISTS
} else {
return err
}
}
defer indexMetaFile.Close()
_, err = indexMetaFile.Write(metaBytes)
if err != nil {
return err
}
return nil
}
func indexMetaPath(path string) string {
return path + string(os.PathSeparator) + metaFilename
}

48
index_meta_test.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package bleve
import (
"os"
"testing"
)
func TestIndexMeta(t *testing.T) {
var testIndexPath = "doesnotexit.bleve"
defer os.RemoveAll(testIndexPath)
// open non-existant meta should error
_, err := OpenIndexMeta(testIndexPath)
if err == nil {
t.Errorf("expected error, got nil")
}
// create meta
im := &indexMeta{Storage: "leveldb"}
err = im.Save(testIndexPath)
if err != nil {
t.Error(err)
}
im = nil
// open a meta that exists
im, err = OpenIndexMeta(testIndexPath)
if err != nil {
t.Error(err)
}
if im.Storage != "leveldb" {
t.Errorf("expected storage 'leveldb', got '%s'", im.Storage)
}
// save a meta that already exists
err = im.Save(testIndexPath)
if err == nil {
t.Errorf("expected error, got nil")
}
}

51
index_test.go
View File

@ -9,6 +9,7 @@
package bleve
import (
"io/ioutil"
"os"
"testing"
"time"
@ -85,10 +86,11 @@ var people = []*Person{
func TestIndex(t *testing.T) {
defer os.RemoveAll("testidx")
index, err := Open("testidx", mapping)
index, err := New("testidx", mapping)
if err != nil {
t.Fatal(err)
}
defer index.Close()
// index all the people
for _, person := range people {
@ -299,3 +301,50 @@ func TestIndex(t *testing.T) {
t.Errorf("expected to find 2 values for tags")
}
}
func TestIndexCreateNewOverExisting(t *testing.T) {
defer os.RemoveAll("testidx")
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
index.Close()
index, err = New("testidx", NewIndexMapping())
if err != ERROR_INDEX_PATH_EXISTS {
t.Fatalf("expected error index path exists, got %v", err)
}
}
func TestIndexOpenNonExisting(t *testing.T) {
_, err := Open("doesnotexist")
if err != ERROR_INDEX_PATH_DOES_NOT_EXIST {
t.Fatalf("expected error index path does not exist, got %v", err)
}
}
func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
defer os.RemoveAll("testidx")
index, err := New("testidx", NewIndexMapping())
if err != nil {
t.Fatal(err)
}
index.Close()
// now intentionally corrupt the metadata
ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666)
index, err = Open("testidx")
if err != ERROR_INDEX_META_CORRUPT {
t.Fatalf("expected error index metadata corrupted, got %v", err)
}
// no intentionally remove the metadata
os.Remove("testidx/index_meta.json")
index, err = Open("testidx")
if err != ERROR_INDEX_META_MISSING {
t.Fatalf("expected error index metadata missing, got %v", err)
}
}

14
registry/registry.go
View File

@ -15,6 +15,9 @@ import (
"github.com/couchbaselabs/bleve/analysis"
)
var stores = make(KVStoreRegistry, 0)
// analysis
var charFilters = make(CharFilterRegistry, 0)
var tokenizers = make(TokenizerRegistry, 0)
var tokenMaps = make(TokenMapRegistry, 0)
@ -156,4 +159,15 @@ func PrintRegistry() {
fmt.Printf("\t%s\n", name)
}
fmt.Println()
sorted = make(sort.StringSlice, 0, len(stores))
for name, _ := range stores {
sorted = append(sorted, name)
}
sorted.Sort()
fmt.Printf("KV Stores:\n")
for _, name := range sorted {
fmt.Printf("\t%s\n", name)
}
fmt.Println()
}

30
registry/store.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package registry
import (
"fmt"
"github.com/couchbaselabs/bleve/index/store"
)
func RegisterKVStore(name string, constructor KVStoreConstructor) {
_, exists := stores[name]
if exists {
panic(fmt.Errorf("attempted to register duplicate store named '%s'", name))
}
stores[name] = constructor
}
type KVStoreConstructor func(config map[string]interface{}) (store.KVStore, error)
type KVStoreRegistry map[string]KVStoreConstructor
func KVStoreConstructorByName(name string) KVStoreConstructor {
return stores[name]
}

8
utils/bleve_dump/main.go
View File

@ -17,16 +17,18 @@ import (
"github.com/couchbaselabs/bleve/index/upside_down"
)
var indexDir = flag.String("indexDir", "index", "index directory")
var indexPath = flag.String("index", "", "index path")
var fieldsOnly = flag.Bool("fields", false, "fields only")
var docId = flag.String("docId", "", "docId to dump")
func main() {
flag.Parse()
if *indexPath == "" {
log.Fatal("specify index to dump")
}
bleve.Config.CreateIfMissing = false
index, err := bleve.Open(*indexDir, bleve.NewIndexMapping())
index, err := bleve.Open(*indexPath)
if err != nil {
log.Fatal(err)
}