From 8ad0f6445976c2b11661c36ed26c9d9eadcc934a Mon Sep 17 00:00:00 2001 From: Marty Schoch Date: Tue, 25 Nov 2014 21:52:35 -0500 Subject: [PATCH] upgrade to current forestdb api --- index/store/forestdb/iterator.go | 6 +- index/store/forestdb/reader.go | 2 +- index/store/forestdb/store.go | 47 +++++--- index/store/forestdb/store_test.go | 187 ++++++++++++++++++++++++++++- 4 files changed, 218 insertions(+), 24 deletions(-) diff --git a/index/store/forestdb/iterator.go b/index/store/forestdb/iterator.go index feb474ad..182efe32 100644 --- a/index/store/forestdb/iterator.go +++ b/index/store/forestdb/iterator.go @@ -17,14 +17,14 @@ import ( type Iterator struct { store *Store - snapshot *forestdb.Database + snapshot *forestdb.KVStore iterator *forestdb.Iterator curr *forestdb.Doc valid bool } func newIterator(store *Store) *Iterator { - itr, err := store.db.IteratorInit([]byte{}, nil, forestdb.ITR_NONE) + itr, err := store.dbkv.IteratorInit([]byte{}, nil, forestdb.ITR_NONE) rv := Iterator{ store: store, iterator: itr, @@ -33,7 +33,7 @@ func newIterator(store *Store) *Iterator { return &rv } -func newIteratorWithSnapshot(store *Store, snapshot *forestdb.Database) *Iterator { +func newIteratorWithSnapshot(store *Store, snapshot *forestdb.KVStore) *Iterator { itr, err := snapshot.IteratorInit([]byte{}, nil, forestdb.ITR_NONE) rv := Iterator{ store: store, diff --git a/index/store/forestdb/reader.go b/index/store/forestdb/reader.go index ac882d91..e310cea8 100644 --- a/index/store/forestdb/reader.go +++ b/index/store/forestdb/reader.go @@ -18,7 +18,7 @@ import ( type Reader struct { store *Store - snapshot *forestdb.Database + snapshot *forestdb.KVStore } func newReader(store *Store) (*Reader, error) { diff --git a/index/store/forestdb/store.go b/index/store/forestdb/store.go index 7908da8a..a22004f3 100644 --- a/index/store/forestdb/store.go +++ b/index/store/forestdb/store.go @@ -25,24 +25,32 @@ import ( const Name = "leveldb" type Store struct { - path string - config *forestdb.Config - db *forestdb.Database - writer sync.Mutex + path string + config *forestdb.Config + kvconfig *forestdb.KVStoreConfig + dbfile *forestdb.File + dbkv *forestdb.KVStore + writer sync.Mutex } func Open(path string, createIfMissing bool) (*Store, error) { rv := Store{ - path: path, - config: forestdb.DefaultConfig(), + path: path, + config: forestdb.DefaultConfig(), + kvconfig: forestdb.DefaultKVStoreConfig(), } if createIfMissing { - rv.config.SetOpenFlags(forestdb.OPEN_FLAG_CREATE) + rv.kvconfig.SetCreateIfMissing(true) } var err error - rv.db, err = forestdb.Open(rv.path, rv.config) + rv.dbfile, err = forestdb.Open(rv.path, rv.config) + if err != nil { + return nil, err + } + + rv.dbkv, err = rv.dbfile.OpenKVStoreDefault(rv.kvconfig) if err != nil { return nil, err } @@ -51,7 +59,7 @@ func Open(path string, createIfMissing bool) (*Store, error) { } func (s *Store) get(key []byte) ([]byte, error) { - res, err := s.db.GetKV(key) + res, err := s.dbkv.GetKV(key) if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND { return nil, err } @@ -65,7 +73,7 @@ func (s *Store) set(key, val []byte) error { } func (s *Store) setlocked(key, val []byte) error { - return s.db.SetKV(key, val) + return s.dbkv.SetKV(key, val) } func (s *Store) delete(key []byte) error { @@ -75,15 +83,20 @@ func (s *Store) delete(key []byte) error { } func (s *Store) deletelocked(key []byte) error { - return s.db.DeleteKV(key) + return s.dbkv.DeleteKV(key) } func (s *Store) commit() error { - return s.db.Commit(forestdb.COMMIT_NORMAL) + return s.dbfile.Commit(forestdb.COMMIT_NORMAL) } func (s *Store) Close() error { - return s.db.Close() + err := s.dbkv.Close() + if err != nil { + return err + } + return s.dbfile.Close() + } func (ldbs *Store) iterator(key []byte) store.KVIterator { @@ -105,19 +118,19 @@ func (ldbs *Store) newBatch() store.KVBatch { } func (s *Store) getSeqNum() (forestdb.SeqNum, error) { - dbinfo, err := s.db.DbInfo() + dbinfo, err := s.dbkv.Info() if err != nil { return 0, err } return dbinfo.LastSeqNum(), nil } -func (s *Store) newSnapshot() (*forestdb.Database, error) { +func (s *Store) newSnapshot() (*forestdb.KVStore, error) { seqNum, err := s.getSeqNum() if err != nil { return nil, err } - return s.db.SnapshotOpen(seqNum) + return s.dbkv.SnapshotOpen(seqNum) } func (s *Store) getRollbackID() ([]byte, error) { @@ -142,7 +155,7 @@ func (s *Store) rollbackTo(rollbackId []byte) error { if err != nil { return err } - err = s.db.Rollback(seqNum) + err = s.dbkv.Rollback(seqNum) if err != nil { return err } diff --git a/index/store/forestdb/store_test.go b/index/store/forestdb/store_test.go index 85b5e0c0..2909fca9 100644 --- a/index/store/forestdb/store_test.go +++ b/index/store/forestdb/store_test.go @@ -13,9 +13,10 @@ package forestdb import ( "os" + "reflect" "testing" - "github.com/blevesearch/bleve/index/store/test" + "github.com/blevesearch/bleve/index/store" ) func TestLevelDBStore(t *testing.T) { @@ -27,7 +28,7 @@ func TestLevelDBStore(t *testing.T) { } defer s.Close() - store_test.CommonTestKVStore(t, s) + CommonTestKVStore(t, s) } func TestReaderIsolation(t *testing.T) { @@ -39,7 +40,7 @@ func TestReaderIsolation(t *testing.T) { } defer s.Close() - store_test.CommonTestReaderIsolation(t, s) + CommonTestReaderIsolation(t, s) } // TestRollbackSameHandle tries to rollback a handle @@ -295,3 +296,183 @@ func TestRollbackOtherHandle(t *testing.T) { } reader2.Close() } + +func CommonTestKVStore(t *testing.T, s store.KVStore) { + + writer, err := s.Writer() + if err != nil { + t.Error(err) + } + err = writer.Set([]byte("a"), []byte("val-a")) + if err != nil { + t.Fatal(err) + } + err = writer.Set([]byte("z"), []byte("val-z")) + if err != nil { + t.Fatal(err) + } + err = writer.Delete([]byte("z")) + if err != nil { + t.Fatal(err) + } + + batch := writer.NewBatch() + batch.Set([]byte("b"), []byte("val-b")) + batch.Set([]byte("c"), []byte("val-c")) + batch.Set([]byte("d"), []byte("val-d")) + batch.Set([]byte("e"), []byte("val-e")) + batch.Set([]byte("f"), []byte("val-f")) + batch.Set([]byte("g"), []byte("val-g")) + batch.Set([]byte("h"), []byte("val-h")) + batch.Set([]byte("i"), []byte("val-i")) + batch.Set([]byte("j"), []byte("val-j")) + + err = batch.Execute() + if err != nil { + t.Fatal(err) + } + writer.Close() + + reader, err := s.Reader() + if err != nil { + t.Error(err) + } + defer reader.Close() + it := reader.Iterator([]byte("b")) + key, val, valid := it.Current() + if !valid { + t.Fatalf("valid false, expected true") + } + if string(key) != "b" { + t.Fatalf("exepcted key b, got %s", key) + } + if string(val) != "val-b" { + t.Fatalf("expected value val-b, got %s", val) + } + + it.Next() + key, val, valid = it.Current() + if !valid { + t.Fatalf("valid false, expected true") + } + if string(key) != "c" { + t.Fatalf("exepcted key c, got %s", key) + } + if string(val) != "val-c" { + t.Fatalf("expected value val-c, got %s", val) + } + + it.Seek([]byte("i")) + key, val, valid = it.Current() + if !valid { + t.Fatalf("valid false, expected true") + } + if string(key) != "i" { + t.Fatalf("exepcted key i, got %s", key) + } + if string(val) != "val-i" { + t.Fatalf("expected value val-i, got %s", val) + } + + it.Close() +} + +func CommonTestReaderIsolation(t *testing.T, s store.KVStore) { + // insert a kv pair + writer, err := s.Writer() + if err != nil { + t.Error(err) + } + err = writer.Set([]byte("a"), []byte("val-a")) + if err != nil { + t.Fatal(err) + } + writer.Close() + + // create an isoalted reader + reader, err := s.Reader() + if err != nil { + t.Error(err) + } + defer reader.Close() + + // verify we see the value already inserted + val, err := reader.Get([]byte("a")) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(val, []byte("val-a")) { + t.Errorf("expected val-a, got nil") + } + + // verify that an iterator sees it + count := 0 + it := reader.Iterator([]byte{0}) + defer it.Close() + for it.Valid() { + it.Next() + count++ + } + if count != 1 { + t.Errorf("expected iterator to see 1, saw %d", count) + } + + // add something after the reader was created + writer, err = s.Writer() + if err != nil { + t.Error(err) + } + err = writer.Set([]byte("b"), []byte("val-b")) + if err != nil { + t.Fatal(err) + } + writer.Close() + + // ensure that a newer reader sees it + newReader, err := s.Reader() + if err != nil { + t.Error(err) + } + defer newReader.Close() + val, err = newReader.Get([]byte("b")) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(val, []byte("val-b")) { + t.Errorf("expected val-b, got nil") + } + + // ensure director iterator sees it + count = 0 + it = newReader.Iterator([]byte{0}) + defer it.Close() + for it.Valid() { + it.Next() + count++ + } + if count != 2 { + t.Errorf("expected iterator to see 2, saw %d", count) + } + + // but that the isolated reader does not + val, err = reader.Get([]byte("b")) + if err != nil { + t.Error(err) + } + if val != nil { + t.Errorf("expected nil, got %v", val) + } + + // and ensure that the iterator on the isolated reader also does not + count = 0 + it = reader.Iterator([]byte{0}) + defer it.Close() + for it.Valid() { + it.Next() + count++ + } + if count != 1 { + t.Errorf("expected iterator to see 1, saw %d", count) + } + +}